diff --git a/.dockerignore b/.dockerignore index f98663d..a9bcbf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ .git .task +.cache build **/target +packages packages-* diff --git a/.github/workflows/build-dependency-image.yaml b/.github/workflows/build-dependency-image.yaml index a2e0d05..2fb43bf 100644 --- a/.github/workflows/build-dependency-image.yaml +++ b/.github/workflows/build-dependency-image.yaml @@ -17,6 +17,9 @@ on: - "taskfiles/**" - "tools/build-packages/build-dependency-image.sh" - "tools/build-packages/dependency-image/**" + - "tools/build-packages/internal/ca-trust/container.sh" + - "tools/build-packages/internal/ca-trust/host.sh" + - "tools/build-packages/internal/host/**" - "tools/yscope-dev-utils" workflow_call: outputs: @@ -51,7 +54,7 @@ jobs: - name: "Resolve dependency image" id: "image" run: |- - source tools/build-packages/dependency-image/utils.sh + source tools/build-packages/internal/host/build-env.sh build_env_hash=$(derive_build_env_hash) image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" image=$(image_ref "${image_repo}" "${IMAGE_NAME}" "${build_env_hash}") @@ -121,7 +124,7 @@ jobs: IMAGE: "${{needs.plan.outputs.image}}" ARCH: "${{matrix.arch}}" run: |- - source tools/build-packages/dependency-image/utils.sh + source tools/build-packages/internal/host/build-env.sh output="--push" if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then output="--load" diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml new file mode 100644 index 0000000..46e2ca1 --- /dev/null +++ b/.github/workflows/build-packages.yaml @@ -0,0 +1,150 @@ +# Builds CLP Presto connector packages for amd64 and arm64. +# +# Flow: +# resolve-version use the input version or derive it from pom.xml +# build-dependency-image ensure the hash-tagged build-env image exists +# build build .deb / .rpm / .tar.gz artifacts per architecture +# +# See tools/build-packages/README.md for package-build details. + +name: "build-packages" + +on: + workflow_dispatch: + inputs: + version: + description: "Package version override (derived from presto-connector/pom.xml when blank)" + required: false + default: "" + push: + paths: + - ".dockerignore" + - ".github/workflows/build-packages.yaml" + - ".github/workflows/build-dependency-image.yaml" + - "presto-connector/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/build-packages/**" + - "tools/yscope-dev-utils" + - "velox-connector/**" + # Markdown files don't change the artifact, so skip rebuilds on documentation-only changes. + - "!**/*.md" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + cancel-in-progress: true + +permissions: + contents: "read" + +jobs: + resolve-version: + runs-on: "ubuntu-24.04" + outputs: + version: "${{steps.version.outputs.version}}" + steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + + - name: "Resolve package version" + id: "version" + env: + INPUT_VERSION: "${{inputs.version || ''}}" + run: |- + version="${INPUT_VERSION}" + if [[ -z "${version}" ]]; then + if ! version=$( + mvn --quiet --file presto-connector/pom.xml \ + help:evaluate \ + -Dexpression=project.version \ + -DforceStdout + ); then + echo "::error::Failed to derive project.version with Maven" + exit 1 + fi + if [[ -z "${version}" ]]; then + echo "::error::Maven returned an empty project.version" + exit 1 + fi + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "::notice::Package version: ${version}" + + build-dependency-image: + uses: "./.github/workflows/build-dependency-image.yaml" + permissions: + contents: "read" + packages: "write" + + build: + needs: ["build-dependency-image", "resolve-version"] + permissions: + contents: "read" + packages: "read" + strategy: + # Build both architectures so one failure does not hide the other result. + fail-fast: false + matrix: + include: + - arch: "amd64" + runner: "ubuntu-24.04" + - arch: "arm64" + runner: "ubuntu-24.04-arm" + runs-on: "${{matrix.runner}}" + timeout-minutes: 120 + env: + VERSION: "${{needs.resolve-version.outputs.version}}" + # Run package steps inside the build-env image, which provides Velox deps, + # JDK 17, go-task, and packaging tools. + container: + image: "${{needs.build-dependency-image.outputs.image}}" + credentials: + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + with: + # Required because taskfiles include shared utilities from the submodule. + submodules: true + + - name: "Build packages" + id: "packages" + shell: "bash" + run: |- + bash tools/build-packages/internal/container/build-artifacts.sh --version "${VERSION}" + + shopt -s nullglob + for extension in deb rpm tar.gz; do + artifact_files=(packages/clp-plugin-presto-connector*."${extension}") + if (( ${#artifact_files[@]} != 1 )); then + echo "::error::Expected exactly one .${extension} artifact, found ${#artifact_files[@]}" + exit 1 + fi + printf '%s_filename=%s\n' \ + "${extension//./_}" "${artifact_files[0]##*/}" >> "$GITHUB_OUTPUT" + done + + - name: "Upload deb package" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "${{steps.packages.outputs.deb_filename}}" + path: "packages/${{steps.packages.outputs.deb_filename}}" + if-no-files-found: "error" + retention-days: 14 + + - name: "Upload rpm package" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "${{steps.packages.outputs.rpm_filename}}" + path: "packages/${{steps.packages.outputs.rpm_filename}}" + if-no-files-found: "error" + retention-days: 14 + + - name: "Upload tarball" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "${{steps.packages.outputs.tar_gz_filename}}" + path: "packages/${{steps.packages.outputs.tar_gz_filename}}" + if-no-files-found: "error" + retention-days: 14 diff --git a/.gitignore b/.gitignore index 774f7b4..cead14b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /.task/ +/.cache/ /build/ +/packages/ diff --git a/README.md b/README.md index 2d555f2..e7862da 100644 --- a/README.md +++ b/README.md @@ -38,4 +38,8 @@ one of the tasks in the table below. | `lint:check-yaml` | Runs the YAML linters. | | `lint:fix-yaml` | Runs the YAML linters and fixes issues. | +## Building installable packages + +To build `.deb`, `.rpm`, and `.tar.gz` artifacts of the plugin locally, see [tools/build-packages/README.md](tools/build-packages/README.md). + [Task]: https://taskfile.dev diff --git a/presto-connector/.gitignore b/presto-connector/.gitignore new file mode 100644 index 0000000..654fccf --- /dev/null +++ b/presto-connector/.gitignore @@ -0,0 +1,3 @@ +# Build artifacts +/dependency-reduced-pom.xml +/target/ diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml index 61f0886..b7c646b 100644 --- a/taskfiles/velox-connector/main.yaml +++ b/taskfiles/velox-connector/main.yaml @@ -9,12 +9,23 @@ includes: vars: G_VELOX_CONNECTOR_BUILD_DIR: "{{.G_BUILD_DIR}}/velox-connector" G_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" + DEFAULT_FETCHCONTENT_BASE_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/_deps" + G_FETCHCONTENT_BASE_DIR: '{{env "FETCHCONTENT_BASE_DIR" | default .DEFAULT_FETCHCONTENT_BASE_DIR}}' tasks: build: deps: - - "generate" + - "deps:install-all" + cmds: + - task: "build-with-installed-deps" + + build-with-installed-deps: + desc: Builds the connector using dependencies already installed in G_DEPS_CPP_DIR. + preconditions: + - sh: "test -f '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake'" + msg: "Installed dependency settings not found; run velox-connector:deps:install-all first." cmds: + - task: "generate" - task: "utils:cmake:build" vars: BUILD_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}" @@ -25,8 +36,6 @@ tasks: generate: internal: true - deps: - - "deps:install-all" cmds: - task: "utils:cmake:generate" vars: @@ -34,5 +43,6 @@ tasks: EXTRA_ARGS: - "-DLIBCLP_PLUGIN_VELOX_CONNECTOR_DEPS_CMAKE_SETTINGS=\ {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake" + - "-DFETCHCONTENT_BASE_DIR={{.G_FETCHCONTENT_BASE_DIR}}" - "-DPRESTO_GIT_TAG={{.G_PRESTO_GIT_TAG}}" SOURCE_DIR: "{{.ROOT_DIR}}/velox-connector" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md new file mode 100644 index 0000000..8f07994 --- /dev/null +++ b/tools/build-packages/README.md @@ -0,0 +1,216 @@ +# Packaging + +This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts for the CLP Presto connector. Each artifact contains both plugin components: + +* Java coordinator plugin from `presto-connector/` +* C++ Velox worker plugin from `velox-connector/`, plus its non-system runtime libraries + +The artifacts are built on `manylinux_2_28` (glibc 2.28) and target common Linux distributions with glibc 2.28 or newer, including Debian 11+, Ubuntu 20.04+, RHEL/AlmaLinux/Rocky Linux 8+, and Fedora 29+. + +## Build model + +The package build needs Velox C++ dependencies, JDK 17, go-task, and packaging tools. CI provides these through a hash-tagged **build-env image** based on `manylinux_2_28`. + +The image is tagged as `env-`. The hash covers the repository inputs that affect its contents: the dependency-image Dockerfile and context filtering, the host-side image-build recipe, taskfiles, the container CA helper, and `tools/yscope-dev-utils`. The tag identifies those source and configuration inputs; it is not a digest of upstream images because the base image currently uses the mutable `manylinux_2_28:latest` tag. + +`internal/container/build-artifacts.sh` runs inside the build-env image and performs the actual package build. Local users should run `build-packages.sh`, which resolves the image and invokes the container-side script. + +## Local usage + +```bash +./tools/build-packages/build-packages.sh +``` + +Packages are written to `./packages/` by default. Use `--output DIR` to choose a different output directory, and `--version VER` to override the package version. When no version is provided, `internal/container/build-artifacts.sh` derives the Maven project version from `presto-connector/pom.xml`. Versions must start with a digit and otherwise contain only letters, digits, `.`, `+`, `~`, or `-`. + +On the first run after a build-env input changes, `build-packages.sh` builds the image locally if it is not already available in Docker or in this repository's GHCR package. Later runs reuse the cached image. + +Local builds use `.cache/maven/` for the Maven Wrapper distribution and downloaded artifacts, and `.cache/ccache/` for content-addressed C++ compilation results, with ccache capped at 1 GiB. These caches are shared across build-env revisions. FetchContent uses `.cache/fetchcontent//` so Presto/Velox and CLP source and build trees remain isolated by build-env revision. See the [build-cache helper documentation](internal/build-cache/README.md) for the cache layout and integration API. + +The top-level CMake build tree is ephemeral, while Task-installed C/C++ dependencies are read directly from the build-env image. The package build runs as root inside the container, so cache and intermediate build files may be root-owned on the host. The container writes final artifacts into an isolated staging directory, and the host process copies them into the requested output directory so they use the invoking user's UID/GID. The repository ignores `.cache/`; remove that directory when no package build is running to discard local cache state. + +`build-packages.sh` also initializes submodules on the host before launching Docker. All arguments are forwarded to the container-side build; run `./tools/build-packages/build-packages.sh --help` for the supported options. + +For corporate TLS environments, local builds stage the host CA bundle as temporary PEM and Java PKCS#12 trust stores, then mount them read-only into the package container. Cleanup removes both stores after packaging; neither enters image layers, persistent caches, or generated packages. See the [CA-trust documentation](internal/ca-trust/README.md) for the design, configuration, and extension API. + +### Container privileges and file ownership + +The package build container runs as root so it can use the root-owned dependencies installed in the image. It receives the generated CA trust stores through a read-only mount and does not write certificate material. The requested host output directory is not mounted into the container; the host process copies completed packages out of temporary staging. + +### Build-env image resolution + +`build-dependency-image.sh` resolves the image in this order: + +1. local Docker cache (`docker image inspect`) +2. this repository's GHCR package (`docker pull ghcr.io///build-env:env-`) +3. local image build from `dependency-image/Dockerfile` + +## Prerequisites + +For local builds through `build-packages.sh`: + +* Docker with buildx, usable by the invoking user without `sudo` +* git and `sha256sum` +* about 10 GB of free disk for the build-env image +* roughly 1 GiB of memory per available processor for a clean local image build + +`internal/container/build-artifacts.sh` runs inside the build-env image, which provides all required build-time tools. Submodules are initialized before it starts: by `build-packages.sh` on the host for local builds, and by `actions/checkout` inside the job container in CI. + +## Outputs + +By default, artifacts are written under `./packages/`: + +```text +./packages/clp-plugin-presto-connector_-1_.deb +./packages/clp-plugin-presto-connector--1..rpm +./packages/clp-plugin-presto-connector--linux-.tar.gz +``` + +`` is the Maven version after package naming normalization. `` is the original Maven version. All formats contain the same plugin files. The `.deb` and `.rpm` install to the standard plugin path: + +```text +/opt/clp-plugin-presto-connector/ +├── coordinator/ +│ └── clp-plugin-presto-connector.jar +└── worker/ + ├── libclp-plugin-velox-connector.so + └── lib/ # bundled non-system .so deps +``` + +The `.tar.gz` contains the same `coordinator/` and `worker/` directories under a relocatable top-level directory: + +```text +clp-plugin-presto-connector--linux-/ +├── coordinator/ +└── worker/ +``` + +To install the tarball, extract it and copy the payload into your chosen plugin directory: + +```bash +tar xzf +cp -a clp-plugin-presto-connector-*-linux-*/. //clp-plugin-presto-connector/ +``` + +## CI flow + +`.github/workflows/build-packages.yaml` runs the same package build in CI: + +1. resolve the package version from the workflow input or `presto-connector/pom.xml` +2. ensure the hash-tagged build-env image exists through `.github/workflows/build-dependency-image.yaml` +3. run `internal/container/build-artifacts.sh` inside that image for `amd64` and `arm64` +4. upload `.deb`, `.rpm`, and `.tar.gz` artifacts for each architecture + +Local and CI package builds share `internal/container/build-artifacts.sh`. Local builds use `build-packages.sh` to resolve the image, initialize submodules, configure temporary system and Java trust, copy final artifacts into the host output directory, and configure persistent caches under `.cache/`. GitHub Actions invokes the container-side script directly inside its build-env job container. + +## Troubleshooting + +* **`Installed dependency settings not found`** — the container-side build was invoked without the dependencies supplied by the build-env image. Run `build-packages.sh` for a local package build. +* **`Cannot connect to the Docker daemon`** — start Docker, then rerun `build-packages.sh`. +* **Intermediate files are not writable outside the container** — local package builds run as root, so `.cache/`, `build/`, and `presto-connector/target/` may contain root-owned files. Remove them through a root container or repair their ownership before switching to a native host build. Final files under `packages/` are owned by the invoking user. +* **Out of disk** — the build-env image is about 5 GB, and intermediate build directories add several more GB. `docker system prune -a` and `rm -rf .cache build` reclaim most local build state. + +--- + +The remaining sections are reference material for maintainers. + +## Build-env image + +The dependency image is built from `tools/build-packages/dependency-image/Dockerfile`: + +* **Base image** — `quay.io/pypa/manylinux_2_28:latest` +* **Connector C/C++ dependencies** — installed by `task velox-connector:deps:install-all` under the image build directory +* **JDK 17** — `java-17-openjdk-devel`, required by Maven builds +* **Packaging tools** — `dpkg`, `gettext` (`envsubst`), `patchelf`, and `rpm-build` +* **go-task** — pinned release binary installed under `/opt/go-task/bin` + +The published image reference is `ghcr.io//build-env:env-`. When a hash input changes, CI computes a new tag and builds the image if that tag is not already present in GHCR. + +### Host-side internals + +Host-only support code is split by responsibility: + +* `internal/ca-trust/host.sh` exposes sourceable staging functions for the host CA bundle and each generated format. The Java PKCS#12 implementation lives under `internal/ca-trust/generators/java-pkcs12/`. +* `internal/build-cache/host.sh` prepares persistent cache directories, while `internal/build-cache/container.sh` configures build tools to consume them. +* `internal/host/build-env.sh` derives the build-env hash, formats its image reference, and drives `docker buildx`. It is used by the image resolver and dependency-image workflow; it sources `internal/ca-trust/host.sh` and uses it when building an image. + +## Package build flow + +```text +internal/container/build-artifacts.sh (inside build-env image) + ├── task velox-connector:build-with-installed-deps + │ ├── use C++ deps and all-deps.cmake installed in the build-env image + │ ├── configure with CMake (FetchContent: prestodb/presto + CLP) + │ └── build libclp-plugin-velox-connector.so + ├── resolve package version from --version or Maven project metadata + ├── mvnw package → build clp-plugin-presto-connector.jar + ├── stage payload under /opt/clp-plugin-presto-connector/... + ├── ldd walk → copy non-system .so deps → patch RUNPATHs + ├── strip bundled third-party libraries and normalize file modes + └── emit .deb, .rpm, and .tar.gz artifacts +``` + +`internal/container/build-artifacts.sh` stages the install tree once, then emits all three package formats from that tree. `PRESTO_JAR_DIR` and `VELOX_SO_DIR`, defined near the top of the script, control the `.deb`/`.rpm` install paths and are passed into `rpmbuild` as spec macros. The `.tar.gz` remains relocatable and always uses a `coordinator/` plus `worker/` layout under its top-level directory. + +Before any package is emitted, staged files are normalized: + +* bundled third-party `.so` files are stripped with `strip --strip-unneeded` +* the plugin `.so` is left unstripped so crash backtraces can resolve to source +* the JAR, plugin `.so`, and bundled libraries are set to mode `0644` + +### `.deb` + +`package-specs/deb/clp-plugin-presto-connector.control.in` is a Debian control file template. `envsubst` fills in `$deb_version` and `$PKG_ARCH`, then `dpkg-deb --build --root-owner-group` records files as `root:root` regardless of the user running the build. + +### `.rpm` + +`package-specs/rpm/clp-plugin-presto-connector.spec` builds only the binary package; the binaries already exist before `rpmbuild` runs. The invocation: + +* uses `--target ${rpm_arch}` to set the package architecture +* passes all per-build values through `--define` macros +* uses `--bb` to skip source RPM creation + +The spec disables automatic dependency generation with `AutoReqProv: no` and sets explicit requirements on `glibc >= 2.28` and `libstdc++`. This prevents RPM from adding requirements for every bundled shared-library soname. + +### `.tar.gz` + +`internal/container/build-artifacts.sh` creates tarballs with: + +```bash +tar -C "${staging}" --owner=0 --group=0 --numeric-owner -czf "${tar_file}" "${tar_dirname}" +``` + +The ownership flags normalize all archived entries to UID/GID 0 across build hosts. + +## Shared-library bundling + +The plugin `.so` is built with hidden visibility and without `-z defs`, so it does not carry link-time soname dependencies on Folly, Arrow, fbthrift, and similar libraries. Those symbols resolve at `dlopen()` time from the Presto worker process. + +The package still ships the runtime libraries the plugin needs. `internal/container/build-artifacts.sh` walks `ldd` output, copies each non-system dependency into `/lib/`, and rewrites RUNPATHs with `patchelf`: + +* bundled libraries get `$ORIGIN` prepended to their existing RUNPATH +* the main plugin `.so` gets `$ORIGIN/lib` prepended to its existing RUNPATH + +Existing RUNPATH entries are preserved so any build-time plugin-loader paths remain available. System libraries such as `libc`, `libstdc++`, `libgcc_s`, `libm`, `libdl`, `libpthread`, and the dynamic loader are not bundled; they come from the target distribution. + +### Why bundle? + +The supported distribution range spans incompatible OpenSSL and ICU versions: + +| Distro | OpenSSL | ICU | +|---|---|---| +| RHEL 8, AlmaLinux 8, Rocky Linux 8, Ubuntu 20.04, Debian 11 | 1.1 | 60 | +| RHEL 9, AlmaLinux 9, Rocky Linux 9, Ubuntu 22.04+, Debian 12+ | 3.0 | 70+ | + +Bundling ships the exact library versions used at build time, so one artifact can run across the supported range without per-distribution rebuilds or compat packages. The ABI floor remains glibc 2.28 from the `manylinux_2_28` base; newer dependencies such as OpenSSL, ICU, and libcurl are bundled. + +### Inspecting the bundle + +```bash +dpkg-deb --contents packages/clp-plugin-presto-connector_*_.deb | awk '/\/opt\/clp-plugin-presto-connector\/worker\/lib\//{print $NF}' +rpm -qlp packages/clp-plugin-presto-connector-*..rpm | grep /opt/clp-plugin-presto-connector/worker/lib/ +``` + +## Package naming limitations + +Package versions must start with a digit and otherwise use only letters, digits, `.`, `+`, `~`, or `-`, a common safe subset for Debian, RPM, and filenames. RPM forbids `-` in the `Version:` field, so `.deb` and `.rpm` metadata and filenames replace `-` with `~`; `0.1.0-SNAPSHOT` becomes `0.1.0~SNAPSHOT`. The tarball filename keeps the original Maven version. For snapshot-style versions, the `~` form also keeps the package ordered before the eventual release package. diff --git a/tools/build-packages/build-dependency-image.sh b/tools/build-packages/build-dependency-image.sh index 4c73c46..e0735bc 100755 --- a/tools/build-packages/build-dependency-image.sh +++ b/tools/build-packages/build-dependency-image.sh @@ -8,7 +8,7 @@ # Example: # image=$(tools/build-packages/build-dependency-image.sh) # docker run --rm -v "$(pwd):/src" -w /src "${image}" \ -# task velox-connector:build +# task velox-connector:build-with-installed-deps # # Requires: docker (with buildx), git, sha256sum. @@ -17,7 +17,8 @@ set -o nounset set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${script_dir}/dependency-image/utils.sh" +# shellcheck source=tools/build-packages/internal/host/build-env.sh +source "${script_dir}/internal/host/build-env.sh" # Derive this repo's GHCR namespace from its GitHub origin remote. image_repo_from_origin() { @@ -71,7 +72,7 @@ main() { fi echo >&2 " docker pull failed; will build from scratch. Pull error:" - printf '%s\n' "${pull_err}" | sed 's/^/ /' >&2 + printf '%s\n' "${pull_err}" | sed >&2 's/^/ /' echo >&2 "==> Image not available — building from scratch..." build_image "${image}" "${platform}" "--load" diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh new file mode 100755 index 0000000..a098b14 --- /dev/null +++ b/tools/build-packages/build-packages.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +# User-facing entry point for packaging. Resolves the build-env image, then runs +# internal/container/build-artifacts.sh inside it. +# +# Requires: docker (with buildx), git, sha256sum. + +set -o errexit +set -o nounset +set -o pipefail + +src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" +# shellcheck source=tools/build-packages/internal/build-cache/host.sh +source "${src}/tools/build-packages/internal/build-cache/host.sh" +# shellcheck source=tools/build-packages/internal/ca-trust/host.sh +source "${src}/tools/build-packages/internal/ca-trust/host.sh" + +show_help() { + cat <<'EOF' +Usage: ./tools/build-packages/build-packages.sh [OPTIONS] + +User-facing entry point for packaging. Resolves the build-env image, then runs +internal/container/build-artifacts.sh inside it. + +Options are forwarded to internal/container/build-artifacts.sh: + --output DIR Output directory for built packages (default: ./packages) + --version VER Override package version + (default: derived from presto-connector/pom.xml) + VER must start with a digit and use [0-9A-Za-z.+~-] + --help Show this help + +See tools/build-packages/README.md for details. +EOF +} + +# Resolve the output directory on the host before copying completed artifacts +# into it. Other arguments are forwarded unchanged to build-artifacts.sh. +output_dir="${src}/packages" +build_args=() +while [[ $# -gt 0 ]]; do + case $1 in + --output) + [[ -n "${2:-}" ]] || { echo >&2 "ERROR: --output requires a value"; exit 1; } + output_dir="$2" + shift 2 + ;; + --help) + show_help + exit 0 + ;; + *) + build_args+=("$1") + shift + ;; + esac +done + +# Run the wrapper as the intended artifact owner. Using sudo would make the +# staging directories and copied artifacts root-owned. +if (( EUID == 0 )); then + echo >&2 "ERROR: build-packages.sh must run as a non-root user; do not invoke it with sudo." + exit 1 +fi + +if [[ "${output_dir}" != /* ]]; then + output_dir="${src}/${output_dir}" +fi +mkdir -p "${output_dir}" +output_dir="$(cd "${output_dir}" && pwd)" + +# Initialize submodules on the host so the source tree is complete before it is +# bind-mounted into the build container. +echo "==> Initializing submodules..." +git -C "${src}" submodule update --init --recursive + +echo "==> Resolving build-env image..." +image=$("${src}/tools/build-packages/build-dependency-image.sh") +# FetchContent build state is compatible only with the image inputs identified +# by this hash. +image_hash="${image##*:env-}" +if [[ "${image_hash}" == "${image}" ]]; then + echo >&2 "ERROR: build-env image lacks an env- tag: ${image}" + exit 1 +fi + +# Keep root-owned container output in temporary staging, then copy it to the +# requested directory as the invoking host user. +stage_dir=$(mktemp -d) +trap 'rm -rf "${stage_dir}"' EXIT +artifact_stage="${stage_dir}/artifacts" +trust_stage="${stage_dir}/trust" +ca_bundle="${trust_stage}/ca-bundle.pem" +java_trust_store="${trust_stage}/truststore.p12" +mkdir -p "${artifact_stage}" "${trust_stage}" +prepare_build_cache "${src}/.cache" "${image_hash}" + +echo "==> Staging temporary container trust stores..." +stage_host_ca_bundle "${ca_bundle}" +stage_java_pkcs12 "${ca_bundle}" "${java_trust_store}" + +# Use a stable container checkout path so cached CMake state does not embed the +# host checkout path. HOME and Task scratch data remain in the disposable +# container. +echo "==> Running internal/container/build-artifacts.sh inside ${image}..." +docker run --rm \ + --mount "type=bind,src=${src},dst=/repo" \ + --mount "type=bind,src=${artifact_stage},dst=/output" \ + --mount "type=bind,src=${trust_stage},dst=/run/ca-trust,readonly" \ + --env MAVEN_OPTS \ + --env "BUILD_CACHE_KEY=${image_hash}" \ + -w /repo \ + "${image}" \ + bash -c ' + set -o errexit + set -o nounset + set -o pipefail + export BUILD_CACHE_DIR=/repo/.cache + export CA_TRUST_DIR=/run/ca-trust + export HOME=/tmp/clp-plugin-presto-connector-home + export TASK_TEMP_DIR=/tmp/clp-plugin-presto-connector-task + source tools/build-packages/internal/build-cache/container.sh + source tools/build-packages/internal/ca-trust/container.sh + mkdir -p "${HOME}" "${TASK_TEMP_DIR}" + umask 0022 + echo "==> Running the package build as root..." + exec bash tools/build-packages/internal/container/build-artifacts.sh "$@" + ' bash --output /output "${build_args[@]+"${build_args[@]}"}" + +echo "==> Copying package artifacts to ${output_dir}..." +for artifact in "${artifact_stage}"/*; do + rm -f -- "${output_dir}/$(basename -- "${artifact}")" + cp -- "${artifact}" "${output_dir}/" +done diff --git a/tools/build-packages/dependency-image/Dockerfile b/tools/build-packages/dependency-image/Dockerfile index b08cec2..9b8beec 100644 --- a/tools/build-packages/dependency-image/Dockerfile +++ b/tools/build-packages/dependency-image/Dockerfile @@ -8,7 +8,7 @@ ARG HOST_CA_ENV=/usr/local/share/clp-plugin-presto-connector/use-host-ca.sh # When available, expose the host CA bundle only to networked build steps via # tool-specific environment variables. This supports local corporate builds # without baking host CA certificates into the final image. -COPY tools/build-packages/dependency-image/use-host-ca.sh ${HOST_CA_ENV} +COPY tools/build-packages/internal/ca-trust/container.sh ${HOST_CA_ENV} # Install OS packages missing from the manylinux base image. # dnf talks to libcurl directly and ignores CURL_CA_BUNDLE/SSL_CERT_FILE, so @@ -20,7 +20,7 @@ RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ dnf_sslcacert_opt="--setopt=sslcacert=${HOST_CA_BUNDLE}"; \ fi \ && dnf install -y --setopt=install_weak_deps=False ${dnf_sslcacert_opt} \ - dpkg gettext git java-17-openjdk-devel libcurl-devel libevent-devel \ + ccache dpkg gettext git java-17-openjdk-devel libcurl-devel libevent-devel \ libunwind-devel ninja-build openssl-devel patchelf python3-pip rpm-build \ && dnf clean all @@ -43,7 +43,8 @@ ENV PATH=/opt/go-task/bin:${PATH} ENV CLP_PLUGIN_BUILD_DIR=/opt/clp-plugin-presto-connector/build ENV TASK_TEMP_DIR=/opt/clp-plugin-presto-connector/.task -# Build one dependency at a time to avoid excessive memory usage from parallel builds. +# Install dependencies one at a time to avoid overlapping high-memory builds; +# each dependency still compiles in parallel using the available processors. RUN --mount=type=bind,source=.,target=/repo,readonly \ --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ . "${HOST_CA_ENV}" \ diff --git a/tools/build-packages/dependency-image/use-host-ca.sh b/tools/build-packages/dependency-image/use-host-ca.sh deleted file mode 100644 index 5cc9a74..0000000 --- a/tools/build-packages/dependency-image/use-host-ca.sh +++ /dev/null @@ -1,7 +0,0 @@ -if [ -s "${HOST_CA_BUNDLE}" ]; then - export CURL_CA_BUNDLE="${HOST_CA_BUNDLE}" - export GIT_SSL_CAINFO="${HOST_CA_BUNDLE}" - export PIP_CERT="${HOST_CA_BUNDLE}" - export REQUESTS_CA_BUNDLE="${HOST_CA_BUNDLE}" - export SSL_CERT_FILE="${HOST_CA_BUNDLE}" -fi diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh deleted file mode 100644 index 3423057..0000000 --- a/tools/build-packages/dependency-image/utils.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash - -# Shared dependency-image helpers for tag derivation and Docker builds. -# Callers decide whether to use a local image, pull from GHCR, or build. - -_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." &>/dev/null && pwd)" - -# Keep local builds working when the checkout lacks initialized submodules. -ensure_yscope_dev_utils_submodule() { - git -C "${_REPO_ROOT}" submodule update --init --recursive tools/yscope-dev-utils -} - -# ── Image identity ──────────────────────────────────────────────────────────── - -# Builds the canonical `:env-` image reference. -# -# Args: -image_ref() { - echo "$1/$2:env-$3" -} - -# Inputs that should change the build-env image tag. -_BUILD_ENV_HASH_INPUTS=( - ".dockerignore" - ".github/workflows/build-dependency-image.yaml" - "taskfile.yaml" - "taskfiles" - "tools/build-packages/build-dependency-image.sh" - "tools/build-packages/dependency-image" - "tools/yscope-dev-utils" -) - -# Computes the 16-hex-char hash used in the image tag. -# -# Requires: git, sha256sum -derive_build_env_hash() { - ( - cd "${_REPO_ROOT}" || exit - ensure_yscope_dev_utils_submodule >&2 - git ls-files -z --recurse-submodules -- "${_BUILD_ENV_HASH_INPUTS[@]}" \ - | LC_ALL=C sort -z \ - | xargs -0 sha256sum \ - | sha256sum \ - | cut -c1-16 - ) -} - -# Stages the host CA bundle into the temporary Docker build context. -# -# Args: -_stage_host_ca_bundle() { - local dest="${1:?_stage_host_ca_bundle requires a destination path}" - local ca_bundle_candidates=( - "${SSL_CERT_FILE:-}" - /etc/ssl/certs/ca-certificates.crt - /etc/pki/tls/certs/ca-bundle.crt - /etc/ssl/cert.pem - ) - - local src - for src in "${ca_bundle_candidates[@]}"; do - [[ -f "${src}" && -s "${src}" ]] || continue - echo >&2 "==> Staging host CA bundle: ${src} -> ${dest}" - if ! cp "${src}" "${dest}"; then - echo >&2 "ERROR: failed to stage host CA bundle: ${src}" - return 1 - fi - return 0 - done - - echo >&2 "==> No host CA bundle found; continuing without host CA context." - return 1 -} - -# ── Docker build ────────────────────────────────────────────────────────────── - -# Builds the dependency image. -# -# Args: -# $1 image tag — e.g. ghcr.io/owner/build-env:env- -# $2 platform — linux/amd64 or linux/arm64 -# $3 output flag — --push (registry) or --load (local docker) -# -# Requires: docker buildx, git -build_image() { - local tag="$1" platform="$2" output="$3" - - # Expose the host CA bundle as a narrow named build context so the Dockerfile - # can bind-mount it during networked RUN steps without baking it into image - # layers. Use a real context instead of a BuildKit secret because corporate CA - # bundles can exceed BuildKit's 500KiB secret limit. - local ca_stage; ca_stage=$(mktemp -d) - ( - trap 'rm -rf "${ca_stage}"' EXIT - - local ca_bundle="${ca_stage}/host-ca" - _stage_host_ca_bundle "${ca_bundle}" || : > "${ca_bundle}" - - ensure_yscope_dev_utils_submodule - - docker buildx build \ - --platform "${platform}" \ - --build-context "host-ca=${ca_stage}" \ - --tag "${tag}" \ - "${output}" \ - -f "${_REPO_ROOT}/tools/build-packages/dependency-image/Dockerfile" \ - "${_REPO_ROOT}" - ) -} diff --git a/tools/build-packages/internal/build-cache/README.md b/tools/build-packages/internal/build-cache/README.md new file mode 100644 index 0000000..ac48dbc --- /dev/null +++ b/tools/build-packages/internal/build-cache/README.md @@ -0,0 +1,45 @@ +# Build cache + +This directory provides reusable host and container helpers for persistent build caches. The helpers keep cache layout and tool configuration consistent while leaving cache location, cache identity, and container mounting under the caller's control. + +## Layout + +Given a cache directory and key, the helpers use: + +```text +/ +├── ccache/ +├── fetchcontent// +└── maven/ +``` + +Maven and ccache content is shared across cache keys. FetchContent contains generated CMake state and is isolated by a caller-provided key, such as a build-environment hash. + +## Host API + +Source `host.sh`, then prepare the cache before mounting it: + +```bash +source tools/build-packages/internal/build-cache/host.sh +prepare_build_cache ./.cache "${build_env_hash}" +``` + +The caller is responsible for choosing a nonempty cache key that is safe to use as a path component. + +## Container API + +Mount the prepared cache, set its container path and key, then source `container.sh`: + +```bash +BUILD_CACHE_DIR=/var/cache/build +BUILD_CACHE_KEY="${build_env_hash}" +source tools/build-packages/internal/build-cache/container.sh +``` + +The container helper creates missing subdirectories and supplies defaults for Maven, ccache, CMake compiler launchers, and `FETCHCONTENT_BASE_DIR`. Existing tool-specific environment variables take precedence over those defaults. + +`FETCHCONTENT_BASE_DIR` is an integration variable: projects must pass it to CMake as `-DFETCHCONTENT_BASE_DIR=...`. The helper does not modify project CMake files or command lines. + +## Lifecycle + +Build caches are persistent and may contain downloaded source, compiled objects, and generated build state. They are not temporary staging material and are not removed automatically. The caller decides when to delete or rotate the cache. diff --git a/tools/build-packages/internal/build-cache/container.sh b/tools/build-packages/internal/build-cache/container.sh new file mode 100644 index 0000000..c175378 --- /dev/null +++ b/tools/build-packages/internal/build-cache/container.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +# Container-side configuration for consuming a prepared build cache. + +if [ -n "${BUILD_CACHE_DIR:-}" ]; then + if [ -z "${BUILD_CACHE_KEY:-}" ]; then + echo >&2 "ERROR: BUILD_CACHE_KEY must be set when BUILD_CACHE_DIR is set" + return 1 + fi + + export CCACHE_DIR="${CCACHE_DIR:-${BUILD_CACHE_DIR}/ccache}" + export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-1G}" + export CMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER:-ccache}" + export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-ccache}" + export FETCHCONTENT_BASE_DIR="${FETCHCONTENT_BASE_DIR:-${BUILD_CACHE_DIR}/fetchcontent/${BUILD_CACHE_KEY}}" + export MAVEN_USER_HOME="${MAVEN_USER_HOME:-${BUILD_CACHE_DIR}/maven}" + + mkdir -p \ + "${CCACHE_DIR}" \ + "${FETCHCONTENT_BASE_DIR}" \ + "${MAVEN_USER_HOME}" +fi diff --git a/tools/build-packages/internal/build-cache/host.sh b/tools/build-packages/internal/build-cache/host.sh new file mode 100644 index 0000000..985da70 --- /dev/null +++ b/tools/build-packages/internal/build-cache/host.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Host-side preparation for persistent container build caches. + +if [[ "${_BUILD_CACHE_HOST_SH_LOADED:-}" == "1" ]]; then + return 0 +fi +readonly _BUILD_CACHE_HOST_SH_LOADED=1 + +# Creates the shared tool caches and a namespaced FetchContent cache. +# +# Args: +prepare_build_cache() { + if (( $# != 2 )) || [[ -z "$1" || -z "$2" ]]; then + echo >&2 "ERROR: prepare_build_cache requires a cache directory and key" + return 2 + fi + + local cache_dir="$1" + local cache_key="$2" + mkdir -p \ + "${cache_dir}/ccache" \ + "${cache_dir}/fetchcontent/${cache_key}" \ + "${cache_dir}/maven" +} diff --git a/tools/build-packages/internal/ca-trust/README.md b/tools/build-packages/internal/ca-trust/README.md new file mode 100644 index 0000000..e341b8c --- /dev/null +++ b/tools/build-packages/internal/ca-trust/README.md @@ -0,0 +1,43 @@ +# CA trust + +This directory provides reusable CA-trust support for containerized builds in corporate environments. It makes the host's trusted certificates available to build tools without installing them in an image or persisting them in image layers, caches, or build artifacts. + +## Design + +Trust preparation and consumption are deliberately separate: + +1. `host.sh` discovers the host CA bundle and stages temporary trust files. +2. Format-specific backends under `generators/` produce any additional trust-store formats required by build tools. +3. The caller mounts the staged files read-only into the build container. +4. `container.sh` configures tools in the container to use those files. +5. The caller removes the staging directory when the build finishes. + +The scripts do not modify either the host or container trust store. The staged files are snapshots used only for the current build. + +## Host API + +Source `host.sh`, then stage the formats needed by the container: + +```bash +source tools/build-packages/internal/ca-trust/host.sh + +stage_host_ca_bundle ./trust/ca-bundle.pem +stage_java_pkcs12 ./trust/ca-bundle.pem ./trust/truststore.p12 +``` + +`stage_host_ca_bundle` uses `SSL_CERT_FILE` when it is set. Otherwise, it searches common Linux CA-bundle locations. `stage_java_pkcs12` runs the Java generator in a temporary container, so Java does not need to be installed on the host. + +## Container API + +After mounting the staging directory, set its container path and source `container.sh`: + +```bash +CA_TRUST_DIR=/trusted +source tools/build-packages/internal/ca-trust/container.sh +``` + +The directory must contain `ca-bundle.pem` and `truststore.p12`. Callers may instead set `HOST_CA_BUNDLE` and `HOST_CA_JAVA_TRUST_STORE` to use different paths. For the PEM bundle, the script exports the standard environment variables used by curl, Git, pip, Python Requests, and OpenSSL-based clients. For the PKCS#12 store, it appends Java trust-store properties to `MAVEN_OPTS`. + +## Extensibility + +Add a backend under `generators/` when a tool requires a trust format that cannot consume the staged PEM bundle directly. Keep host discovery and lifecycle management in `host.sh`, and keep format-specific conversion in the backend. See `generators/java-pkcs12/README.md` for the current implementation. diff --git a/tools/build-packages/internal/ca-trust/container.sh b/tools/build-packages/internal/ca-trust/container.sh new file mode 100644 index 0000000..94d47e1 --- /dev/null +++ b/tools/build-packages/internal/ca-trust/container.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env sh + +# Container-side configuration for consuming prepared CA trust stores. +if [ -n "${CA_TRUST_DIR:-}" ]; then + HOST_CA_BUNDLE="${HOST_CA_BUNDLE:-${CA_TRUST_DIR}/ca-bundle.pem}" + HOST_CA_JAVA_TRUST_STORE="${HOST_CA_JAVA_TRUST_STORE:-${CA_TRUST_DIR}/truststore.p12}" +fi + +if [ -s "${HOST_CA_BUNDLE:-}" ]; then + export CURL_CA_BUNDLE="${HOST_CA_BUNDLE}" + export GIT_SSL_CAINFO="${HOST_CA_BUNDLE}" + export PIP_CERT="${HOST_CA_BUNDLE}" + export REQUESTS_CA_BUNDLE="${HOST_CA_BUNDLE}" + export SSL_CERT_FILE="${HOST_CA_BUNDLE}" +fi + +# Configure Java only when the caller explicitly provides a trust store. +if [ "${HOST_CA_JAVA_TRUST_STORE+x}" = x ]; then + if [ ! -s "${HOST_CA_JAVA_TRUST_STORE}" ]; then + echo >&2 "ERROR: Java trust store is not readable: ${HOST_CA_JAVA_TRUST_STORE}" + return 1 + fi + + # Preserve any Maven options supplied by the caller. + _host_ca_maven_opts="${MAVEN_OPTS:-}" + [ -n "${_host_ca_maven_opts}" ] && _host_ca_maven_opts="${_host_ca_maven_opts} " + _host_ca_maven_opts="${_host_ca_maven_opts}-Djavax.net.ssl.trustStore=${HOST_CA_JAVA_TRUST_STORE}" + _host_ca_maven_opts="${_host_ca_maven_opts} -Djavax.net.ssl.trustStoreType=PKCS12" + # The store contains only public certificates; this is an integrity password, not a secret. + _host_ca_maven_opts="${_host_ca_maven_opts} -Djavax.net.ssl.trustStorePassword=changeit" + export MAVEN_OPTS="${_host_ca_maven_opts}" + unset _host_ca_maven_opts +fi diff --git a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/CreateJavaTrustStore.java b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/CreateJavaTrustStore.java new file mode 100644 index 0000000..754d1f4 --- /dev/null +++ b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/CreateJavaTrustStore.java @@ -0,0 +1,50 @@ +// Creates the Java PKCS#12 store inside the temporary JDK generator container. +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.MessageDigest; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.util.Enumeration; +import java.util.HexFormat; + +final class CreateJavaTrustStore { + private CreateJavaTrustStore() {} + + public static void main(String[] args) throws Exception { + if (args.length != 4) { + throw new IllegalArgumentException( + "Usage: CreateJavaTrustStore "); + } + + Path hostCa = Path.of(args[0]); + Path baseTrustStore = Path.of(args[1]); + Path outputTrustStore = Path.of(args[2]); + char[] password = args[3].toCharArray(); + + KeyStore base = KeyStore.getInstance(baseTrustStore.toFile(), password); + KeyStore trustStore = KeyStore.getInstance("PKCS12"); + trustStore.load(null, password); + for (Enumeration aliases = base.aliases(); aliases.hasMoreElements(); ) { + String alias = aliases.nextElement(); + if (base.isCertificateEntry(alias)) { + trustStore.setCertificateEntry(alias, base.getCertificate(alias)); + } + } + + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); + try (InputStream input = Files.newInputStream(hostCa)) { + for (Certificate certificate : certificateFactory.generateCertificates(input)) { + String fingerprint = HexFormat.of().formatHex(sha256.digest(certificate.getEncoded())); + trustStore.setCertificateEntry("host-ca-" + fingerprint, certificate); + } + } + + try (OutputStream output = Files.newOutputStream(outputTrustStore)) { + trustStore.store(output, password); + } + } +} diff --git a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md new file mode 100644 index 0000000..f324608 --- /dev/null +++ b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md @@ -0,0 +1,33 @@ +# Java PKCS#12 generator + +This backend creates a Java PKCS#12 trust store from a staged PEM CA bundle. + +## Why it is needed + +Java's TLS implementation does not use environment variables such as `SSL_CERT_FILE`, `CURL_CA_BUNDLE`, or `REQUESTS_CA_BUNDLE`. It normally reads the JDK trust store, or a custom store selected with Java system properties. The packaging flow supplies the generated store to Maven with: + +```text +-Djavax.net.ssl.trustStore=/path/to/truststore.p12 +-Djavax.net.ssl.trustStoreType=PKCS12 +-Djavax.net.ssl.trustStorePassword=changeit +``` + +This avoids modifying the JDK's installed `cacerts` file. + +## What it does + +Given `generate.sh `, the backend: + +1. Detects the base JDK trust store (`jssecacerts` if present, otherwise `cacerts`). +2. Copies its trusted certificates into a new PKCS#12 store. +3. Adds certificates from the staged PEM bundle using SHA-256-based aliases. +4. Writes the result to the requested path. + +PKCS#12 is a standard format rather than a JDK-specific format. The generator currently uses a pinned Temurin JDK 17 image to keep the generator environment and base certificate set stable. Set `CA_TRUST_JAVA_PKCS12_GENERATOR_IMAGE` to use a different generator image. + +The caller mounts the generated file read-only for the packaging build and removes it afterward. It is not copied into the dependency image, caches, packages, or image layers. + +## Files + +- `generate.sh` validates the inputs, locates the JDK trust store, and invokes the converter. +- `CreateJavaTrustStore.java` copies the base certificates and imports the staged PEM certificates. diff --git a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh new file mode 100644 index 0000000..dac1fdc --- /dev/null +++ b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Generates a Java PKCS#12 trust store inside a temporary JDK 17 container. + +set -o errexit +set -o nounset +set -o pipefail + +if (( $# != 2 )) || [[ -z "$1" || -z "$2" ]]; then + echo >&2 "ERROR: generate.sh requires an input CA bundle and output path" + exit 2 +fi + +input_bundle="$1" +output_trust_store="$2" +if [[ ! -f "${input_bundle}" || ! -r "${input_bundle}" ]]; then + echo >&2 "ERROR: input CA bundle is not a readable regular file: ${input_bundle}" + exit 1 +fi +output_dir="$(dirname "${output_trust_store}")" +if [[ ! -d "${output_dir}" || ! -w "${output_dir}" ]]; then + echo >&2 "ERROR: output directory is not writable: ${output_dir}" + exit 1 +fi +if [[ -e "${output_trust_store}" && ! -f "${output_trust_store}" ]]; then + echo >&2 "ERROR: output path is not a regular file: ${output_trust_store}" + exit 1 +fi + +java_home="${JAVA_HOME:-}" +if [[ -z "${java_home}" ]]; then + java_executable="$(command -v java)" || { + echo >&2 "ERROR: java was not found in PATH" + exit 1 + } + java_executable="$(readlink -f "${java_executable}")" + java_home="${java_executable%/bin/java}" +else + java_executable="${java_home}/bin/java" +fi +if [[ ! -x "${java_executable}" ]]; then + echo >&2 "ERROR: Java executable is not available: ${java_executable}" + exit 1 +fi + +java_security_dir="${java_home}/lib/security" +base_java_trust_store="${java_security_dir}/cacerts" +# Match Java's trust-store lookup order: jssecacerts overrides cacerts. +if [[ -f "${java_security_dir}/jssecacerts" && -s "${java_security_dir}/jssecacerts" ]]; then + base_java_trust_store="${java_security_dir}/jssecacerts" +fi +if [[ ! -f "${base_java_trust_store}" || ! -r "${base_java_trust_store}" \ + || ! -s "${base_java_trust_store}" ]]; then + echo >&2 "ERROR: Java trust store is not readable: ${base_java_trust_store}" + exit 1 +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +# Use Java source-file mode to avoid a separate compilation step and class files. +"${java_executable}" "${script_dir}/CreateJavaTrustStore.java" \ + "${input_bundle}" "${base_java_trust_store}" "${output_trust_store}" changeit diff --git a/tools/build-packages/internal/ca-trust/host.sh b/tools/build-packages/internal/ca-trust/host.sh new file mode 100644 index 0000000..242c918 --- /dev/null +++ b/tools/build-packages/internal/ca-trust/host.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +# Host-side CA discovery and staging shared by Docker build and run workflows. + +if [[ "${_CA_TRUST_HOST_SH_LOADED:-}" == "1" ]]; then + return 0 +fi +readonly _CA_TRUST_HOST_SH_LOADED=1 + +_CA_TRUST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +readonly _CA_TRUST_DIR + +# Stages the host CA bundle for a temporary Docker mount. Creates an empty +# destination when the host has no CA bundle; returns nonzero only on an error. +# +# Args: +stage_host_ca_bundle() { + if (( $# != 1 )) || [[ -z "$1" ]]; then + echo >&2 "ERROR: stage_host_ca_bundle requires a destination path" + return 2 + fi + local dest="$1" + if [[ -L "${dest}" || ( -e "${dest}" && ! -f "${dest}" ) ]]; then + echo >&2 "ERROR: host CA bundle destination is not a regular file: ${dest}" + return 1 + fi + local dest_dir + dest_dir="$(dirname "${dest}")" + if ! mkdir -p "${dest_dir}"; then + echo >&2 "ERROR: failed to create host CA bundle directory: ${dest_dir}" + return 1 + fi + local source_path="" + local candidates=() + + if [[ -n "${SSL_CERT_FILE:-}" ]]; then + if [[ ! -f "${SSL_CERT_FILE}" || ! -s "${SSL_CERT_FILE}" ]]; then + echo >&2 "ERROR: SSL_CERT_FILE is not a nonempty regular file: ${SSL_CERT_FILE}" + return 1 + fi + candidates=("${SSL_CERT_FILE}") + else + candidates=( + /etc/ssl/certs/ca-certificates.crt + /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + /etc/pki/tls/certs/ca-bundle.crt + /etc/ssl/ca-bundle.pem + /etc/pki/tls/cacert.pem + /etc/ssl/cert.pem + ) + fi + + local candidate + for candidate in "${candidates[@]}"; do + if [[ -f "${candidate}" && -s "${candidate}" ]]; then + source_path="${candidate}" + break + fi + done + + if [[ -n "${source_path}" && -e "${dest}" && "${source_path}" -ef "${dest}" ]]; then + echo >&2 "ERROR: host CA bundle source and destination must differ: ${dest}" + return 1 + fi + + local staged_bundle + if ! staged_bundle="$(mktemp "${dest_dir}/.ca-bundle.XXXXXX")"; then + echo >&2 "ERROR: failed to create temporary host CA bundle in: ${dest_dir}" + return 1 + fi + if [[ -n "${source_path}" ]]; then + echo >&2 "==> Staging host CA bundle: ${source_path} -> ${dest}" + if ! cp "${source_path}" "${staged_bundle}"; then + rm -f "${staged_bundle}" + echo >&2 "ERROR: failed to stage host CA bundle: ${source_path}" + return 1 + fi + else + echo >&2 "==> No host CA bundle found; continuing without host CA context." + fi + + # BuildKit and runtime containers consume the staged bundle read-only. + if ! chmod 0444 "${staged_bundle}"; then + rm -f "${staged_bundle}" + echo >&2 "ERROR: failed to set host CA bundle permissions: ${dest}" + return 1 + fi + if ! mv -f "${staged_bundle}" "${dest}"; then + rm -f "${staged_bundle}" + echo >&2 "ERROR: failed to replace host CA bundle: ${dest}" + return 1 + fi +} + +# Stages a Java PKCS#12 trust store containing the selected JDK's default +# certificates plus those from an input CA bundle. +# +# Args: +# The subshell keeps the cleanup trap local to this invocation. +stage_java_pkcs12() ( + if (( $# != 2 )) || [[ -z "$1" || -z "$2" ]]; then + echo >&2 "ERROR: stage_java_pkcs12 requires an input bundle and destination path" + return 2 + fi + + local input_bundle="$1" + local dest="$2" + if [[ ! -f "${input_bundle}" || ! -r "${input_bundle}" ]]; then + echo >&2 "ERROR: input CA bundle is not a readable regular file: ${input_bundle}" + return 1 + fi + local input_dir + input_dir="$(cd "$(dirname "${input_bundle}")" &>/dev/null && pwd)" || return + input_bundle="${input_dir}/$(basename "${input_bundle}")" + if [[ -L "${dest}" || ( -e "${dest}" && ! -f "${dest}" ) ]]; then + echo >&2 "ERROR: Java PKCS#12 destination is not a regular file: ${dest}" + return 1 + fi + + local output_dir + output_dir="$(dirname "${dest}")" + if ! mkdir -p "${output_dir}"; then + echo >&2 "ERROR: failed to create Java PKCS#12 directory: ${output_dir}" + return 1 + fi + output_dir="$(cd "${output_dir}" &>/dev/null && pwd)" || return + local output_name + output_name="$(basename "${dest}")" + dest="${output_dir}/${output_name}" + if [[ "${input_bundle}" == "${dest}" ]] \ + || [[ -e "${dest}" && "${input_bundle}" -ef "${dest}" ]]; then + echo >&2 "ERROR: input CA bundle and Java PKCS#12 destination must differ" + return 1 + fi + + local generator_dir + generator_dir="$(cd "${_CA_TRUST_DIR}/generators/java-pkcs12" &>/dev/null && pwd)" || return + local generator_image="${CA_TRUST_JAVA_PKCS12_GENERATOR_IMAGE:-docker.io/library/eclipse-temurin:17.0.19_10-jdk-jammy@sha256:723151f3fc88ca2060153ee08ab8dbbea7983d6ed6f2622fe440acf178737c94}" + local container_assets="/opt/clp-trust-store" + local container_input="/run/secrets/host-ca" + local container_output_dir="/tmp/clp-java-trust" + local container_output="${container_output_dir}/truststore.p12" + local generator_stage="" + trap '[[ -z "${generator_stage}" ]] || rm -rf "${generator_stage}"' EXIT + if ! generator_stage="$(mktemp -d "${output_dir}/.java-pkcs12.XXXXXX")"; then + echo >&2 "ERROR: failed to create private Java PKCS#12 staging directory in: ${output_dir}" + return 1 + fi + local staged_trust_store + staged_trust_store="${generator_stage}/truststore.p12" + # Use the host identity so bind-mounted output remains owned by the caller. + if ! docker run --rm \ + --network none \ + --user "$(id -u):$(id -g)" \ + --entrypoint bash \ + --mount "type=bind,src=${generator_dir},dst=${container_assets},readonly" \ + --mount "type=bind,src=${input_bundle},dst=${container_input},readonly" \ + --mount "type=bind,src=${generator_stage},dst=${container_output_dir}" \ + "${generator_image}" \ + "${container_assets}/generate.sh" "${container_input}" "${container_output}"; then + echo >&2 "ERROR: failed to generate Java PKCS#12 trust store" + return 1 + fi + if [[ ! -s "${staged_trust_store}" ]]; then + echo >&2 "ERROR: Java PKCS#12 generator produced no output: ${dest}" + return 1 + fi + # Publish only a complete, read-only trust store. + if ! chmod 0444 "${staged_trust_store}"; then + echo >&2 "ERROR: failed to set Java PKCS#12 permissions: ${dest}" + return 1 + fi + if ! mv -f "${staged_trust_store}" "${dest}"; then + echo >&2 "ERROR: failed to replace Java PKCS#12 trust store: ${dest}" + return 1 + fi +) diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh new file mode 100755 index 0000000..92e6898 --- /dev/null +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -0,0 +1,370 @@ +#!/usr/bin/env bash + +# Build installable .deb / .rpm / .tar.gz packages. +# +# Must run inside the build-env image. See tools/build-packages/README.md +# for local and CI entry points. + +set -o errexit +set -o nounset +set -o pipefail + +src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." &>/dev/null && pwd)" + +# Install layout. Overridable via env; also exposed to rpmbuild via --define. +readonly PLUGIN_ROOT="${PLUGIN_ROOT:-/opt/clp-plugin-presto-connector}" +readonly PRESTO_JAR_DIR="${PRESTO_JAR_DIR:-${PLUGIN_ROOT}/coordinator}" +readonly VELOX_SO_DIR="${VELOX_SO_DIR:-${PLUGIN_ROOT}/worker}" +readonly PACKAGE_RELEASE=1 + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +show_help() { + cat <<'EOF' +Usage: ./tools/build-packages/internal/container/build-artifacts.sh [OPTIONS] + +Builds .deb / .rpm / .tar.gz packages for the current architecture. Must run +inside the build-env image. + +Options: + --output DIR Output directory for built packages (default: ./packages) + --version VER Override package version + (default: derived from presto-connector/pom.xml) + VER must start with a digit and use [0-9A-Za-z.+~-] + --help Show this help + +See tools/build-packages/README.md for the recommended local entry point. +EOF +} + +die() { + echo >&2 "ERROR: $*" + exit 1 +} + +require_value() { + [[ -n "${2:-}" ]] || die "$1 requires a value" +} + +validate_package_version() { + local candidate="$1" + [[ "${candidate}" =~ ^[0-9][0-9A-Za-z.+~-]*$ ]] \ + || die "invalid package version '${candidate}'; expected a digit followed by letters, digits, '.', '+', '~', or '-'" +} + +# ── Parse arguments ─────────────────────────────────────────────────────────── + +output_dir="${src}/packages" +version="" + +while [[ $# -gt 0 ]]; do + case $1 in + --output) require_value "$1" "${2:-}"; output_dir="$2"; shift 2 ;; + --version) require_value "$1" "${2:-}"; version="$2"; shift 2 ;; + --help) show_help; exit 0 ;; + *) die "unknown option: $1 (use --help for usage)" ;; + esac +done + +[[ -z "${version}" ]] || validate_package_version "${version}" + +# ── Configure Java and Maven ────────────────────────────────────────────────── + +if [[ -z "${JAVA_HOME:-}" ]]; then + javac_path=$(readlink -f "$(command -v javac)") + export JAVA_HOME="${javac_path%/bin/javac}" +fi + +maven_opts="${MAVEN_OPTS:-}" +if [[ -n "${MAVEN_USER_HOME:-}" ]]; then + # MAVEN_USER_HOME also caches the Maven Wrapper distribution, which is + # separate from Maven's local artifact repository. + mkdir -p "${MAVEN_USER_HOME}/repository" + [[ -n "${maven_opts}" ]] && maven_opts+=" " + maven_opts+="-Dmaven.repo.local=${MAVEN_USER_HOME}/repository" +fi + +# ── Resolve architecture ────────────────────────────────────────────────────── + +case "$(uname -m)" in + x86_64) arch="amd64"; rpm_arch="x86_64" ;; + aarch64) arch="arm64"; rpm_arch="aarch64" ;; + *) die "unsupported architecture: $(uname -m)" ;; +esac + +pkg_specs_dir="${src}/tools/build-packages/package-specs" +project_build_dir="${CLP_PLUGIN_BUILD_DIR:-${src}/build}" +velox_build_dir="${project_build_dir}/velox-connector" +fetchcontent_base_dir="${FETCHCONTENT_BASE_DIR:-${velox_build_dir}/_deps}" +build_root="${src}/build/packaging" +payload="${build_root}/payload" +artifacts=() + +mkdir -p "${output_dir}" +output_dir="$(cd "${output_dir}" && pwd)" +if [[ "${output_dir}" == "${build_root}" || "${output_dir}" == "${build_root}/"* ]]; then + die "--output must not be ${build_root} or one of its subdirectories" +fi + +# Start each run from a clean build_root so stale artifacts from prior runs +# (e.g. a previous --version) cannot leak into the generated packages. +rm -rf "${build_root}" +mkdir -p "${build_root}" + +# ── Build velox-connector .so ───────────────────────────────────────────────── + +# Submodules are initialized by the caller. Running git inside the container can +# trigger ownership checks because the source tree is bind-mounted from the host. + +echo "==> Building velox-connector .so with image-installed dependencies..." +# The build-env image already contains the C++ dependency installations and +# generated all-deps.cmake. Configure and build the plugin without rerunning +# the dependency installation workflow. +task --concurrency 1 -d "${src}" velox-connector:build-with-installed-deps +so_file="${velox_build_dir}/libclp-plugin-velox-connector.so" +[[ -f "${so_file}" ]] || die "expected .so not found at ${so_file}" +echo " -> ${so_file}" + +# ── Derive version ──────────────────────────────────────────────────────────── + +# Use the Maven wrapper from the Presto source fetched by CMake. +presto_src="${fetchcontent_base_dir}/presto_native_execution-src" +maven_wrapper="${presto_src}/mvnw" +[[ -x "${maven_wrapper}" ]] \ + || die "expected Maven wrapper not found or not executable at ${maven_wrapper}" + +run_maven() { + MAVEN_PROJECTBASEDIR="${presto_src}" \ + MAVEN_OPTS="${maven_opts}" \ + "${maven_wrapper}" -f "${src}/presto-connector/pom.xml" "$@" +} + +if [[ -z "${version}" ]]; then + echo "==> Deriving version from presto-connector/pom.xml via mvnw..." + version=$(run_maven \ + -q help:evaluate -Dexpression=project.version -DforceStdout) \ + || die "mvnw help:evaluate failed" + [[ -n "${version}" ]] || die "derived project.version is empty" +fi + +validate_package_version "${version}" + +# rpm Version: forbids '-'. Debian and rpm both sort '~' before the empty +# string, so dash-qualified versions sort before their base release. +pkg_version_normalized="${version//-/\~}" + +echo "" +echo "==> CLP Plugin Presto Connector package build" +echo " Arch: ${arch}" +echo " Version: ${version}" +echo " Output: ${output_dir}" +echo "" + +# ── Build presto-connector .jar ─────────────────────────────────────────────── + +echo "==> Building presto-connector .jar via fetched mvnw..." +run_maven clean package -DskipTests -B + +jar_file=$(find "${src}/presto-connector/target" -maxdepth 1 \ + -name 'clp-plugin-presto-connector-*.jar' \ + ! -name '*-sources.jar' ! -name '*-javadoc.jar' \ + -print -quit) +[[ -f "${jar_file}" ]] \ + || die "expected .jar not found in presto-connector/target/" +echo " -> ${jar_file}" + +# ── Stage payload ───────────────────────────────────────────────────────────── + +prepend_runpath() { + local entry="$1" + local file="$2" + local old_runpath new_runpath="${entry}" + + old_runpath=$(patchelf --print-rpath "${file}" 2>/dev/null || true) + if [[ "${old_runpath}" == "${entry}" || "${old_runpath}" == "${entry}:"* ]]; then + new_runpath="${old_runpath}" + elif [[ -n "${old_runpath}" ]]; then + new_runpath+=":${old_runpath}" + fi + patchelf --set-rpath "${new_runpath}" "${file}" +} + +bundle_velox_shared_libraries() { + local installed_so="$1" + # Bundled libraries live next to the worker plugin under lib/. + local bundled_dir="${payload}${VELOX_SO_DIR}/lib" + mkdir -p "${bundled_dir}" + + # System libs that come from the target distro at install time. Don't + # bundle these — they must come from the host so the package works + # across glibc/libstdc++ versions. + local system_libs=( + linux-vdso libc libm libmvec libanl libutil libnsl + libpthread libdl librt libresolv 'libstdc\+\+' libgcc_s + ) + # Dynamic loaders ship as ld-linux-.so. / + # libc.musl-.so. — the `-[^.]+` in the regex below matches + # the `-` suffix on these only. + local ld_loaders=(ld-linux ld-musl 'libc\.musl') + + # IFS='|' is scoped to the subshell so the rest of the function's + # `read` loop still splits on whitespace. Anchoring on the full + # soname stops e.g. libdleak.so.1 from masquerading as libdl. + local skip_re + skip_re=$(IFS='|'; echo "^((${system_libs[*]})|(${ld_loaders[*]})-[^.]+)\.so(\.[0-9]+)*$") + + local ldd_out + ldd_out=$(LC_ALL=C ldd "${installed_so}" 2>&1) || { + echo >&2 "ERROR: ldd failed for ${installed_so}:" + echo >&2 "${ldd_out}" + exit 1 + } + + echo "==> Bundling velox-connector shared library dependencies..." + # ldd line shapes: + # => () resolved — bundle if not system + # => not found unresolved — fail at end + # Anything else (vdso, ld-linux without `=>`, blanks) is silently skipped. + local soname op target _rest missing=() + while read -r soname op target _rest; do + case "${op}${target}" in + "=>not") missing+=("${soname}"); continue ;; + "=>"/*) ;; + *) continue ;; + esac + [[ "${soname}" =~ ${skip_re} ]] && continue + [[ -f "${bundled_dir}/${soname}" ]] && continue + cp --dereference "${target}" "${bundled_dir}/${soname}" + echo " Bundled: ${soname}" + done <<< "${ldd_out}" + + if (( ${#missing[@]} > 0 )); then + echo >&2 "ERROR: unresolved shared library deps for ${installed_so}:" + printf >&2 ' %s\n' "${missing[@]}" + exit 1 + fi + + echo "==> Patching velox-connector RUNPATHs..." + # PREPEND $ORIGIN-relative paths on both bundled libs and the main .so, + # preserving any build-time RUNPATH so dlopen() backends (codec engines, + # plugin loaders) keep finding their resources. + for lib in "${bundled_dir}"/*.so*; do + [[ -f "${lib}" ]] || continue + prepend_runpath '$ORIGIN' "${lib}" + done + # Main .so: $ORIGIN/lib points at the bundled-libs dir alongside it. + prepend_runpath '$ORIGIN/lib' "${installed_so}" +} + +echo "==> Staging payload..." +presto_jar_install="${payload}${PRESTO_JAR_DIR}" +velox_so_install="${payload}${VELOX_SO_DIR}" +mkdir -p "${presto_jar_install}" "${velox_so_install}" + +cp "${jar_file}" "${presto_jar_install}/clp-plugin-presto-connector.jar" +cp "${so_file}" "${velox_so_install}/libclp-plugin-velox-connector.so" + +bundle_velox_shared_libraries "${velox_so_install}/libclp-plugin-velox-connector.so" + +# Strip bundled third-party libs (folly, glog, boost, etc.) to keep packages +# smaller. Leave the plugin .so unstripped so crash backtraces resolve to source. +find "${velox_so_install}/lib" -type f -name '*.so*' \ + -exec strip --strip-unneeded {} + + +# cp may preserve source mode; force 0644 on installed files. +chmod 0644 \ + "${presto_jar_install}/clp-plugin-presto-connector.jar" \ + "${velox_so_install}/libclp-plugin-velox-connector.so" +find "${velox_so_install}/lib" -type f -exec chmod 0644 {} + + +# ── Emit packages ───────────────────────────────────────────────────────────── + +build_deb() { + local deb_version="${pkg_version_normalized}-${PACKAGE_RELEASE}" + local staging="${build_root}/staging-deb" + local deb_file="${build_root}/clp-plugin-presto-connector_${deb_version}_${arch}.deb" + + rm -rf "${staging}" + cp -a "${payload}" "${staging}" + mkdir -p "${staging}/DEBIAN" + PKG_ARCH="${arch}" deb_version="${deb_version}" \ + envsubst '$deb_version $PKG_ARCH' \ + < "${pkg_specs_dir}/deb/clp-plugin-presto-connector.control.in" \ + > "${staging}/DEBIAN/control" + + echo "==> Building .deb..." + dpkg-deb --build --root-owner-group "${staging}" "${deb_file}" + artifacts+=("${deb_file}") + echo " -> ${deb_file}" +} + +build_rpm() { + local rpm_version="${pkg_version_normalized}" + local rpmbuild_dir="${build_root}/rpmbuild" + local rpm_filename="clp-plugin-presto-connector-${rpm_version}-${PACKAGE_RELEASE}.${rpm_arch}.rpm" + local rpm_file_out="${build_root}/${rpm_filename}" + + rm -rf "${rpmbuild_dir}" + # rpmbuild creates BUILD/SOURCES/SRPMS on demand; only SPECS+RPMS needed. + mkdir -p "${rpmbuild_dir}"/{SPECS,RPMS} + cp "${pkg_specs_dir}/rpm/clp-plugin-presto-connector.spec" \ + "${rpmbuild_dir}/SPECS/clp-plugin-presto-connector.spec" + + echo "==> Building .rpm..." + rpmbuild \ + --define "_topdir ${rpmbuild_dir}" \ + --define "pkg_version ${rpm_version}" \ + --define "pkg_release ${PACKAGE_RELEASE}" \ + --define "payload_dir ${payload}" \ + --define "plugin_root ${PLUGIN_ROOT}" \ + --define "presto_jar_dir ${PRESTO_JAR_DIR}" \ + --define "velox_so_dir ${VELOX_SO_DIR}" \ + --target "${rpm_arch}" \ + --bb "${rpmbuild_dir}/SPECS/clp-plugin-presto-connector.spec" + + cp "${rpmbuild_dir}/RPMS/${rpm_arch}/${rpm_filename}" "${rpm_file_out}" + artifacts+=("${rpm_file_out}") + echo " -> ${rpm_file_out}" +} + +build_tarball() { + local tar_dirname="clp-plugin-presto-connector-${version}-linux-${arch}" + local staging="${build_root}/staging-tar" + local tar_file="${build_root}/${tar_dirname}.tar.gz" + local tar_root="${staging}/${tar_dirname}" + + # The tarball is the portable artifact and uses a canonical + # `coordinator/` + `worker/` layout. PRESTO_JAR_DIR / VELOX_SO_DIR govern + # only where the .deb/.rpm install to. + rm -rf "${staging}" + mkdir -p "${tar_root}/coordinator" "${tar_root}/worker" + cp -a "${payload}${PRESTO_JAR_DIR}/." "${tar_root}/coordinator/" + cp -a "${payload}${VELOX_SO_DIR}/." "${tar_root}/worker/" + + echo "==> Building .tar.gz..." + tar -C "${staging}" --owner=0 --group=0 --numeric-owner \ + -czf "${tar_file}" "${tar_dirname}" + artifacts+=("${tar_file}") + echo " -> ${tar_file}" +} + +build_deb +build_rpm +build_tarball + +# ── Copy artifacts ──────────────────────────────────────────────────────────── + +echo "" +echo "==> Copying artifacts to ${output_dir}..." +cp "${artifacts[@]}" "${output_dir}/" + +output_artifacts=() +for artifact in "${artifacts[@]}"; do + output_artifacts+=("${output_dir}/${artifact##*/}") +done + +echo "" +echo "========================================" +echo "Build complete" +echo "========================================" +ls -lh "${output_artifacts[@]}" diff --git a/tools/build-packages/internal/host/build-env.sh b/tools/build-packages/internal/host/build-env.sh new file mode 100644 index 0000000..634d39c --- /dev/null +++ b/tools/build-packages/internal/host/build-env.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +# Host-side helpers for build-env image identity and Docker builds. + +if [[ "${_BUILD_ENV_SH_LOADED:-}" == "1" ]]; then + return 0 +fi +readonly _BUILD_ENV_SH_LOADED=1 + +_BUILD_ENV_HOST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +readonly _BUILD_ENV_HOST_DIR +# shellcheck source=tools/build-packages/internal/ca-trust/host.sh +source "${_BUILD_ENV_HOST_DIR}/../ca-trust/host.sh" + +_REPO_ROOT="$(cd "${_BUILD_ENV_HOST_DIR}/../../../.." &>/dev/null && pwd)" +readonly _REPO_ROOT + +# Keep local builds working when the checkout lacks initialized submodules. +_ensure_build_env_submodules() { + git -C "${_REPO_ROOT}" \ + submodule update --init --recursive tools/yscope-dev-utils +} + +# Formats the canonical image reference. +# +# Args: +image_ref() { + if (( $# != 3 )) || [[ -z "$1" || -z "$2" || -z "$3" ]]; then + echo >&2 "ERROR: image_ref requires a repository, image name, and build-env hash" + return 2 + fi + local repository="$1" + local image_name="$2" + local build_env_hash="$3" + printf '%s/%s:env-%s\n' "${repository}" "${image_name}" "${build_env_hash}" +} + +# Computes the 16-character source/configuration hash used in the image tag. +derive_build_env_hash() ( + set -o errexit + set -o pipefail + export LC_ALL=C + + local hash_inputs=( + ".dockerignore" + "taskfile.yaml" + "taskfiles" + "tools/build-packages/dependency-image" + "tools/build-packages/internal/ca-trust/container.sh" + "tools/build-packages/internal/host/build-env.sh" + "tools/yscope-dev-utils" + ) + + cd "${_REPO_ROOT}" + _ensure_build_env_submodules >&2 + + local stat_mode_cmd=(stat -c '%a') + if ! "${stat_mode_cmd[@]}" -- "${_REPO_ROOT}" &>/dev/null; then + stat_mode_cmd=(stat -f '%A') + fi + + local sha256_cmd=(sha256sum) + if ! command -v sha256sum &>/dev/null; then + sha256_cmd=(shasum -a 256) + fi + + { + git ls-files -z --cached --recurse-submodules -- "${hash_inputs[@]}" + git ls-files -z --others --exclude-standard -- "${hash_inputs[@]}" + git -C tools/yscope-dev-utils ls-files -z --others --exclude-standard \ + | while IFS= read -r -d '' rel_file; do + printf '%s\0' "tools/yscope-dev-utils/${rel_file}" + done + } \ + | sort -zu \ + | while IFS= read -r -d '' file; do + # Skip tracked files deleted in the working tree. Include untracked, + # non-ignored inputs so local image tags reflect pre-commit work. + if [[ -L "${file}" ]]; then + printf 'symlink %q ' "${file}" + readlink -- "${file}" + elif [[ -f "${file}" ]]; then + file_mode=$("${stat_mode_cmd[@]}" -- "${file}") + executable_mode=$((8#${file_mode} & 8#111)) + printf 'file %03o ' "${executable_mode}" + "${sha256_cmd[@]}" -- "${file}" + fi + done \ + | "${sha256_cmd[@]}" \ + | cut -c1-16 +) + +# Builds the build-env image and writes build progress to stderr. +# +# Args: +# $1 image tag — e.g. ghcr.io/owner/repo/build-env:env- +# $2 platform — linux/amd64 or linux/arm64 +# $3 output mode — --push or --load +build_image() { + if (( $# != 3 )) || [[ -z "$1" || -z "$2" || -z "$3" ]]; then + echo >&2 "ERROR: build_image requires an image tag, platform, and output mode" + return 2 + fi + local tag="$1" + local platform="$2" + local output_mode="$3" + + case "${platform}" in + linux/amd64|linux/arm64) ;; + *) echo >&2 "ERROR: unsupported build-env platform: ${platform}"; return 1 ;; + esac + case "${output_mode}" in + --push|--load) ;; + *) echo >&2 "ERROR: unsupported build-env output mode: ${output_mode}"; return 1 ;; + esac + + local ca_stage + ca_stage=$(mktemp -d) || return + ( + set -o errexit + trap 'rm -rf "${ca_stage}"' EXIT + + local ca_bundle="${ca_stage}/host-ca" + stage_host_ca_bundle "${ca_bundle}" + _ensure_build_env_submodules + + # A named context avoids BuildKit's 500 KiB secret limit while the + # Dockerfile mounts the bundle only into networked RUN steps. + docker buildx build \ + --platform "${platform}" \ + --build-context "host-ca=${ca_stage}" \ + --tag "${tag}" \ + "${output_mode}" \ + -f "${_REPO_ROOT}/tools/build-packages/dependency-image/Dockerfile" \ + "${_REPO_ROOT}" + ) >&2 +} diff --git a/tools/build-packages/package-specs/deb/clp-plugin-presto-connector.control.in b/tools/build-packages/package-specs/deb/clp-plugin-presto-connector.control.in new file mode 100644 index 0000000..25cfc2b --- /dev/null +++ b/tools/build-packages/package-specs/deb/clp-plugin-presto-connector.control.in @@ -0,0 +1,21 @@ +Package: clp-plugin-presto-connector +Version: ${deb_version} +Architecture: ${PKG_ARCH} +Section: database +Priority: optional +Maintainer: YScope Inc. +Homepage: https://github.com/y-scope/clp-plugin-presto-connector +Depends: libc6 (>= 2.28), libstdc++6 +Description: CLP plugins for Presto coordinator (Java) and Velox worker (C++) + Bundles both halves of the CLP Presto integration in a single package: + . + * Java JAR - Presto coordinator connector that exposes CLP datasets as + Presto tables. + * C++ .so - Velox worker plugin that pushes CLP-format scans down into + the native execution engine. + . + Compiled on manylinux_2_28 (glibc >= 2.28); compatible with Debian 11+, + Ubuntu 20.04+, and other glibc-based Debian-family distros. Non-system + native runtime libraries are bundled beside the Velox plugin and loaded + through relative RUNPATHs, so the package does not depend on distro + OpenSSL/libcurl versions. diff --git a/tools/build-packages/package-specs/rpm/clp-plugin-presto-connector.spec b/tools/build-packages/package-specs/rpm/clp-plugin-presto-connector.spec new file mode 100644 index 0000000..7d59165 --- /dev/null +++ b/tools/build-packages/package-specs/rpm/clp-plugin-presto-connector.spec @@ -0,0 +1,43 @@ +Name: clp-plugin-presto-connector +Version: %{pkg_version} +Release: %{pkg_release} +Summary: CLP plugins for Presto coordinator (Java) and Velox worker (C++) +License: Apache-2.0 +URL: https://github.com/y-scope/clp-plugin-presto-connector +Packager: YScope Inc. + +# internal/container/build-artifacts.sh passes the target arch to `rpmbuild --target` +# (x86_64 for amd64, aarch64 for arm64); no BuildArch: directive needed here. +# +# Loose deps to match the .deb's Depends:; don't let rpm auto-scan the JAR/.so. +AutoReqProv: no +Requires: glibc >= 2.28 +Requires: libstdc++ + +%description +Bundles both halves of the CLP Presto integration in a single package: + + * Java JAR - Presto coordinator connector that exposes CLP datasets as + Presto tables. + * C++ .so - Velox worker plugin that pushes CLP-format scans down into + the native execution engine. + +Compiled on manylinux_2_28 (glibc >= 2.28); compatible with RHEL 8+, +AlmaLinux 8+, Rocky 8+, Fedora 29+, and other glibc-based RPM distros. +Non-system native runtime libraries are bundled beside the Velox plugin and +loaded through relative RUNPATHs, so the package does not depend on distro +OpenSSL/libcurl versions. + +%install +cp -a %{payload_dir}/. %{buildroot}/ + +# Explicitly own every install dir, including the package root, so rpm +# tracks and removes all of them on uninstall. +%files +%dir %{plugin_root} +%dir %{presto_jar_dir} +%{presto_jar_dir}/clp-plugin-presto-connector.jar +%dir %{velox_so_dir} +%{velox_so_dir}/libclp-plugin-velox-connector.so +%dir %{velox_so_dir}/lib +%{velox_so_dir}/lib/*