diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..95fd1afe51 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +# Self-hosted runner labels used by this repo's workflows so actionlint does +# not flag them as unknown. The prod-nixl-*/stg-nixl-* runners are velonix ARC +# runner scale sets (see velonix flux-apps/.../runner-scale-sets/nixl). +self-hosted-runner: + labels: + - gitlab + - blossom + - prod-nixl-builder-amd-v1 + - prod-nixl-builder-arm-v1 + - prod-nixl-tester-gpu-v1 + - stg-nixl-builder-amd-v1 + - stg-nixl-builder-arm-v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..d05dfc8d1b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,345 @@ +name: NIXL CI + +# Native GitHub Actions replacement for the GitLab pipeline that previously ran +# in nixl-ci (.gitlab-ci.yml). Builds run on self-hosted velonix ARC runners +# (prod-nixl-builder-amd-v1 / prod-nixl-builder-arm-v1), which provide an +# in-pod Docker daemon (dind sidecar), so the build/test docker commands below +# work just as they did under GitLab. +# +# Repository configuration required (Settings -> Secrets and variables -> Actions): +# Variables: +# NIXL_ECR_IMAGE - ECR image base, e.g. +# 210086341041.dkr.ecr.us-west-2.amazonaws.com/nixl-ci +# ENABLE_GPU_CI - set to "true" to enable the (currently deferred) +# GPU test/verify jobs once GPU runners exist. +# Secrets: +# GITLAB_REGISTRY_USER - user for gitlab-master.nvidia.com:5005 (manylinux +# GITLAB_REGISTRY_TOKEN base images + wheeltamer scan image) +# ARTIFACTORY_URL - JFrog Artifactory base URL (release uploads) +# ARTIFACTORY_PYPI_TOKEN +# ARTIFACTORY_CARGO_TOKEN +# AWS/ECR push auth comes from the runner pod's IRSA service account, not a secret. + +on: + pull_request: + push: + branches: [main, 'release/**'] + tags: ['v*'] + workflow_dispatch: + inputs: + release_build: + description: "Build/publish release artifacts (maps to GitLab RELEASE_BUILD)" + type: boolean + default: false + security_scan: + description: "Run the wheel security scan (maps to GitLab SECURITY_SCAN)" + type: boolean + default: false + +permissions: + contents: read + +# Cancel superseded runs on the same ref. PRs cancel-in-progress (latest push +# wins); release/** + main pushes do NOT cancel, so an in-flight RC upload isn't +# interrupted mid-publish. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + AWS_REGION: us-west-2 + REPO_NAME: nixl + WHL_PYTHON_VERSIONS: "3.10,3.11,3.12,3.13,3.14" + IMAGE_BASE: ${{ vars.NIXL_ECR_IMAGE }} + # Release flag, normalized to a plain "true"/"false" string usable in shells. + # True on a push to a release/** branch (e.g. a PR merged into release/1.3.0 triggers + # RC generation on the merge commit) or an explicit workflow_dispatch release_build. + RELEASE_BUILD: ${{ github.event.inputs.release_build == true || github.event.inputs.release_build == 'true' || startsWith(github.ref, 'refs/heads/release/') }} + +jobs: + # ---------------------------------------------------------------------------- + # version: replicate the GitLab before_script version computation. + # ---------------------------------------------------------------------------- + version: + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-amd-v1 + outputs: + version: ${{ steps.compute.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Compute version + id: compute + run: | + set -e + git fetch --tags --force || true + RELEASE_TAG=$(git tag --sort=-v:refname | head -n 1 | sed 's/^v//' | tr -d '\n') + if [ -z "$RELEASE_TAG" ]; then RELEASE_TAG="0.0.1"; fi + if [ "${RELEASE_BUILD}" != "true" ]; then + BASE_VERSION=$(echo "$RELEASE_TAG" | awk -F. '{$NF = $NF + 1;} 1' OFS=.) + VERSION="${BASE_VERSION}.dev${{ github.run_id }}+$(git rev-parse --short HEAD)" + else + VERSION="${RELEASE_TAG}" + fi + echo -n "$VERSION" > version.txt + echo "Computed VERSION=$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v4 + with: + name: version + path: version.txt + retention-days: 1 + + # ---------------------------------------------------------------------------- + # build: five container builds (the GitLab build stage), pushed to ECR with a + # unique per-variant tag; dist/ extracted and uploaded as an artifact. + # ---------------------------------------------------------------------------- + build: + needs: version + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-${{ matrix.runner }}-v1 + timeout-minutes: 120 # ARM manylinux builds everything from source (~60min); was timing out at the push step + strategy: + fail-fast: false + matrix: + include: + - name: build-nixl + dockerfile: contrib/Dockerfile + base_image: nvcr.io/nvidia/cuda-dl-base + base_image_tag: 25.06-cuda12.9-devel-ubuntu24.04 + whl_base: manylinux_2_39 + cuda_version: "12.9" + arch: x86_64 + runner: amd + # Option B: manylinux jobs build on the public PyPA manylinux_2_28 base + # (Dockerfile.manylinux) and pull CUDA from a public NGC image — no GitLab. + # VERIFY the nvcr.io/nvidia/cuda el8/ubi8 devel tags below actually exist. + - name: build-nixl-manylinux + dockerfile: contrib/Dockerfile.manylinux + base_image: nvcr.io/nvidia/cuda + base_image_tag: 12.9.1-devel-ubi8 + whl_base: manylinux_2_28 + cuda_version: "12.9" + arch: x86_64 + runner: amd + - name: build-nixl-manylinux-cuda13 + dockerfile: contrib/Dockerfile.manylinux + base_image: nvcr.io/nvidia/cuda + base_image_tag: 13.0.1-devel-ubi8 + whl_base: manylinux_2_28 + cuda_version: "13.0" + arch: x86_64 + runner: amd + - name: build-nixl-arm-manylinux + dockerfile: contrib/Dockerfile.manylinux + base_image: nvcr.io/nvidia/cuda + base_image_tag: 12.9.1-devel-ubi8 + whl_base: manylinux_2_28 + cuda_version: "12.9" + arch: aarch64 + runner: arm + - name: build-nixl-arm-manylinux-cuda13 + dockerfile: contrib/Dockerfile.manylinux + base_image: nvcr.io/nvidia/cuda + base_image_tag: 13.0.1-devel-ubi8 + whl_base: manylinux_2_28 + cuda_version: "13.0" + arch: aarch64 + runner: arm + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Log in to ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Build and push image + run: | + set -e + IMAGE_NAME="${IMAGE_BASE}:${{ matrix.name }}-${{ github.sha }}-${{ github.run_id }}" + echo "IMAGE_NAME=$IMAGE_NAME" >> "$GITHUB_ENV" + chmod +x contrib/build-container.sh + bash contrib/build-container.sh \ + --base-image "${{ matrix.base_image }}" \ + --base-image-tag "${{ matrix.base_image_tag }}" \ + --cuda-version "${{ matrix.cuda_version }}" \ + --wheel-base "${{ matrix.whl_base }}" \ + --python-versions "${WHL_PYTHON_VERSIONS}" \ + --tag "${IMAGE_NAME}" \ + --os "ubuntu24" \ + --arch "${{ matrix.arch }}" \ + --dockerfile "${{ matrix.dockerfile }}" + docker push "$IMAGE_NAME" + - name: Extract build artifacts + run: | + set -e + CN="nixl-extract-${{ github.run_id }}-${{ strategy.job-index }}" + docker rm -f "$CN" || true + docker create --name "$CN" "$IMAGE_NAME" + # Don't mask a build that produced no wheels: fail if dist is absent/empty. + docker cp "$CN:/workspace/nixl/dist" ./dist + docker cp "$CN:/usr/local/nixl" ./nixl_install || true + docker rm -f "$CN" || true + ls dist/*.whl >/dev/null 2>&1 || { echo "ERROR: no wheels in dist/"; exit 1; } + - uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.name }} + path: dist + retention-days: 1 + if-no-files-found: error + + # ---------------------------------------------------------------------------- + # scan-wheels: GitLab "security scan" stage. Runs only on release builds or + # when security_scan is explicitly requested. + # ---------------------------------------------------------------------------- + scan-wheels: + needs: build + if: ${{ github.event.inputs.security_scan == 'true' || github.event.inputs.release_build == 'true' || startsWith(github.ref, 'refs/heads/release/') }} + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-amd-v1 + env: + PACKAGE_LICENSE: "Apache-2.0" + SKIPPED_SECURITY_RULES: "B404,B603,B108" + ALLOWED_NOSEC_COUNT: "0" + IGNORE_FAILED_PIP_INSTALL: "1" + steps: + - name: Download manylinux wheels + uses: actions/download-artifact@v4 + with: + name: dist-build-nixl-manylinux + path: dist + - name: Log in to ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Scan wheels with wheeltamer + run: | + set -e + # wheeltamer is mirrored into ECR (gitlab-master is unreachable from the + # AWS runners). Mirror with: crane copy + # gitlab-master.nvidia.com:5005/dl/pypi/wheel-ci-cd:wheeltamer + # ${NIXL_ECR_IMAGE}:wheeltamer + CN="wheeltamer_${{ github.run_id }}" + docker rm -f "$CN" || true + docker create --name="$CN" \ + -e EXPECTED_PKG_LICENSE="${PACKAGE_LICENSE}" \ + -e SKIPPED_SECURITY_RULES="${SKIPPED_SECURITY_RULES}" \ + -e ALLOWED_NOSEC_COUNT="${ALLOWED_NOSEC_COUNT}" \ + -e IGNORE_FAILED_PIP_INSTALL="${IGNORE_FAILED_PIP_INSTALL}" \ + --pull=always \ + "${{ vars.NIXL_ECR_IMAGE }}:wheeltamer" + docker cp "$PWD/dist/." "$CN:/workspace" + docker start -a "$CN" + - name: Cleanup + if: always() + run: docker rm -f "wheeltamer_${{ github.run_id }}" || true + + # ---------------------------------------------------------------------------- + # upload: GitLab upload stage. Release-only. Wheels -> Artifactory (JFrog CLI), + # crates -> Artifactory cargo registry (manual approval via environment). + # ---------------------------------------------------------------------------- + upload-x86-wheels: + needs: build + if: ${{ github.event.inputs.release_build == 'true' || startsWith(github.ref, 'refs/heads/release/') }} + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-amd-v1 + timeout-minutes: 30 + env: + ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} + ARTIFACTORY_PYPI_TOKEN: ${{ secrets.ARTIFACTORY_PYPI_TOKEN }} + ARCH: x86_64 + steps: + - name: Download x86 wheels + uses: actions/download-artifact@v4 + with: + pattern: dist-build-nixl-manylinux* + path: dist + merge-multiple: true + - name: Upload wheels to Artifactory + run: | + set -e + cd dist + ls -la *.whl + WHEEL_VERSION=$(ls nixl*.whl | head -n 1 | cut -d'-' -f2) + CN="upload_nixl_build_${{ github.run_id }}" + docker rm -f "$CN" || true + docker create --name "$CN" -w /workspace -e CI=true -e JFROG_CLI_LOG_LEVEL=INFO \ + -e ARTIFACTORY_PYPI_TOKEN -e ARTIFACTORY_URL \ + releases-docker.jfrog.io/jfrog/jfrog-cli-v2-jf bash -c " + TARGET_PROPS=\"CI_PIPELINE_ID=${{ github.run_id }};component_name=nixl;os=linux;arch=${ARCH};version=${WHEEL_VERSION}\" && + jf rt upload '*.whl' 'sw-dynamo-nixl-pypi-local/release/${WHEEL_VERSION}/${{ github.run_id }}/${ARCH}/' \ + --target-props=\"\$TARGET_PROPS\" \ + --access-token \"\$ARTIFACTORY_PYPI_TOKEN\" --url \"\$ARTIFACTORY_URL\" \ + --flat --fail-no-op=true --detailed-summary + " + docker cp . "$CN:/workspace/" + docker start -a "$CN" + - name: Cleanup + if: always() + run: docker rm -f "upload_nixl_build_${{ github.run_id }}" || true + + upload-arm-wheels: + needs: build + if: ${{ github.event.inputs.release_build == 'true' || startsWith(github.ref, 'refs/heads/release/') }} + # amd runner: the jfrog-cli-v2-jf image is amd64-only. This job only uploads the + # already-built arm wheel files (downloaded as artifacts), so the host arch is irrelevant. + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-amd-v1 + timeout-minutes: 30 + env: + ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} + ARTIFACTORY_PYPI_TOKEN: ${{ secrets.ARTIFACTORY_PYPI_TOKEN }} + ARCH: aarch64 + steps: + - name: Download arm wheels + uses: actions/download-artifact@v4 + with: + pattern: dist-build-nixl-arm-manylinux* + path: dist + merge-multiple: true + - name: Upload wheels to Artifactory + run: | + set -e + cd dist + ls -la *.whl + WHEEL_VERSION=$(ls nixl*.whl | head -n 1 | cut -d'-' -f2) + CN="upload_arm_nixl_build_${{ github.run_id }}" + docker rm -f "$CN" || true + docker create --name "$CN" -w /workspace -e CI=true -e JFROG_CLI_LOG_LEVEL=INFO \ + -e ARTIFACTORY_PYPI_TOKEN -e ARTIFACTORY_URL \ + releases-docker.jfrog.io/jfrog/jfrog-cli-v2-jf bash -c " + TARGET_PROPS=\"CI_PIPELINE_ID=${{ github.run_id }};component_name=nixl;os=linux;arch=${ARCH};version=${WHEEL_VERSION}\" && + jf rt upload '*.whl' 'sw-dynamo-nixl-pypi-local/release/${WHEEL_VERSION}/${{ github.run_id }}/${ARCH}/' \ + --target-props=\"\$TARGET_PROPS\" \ + --access-token \"\$ARTIFACTORY_PYPI_TOKEN\" --url \"\$ARTIFACTORY_URL\" \ + --flat --fail-no-op=true --detailed-summary + " + docker cp . "$CN:/workspace/" + docker start -a "$CN" + - name: Cleanup + if: always() + run: docker rm -f "upload_arm_nixl_build_${{ github.run_id }}" || true + + upload-crates: + needs: build + # GitLab marked this job `when: manual` on release builds. The `release` + # environment provides the equivalent manual approval gate (configure + # required reviewers under Settings -> Environments -> release). + if: ${{ github.event.inputs.release_build == 'true' || startsWith(github.ref, 'refs/heads/release/') }} + runs-on: ${{ vars.NIXL_RUNNER_PREFIX || 'prod' }}-nixl-builder-amd-v1 + environment: release + env: + ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} + ARTIFACTORY_CARGO_TOKEN: ${{ secrets.ARTIFACTORY_CARGO_TOKEN }} + steps: + - name: Log in to ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Publish crates to Artifactory + run: | + set -e + IMAGE_NAME="${IMAGE_BASE}:build-nixl-${{ github.sha }}-${{ github.run_id }}" + docker run -e ARTIFACTORY_CARGO_TOKEN -e ARTIFACTORY_URL \ + -e CI_PIPELINE_ID="${{ github.run_id }}" "$IMAGE_NAME" /bin/bash -c "set -e && + grep '^version = ' Cargo.toml && + sed -i -E 's/^(version = \"([^\"]+)\")/version = \"\2-rc.${{ github.run_id }}\"/' Cargo.toml && + grep '^version = ' Cargo.toml && + cargo check --manifest-path src/bindings/rust/Cargo.toml && + cargo publish --manifest-path src/bindings/rust/Cargo.toml \ + --token \"Bearer \$ARTIFACTORY_CARGO_TOKEN\" \ + --index \"sparse+\$ARTIFACTORY_URL/api/cargo/sw-dynamo-nixl-cargo-local/index/\" \ + --no-verify --allow-dirty" diff --git a/contrib/Dockerfile.manylinux b/contrib/Dockerfile.manylinux index 5ab1f7a09d..594a4be209 100644 --- a/contrib/Dockerfile.manylinux +++ b/contrib/Dockerfile.manylinux @@ -14,12 +14,93 @@ # limitations under the License. -ARG BASE_IMAGE -ARG BASE_IMAGE_TAG -FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} +# === Option B (DRAFT, UNVALIDATED) =========================================== +# Build wheels on the PUBLIC PyPA manylinux_2_28 image and bring CUDA in from a +# public NGC CUDA image, instead of the internal +# gitlab-master.nvidia.com:5005/dl/dgx/cuda manylinux image (NOT reachable from +# the velonix AWS runners — no DNS/PrivateLink). Mirrors dynamo's +# container/templates/wheel_builder.Dockerfile pattern. Validate on the PR before +# relying on it (gcc-toolset for CUDA, the CUDA COPY, DOCA el8 rpms). +# ARCH must be declared before the FROM lines that interpolate it. +ARG ARCH="x86_64" -ARG DEFAULT_PYTHON_VERSION="3.14" +# CUDA toolkit provider: a public NGC CUDA *devel* image, el8/ubi8 to match the +# AlmaLinux-8 manylinux base (e.g. nvcr.io/nvidia/cuda:12.9.1-devel-ubi8). +# Reachable from the AWS runners; replaces the internal GitLab base image. +ARG BASE_IMAGE="nvcr.io/nvidia/cuda" +ARG BASE_IMAGE_TAG +# INFINIA libs image — declared here in the global (pre-first-FROM) scope because an ARG +# used in a FROM line is only visible if declared before the first FROM. See the INFINIA +# block below for what this is and how it's mirrored. +ARG INFINIA_LIBS_IMAGE="210086341041.dkr.ecr.us-west-2.amazonaws.com/nixl/infinia-libs:v2.4.0-beta.1" +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS cuda + +# --- INFINIA plugin libs (DDN "red") — PRE-STAGED, enable once mirrored to ECR ---- +# The INFINIA plugin only builds when libred_client.so is found at /opt/ddn/red/lib +# (src/plugins/meson.build does cc.find_library('red_client', dirs:['/opt/ddn/red/lib']) +# and silently skips the plugin otherwise — which is why it's missing from the wheel). +# The libs live in harbor.mellanox.com/nixl/infinia-libs (NOT reachable from the AWS CI +# runners), so mirror that image into ECR — one time per INFINIA version, from a host +# that sees both harbor and AWS: +# docker pull harbor.mellanox.com/nixl/infinia-libs:v2.4.0-beta.1 +# ECR=210086341041.dkr.ecr.us-west-2.amazonaws.com/nixl/infinia-libs +# docker tag harbor.mellanox.com/nixl/infinia-libs:v2.4.0-beta.1 $ECR:v2.4.0-beta.1 +# aws ecr get-login-password --region us-west-2 | docker login --username AWS \ +# --password-stdin 210086341041.dkr.ecr.us-west-2.amazonaws.com +# docker push $ECR:v2.4.0-beta.1 +# The dedicated nixl/infinia-libs ECR repo (no-delete lifecycle) is provisioned in +# velonix terraform/aws/envs/staging/ecr.tf. The runner's existing IRSA already grants +# pull (account-wide ECR), so no workflow/IAM change is needed. Bump the tag when DDN +# bumps the INFINIA version (and re-run the mirror push). The ARG is declared in the +# global pre-FROM block at the top of this file (required to use it in a FROM line). +FROM ${INFINIA_LIBS_IMAGE} AS infinia + +# Manylinux build base — official PyPA image (public on quay.io; velonix also has +# a quay ECR pull-through). AlmaLinux 8 / glibc 2.28 => manylinux_2_28 wheels. +FROM quay.io/pypa/manylinux_2_28_${ARCH} + +# Re-declare args needed inside this build stage (args before the first FROM are +# only visible to FROM lines). ARG ARCH="x86_64" +ARG DEFAULT_PYTHON_VERSION="3.14" + +# CUDA version (e.g. 12.9 / 13.0) — was provided as an ENV by the old GitLab base; +# now passed explicitly. Used for the torch cuXXX index and the cu12/cu13 wheel split. +ARG CUDA_VERSION +ENV CUDA_VERSION=${CUDA_VERSION} + +# Bring the CUDA toolkit onto the manylinux base (dynamo wheel_builder pattern). +COPY --from=cuda /usr/local/cuda /usr/local/cuda +ENV CUDA_HOME=/usr/local/cuda \ + PATH=/usr/local/cuda/bin:${PATH} \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH:-} + +# INFINIA libs (paired with the `infinia` stage above). The mirrored image lays the libs +# out per-arch under /infinia/${ARCH}/ (lib/ + include/); copying that to /opt/ddn/red +# puts libred_client.so / libred_async.so + headers where meson looks, so it auto-detects +# and builds the INFINIA plugin into the wheel (default -Dinfinia_path=/opt/ddn/red). +# NOTE: libred_client/libred_async are DDN proprietary and are intentionally NOT +# redistributed — contrib/build-wheel.sh already auditwheel --excludes them, so the wheel +# ships only the INFINIA plugin (libplugin_INFINIA.so), which loads libred_* at runtime +# from a customer's DDN install via its RUNPATH (/opt/ddn/red/lib). The plugin's other +# transitive deps (liburing.so.2, libldap.so.2, etc.) are why this Dockerfile installs/ +# builds them above: they must resolve at link time even though libred itself is excluded. +ARG INFINIA_DEST=/opt/ddn/red +COPY --from=infinia /infinia/${ARCH}/ ${INFINIA_DEST}/ + +# CUDA needs gcc <= 14. The PyPA manylinux base + system "Development Tools" can +# pull in a gcc that CUDA rejects, so pin gcc-toolset-14 on PATH ahead of the +# system gcc (same approach as dynamo's wheel_builder). gcc-toolset-14 is added to +# the dnf install list below. +ENV PATH=/opt/rh/gcc-toolset-14/root/usr/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/rh/gcc-toolset-14/root/usr/lib64:${LD_LIBRARY_PATH:-} + +# The PyPA manylinux_2_28 image ships CMake 4.x, which removed support for +# cmake_minimum_required(VERSION < 3.5). Some bundled deps (e.g. gRPC's c-ares) +# still declare ancient minimums. This tells CMake 4.x to accept them. (Harmless +# on the older CMake that the previous GitLab base shipped.) +ENV CMAKE_POLICY_VERSION_MINIMUM=3.5 +# === end Option B header ====================================================== ARG LIBFABRIC_VERSION="v1.21.0" ARG HWLOC_VERSION="2.12.2" ARG BUILD_TYPE="release" @@ -29,7 +110,7 @@ ARG GRPC_TAG="v1.73.0" ARG NPROC RUN yum groupinstall -y 'Development Tools' && \ - dnf install -y almalinux-release-synergy && \ + dnf install -y almalinux-release-synergy epel-release && \ dnf config-manager --set-enabled powertools && \ dnf install -y \ boost \ @@ -42,9 +123,14 @@ RUN yum groupinstall -y 'Development Tools' && \ gflags \ glibc-headers \ gcc-c++ \ + gcc-toolset-14 \ libaio \ libaio-devel \ libtool-ltdl \ + libuuid-devel \ + jansson \ + libjose \ + xxhash-libs \ ninja-build \ openssl \ openssl-devel \ @@ -97,6 +183,43 @@ ENV OPENSSL_ROOT_DIR="/usr/local/openssl3" ENV OPENSSL_LIBRARIES="/usr/local/openssl3/lib64:/usr/local/openssl3/lib" ENV OPENSSL_INCLUDE_DIR="/usr/local/openssl3/include" +# --- INFINIA (libred_client) transitive deps not packaged at the right soname on EL8 --- +# liburing 2.x: EL8 only ships liburing.so.1 (1.0.7); libred_client needs liburing.so.2. +RUN cd /tmp && \ + LIBURING_VERSION=2.6 && \ + wget -q "https://github.com/axboe/liburing/archive/refs/tags/liburing-${LIBURING_VERSION}.tar.gz" && \ + tar -xzf "liburing-${LIBURING_VERSION}.tar.gz" && \ + cd "liburing-liburing-${LIBURING_VERSION}" && \ + ./configure --prefix=/usr/local && \ + make -j"${NPROC:-$(nproc)}" && \ + make install && \ + echo "/usr/local/lib" > /etc/ld.so.conf.d/usrlocal.conf && \ + echo "/usr/local/lib64" >> /etc/ld.so.conf.d/usrlocal.conf && \ + ldconfig && \ + rm -rf /tmp/liburing* + +# OpenLDAP 2.6 client libs: EL8 ships 2.4 (libldap-2.4.so.2); libred_client needs the +# 2.6 soname libldap.so.2 (+ liblber.so.2). Build client libs only, against openssl3. +RUN cd /tmp && \ + OPENLDAP_VERSION=2.6.8 && \ + wget -q "https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${OPENLDAP_VERSION}.tgz" && \ + tar -xzf "openldap-${OPENLDAP_VERSION}.tgz" && \ + cd "openldap-${OPENLDAP_VERSION}" && \ + ./configure --prefix=/usr/local --enable-shared --disable-static --disable-slapd \ + --without-cyrus-sasl --with-tls=openssl \ + CPPFLAGS="-I/usr/local/openssl3/include" \ + LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + make depend && \ + make -j"${NPROC:-$(nproc)}" && \ + make install && \ + ldconfig && \ + rm -rf /tmp/openldap-${OPENLDAP_VERSION}* + +# Let the linker (gcc LIBRARY_PATH) and loader/auditwheel (LD_LIBRARY_PATH) find the +# INFINIA libs at /opt/ddn/red/lib and the source-built transitive deps in /usr/local. +ENV LD_LIBRARY_PATH="/opt/ddn/red/lib:/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" +ENV LIBRARY_PATH="/opt/ddn/red/lib:/usr/local/lib:/usr/local/lib64:${LIBRARY_PATH:-}" + WORKDIR /workspace # Build Abseil from source (required for absl/log). diff --git a/contrib/build-container.sh b/contrib/build-container.sh index f99400e0a4..d5c6a2e7de 100755 --- a/contrib/build-container.sh +++ b/contrib/build-container.sh @@ -41,6 +41,9 @@ OS="ubuntu24" NPROC=${NPROC:-$(nproc)} GRPC_NPROC=${GRPC_NPROC:-$(nproc)} BUILD_TYPE="release" +# CUDA toolkit version (e.g. 12.9 / 13.0). Option B (manylinux on public PyPA base) +# no longer inherits this from the base image's ENV, so it must be passed in. +CUDA_VERSION=${CUDA_VERSION:-} get_options() { while :; do @@ -92,6 +95,14 @@ get_options() { missing_requirement $1 fi ;; + --cuda-version) + if [ "$2" ]; then + CUDA_VERSION=$2 + shift + else + missing_requirement $1 + fi + ;; --tag) if [ "$2" ]; then TAG="--tag $2" @@ -187,6 +198,7 @@ show_help() { echo "usage: build-container.sh" echo " [--base base image]" echo " [--base-image-tag base image tag]" + echo " [--cuda-version CUDA version, e.g. 12.9 (manylinux builds)]" echo " [--wheel-base base platform for wheel builds]" echo " [--no-cache disable docker build cache]" echo " [--os [ubuntu24|ubuntu22] to select Ubuntu version]" @@ -217,6 +229,7 @@ if [ -d "$NIXL_DIR/build" ]; then fi BUILD_ARGS+=" --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg BASE_IMAGE_TAG=$BASE_IMAGE_TAG" +BUILD_ARGS+=" --build-arg CUDA_VERSION=$CUDA_VERSION" BUILD_ARGS+=" --build-arg WHL_PYTHON_VERSIONS=$WHL_PYTHON_VERSIONS" BUILD_ARGS+=" --build-arg WHL_PLATFORM=$WHL_PLATFORM" BUILD_ARGS+=" --build-arg ARCH=$ARCH" @@ -229,4 +242,7 @@ BUILD_ARGS+=" --build-arg BUILD_TYPE=$BUILD_TYPE" show_build_options -docker build --platform linux/$ARCH -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT +# --provenance/--sbom default to true under buildx and add a slow attestation-manifest +# export/unpack step at push time (which was tipping the ARM build over the job timeout). +# CI images don't need attestations, so disable them. +docker build --provenance=false --sbom=false --platform linux/$ARCH -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT diff --git a/meson.build b/meson.build index 996c592747..7e96528197 100644 --- a/meson.build +++ b/meson.build @@ -445,7 +445,11 @@ plugins_inc_dirs = include_directories('src/plugins') utils_inc_dirs = include_directories('src/utils') subdir('src') -if get_option('build_tests') and get_option('buildtype') != 'release' +# Build tests whenever build_tests is set. (Previously also required a non-release +# buildtype, but CI builds the image as --buildtype=release and test_cpp.sh runs the +# installed test binaries, e.g. /usr/local/nixl/bin/desc_example — so release builds +# must include them.) +if get_option('build_tests') subdir('test') endif