From fceeda7dbafbceabbef06f1759324ccd99f10f94 Mon Sep 17 00:00:00 2001 From: NirWolfer Date: Wed, 30 Sep 2026 19:00:36 +0300 Subject: [PATCH] ci: authenticate github.com clones in the ci-demo-built base images Dockerfile.base (via .gitlab/build.sh and the vLLM clone) and the manylinux wheel_base clone their dependencies from github.com anonymously. github.com intermittently answers an anonymous clone with an HTTP 401; with no credential git tries to prompt, finds no TTY, and fails the image build as "could not read Username". These images are built by ci-demo outside any step, so their runs_on_dockers entries carry credentialsId: 'svc-nixl-github-read-only-token', which ci-demo binds around the build. A pipeline_on_image_build hook calls write_github_gitconfig (common.sh, added by the container-build PR) to write an http.extraheader git config at /tmp/ghconfig in the build pod, or an empty file when nothing is bound, and build_args hand it to docker build as the "ghconfig" secret. Dockerfile.base mounts it at ~/.gitconfig for the build.sh and vLLM RUNs, so it never lands in a layer; GIT_TERMINAL_PROMPT=0 is an ARG so it does not persist into the image. The hook runs under bash explicitly: common.sh uses $RANDOM, which a POSIX sh lacks. The token is a fine-grained one with read-only access to public repositories and no other permissions, since the build runs PR-controlled code with it available. Because the header is sent on every request, an invalid token fails every clone in these builds rather than falling back to anonymous. Unbound, the file is empty and the clones stay anonymous as before, so this is safe ahead of ci-demo support for image credentialsId. The manylinux wheel_base has mounted the secret since the container-build PR, so its clones are authenticated too. Signed-off-by: NirWolfer --- .ci/dockerfiles/Dockerfile.base | 8 ++++++-- .ci/docs/ci-overview.md | 24 +++++++++++++++++++++- .ci/jenkins/lib/build-matrix.yaml | 23 ++++++++++++++++++--- .ci/jenkins/lib/build-wheel-matrix.yaml | 15 +++++++++++++- .ci/jenkins/lib/test-dl-ep-matrix.yaml | 15 +++++++++++++- .ci/jenkins/lib/test-dl-matrix.yaml | 15 +++++++++++++- .ci/jenkins/lib/test-matrix.yaml | 15 +++++++++++++- .ci/jenkins/lib/test-sanitizer-matrix.yaml | 20 ++++++++++++++++-- .ci/jenkins/lib/test-vr-matrix.yaml | 15 +++++++++++++- 9 files changed, 137 insertions(+), 13 deletions(-) diff --git a/.ci/dockerfiles/Dockerfile.base b/.ci/dockerfiles/Dockerfile.base index a0d8c686b7..2655ef3d98 100644 --- a/.ci/dockerfiles/Dockerfile.base +++ b/.ci/dockerfiles/Dockerfile.base @@ -85,12 +85,16 @@ ENV HOME=${_HOME} ENV USER=${_LOGIN} ENV VLLM_ELASTIC_TEST_DIR=${_HOME}/vllm -RUN /.gitlab/build.sh ${NIXL_INSTALL_DIR} +# CI supplies a github.com credential as the optional "ghconfig" build secret +# (see pipeline_on_image_build in the matrices); without it these clones stay anonymous. +# mode=0444 because this stage runs as a non-root user. +ARG GIT_TERMINAL_PROMPT=0 +RUN --mount=type=secret,id=ghconfig,target=${_HOME}/.gitconfig,mode=0444 /.gitlab/build.sh ${NIXL_INSTALL_DIR} RUN sudo rm -rf /.gitlab/build.sh /.ci/scripts/ # Install editable vLLM source with matching CUDA 13 precompiled binaries. # VLLM_REF must name the release; never query GitHub for "latest". -RUN if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \ +RUN --mount=type=secret,id=ghconfig,target=${_HOME}/.gitconfig,mode=0444 if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \ UV="${HOME}/.local/bin/uv"; \ VENV_PY="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"; \ CUDA_VARIANT=cu130; \ diff --git a/.ci/docs/ci-overview.md b/.ci/docs/ci-overview.md index 664a0543bc..adf1a8ed2f 100644 --- a/.ci/docs/ci-overview.md +++ b/.ci/docs/ci-overview.md @@ -369,7 +369,29 @@ credential is bound. `build-wheel-matrix.yaml` needs nothing: it passes `--wheel-base-image`, which skips the `wheel_base` stage that holds the manylinux clones. That base, and `Dockerfile.base`, -are built by ci-demo and are not covered here. +are built by ci-demo; see the next section. + +## Authenticated clones in ci-demo-built images + +ci-demo builds the images declared with `file:` in a matrix (`Dockerfile.base`, +the manylinux `wheel_base`) outside any step. github.com intermittently answers +anonymous clones with an HTTP 401, which git reports as `could not read Username for +'https://github.com'` — the missing terminal git tried to prompt on, not DNS. + +- Each such `runs_on_dockers` entry sets `credentialsId: 'svc-nixl-github-read-only-token'`, + which ci-demo binds around that image's build. +- `pipeline_on_image_build` calls `write_github_gitconfig /tmp/ghconfig` + (`.ci/scripts/common.sh`) in the build pod: a git config whose + `http.https://github.com/.extraheader` authenticates every github.com request + (tracing off, mode 600), or an empty file when nothing is bound. +- The entry's `build_args` pass it as `--secret id=ghconfig,src=/tmp/ghconfig`, and + `Dockerfile.base` mounts it at `${_HOME}/.gitconfig` (`mode=0444`, non-root user) + on the `build.sh` and vLLM `RUN`s. It never lands in a layer or `podman history`. + `GIT_TERMINAL_PROMPT=0` is an `ARG`, so it does not persist into the image. + +An unbound or empty secret leaves the clones anonymous, as before. A revoked, expired +or org-blocked token, however, fails every github.com clone in these builds, since the +header is sent on every request. ## Related docs diff --git a/.ci/jenkins/lib/build-matrix.yaml b/.ci/jenkins/lib/build-matrix.yaml index 0c64450018..264fcdfd8f 100644 --- a/.ci/jenkins/lib/build-matrix.yaml +++ b/.ci/jenkins/lib/build-matrix.yaml @@ -49,18 +49,35 @@ env: # the "CI_MANAGED" placeholder must not be hand-edited. CI_IMAGE_TAG: "CI_MANAGED" +credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' + +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + runs_on_dockers: - { file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base-cuda12.9", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.06-cuda12.9-devel-ubuntu24.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true' + build_args: '--build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.06-cuda12.9-devel-ubuntu24.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - { file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - { file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base-ubuntu22", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/nvidia/cuda:13.0.1-devel-ubuntu22.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true' + build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/nvidia/cuda:13.0.1-devel-ubuntu22.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } matrix: diff --git a/.ci/jenkins/lib/build-wheel-matrix.yaml b/.ci/jenkins/lib/build-wheel-matrix.yaml index 39d13abe86..3328867bad 100644 --- a/.ci/jenkins/lib/build-wheel-matrix.yaml +++ b/.ci/jenkins/lib/build-wheel-matrix.yaml @@ -73,6 +73,9 @@ registry_auth: svc-nixl-new-artifactory-token registry_path: /sw-nbu-swx-nixl-docker-local/ci credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' - credentialsId: 'svc-nixl-new-artifactory-token' usernameVariable: 'ARTIFACTORY_USER' passwordVariable: 'ARTIFACTORY_TOKEN' @@ -98,6 +101,15 @@ empty_volumes: pvc_volumes: - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + # ============================================================================= # Docker Build Configuration # ============================================================================= @@ -119,7 +131,8 @@ runs_on_dockers: file: 'contrib/Dockerfile.manylinux', name: "${WHEEL_BASE_IMAGE_NAME}", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg ARCH=${arch} --build-arg NPROC=${NPROC} --build-arg GRPC_NPROC=${GRPC_NPROC} --target wheel_base' + build_args: '--build-arg ARCH=${arch} --build-arg NPROC=${NPROC} --build-arg GRPC_NPROC=${GRPC_NPROC} --target wheel_base --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } # The custom 'fw' key is injected as an env var into every step of the branch # (ci-demo merges the docker entry into the branch axis), so the shared sanity diff --git a/.ci/jenkins/lib/test-dl-ep-matrix.yaml b/.ci/jenkins/lib/test-dl-ep-matrix.yaml index 5312d53199..93d4bbad0c 100644 --- a/.ci/jenkins/lib/test-dl-ep-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-ep-matrix.yaml @@ -43,6 +43,9 @@ kubernetes: privileged: true credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'} env: @@ -72,6 +75,15 @@ empty_volumes: pvc_volumes: - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + # Docker images for DL EP testing. # Dedicated EP base + build_helper images (separate artifactory paths from the # nixl-ci-dl-gpu job). @@ -81,7 +93,8 @@ runs_on_dockers: name: 'nixl-ci-dl-gpu-ep-base', tag: "${CI_IMAGE_TAG}", arch: "aarch64", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg VLLM_REF=v0.30.0 --build-arg ARCH=${arch} --pull --no-cache' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg VLLM_REF=v0.30.0 --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - { diff --git a/.ci/jenkins/lib/test-dl-matrix.yaml b/.ci/jenkins/lib/test-dl-matrix.yaml index 1a6aa7521b..5db8c5a496 100644 --- a/.ci/jenkins/lib/test-dl-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-matrix.yaml @@ -37,6 +37,9 @@ kubernetes: privileged: true credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'} env: @@ -64,6 +67,15 @@ empty_volumes: pvc_volumes: - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + # Docker images for DL testing runs_on_dockers: - { @@ -71,7 +83,8 @@ runs_on_dockers: name: 'nixl-ci-dl-gpu-base', tag: "${CI_IMAGE_TAG}", arch: "aarch64", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - { diff --git a/.ci/jenkins/lib/test-matrix.yaml b/.ci/jenkins/lib/test-matrix.yaml index 2a927b7ad4..8574578e09 100644 --- a/.ci/jenkins/lib/test-matrix.yaml +++ b/.ci/jenkins/lib/test-matrix.yaml @@ -37,6 +37,9 @@ kubernetes: privileged: true credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'} env: @@ -67,13 +70,23 @@ empty_volumes: pvc_volumes: - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + # label is defined at jenkins slave configuration, we want to run the job on a gpu agent and be able to esaly replace it without having to change this file runs_on_dockers: - { file: '.ci/dockerfiles/Dockerfile.base', name: 'nixl-ci-gpu-base', tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - { file: '.ci/dockerfiles/Dockerfile.build_helper', diff --git a/.ci/jenkins/lib/test-sanitizer-matrix.yaml b/.ci/jenkins/lib/test-sanitizer-matrix.yaml index 9b8fe625b1..90201e203d 100644 --- a/.ci/jenkins/lib/test-sanitizer-matrix.yaml +++ b/.ci/jenkins/lib/test-sanitizer-matrix.yaml @@ -45,17 +45,33 @@ env: STORAGE_DRIVER: 'overlay' CI_IMAGE_TAG: "CI_MANAGED" +credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' + +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + runs_on_dockers: # TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread # so abseil's absl::Mutex annotations are visible to ThreadSanitizer). - { file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-test-sanitizers-base-tsan", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=thread' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=thread --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } # ASan-instrumented dependency image for the asan_ubsan build. - { file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-test-sanitizers-base-asan", tag: "${CI_IMAGE_TAG}", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=address' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=address --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } matrix: diff --git a/.ci/jenkins/lib/test-vr-matrix.yaml b/.ci/jenkins/lib/test-vr-matrix.yaml index aee3596b21..3e0b497958 100644 --- a/.ci/jenkins/lib/test-vr-matrix.yaml +++ b/.ci/jenkins/lib/test-vr-matrix.yaml @@ -45,6 +45,9 @@ kubernetes: privileged: true credentials: + - credentialsId: 'svc-nixl-github-read-only-token' + usernameVariable: 'NIXL_GITHUB_USER' + passwordVariable: 'NIXL_GITHUB_TOKEN' - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'} env: @@ -73,6 +76,15 @@ empty_volumes: pvc_volumes: - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} +# ci-demo builds the images below outside any step; an entry's credentialsId binds +# the token around its build, and this hook writes the git config the Dockerfile +# mounts as the "ghconfig" secret. Left empty when nothing is bound. +pipeline_on_image_build: + run: | + #!/bin/bash -e + . .ci/scripts/common.sh + write_github_gitconfig /tmp/ghconfig + # Docker images for DL testing runs_on_dockers: - { @@ -80,7 +92,8 @@ runs_on_dockers: name: 'nixl-ci-dl-gpu-base', tag: "${CI_IMAGE_TAG}", arch: "aarch64", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig', + credentialsId: 'svc-nixl-github-read-only-token' } - {