Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .ci/dockerfiles/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,16 @@ ENV HOME=${_HOME}
ENV USER=${_LOGIN}
ENV VLLM_ELASTIC_TEST_DIR=${_HOME}/vllm

RUN /.gitlab/build.sh ${NIXL_INSTALL_DIR}
# CI supplies a github.com credential as the optional "ghconfig" build secret
# (see pipeline_on_image_build in the matrices); without it these clones stay anonymous.
# mode=0444 because this stage runs as a non-root user.
ARG GIT_TERMINAL_PROMPT=0
Comment thread
NirWolfer marked this conversation as resolved.
RUN --mount=type=secret,id=ghconfig,target=${_HOME}/.gitconfig,mode=0444 /.gitlab/build.sh ${NIXL_INSTALL_DIR}
RUN sudo rm -rf /.gitlab/build.sh /.ci/scripts/

# Install editable vLLM source with matching CUDA 13 precompiled binaries.
# VLLM_REF must name the release; never query GitHub for "latest".
RUN if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \
RUN --mount=type=secret,id=ghconfig,target=${_HOME}/.gitconfig,mode=0444 if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \
UV="${HOME}/.local/bin/uv"; \
VENV_PY="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"; \
CUDA_VARIANT=cu130; \
Expand Down
24 changes: 23 additions & 1 deletion .ci/docs/ci-overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,29 @@ credential is bound.

`build-wheel-matrix.yaml` needs nothing: it passes `--wheel-base-image`, which skips the
`wheel_base` stage that holds the manylinux clones. That base, and `Dockerfile.base`,
are built by ci-demo and are not covered here.
are built by ci-demo; see the next section.

## Authenticated clones in ci-demo-built images

ci-demo builds the images declared with `file:` in a matrix (`Dockerfile.base`,
the manylinux `wheel_base`) outside any step. github.com intermittently answers
anonymous clones with an HTTP 401, which git reports as `could not read Username for
'https://github.com'` — the missing terminal git tried to prompt on, not DNS.

- Each such `runs_on_dockers` entry sets `credentialsId: 'svc-nixl-github-read-only-token'`,
which ci-demo binds around that image's build.
- `pipeline_on_image_build` calls `write_github_gitconfig /tmp/ghconfig`
(`.ci/scripts/common.sh`) in the build pod: a git config whose
`http.https://github.com/.extraheader` authenticates every github.com request
(tracing off, mode 600), or an empty file when nothing is bound.
- The entry's `build_args` pass it as `--secret id=ghconfig,src=/tmp/ghconfig`, and
`Dockerfile.base` mounts it at `${_HOME}/.gitconfig` (`mode=0444`, non-root user)
on the `build.sh` and vLLM `RUN`s. It never lands in a layer or `podman history`.
`GIT_TERMINAL_PROMPT=0` is an `ARG`, so it does not persist into the image.

An unbound or empty secret leaves the clones anonymous, as before. A revoked, expired
or org-blocked token, however, fails every github.com clone in these builds, since the
header is sent on every request.

## Related docs

Expand Down
23 changes: 20 additions & 3 deletions .ci/jenkins/lib/build-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,35 @@ env:
# the "CI_MANAGED" placeholder must not be hand-edited.
CI_IMAGE_TAG: "CI_MANAGED"

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

runs_on_dockers:
- {
file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base-cuda12.9", tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.06-cuda12.9-devel-ubuntu24.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true'
build_args: '--build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.06-cuda12.9-devel-ubuntu24.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}
- {
file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base", tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}
- {
file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-non-gpu-base-ubuntu22", tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/nvidia/cuda:13.0.1-devel-ubuntu22.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true'
build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/nvidia/cuda:13.0.1-devel-ubuntu22.04 --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg ARCH=${arch} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}

matrix:
Expand Down
15 changes: 14 additions & 1 deletion .ci/jenkins/lib/build-wheel-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ registry_auth: svc-nixl-new-artifactory-token
registry_path: /sw-nbu-swx-nixl-docker-local/ci

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'
- credentialsId: 'svc-nixl-new-artifactory-token'
usernameVariable: 'ARTIFACTORY_USER'
passwordVariable: 'ARTIFACTORY_TOKEN'
Expand All @@ -98,6 +101,15 @@ empty_volumes:
pvc_volumes:
- {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

# =============================================================================
# Docker Build Configuration
# =============================================================================
Expand All @@ -119,7 +131,8 @@ runs_on_dockers:
file: 'contrib/Dockerfile.manylinux',
name: "${WHEEL_BASE_IMAGE_NAME}",
tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg ARCH=${arch} --build-arg NPROC=${NPROC} --build-arg GRPC_NPROC=${GRPC_NPROC} --target wheel_base'
build_args: '--build-arg ARCH=${arch} --build-arg NPROC=${NPROC} --build-arg GRPC_NPROC=${GRPC_NPROC} --target wheel_base --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}
# The custom 'fw' key is injected as an env var into every step of the branch
# (ci-demo merges the docker entry into the branch axis), so the shared sanity
Expand Down
15 changes: 14 additions & 1 deletion .ci/jenkins/lib/test-dl-ep-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ kubernetes:
privileged: true

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'
- {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}

env:
Expand Down Expand Up @@ -72,6 +75,15 @@ empty_volumes:
pvc_volumes:
- {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

# Docker images for DL EP testing.
# Dedicated EP base + build_helper images (separate artifactory paths from the
# nixl-ci-dl-gpu job).
Expand All @@ -81,7 +93,8 @@ runs_on_dockers:
name: 'nixl-ci-dl-gpu-ep-base',
tag: "${CI_IMAGE_TAG}",
arch: "aarch64",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg VLLM_REF=v0.30.0 --build-arg ARCH=${arch} --pull --no-cache'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg VLLM_REF=v0.30.0 --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}

- {
Expand Down
15 changes: 14 additions & 1 deletion .ci/jenkins/lib/test-dl-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ kubernetes:
privileged: true

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'
- {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}

env:
Expand Down Expand Up @@ -64,14 +67,24 @@ empty_volumes:
pvc_volumes:
- {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

# Docker images for DL testing
runs_on_dockers:
- {
file: '.ci/dockerfiles/Dockerfile.base',
name: 'nixl-ci-dl-gpu-base',
tag: "${CI_IMAGE_TAG}",
arch: "aarch64",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}

- {
Expand Down
15 changes: 14 additions & 1 deletion .ci/jenkins/lib/test-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ kubernetes:
privileged: true

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'
- {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}

env:
Expand Down Expand Up @@ -67,13 +70,23 @@ empty_volumes:
pvc_volumes:
- {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

# label is defined at jenkins slave configuration, we want to run the job on a gpu agent and be able to esaly replace it without having to change this file
runs_on_dockers:
- {
file: '.ci/dockerfiles/Dockerfile.base',
name: 'nixl-ci-gpu-base',
tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}
- {
file: '.ci/dockerfiles/Dockerfile.build_helper',
Expand Down
20 changes: 18 additions & 2 deletions .ci/jenkins/lib/test-sanitizer-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,33 @@ env:
STORAGE_DRIVER: 'overlay'
CI_IMAGE_TAG: "CI_MANAGED"

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

runs_on_dockers:
# TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread
# so abseil's absl::Mutex annotations are visible to ThreadSanitizer).
- {
file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-test-sanitizers-base-tsan", tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=thread'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=thread --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}
# ASan-instrumented dependency image for the asan_ubsan build.
- {
file: '.ci/dockerfiles/Dockerfile.base', name: "nixl-ci-test-sanitizers-base-asan", tag: "${CI_IMAGE_TAG}",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=address'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --pull --no-cache --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg DEPS_SANITIZE=address --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}

matrix:
Expand Down
15 changes: 14 additions & 1 deletion .ci/jenkins/lib/test-vr-matrix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ kubernetes:
privileged: true

credentials:
- credentialsId: 'svc-nixl-github-read-only-token'
usernameVariable: 'NIXL_GITHUB_USER'
passwordVariable: 'NIXL_GITHUB_TOKEN'
- {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}

env:
Expand Down Expand Up @@ -73,14 +76,24 @@ empty_volumes:
pvc_volumes:
- {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}

# ci-demo builds the images below outside any step; an entry's credentialsId binds
# the token around its build, and this hook writes the git config the Dockerfile
# mounts as the "ghconfig" secret. Left empty when nothing is bound.
pipeline_on_image_build:
run: |
#!/bin/bash -e
. .ci/scripts/common.sh
write_github_gitconfig /tmp/ghconfig

# Docker images for DL testing
runs_on_dockers:
- {
file: '.ci/dockerfiles/Dockerfile.base',
name: 'nixl-ci-dl-gpu-base',
tag: "${CI_IMAGE_TAG}",
arch: "aarch64",
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache --secret id=ghconfig,src=/tmp/ghconfig',
credentialsId: 'svc-nixl-github-read-only-token'
}

- {
Expand Down
Loading