From 0a7d7ce454aaf4278493cd15b7a4e7cc693546ef Mon Sep 17 00:00:00 2001 From: Wenxin Ding Date: Wed, 12 Aug 2026 22:43:52 -0700 Subject: [PATCH 1/3] fix(ci): authenticate nightly vLLM image pulls Signed-off-by: Wenxin Ding --- .github/workflows/nightly-xr-ai-test.yml | 23 +++++++++++++++++++++++ docs/changelog.md | 9 +++++++++ 2 files changed, 32 insertions(+) diff --git a/.github/workflows/nightly-xr-ai-test.yml b/.github/workflows/nightly-xr-ai-test.yml index d69ea2996..e58eeb829 100644 --- a/.github/workflows/nightly-xr-ai-test.yml +++ b/.github/workflows/nightly-xr-ai-test.yml @@ -86,6 +86,22 @@ jobs: nvcc --version nvcc --list-gpu-arch | grep compute_89 + - name: Authenticate to NGC + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + run: | + set -euo pipefail + docker_config="$RUNNER_TEMP/xr-ai-docker-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "DOCKER_CONFIG=$docker_config" >> "$GITHUB_ENV" + install -d -m 0700 "$docker_config" + + if [[ -z "${NGC_API_KEY:-}" ]]; then + echo "::error::NGC_API_KEY repository secret is required for nvcr.io images" + exit 1 + fi + printf '%s' "$NGC_API_KEY" | docker login nvcr.io \ + --username '$oauthtoken' --password-stdin + - name: Install uv uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 with: @@ -169,6 +185,13 @@ jobs: echo "No xr-ai-vllm-* containers to remove." fi + - name: Remove NGC credentials + if: always() + run: | + if [[ -n "${DOCKER_CONFIG:-}" ]]; then + rm -rf "$DOCKER_CONFIG" + fi + notify: name: alert on failure needs: [pytest] diff --git a/docs/changelog.md b/docs/changelog.md index 9cabc2e36..a9eabf7fe 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -18,6 +18,15 @@ allowing a valid command to survive noisy STT preambles without sending background speech to the agent. Commas, semicolons, URLs, and ordinary mid-sentence mentions remain non-matches to limit false wakes. +### 2026-08-12 — Nightly GPU CI authenticates to NGC explicitly + +The nightly GPU workflow logs in to `nvcr.io` with the repository's +`NGC_API_KEY` before setup and tests begin. The login uses a per-run Docker +configuration under `RUNNER_TEMP`, inherited by the model-service subprocesses +without exposing the API key to pytest, and removes that configuration during +always-run cleanup. Nightly correctness therefore does not depend on a cached +vLLM image or mutable Docker credentials on the self-hosted runner. + ### 2026-08-12 — NeMo Agent Toolkit compatibility is retired All surviving capabilities now use native `xr-ai-tools` contracts. OpenXR, From 848064623dba76cea8d72de4d27350b90f082b37 Mon Sep 17 00:00:00 2001 From: Wenxin Ding Date: Fri, 14 Aug 2026 13:30:26 -0700 Subject: [PATCH 2/3] Fix nightly NGC credential isolation Signed-off-by: Wenxin Ding --- .github/workflows/nightly-xr-ai-test.yml | 27 +++++++++++++++------- docs/changelog.md | 12 ++++++---- docs/credentials.md | 5 ++++ docs/source/getting_started/credentials.md | 5 ++++ 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/nightly-xr-ai-test.yml b/.github/workflows/nightly-xr-ai-test.yml index e58eeb829..687f7bc94 100644 --- a/.github/workflows/nightly-xr-ai-test.yml +++ b/.github/workflows/nightly-xr-ai-test.yml @@ -8,9 +8,11 @@ name: nightly XR AI test # the default `tests` workflow (ubuntu-latest). Mirrors the local # `tests/run_local_gpu_tests.sh` entry point. # -# Setup note: the GPU runner's Docker needs the NVIDIA runtime registered -# (`sudo nvidia-ctk runtime configure --runtime=docker`); without it the -# vLLM tests fail at container launch. +# Setup notes: +# - The GPU runner's Docker needs the NVIDIA runtime registered +# (`sudo nvidia-ctk runtime configure --runtime=docker`). +# - The repository needs an Actions secret named `NGC_API_KEY` so trusted +# events can authenticate pulls from `nvcr.io`. on: schedule: @@ -44,6 +46,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Configure isolated Docker credentials + run: | + set -euo pipefail + docker_config="$RUNNER_TEMP/xr-ai-docker-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + install -d -m 0700 "$docker_config" + echo "DOCKER_CONFIG=$docker_config" >> "$GITHUB_ENV" + - name: Discover CUDA toolkit # Prefer CUDA 13.0 (current target), but fall back through the # usual symlinks and finally whatever `nvcc` is on PATH so we @@ -87,20 +96,20 @@ jobs: nvcc --list-gpu-arch | grep compute_89 - name: Authenticate to NGC + if: >- + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository env: NGC_API_KEY: ${{ secrets.NGC_API_KEY }} run: | set -euo pipefail - docker_config="$RUNNER_TEMP/xr-ai-docker-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "DOCKER_CONFIG=$docker_config" >> "$GITHUB_ENV" - install -d -m 0700 "$docker_config" - if [[ -z "${NGC_API_KEY:-}" ]]; then echo "::error::NGC_API_KEY repository secret is required for nvcr.io images" exit 1 fi printf '%s' "$NGC_API_KEY" | docker login nvcr.io \ --username '$oauthtoken' --password-stdin + docker manifest inspect nvcr.io/nvidia/vllm:26.04-py3 >/dev/null - name: Install uv uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 @@ -188,9 +197,11 @@ jobs: - name: Remove NGC credentials if: always() run: | + set -u if [[ -n "${DOCKER_CONFIG:-}" ]]; then - rm -rf "$DOCKER_CONFIG" + rm -rf -- "$DOCKER_CONFIG" fi + DOCKER_CONFIG="$HOME/.docker" docker logout nvcr.io || true notify: name: alert on failure diff --git a/docs/changelog.md b/docs/changelog.md index a9eabf7fe..4cb434d9a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -21,11 +21,13 @@ mid-sentence mentions remain non-matches to limit false wakes. ### 2026-08-12 — Nightly GPU CI authenticates to NGC explicitly The nightly GPU workflow logs in to `nvcr.io` with the repository's -`NGC_API_KEY` before setup and tests begin. The login uses a per-run Docker -configuration under `RUNNER_TEMP`, inherited by the model-service subprocesses -without exposing the API key to pytest, and removes that configuration during -always-run cleanup. Nightly correctness therefore does not depend on a cached -vLLM image or mutable Docker credentials on the self-hosted runner. +`NGC_API_KEY` before setup and tests begin, then verifies access to the vLLM +image manifest. A job-scoped, per-run `DOCKER_CONFIG` is inherited by model +subprocesses without exposing the key to pytest; fork pull requests run without +the secret. CI authenticates directly because `_maybe_ngc_login()` checks the +default Docker config rather than this isolated path. Always-run cleanup deletes +the isolated config and removes credentials leaked to the runner-global config +by earlier workflow revisions. ### 2026-08-12 — NeMo Agent Toolkit compatibility is retired diff --git a/docs/credentials.md b/docs/credentials.md index ce5c88444..6385c0f94 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -71,6 +71,11 @@ set it ahead of time: export NGC_API_KEY=nvapi-xxx ``` +Nightly GPU CI reads the same value from a required repository Actions secret +named `NGC_API_KEY`. Scheduled, manual, and same-repository pull-request runs +use it only in the NGC authentication step; fork pull requests do not receive +the secret. + ## How a token is resolved `load_credentials()` (always) and `ensure_credentials()` (NGC only) resolve in diff --git a/docs/source/getting_started/credentials.md b/docs/source/getting_started/credentials.md index e720250c2..0807bebe9 100644 --- a/docs/source/getting_started/credentials.md +++ b/docs/source/getting_started/credentials.md @@ -72,6 +72,11 @@ set it ahead of time: export NGC_API_KEY=nvapi-xxx ``` +Nightly GPU CI reads the same value from a required repository Actions secret +named `NGC_API_KEY`. Scheduled, manual, and same-repository pull-request runs +use it only in the NGC authentication step; fork pull requests do not receive +the secret. + ## How a token is resolved `load_credentials()` (always) and `ensure_credentials()` (NGC only) resolve in From e48b5ddc98bb73c34f5dc96832540f3cc153c469 Mon Sep 17 00:00:00 2001 From: Wenxin Ding Date: Fri, 14 Aug 2026 13:34:15 -0700 Subject: [PATCH 3/3] Document NGC Catalog key requirement Signed-off-by: Wenxin Ding --- .github/workflows/nightly-xr-ai-test.yml | 9 ++++++--- docs/credentials.md | 7 ++++--- docs/source/getting_started/credentials.md | 7 ++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-xr-ai-test.yml b/.github/workflows/nightly-xr-ai-test.yml index 687f7bc94..41f7d4800 100644 --- a/.github/workflows/nightly-xr-ai-test.yml +++ b/.github/workflows/nightly-xr-ai-test.yml @@ -11,8 +11,8 @@ name: nightly XR AI test # Setup notes: # - The GPU runner's Docker needs the NVIDIA runtime registered # (`sudo nvidia-ctk runtime configure --runtime=docker`). -# - The repository needs an Actions secret named `NGC_API_KEY` so trusted -# events can authenticate pulls from `nvcr.io`. +# - The repository needs an Actions secret named `NGC_API_KEY`, backed by an +# NGC Personal or Service Key with NGC Catalog access. on: schedule: @@ -109,7 +109,10 @@ jobs: fi printf '%s' "$NGC_API_KEY" | docker login nvcr.io \ --username '$oauthtoken' --password-stdin - docker manifest inspect nvcr.io/nvidia/vllm:26.04-py3 >/dev/null + if ! docker manifest inspect nvcr.io/nvidia/vllm:26.04-py3 >/dev/null; then + echo "::error::NGC_API_KEY must grant NGC Catalog access to nvidia/vllm" + exit 1 + fi - name: Install uv uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 diff --git a/docs/credentials.md b/docs/credentials.md index 6385c0f94..1c473722a 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -72,9 +72,10 @@ export NGC_API_KEY=nvapi-xxx ``` Nightly GPU CI reads the same value from a required repository Actions secret -named `NGC_API_KEY`. Scheduled, manual, and same-repository pull-request runs -use it only in the NGC authentication step; fork pull requests do not receive -the secret. +named `NGC_API_KEY`. Use an NGC Personal or Service Key authorized for **NGC +Catalog**; an endpoint-only NVIDIA API key cannot pull `nvcr.io` images. +Scheduled, manual, and same-repository pull-request runs use the key only in the +NGC authentication step; fork pull requests do not receive the secret. ## How a token is resolved diff --git a/docs/source/getting_started/credentials.md b/docs/source/getting_started/credentials.md index 0807bebe9..4d94423f8 100644 --- a/docs/source/getting_started/credentials.md +++ b/docs/source/getting_started/credentials.md @@ -73,9 +73,10 @@ export NGC_API_KEY=nvapi-xxx ``` Nightly GPU CI reads the same value from a required repository Actions secret -named `NGC_API_KEY`. Scheduled, manual, and same-repository pull-request runs -use it only in the NGC authentication step; fork pull requests do not receive -the secret. +named `NGC_API_KEY`. Use an NGC Personal or Service Key authorized for **NGC +Catalog**; an endpoint-only NVIDIA API key cannot pull `nvcr.io` images. +Scheduled, manual, and same-repository pull-request runs use the key only in the +NGC authentication step; fork pull requests do not receive the secret. ## How a token is resolved