From ce4613dc72a0d5081eb45712cac0b149fd1eade8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 29 Jun 2026 16:23:37 -0700 Subject: [PATCH 1/7] fix(kube): Update kube scripts to pull from ghcr Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/install_helm_e2e.sh | 6 ++++++ e2e/k8s/scripts/setup_local_kind_cpu.sh | 11 +++++++++++ e2e/k8s/scripts/setup_local_minikube_cpu.sh | 9 +++++++++ e2e/k8s/scripts/setup_local_minikube_gpu.sh | 11 +++++++++++ 4 files changed, 37 insertions(+) diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh index 07f4f34baf..e1374abd85 100755 --- a/e2e/k8s/scripts/install_helm_e2e.sh +++ b/e2e/k8s/scripts/install_helm_e2e.sh @@ -239,6 +239,12 @@ if [ -n "${NMP_E2E_PULL_POLICY}" ]; then ) fi +if [ -n "${GITHUB_TOKEN:-}" ]; then + HELM_ARGS+=( + --set "imagePullSecrets[0].name=ghcr-pull" + ) +fi + log_info "Helm install inputs:" printf ' release: %s\n' "${HELM_RELEASE_NAME}" printf ' namespace: %s\n' "${NAMESPACE}" diff --git a/e2e/k8s/scripts/setup_local_kind_cpu.sh b/e2e/k8s/scripts/setup_local_kind_cpu.sh index e770431d7a..22e23f1e27 100755 --- a/e2e/k8s/scripts/setup_local_kind_cpu.sh +++ b/e2e/k8s/scripts/setup_local_kind_cpu.sh @@ -103,6 +103,17 @@ log_info "Creating Kubernetes secrets in namespace ${KUBE_NAMESPACE}..." --docker-password="${NGC_API_KEY}" \ --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - +if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Creating GHCR image pull secret..." + "${KUBECTL_NS[@]}" create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io \ + --docker-username=x-access-token \ + --docker-password="${GITHUB_TOKEN}" \ + --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - +else + log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" +fi + if [ -n "${HF_TOKEN:-}" ]; then log_info "Creating HuggingFace token secret..." "${KUBECTL_NS[@]}" create secret generic huggingface-token \ diff --git a/e2e/k8s/scripts/setup_local_minikube_cpu.sh b/e2e/k8s/scripts/setup_local_minikube_cpu.sh index 4bcb4e87cb..b2e0ee2c03 100755 --- a/e2e/k8s/scripts/setup_local_minikube_cpu.sh +++ b/e2e/k8s/scripts/setup_local_minikube_cpu.sh @@ -101,6 +101,15 @@ ${KUBECTL_NS[@]} create secret docker-registry nvcrimagepullsecret \ --docker-password="${NVCI_DOCKER_PASSWORD:-local-dev-placeholder}" \ --dry-run=client -o yaml | ${KUBECTL_NS[@]} apply -f - +if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Creating GHCR image pull secret..." + ${KUBECTL_NS[@]} create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io \ + --docker-username=x-access-token \ + --docker-password="${GITHUB_TOKEN}" \ + --dry-run=client -o yaml | ${KUBECTL_NS[@]} apply -f - +fi + if [ -n "${HF_TOKEN:-}" ]; then log_info "Creating HuggingFace token secret..." ${KUBECTL_NS[@]} create secret generic huggingface-token \ diff --git a/e2e/k8s/scripts/setup_local_minikube_gpu.sh b/e2e/k8s/scripts/setup_local_minikube_gpu.sh index 297c5df279..34864b310b 100755 --- a/e2e/k8s/scripts/setup_local_minikube_gpu.sh +++ b/e2e/k8s/scripts/setup_local_minikube_gpu.sh @@ -144,6 +144,17 @@ ${KUBECTL_NS} create secret docker-registry nvcrimagepullsecret \ --docker-password="$NGC_API_KEY" \ --dry-run=client -o yaml | ${KUBECTL_NS} apply -f - +if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Creating GHCR image pull secret..." + ${KUBECTL_NS} create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io \ + --docker-username=x-access-token \ + --docker-password="$GITHUB_TOKEN" \ + --dry-run=client -o yaml | ${KUBECTL_NS} apply -f - +else + log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" +fi + if [ -n "${HF_TOKEN:-}" ]; then log_info "Creating HuggingFace token secret..." ${KUBECTL_NS} create secret generic huggingface-token \ From 438763f3d345e49b79bf27cac8f02fdffa60a764 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 29 Jun 2026 20:19:53 -0700 Subject: [PATCH 2/7] add nvcrimagepullsecret Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/install_helm_e2e.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh index e1374abd85..e835faa308 100755 --- a/e2e/k8s/scripts/install_helm_e2e.sh +++ b/e2e/k8s/scripts/install_helm_e2e.sh @@ -242,6 +242,7 @@ fi if [ -n "${GITHUB_TOKEN:-}" ]; then HELM_ARGS+=( --set "imagePullSecrets[0].name=ghcr-pull" + --set "imagePullSecrets[1].name=nvcrimagepullsecret" ) fi From 101ad8940a358141e838f7efb093acfd3b065d92 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 29 Jun 2026 20:28:17 -0700 Subject: [PATCH 3/7] consolidate secrets Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/create_secrets.sh | 56 +++++++++++++++++++++ e2e/k8s/scripts/setup_local_kind_cpu.sh | 33 ++---------- e2e/k8s/scripts/setup_local_minikube_cpu.sh | 31 ++---------- e2e/k8s/scripts/setup_local_minikube_gpu.sh | 36 ++----------- 4 files changed, 69 insertions(+), 87 deletions(-) create mode 100644 e2e/k8s/scripts/create_secrets.sh diff --git a/e2e/k8s/scripts/create_secrets.sh b/e2e/k8s/scripts/create_secrets.sh new file mode 100644 index 0000000000..efc1175518 --- /dev/null +++ b/e2e/k8s/scripts/create_secrets.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Shared helper for creating platform K8s secrets. +# Sourced by setup_local_minikube_cpu.sh, setup_local_minikube_gpu.sh, and setup_local_kind_cpu.sh. + +# create_platform_secrets NAMESPACE [NGC_API_KEY_VALUE] +# +# Creates the standard set of platform secrets in the given namespace: +# - ngc-api: NGC API key (generic secret) +# - nvcrimagepullsecret: NGC container registry pull secret +# - ghcr-pull: GHCR pull secret (when GITHUB_TOKEN is set) +# - huggingface-token: HF token (when HF_TOKEN is set) +# +# The NGC_API_KEY_VALUE argument defaults to $NGC_API_KEY. Pass a placeholder +# value explicitly for setups that don't require a real key. +create_platform_secrets() { + local namespace="${1:?namespace is required}" + local ngc_key="${2:-${NGC_API_KEY:-}}" + local kubectl_ns=(kubectl -n "${namespace}") + + if [ -z "${ngc_key}" ]; then + log_warn "NGC_API_KEY not set, using placeholder for ngc-api secret" + ngc_key="local-dev-placeholder" + fi + + log_info "Creating NGC API secret..." + "${kubectl_ns[@]}" create secret generic ngc-api \ + --from-literal=NGC_API_KEY="${ngc_key}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + + log_info "Creating NGC image pull secret..." + "${kubectl_ns[@]}" create secret docker-registry nvcrimagepullsecret \ + --docker-server=nvcr.io \ + --docker-username='$oauthtoken' \ + --docker-password="${ngc_key}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + + if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Creating GHCR image pull secret..." + "${kubectl_ns[@]}" create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io \ + --docker-username=x-access-token \ + --docker-password="${GITHUB_TOKEN}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + else + log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" + fi + + if [ -n "${HF_TOKEN:-}" ]; then + log_info "Creating HuggingFace token secret..." + "${kubectl_ns[@]}" create secret generic huggingface-token \ + --from-literal=HF_TOKEN="${HF_TOKEN}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + else + log_warn "HF_TOKEN not set, skipping HuggingFace token secret" + fi +} diff --git a/e2e/k8s/scripts/setup_local_kind_cpu.sh b/e2e/k8s/scripts/setup_local_kind_cpu.sh index 22e23f1e27..05251e80c7 100755 --- a/e2e/k8s/scripts/setup_local_kind_cpu.sh +++ b/e2e/k8s/scripts/setup_local_kind_cpu.sh @@ -4,6 +4,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/create_secrets.sh" + KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-nmp-e2e}" KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.33.7@sha256:d26ef333bdb2cbe9862a0f7c3803ecc7b4303d8cea8e814b481b09949d353040}" KUBE_NAMESPACE="${KUBE_NAMESPACE:-default}" @@ -93,35 +96,7 @@ fi KUBECTL_NS=(kubectl -n "${KUBE_NAMESPACE}") log_info "Creating Kubernetes secrets in namespace ${KUBE_NAMESPACE}..." -"${KUBECTL_NS[@]}" create secret generic ngc-api \ - --from-literal=NGC_API_KEY="${NGC_API_KEY}" \ - --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - - -"${KUBECTL_NS[@]}" create secret docker-registry nvcrimagepullsecret \ - --docker-server=nvcr.io \ - --docker-username="\$oauthtoken" \ - --docker-password="${NGC_API_KEY}" \ - --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - - -if [ -n "${GITHUB_TOKEN:-}" ]; then - log_info "Creating GHCR image pull secret..." - "${KUBECTL_NS[@]}" create secret docker-registry ghcr-pull \ - --docker-server=ghcr.io \ - --docker-username=x-access-token \ - --docker-password="${GITHUB_TOKEN}" \ - --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - -else - log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" -fi - -if [ -n "${HF_TOKEN:-}" ]; then - log_info "Creating HuggingFace token secret..." - "${KUBECTL_NS[@]}" create secret generic huggingface-token \ - --from-literal=HF_TOKEN="${HF_TOKEN}" \ - --dry-run=client -o yaml | "${KUBECTL_NS[@]}" apply -f - -else - log_warn "HF_TOKEN not set, skipping HuggingFace token secret" -fi +create_platform_secrets "${KUBE_NAMESPACE}" log_info "Creating Gateway ${KUBE_NAMESPACE}/${KUBE_GATEWAY_NAME}..." kubectl apply -f - < Date: Mon, 29 Jun 2026 20:34:21 -0700 Subject: [PATCH 4/7] more consolidation Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/create_secrets.sh | 56 ------------------- e2e/k8s/scripts/install_helm_e2e.sh | 17 +----- e2e/k8s/scripts/lib.sh | 62 +++++++++++++++++++++ e2e/k8s/scripts/setup_local_kind_cpu.sh | 19 +------ e2e/k8s/scripts/setup_local_minikube_cpu.sh | 19 +------ e2e/k8s/scripts/setup_local_minikube_gpu.sh | 19 +------ 6 files changed, 66 insertions(+), 126 deletions(-) delete mode 100644 e2e/k8s/scripts/create_secrets.sh create mode 100644 e2e/k8s/scripts/lib.sh diff --git a/e2e/k8s/scripts/create_secrets.sh b/e2e/k8s/scripts/create_secrets.sh deleted file mode 100644 index efc1175518..0000000000 --- a/e2e/k8s/scripts/create_secrets.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Shared helper for creating platform K8s secrets. -# Sourced by setup_local_minikube_cpu.sh, setup_local_minikube_gpu.sh, and setup_local_kind_cpu.sh. - -# create_platform_secrets NAMESPACE [NGC_API_KEY_VALUE] -# -# Creates the standard set of platform secrets in the given namespace: -# - ngc-api: NGC API key (generic secret) -# - nvcrimagepullsecret: NGC container registry pull secret -# - ghcr-pull: GHCR pull secret (when GITHUB_TOKEN is set) -# - huggingface-token: HF token (when HF_TOKEN is set) -# -# The NGC_API_KEY_VALUE argument defaults to $NGC_API_KEY. Pass a placeholder -# value explicitly for setups that don't require a real key. -create_platform_secrets() { - local namespace="${1:?namespace is required}" - local ngc_key="${2:-${NGC_API_KEY:-}}" - local kubectl_ns=(kubectl -n "${namespace}") - - if [ -z "${ngc_key}" ]; then - log_warn "NGC_API_KEY not set, using placeholder for ngc-api secret" - ngc_key="local-dev-placeholder" - fi - - log_info "Creating NGC API secret..." - "${kubectl_ns[@]}" create secret generic ngc-api \ - --from-literal=NGC_API_KEY="${ngc_key}" \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - - log_info "Creating NGC image pull secret..." - "${kubectl_ns[@]}" create secret docker-registry nvcrimagepullsecret \ - --docker-server=nvcr.io \ - --docker-username='$oauthtoken' \ - --docker-password="${ngc_key}" \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - - if [ -n "${GITHUB_TOKEN:-}" ]; then - log_info "Creating GHCR image pull secret..." - "${kubectl_ns[@]}" create secret docker-registry ghcr-pull \ - --docker-server=ghcr.io \ - --docker-username=x-access-token \ - --docker-password="${GITHUB_TOKEN}" \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - else - log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" - fi - - if [ -n "${HF_TOKEN:-}" ]; then - log_info "Creating HuggingFace token secret..." - "${kubectl_ns[@]}" create secret generic huggingface-token \ - --from-literal=HF_TOKEN="${HF_TOKEN}" \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - else - log_warn "HF_TOKEN not set, skipping HuggingFace token secret" - fi -} diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh index e835faa308..fec50f4395 100755 --- a/e2e/k8s/scripts/install_helm_e2e.sh +++ b/e2e/k8s/scripts/install_helm_e2e.sh @@ -9,6 +9,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" NAMESPACE="${NAMESPACE:-${KUBE_NAMESPACE:-default}}" @@ -31,22 +32,6 @@ RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" EXTRA_HELM_ARGS=() -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} require_non_empty() { local name="$1" diff --git a/e2e/k8s/scripts/lib.sh b/e2e/k8s/scripts/lib.sh new file mode 100644 index 0000000000..3018185453 --- /dev/null +++ b/e2e/k8s/scripts/lib.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Shared utilities for e2e K8s setup scripts. + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- + +log_info() { echo -e "\033[0;32m[INFO]\033[0m $*"; } +log_warn() { echo -e "\033[1;33m[WARN]\033[0m $*"; } +log_error() { echo -e "\033[0;31m[ERROR]\033[0m $*"; } + +# --------------------------------------------------------------------------- +# Secrets +# --------------------------------------------------------------------------- + +# create_platform_secrets NAMESPACE +# +# Creates the standard set of platform secrets in the given namespace. +# Each secret is created only when its corresponding env var is set: +# - ngc-api + nvcrimagepullsecret: when NGC_API_KEY is set +# - ghcr-pull: when GITHUB_TOKEN is set +# - huggingface-token: when HF_TOKEN is set +create_platform_secrets() { + local namespace="${1:?namespace is required}" + local kubectl_ns=(kubectl -n "${namespace}") + + if [ -n "${NGC_API_KEY:-}" ]; then + log_info "Creating NGC API secret..." + "${kubectl_ns[@]}" create secret generic ngc-api \ + --from-literal=NGC_API_KEY="${NGC_API_KEY}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + + log_info "Creating NGC image pull secret..." + "${kubectl_ns[@]}" create secret docker-registry nvcrimagepullsecret \ + --docker-server=nvcr.io \ + --docker-username='$oauthtoken' \ + --docker-password="${NGC_API_KEY}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + else + log_warn "NGC_API_KEY not set, skipping NGC secrets" + fi + + if [ -n "${GITHUB_TOKEN:-}" ]; then + log_info "Creating GHCR image pull secret..." + "${kubectl_ns[@]}" create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io \ + --docker-username=x-access-token \ + --docker-password="${GITHUB_TOKEN}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + else + log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" + fi + + if [ -n "${HF_TOKEN:-}" ]; then + log_info "Creating HuggingFace token secret..." + "${kubectl_ns[@]}" create secret generic huggingface-token \ + --from-literal=HF_TOKEN="${HF_TOKEN}" \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + else + log_warn "HF_TOKEN not set, skipping HuggingFace token secret" + fi +} diff --git a/e2e/k8s/scripts/setup_local_kind_cpu.sh b/e2e/k8s/scripts/setup_local_kind_cpu.sh index 05251e80c7..9300ec65ff 100755 --- a/e2e/k8s/scripts/setup_local_kind_cpu.sh +++ b/e2e/k8s/scripts/setup_local_kind_cpu.sh @@ -5,7 +5,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/create_secrets.sh" +source "${SCRIPT_DIR}/lib.sh" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-nmp-e2e}" KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.33.7@sha256:d26ef333bdb2cbe9862a0f7c3803ecc7b4303d8cea8e814b481b09949d353040}" @@ -15,23 +15,6 @@ CLOUD_PROVIDER_KIND_VERSION="${CLOUD_PROVIDER_KIND_VERSION:-v0.10.0}" GATEWAY_API_VERSION="${GATEWAY_API_VERSION:-v1.4.1}" CLOUD_PROVIDER_KIND_CONTAINER="cloud-provider-kind-${KIND_CLUSTER_NAME}" -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - for tool in kind docker kubectl helm; do if ! command -v "$tool" >/dev/null 2>&1; then log_error "$tool is not installed. Please install it first." diff --git a/e2e/k8s/scripts/setup_local_minikube_cpu.sh b/e2e/k8s/scripts/setup_local_minikube_cpu.sh index 6b3cb69516..92ce8c04e5 100755 --- a/e2e/k8s/scripts/setup_local_minikube_cpu.sh +++ b/e2e/k8s/scripts/setup_local_minikube_cpu.sh @@ -5,7 +5,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/create_secrets.sh" +source "${SCRIPT_DIR}/lib.sh" MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube-auth}" KUBE_NAMESPACE="${KUBE_NAMESPACE:-default}" @@ -14,23 +14,6 @@ INGRESS_HOST_PORT="${INGRESS_HOST_PORT:-30080}" MINIKUBE_CPUS="${MINIKUBE_CPUS:-4}" MINIKUBE_MEMORY_MB="${MINIKUBE_MEMORY_MB:-6144}" -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - for tool in minikube docker kubectl helm; do if ! command -v "$tool" >/dev/null 2>&1; then log_error "$tool is not installed. Please install it first." diff --git a/e2e/k8s/scripts/setup_local_minikube_gpu.sh b/e2e/k8s/scripts/setup_local_minikube_gpu.sh index c9da4860ca..e0fbfa601c 100755 --- a/e2e/k8s/scripts/setup_local_minikube_gpu.sh +++ b/e2e/k8s/scripts/setup_local_minikube_gpu.sh @@ -11,27 +11,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/create_secrets.sh" +source "${SCRIPT_DIR}/lib.sh" MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - log_info "Validating environment..." if [ -z "${NGC_API_KEY:-}" ]; then From 3d2d1b4ecaed76beb57557dff726347f37258e53 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 29 Jun 2026 20:38:27 -0700 Subject: [PATCH 5/7] update Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/install_helm_e2e.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh index fec50f4395..5cdd785051 100755 --- a/e2e/k8s/scripts/install_helm_e2e.sh +++ b/e2e/k8s/scripts/install_helm_e2e.sh @@ -224,11 +224,16 @@ if [ -n "${NMP_E2E_PULL_POLICY}" ]; then ) fi +IMAGE_PULL_SECRET_INDEX=0 + +if [ -n "${NGC_API_KEY:-}" ]; then + HELM_ARGS+=(--set "imagePullSecrets[${IMAGE_PULL_SECRET_INDEX}].name=nvcrimagepullsecret") + IMAGE_PULL_SECRET_INDEX=$((IMAGE_PULL_SECRET_INDEX + 1)) +fi + if [ -n "${GITHUB_TOKEN:-}" ]; then - HELM_ARGS+=( - --set "imagePullSecrets[0].name=ghcr-pull" - --set "imagePullSecrets[1].name=nvcrimagepullsecret" - ) + HELM_ARGS+=(--set "imagePullSecrets[${IMAGE_PULL_SECRET_INDEX}].name=ghcr-pull") + IMAGE_PULL_SECRET_INDEX=$((IMAGE_PULL_SECRET_INDEX + 1)) fi log_info "Helm install inputs:" From 6d123672ef9459c24d5a4facc03eb870fc98375d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 30 Jun 2026 10:36:01 -0700 Subject: [PATCH 6/7] comment about argv Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/lib.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/e2e/k8s/scripts/lib.sh b/e2e/k8s/scripts/lib.sh index 3018185453..a1ef1b415b 100644 --- a/e2e/k8s/scripts/lib.sh +++ b/e2e/k8s/scripts/lib.sh @@ -13,6 +13,14 @@ log_error() { echo -e "\033[0;31m[ERROR]\033[0m $*"; } # Secrets # --------------------------------------------------------------------------- +# _dockerconfigjson SERVER USERNAME PASSWORD +# +# Prints a dockerconfigjson blob for a single registry. +_dockerconfigjson() { + printf '{"auths":{"%s":{"username":"%s","password":"%s","auth":"%s"}}}' \ + "$1" "$2" "$3" "$(printf '%s:%s' "$2" "$3" | base64 | tr -d '\n')" +} + # create_platform_secrets NAMESPACE # # Creates the standard set of platform secrets in the given namespace. @@ -20,6 +28,9 @@ log_error() { echo -e "\033[0;31m[ERROR]\033[0m $*"; } # - ngc-api + nvcrimagepullsecret: when NGC_API_KEY is set # - ghcr-pull: when GITHUB_TOKEN is set # - huggingface-token: when HF_TOKEN is set +# +# Secret values are passed via process substitution to keep them out of +# process arguments visible in /proc. create_platform_secrets() { local namespace="${1:?namespace is required}" local kubectl_ns=(kubectl -n "${namespace}") @@ -27,14 +38,13 @@ create_platform_secrets() { if [ -n "${NGC_API_KEY:-}" ]; then log_info "Creating NGC API secret..." "${kubectl_ns[@]}" create secret generic ngc-api \ - --from-literal=NGC_API_KEY="${NGC_API_KEY}" \ + --from-file=NGC_API_KEY=<(printf '%s' "${NGC_API_KEY}") \ --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - log_info "Creating NGC image pull secret..." - "${kubectl_ns[@]}" create secret docker-registry nvcrimagepullsecret \ - --docker-server=nvcr.io \ - --docker-username='$oauthtoken' \ - --docker-password="${NGC_API_KEY}" \ + "${kubectl_ns[@]}" create secret generic nvcrimagepullsecret \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson=<(_dockerconfigjson nvcr.io '$oauthtoken' "${NGC_API_KEY}") \ --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - else log_warn "NGC_API_KEY not set, skipping NGC secrets" @@ -42,10 +52,9 @@ create_platform_secrets() { if [ -n "${GITHUB_TOKEN:-}" ]; then log_info "Creating GHCR image pull secret..." - "${kubectl_ns[@]}" create secret docker-registry ghcr-pull \ - --docker-server=ghcr.io \ - --docker-username=x-access-token \ - --docker-password="${GITHUB_TOKEN}" \ + "${kubectl_ns[@]}" create secret generic ghcr-pull \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson=<(_dockerconfigjson ghcr.io x-access-token "${GITHUB_TOKEN}") \ --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - else log_warn "GITHUB_TOKEN not set, skipping GHCR image pull secret" @@ -54,7 +63,7 @@ create_platform_secrets() { if [ -n "${HF_TOKEN:-}" ]; then log_info "Creating HuggingFace token secret..." "${kubectl_ns[@]}" create secret generic huggingface-token \ - --from-literal=HF_TOKEN="${HF_TOKEN}" \ + --from-file=HF_TOKEN=<(printf '%s' "${HF_TOKEN}") \ --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - else log_warn "HF_TOKEN not set, skipping HuggingFace token secret" From 6e0dd22fe60965959242fd21991098e7f2bf2ea4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 30 Jun 2026 10:42:47 -0700 Subject: [PATCH 7/7] placeholder Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/lib.sh | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/e2e/k8s/scripts/lib.sh b/e2e/k8s/scripts/lib.sh index a1ef1b415b..6df829398c 100644 --- a/e2e/k8s/scripts/lib.sh +++ b/e2e/k8s/scripts/lib.sh @@ -24,8 +24,9 @@ _dockerconfigjson() { # create_platform_secrets NAMESPACE # # Creates the standard set of platform secrets in the given namespace. -# Each secret is created only when its corresponding env var is set: -# - ngc-api + nvcrimagepullsecret: when NGC_API_KEY is set +# NGC secrets are always created (with a placeholder if NGC_API_KEY is unset) +# because the helm chart expects the ngc-api secret to exist. Other secrets +# are created only when their corresponding env var is set: # - ghcr-pull: when GITHUB_TOKEN is set # - huggingface-token: when HF_TOKEN is set # @@ -34,22 +35,23 @@ _dockerconfigjson() { create_platform_secrets() { local namespace="${1:?namespace is required}" local kubectl_ns=(kubectl -n "${namespace}") + local ngc_key="${NGC_API_KEY:-placeholder}" - if [ -n "${NGC_API_KEY:-}" ]; then - log_info "Creating NGC API secret..." - "${kubectl_ns[@]}" create secret generic ngc-api \ - --from-file=NGC_API_KEY=<(printf '%s' "${NGC_API_KEY}") \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - - log_info "Creating NGC image pull secret..." - "${kubectl_ns[@]}" create secret generic nvcrimagepullsecret \ - --type=kubernetes.io/dockerconfigjson \ - --from-file=.dockerconfigjson=<(_dockerconfigjson nvcr.io '$oauthtoken' "${NGC_API_KEY}") \ - --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - - else - log_warn "NGC_API_KEY not set, skipping NGC secrets" + if [ -z "${NGC_API_KEY:-}" ]; then + log_warn "NGC_API_KEY not set, creating NGC secrets with placeholder" fi + log_info "Creating NGC API secret..." + "${kubectl_ns[@]}" create secret generic ngc-api \ + --from-file=NGC_API_KEY=<(printf '%s' "${ngc_key}") \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + + log_info "Creating NGC image pull secret..." + "${kubectl_ns[@]}" create secret generic nvcrimagepullsecret \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson=<(_dockerconfigjson nvcr.io '$oauthtoken' "${ngc_key}") \ + --dry-run=client -o yaml | "${kubectl_ns[@]}" apply -f - + if [ -n "${GITHUB_TOKEN:-}" ]; then log_info "Creating GHCR image pull secret..." "${kubectl_ns[@]}" create secret generic ghcr-pull \