From 2974474c3e0cc9f8a8555538e7ad8889456ee007 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 14:06:36 -0700 Subject: [PATCH 1/7] fix(e2e): Update scripts for minikube Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/install_helm_e2e.sh | 8 +++ e2e/k8s/scripts/install_nmp_auth_e2e.sh | 6 +- e2e/k8s/scripts/local_build_and_upgrade.sh | 64 ++++++++++--------- e2e/k8s/values/minikube-auth-portforward.yaml | 42 +++--------- e2e/k8s/values/minikube-auth.yaml | 36 ++--------- e2e/k8s/values/minikube.yaml | 25 +++++++- 6 files changed, 87 insertions(+), 94 deletions(-) diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh index c5c0869dc0..07f4f34baf 100755 --- a/e2e/k8s/scripts/install_helm_e2e.sh +++ b/e2e/k8s/scripts/install_helm_e2e.sh @@ -18,6 +18,7 @@ HELM_VALUES="${HELM_VALUES:-${HELM_VALUES_FILE:-${REPO_ROOT}/e2e/k8s/values/defa HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:-}" NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-}" NMP_E2E_TAG="${NMP_E2E_TAG:-}" +NMP_E2E_PULL_POLICY="${NMP_E2E_PULL_POLICY:-}" REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-false}" POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}" BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox}" @@ -231,6 +232,13 @@ if [ -n "${NMP_E2E_TAG}" ]; then ) fi +if [ -n "${NMP_E2E_PULL_POLICY}" ]; then + HELM_ARGS+=( + --set api.image.pullPolicy="${NMP_E2E_PULL_POLICY}" + --set core.image.pullPolicy="${NMP_E2E_PULL_POLICY}" + ) +fi + log_info "Helm install inputs:" printf ' release: %s\n' "${HELM_RELEASE_NAME}" printf ' namespace: %s\n' "${NAMESPACE}" diff --git a/e2e/k8s/scripts/install_nmp_auth_e2e.sh b/e2e/k8s/scripts/install_nmp_auth_e2e.sh index ceff9c1488..84dadcbfdd 100755 --- a/e2e/k8s/scripts/install_nmp_auth_e2e.sh +++ b/e2e/k8s/scripts/install_nmp_auth_e2e.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash # Install the auth-enabled local E2E harness on minikube. +# +# Layers minikube-auth.yaml on top of minikube.yaml so auth-specific +# config stays minimal and doesn't duplicate base minikube values. set -euo pipefail @@ -8,7 +11,8 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" export NAMESPACE="${NAMESPACE:-${KUBE_NAMESPACE:-default}}" export HELM_RELEASE_NAME="${HELM_RELEASE_NAME:-nemo-platform}" -export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml}" +export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube.yaml}" +export HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:-} -f ${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml" export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}" export NMP_E2E_TAG="${NMP_E2E_TAG:-local}" export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}" diff --git a/e2e/k8s/scripts/local_build_and_upgrade.sh b/e2e/k8s/scripts/local_build_and_upgrade.sh index 70785b78a3..87ad05d541 100755 --- a/e2e/k8s/scripts/local_build_and_upgrade.sh +++ b/e2e/k8s/scripts/local_build_and_upgrade.sh @@ -1,61 +1,65 @@ #!/usr/bin/env bash +# Build Docker images locally and deploy to minikube via Helm. +# +# This script handles the build step, then delegates the Helm install to +# install_helm_e2e.sh so install logic lives in one place. +# +# Environment variables: +# MINIKUBE_PROFILE - minikube profile name (default: minikube) +# NMP_REGISTRY - image registry (default: docker.io/my-registry) +# IMAGE_TAG - image tag (default: local-) +# BUILD_ARCH - target platform (default: auto-detected from host) +# HELM_VALUES - values file (default: e2e/k8s/values/minikube.yaml) + set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" + # Check if minikube is running -if ! minikube status &>/dev/null; then - echo "Minikube is not running. Starting minikube..." - # Use the setup_local_minikube_gpu.sh script to start minikube, - # and ensure the script is in the same directory as this script. - - "$SCRIPT_DIR/setup_local_minikube_gpu.sh" +if ! minikube status -p "${MINIKUBE_PROFILE}" &>/dev/null; then + echo "Minikube profile ${MINIKUBE_PROFILE} is not running. Starting..." + MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh" fi # Wait for minikube to be ready -minikube status +minikube status -p "${MINIKUBE_PROFILE}" -# Build the images with a local tag and then load them -# Use epoch seconds (date +%s) so each run gets a unique tag and upgrades pick up new images +# Use epoch seconds so each run gets a unique tag and upgrades pick up new images IMAGE_TAG="${IMAGE_TAG:-local-$(date +%s)}" # Detect platform for build (match host arch) BUILD_ARCH="${BUILD_ARCH:-linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')}" GIT_SHA=$(git -C "${REPO_ROOT}" rev-parse HEAD) +NMP_REGISTRY="${NMP_REGISTRY:-docker.io/my-registry}" + echo "Building docker-cpu images with tag $IMAGE_TAG (platform=$BUILD_ARCH)..." -# Allow building directly into minikube's docker daemon -eval "$(minikube docker-env)" +# Build directly into minikube's docker daemon +eval "$(minikube -p "${MINIKUBE_PROFILE}" docker-env)" -# Set the image tag to the git sha ( cd "${REPO_ROOT}" CI_COMMIT_SHA="$GIT_SHA" \ BAKE_TAG="$IMAGE_TAG" \ - IMAGE_REGISTRY="docker.io/my-registry" \ + IMAGE_REGISTRY="${NMP_REGISTRY}" \ BUILD_ARCH="$BUILD_ARCH" \ docker buildx bake docker-cpu --set "*.platform=$BUILD_ARCH" ) - -# Echo the image tags and an example script to run end-to-end tests -echo "Image tags:" -echo " nmp-api: $IMAGE_TAG" -echo " nmp-cpu-tasks: $IMAGE_TAG" -echo " platform: $IMAGE_TAG" -echo "----------------------------------------" -echo "Example script to run end-to-end jobs tests:" -echo " NMP_E2E_INTERNAL_HOST=nemo-platform-api:8080 NMP_E2E_REGISTRY=docker.io/my-registry NMP_E2E_TAG=$IMAGE_TAG uv run pytest e2e --kubernetes --cluster-url=http://localhost:80" echo "----------------------------------------" -echo "To rerun the helm install/upgrade, run:" -echo " helm upgrade --install nemo-platform k8s/helm/ -f e2e/k8s/values/local.yaml --set \"api.image.tag=$IMAGE_TAG\" --set \"core.image.tag=$IMAGE_TAG\" --set \"platformConfig.platform.image_tag=$IMAGE_TAG\"" +echo "Images built with tag: $IMAGE_TAG" echo "----------------------------------------" -# Install/upgrade the helm chart with image tags -helm upgrade --install nemo-platform k8s/helm/ \ - -f e2e/k8s/values/local.yaml \ - --set "api.image.tag=$IMAGE_TAG" \ - --set "core.image.tag=$IMAGE_TAG" \ - --set "platformConfig.platform.image_tag=$IMAGE_TAG" +# Delegate helm install to install_helm_e2e.sh +export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube.yaml}" +export NMP_E2E_REGISTRY="${NMP_REGISTRY}" +export NMP_E2E_TAG="${IMAGE_TAG}" +export NMP_E2E_PULL_POLICY="Never" +export MINIKUBE_PROFILE +export REQUIRE_NMP_E2E_IMAGES=true + +exec "$SCRIPT_DIR/install_helm_e2e.sh" diff --git a/e2e/k8s/values/minikube-auth-portforward.yaml b/e2e/k8s/values/minikube-auth-portforward.yaml index 0a262c289c..41f91713e1 100644 --- a/e2e/k8s/values/minikube-auth-portforward.yaml +++ b/e2e/k8s/values/minikube-auth-portforward.yaml @@ -1,42 +1,18 @@ -# CPU-only minikube values for local auth E2E verification without ingress. +# Port-forward overlay — use on top of minikube.yaml + minikube-auth.yaml. # # Usage: -# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth-portforward.yaml +# helm upgrade -i nemo-platform k8s/helm \ +# -f e2e/k8s/values/minikube.yaml \ +# -f e2e/k8s/values/minikube-auth.yaml \ +# -f e2e/k8s/values/minikube-auth-portforward.yaml \ +# --set api.image.tag= ... # -# This keeps the auth-enabled local harness but disables ingress so the stack -# can be validated through kubectl port-forward on machines where registry.k8s.io -# is blocked and the ingress addon cannot bootstrap. The chart still keeps split -# pods on the API service URL while the API pod itself loops back to localhost. - -k8s-nim-operator: - enabled: false - -postgresql: - persistence: - storageClass: standard - -core: - storage: - storageClass: standard - volumePermissionsImage: busybox +# Disables ingress and envoy proxy for machines where registry.k8s.io is +# blocked and the ingress addon cannot bootstrap. Use kubectl port-forward +# instead. envoyProxy: enabled: false ingress: enabled: false - -platformConfig: - auth: - enabled: true - policy_decision_point_provider: embedded - policy_data_refresh_interval: 2 - bundle_cache_seconds: 2 - admin_email: "admin@example.com" - inference_gateway: - mock_provider_prefix: igw-mock- - models: - controller: - backends: - nim_operator: - enabled: false diff --git a/e2e/k8s/values/minikube-auth.yaml b/e2e/k8s/values/minikube-auth.yaml index a3f7eab3aa..5aa3b36251 100644 --- a/e2e/k8s/values/minikube-auth.yaml +++ b/e2e/k8s/values/minikube-auth.yaml @@ -1,37 +1,17 @@ -# CPU-only minikube values for local auth E2E verification. +# Auth overlay for minikube — use on top of minikube.yaml. # # Usage: -# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth.yaml +# helm upgrade -i nemo-platform k8s/helm \ +# -f e2e/k8s/values/minikube.yaml \ +# -f e2e/k8s/values/minikube-auth.yaml \ +# --set api.image.tag= ... # -# This values file is intentionally local-friendly: -# - disables the NIM operator dependency -# - enables platform auth without requiring external OIDC -# - relies on the chart default that keeps split pods on the API service URL -# while the API pod itself loops back to localhost -# - keeps ingress enabled for browser/curl access through minikube ingress +# Disables the NIM operator and enables embedded auth with short +# cache/refresh intervals for fast test feedback. k8s-nim-operator: enabled: false -postgresql: - persistence: - storageClass: standard - -core: - storage: - storageClass: standard - volumePermissionsImage: busybox - -ingress: - enabled: true - className: nginx - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "0" - nginx.ingress.kubernetes.io/proxy-read-timeout: "600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "600" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" - nginx.ingress.kubernetes.io/proxy-request-buffering: "off" - platformConfig: auth: enabled: true @@ -39,8 +19,6 @@ platformConfig: policy_data_refresh_interval: 2 bundle_cache_seconds: 2 admin_email: "admin@example.com" - inference_gateway: - mock_provider_prefix: igw-mock- models: controller: backends: diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml index 2570a9b716..78159c0a97 100644 --- a/e2e/k8s/values/minikube.yaml +++ b/e2e/k8s/values/minikube.yaml @@ -1,7 +1,23 @@ # Minikube values for local development # This is a standalone values file - use it in place of ./default.yaml # -# Usage: helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml +# Usage (local build): +# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \ +# --set api.image.repository=docker.io/my-registry/nmp-api \ +# --set api.image.tag=local --set api.image.pullPolicy=Never \ +# --set core.image.repository=docker.io/my-registry/nmp-api \ +# --set core.image.tag=local --set core.image.pullPolicy=Never \ +# --set platformConfig.platform.image_registry=docker.io/my-registry \ +# --set platformConfig.platform.image_tag=local +# +# Usage (GHCR): +# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \ +# --set api.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \ +# --set api.image.tag=latest \ +# --set core.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \ +# --set core.image.tag=latest \ +# --set platformConfig.platform.image_registry=ghcr.io/nvidia-nemo/platform \ +# --set platformConfig.platform.image_tag=latest # Enable NIM operator for local GPU testing k8s-nim-operator: @@ -30,6 +46,13 @@ ingress: nginx.ingress.kubernetes.io/proxy-send-timeout: "600" nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + hosts: + - name: "" + paths: + - path: / + pathType: Prefix + service: '{{ include "nemo-platform.ingressBackendService" . }}' + port: '{{ include "nemo-platform.ingressBackendPort" . }}' platformConfig: inference_gateway: From 5925d1f5e74d6ce7b8cea5cd3de622e94cf34706 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 14:24:55 -0700 Subject: [PATCH 2/7] fix prefixes Signed-off-by: Matthew Grossman --- e2e/k8s/values/minikube.yaml | 7 ------- k8s/helm/values.yaml | 4 ++++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml index 78159c0a97..147345f426 100644 --- a/e2e/k8s/values/minikube.yaml +++ b/e2e/k8s/values/minikube.yaml @@ -46,13 +46,6 @@ ingress: nginx.ingress.kubernetes.io/proxy-send-timeout: "600" nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" - hosts: - - name: "" - paths: - - path: / - pathType: Prefix - service: '{{ include "nemo-platform.ingressBackendService" . }}' - port: '{{ include "nemo-platform.ingressBackendPort" . }}' platformConfig: inference_gateway: diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 031cda827c..85d1593cab 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -410,6 +410,10 @@ ingress: pathType: Exact service: '{{ include "nemo-platform.ingressBackendService" . }}' port: '{{ include "nemo-platform.ingressBackendPort" . }}' + - path: /health + pathType: Prefix + service: '{{ include "nemo-platform.ingressBackendService" . }}' + port: '{{ include "nemo-platform.ingressBackendPort" . }}' - path: /apis pathType: Prefix service: '{{ include "nemo-platform.ingressBackendService" . }}' From f9faa858687d52c454b3594511e7a05c0e06da29 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 15:09:39 -0700 Subject: [PATCH 3/7] fixes Signed-off-by: Matthew Grossman --- e2e/conftest.py | 4 ++-- e2e/k8s/scripts/run_auth_e2e.sh | 4 ++-- k8s/helm/values.yaml | 4 ---- .../src/nemo_platform_ext/cli/commands/quickstart/cli.py | 2 +- .../src/nemo_platform_ext/cli/commands/services/cli.py | 6 +++--- .../src/nemo_platform_ext/cli/commands/setup.py | 2 +- .../src/nemo_platform/cli/commands/quickstart/cli.py | 2 +- .../src/nemo_platform/cli/commands/services/cli.py | 6 +++--- .../nemo-platform/src/nemo_platform/cli/commands/setup.py | 2 +- 9 files changed, 14 insertions(+), 18 deletions(-) diff --git a/e2e/conftest.py b/e2e/conftest.py index 543c39efd3..8c5aecb96a 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -162,11 +162,11 @@ def _find_free_port() -> int: def _wait_for_healthy(url: str, timeout: float = _HEALTH_TIMEOUT) -> bool: - """Poll /health/ready until it returns 200 or timeout expires.""" + """Poll /status until it returns 200 or timeout expires.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: - resp = httpx.get(f"{url}/health/ready", timeout=2.0) + resp = httpx.get(f"{url}/status", timeout=2.0) if resp.status_code == 200: return True except httpx.RequestError: diff --git a/e2e/k8s/scripts/run_auth_e2e.sh b/e2e/k8s/scripts/run_auth_e2e.sh index 41f89f5381..2e5016b7b8 100755 --- a/e2e/k8s/scripts/run_auth_e2e.sh +++ b/e2e/k8s/scripts/run_auth_e2e.sh @@ -43,8 +43,8 @@ wait_for_url() { echo "Using minikube profile: ${MINIKUBE_PROFILE}" echo "Using base URL: ${BASE_URL}" -if ! wait_for_url "${BASE_URL}/health/ready"; then - echo "Platform did not become ready at ${BASE_URL}/health/ready" >&2 +if ! wait_for_url "${BASE_URL}/status"; then + echo "Platform did not become ready at ${BASE_URL}/status" >&2 exit 1 fi diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 85d1593cab..031cda827c 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -410,10 +410,6 @@ ingress: pathType: Exact service: '{{ include "nemo-platform.ingressBackendService" . }}' port: '{{ include "nemo-platform.ingressBackendPort" . }}' - - path: /health - pathType: Prefix - service: '{{ include "nemo-platform.ingressBackendService" . }}' - port: '{{ include "nemo-platform.ingressBackendPort" . }}' - path: /apis pathType: Prefix service: '{{ include "nemo-platform.ingressBackendService" . }}' diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/quickstart/cli.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/quickstart/cli.py index 5fb3ebab04..03e1d59e0c 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/quickstart/cli.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/quickstart/cli.py @@ -91,7 +91,7 @@ def _check_ready_endpoint(port: int, timeout: float = 2.0) -> bool: import httpx try: - response = httpx.get(f"http://localhost:{port}/health/ready", timeout=timeout) + response = httpx.get(f"http://localhost:{port}/status", timeout=timeout) return response.status_code == 200 except Exception: return False diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py index 0d00241d13..f14f462b0c 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py @@ -92,9 +92,9 @@ def _wait_for_healthy( timeout: int = _HEALTH_TIMEOUT_SECONDS, poll_interval: float = _HEALTH_POLL_INTERVAL, ) -> bool: - """Poll the platform health endpoint until it responds or timeout.""" + """Poll the platform status endpoint until it responds or timeout.""" effective_host = "localhost" if host in ("0.0.0.0", "::") else host # noqa: S104 - url = str(httpx.URL(scheme="http", host=effective_host, port=port, path="/health/ready")) + url = str(httpx.URL(scheme="http", host=effective_host, port=port, path="/status")) deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: @@ -335,7 +335,7 @@ def start_services( ) -> None: """Start platform services in the background. - Detaches the process, polls /health/ready, then returns. + Detaches the process, polls /status, then returns. Examples: nemo services start diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py index 02cf2d7acc..ea72e45b66 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py @@ -283,7 +283,7 @@ def _bootstrap_config_if_missing(base_url: str, workspace: str) -> None: def _check_platform_reachable(base_url: str, timeout: float = 5.0) -> bool: """Return True if the platform health endpoint responds.""" try: - resp = httpx.get(f"{base_url.rstrip('/')}/health/ready", timeout=timeout) + resp = httpx.get(f"{base_url.rstrip('/')}/status", timeout=timeout) return resp.status_code == 200 except Exception: return False diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/cli.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/cli.py index 626c107aaf..456b074727 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/cli.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/quickstart/cli.py @@ -91,7 +91,7 @@ def _check_ready_endpoint(port: int, timeout: float = 2.0) -> bool: import httpx try: - response = httpx.get(f"http://localhost:{port}/health/ready", timeout=timeout) + response = httpx.get(f"http://localhost:{port}/status", timeout=timeout) return response.status_code == 200 except Exception: return False diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py index 35bc857140..e9ee1b915e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py @@ -92,9 +92,9 @@ def _wait_for_healthy( timeout: int = _HEALTH_TIMEOUT_SECONDS, poll_interval: float = _HEALTH_POLL_INTERVAL, ) -> bool: - """Poll the platform health endpoint until it responds or timeout.""" + """Poll the platform status endpoint until it responds or timeout.""" effective_host = "localhost" if host in ("0.0.0.0", "::") else host # noqa: S104 - url = str(httpx.URL(scheme="http", host=effective_host, port=port, path="/health/ready")) + url = str(httpx.URL(scheme="http", host=effective_host, port=port, path="/status")) deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: @@ -335,7 +335,7 @@ def start_services( ) -> None: """Start platform services in the background. - Detaches the process, polls /health/ready, then returns. + Detaches the process, polls /status, then returns. Examples: nemo services start diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py index 37ce10f7b5..dc1b408f94 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py @@ -283,7 +283,7 @@ def _bootstrap_config_if_missing(base_url: str, workspace: str) -> None: def _check_platform_reachable(base_url: str, timeout: float = 5.0) -> bool: """Return True if the platform health endpoint responds.""" try: - resp = httpx.get(f"{base_url.rstrip('/')}/health/ready", timeout=timeout) + resp = httpx.get(f"{base_url.rstrip('/')}/status", timeout=timeout) return resp.status_code == 200 except Exception: return False From 1b9377bc8b9b4e38f87cd55dcb565d246e657f79 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 15:12:39 -0700 Subject: [PATCH 4/7] remove the file chagnes Signed-off-by: Matthew Grossman --- e2e/k8s/scripts/install_nmp_auth_e2e.sh | 6 +-- e2e/k8s/values/minikube-auth-portforward.yaml | 42 +++++++++++++++---- e2e/k8s/values/minikube-auth.yaml | 36 ++++++++++++---- 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/e2e/k8s/scripts/install_nmp_auth_e2e.sh b/e2e/k8s/scripts/install_nmp_auth_e2e.sh index 84dadcbfdd..ceff9c1488 100755 --- a/e2e/k8s/scripts/install_nmp_auth_e2e.sh +++ b/e2e/k8s/scripts/install_nmp_auth_e2e.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash # Install the auth-enabled local E2E harness on minikube. -# -# Layers minikube-auth.yaml on top of minikube.yaml so auth-specific -# config stays minimal and doesn't duplicate base minikube values. set -euo pipefail @@ -11,8 +8,7 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" export NAMESPACE="${NAMESPACE:-${KUBE_NAMESPACE:-default}}" export HELM_RELEASE_NAME="${HELM_RELEASE_NAME:-nemo-platform}" -export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube.yaml}" -export HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:-} -f ${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml" +export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yaml}" export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}" export NMP_E2E_TAG="${NMP_E2E_TAG:-local}" export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}" diff --git a/e2e/k8s/values/minikube-auth-portforward.yaml b/e2e/k8s/values/minikube-auth-portforward.yaml index 41f91713e1..0a262c289c 100644 --- a/e2e/k8s/values/minikube-auth-portforward.yaml +++ b/e2e/k8s/values/minikube-auth-portforward.yaml @@ -1,18 +1,42 @@ -# Port-forward overlay — use on top of minikube.yaml + minikube-auth.yaml. +# CPU-only minikube values for local auth E2E verification without ingress. # # Usage: -# helm upgrade -i nemo-platform k8s/helm \ -# -f e2e/k8s/values/minikube.yaml \ -# -f e2e/k8s/values/minikube-auth.yaml \ -# -f e2e/k8s/values/minikube-auth-portforward.yaml \ -# --set api.image.tag= ... +# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth-portforward.yaml # -# Disables ingress and envoy proxy for machines where registry.k8s.io is -# blocked and the ingress addon cannot bootstrap. Use kubectl port-forward -# instead. +# This keeps the auth-enabled local harness but disables ingress so the stack +# can be validated through kubectl port-forward on machines where registry.k8s.io +# is blocked and the ingress addon cannot bootstrap. The chart still keeps split +# pods on the API service URL while the API pod itself loops back to localhost. + +k8s-nim-operator: + enabled: false + +postgresql: + persistence: + storageClass: standard + +core: + storage: + storageClass: standard + volumePermissionsImage: busybox envoyProxy: enabled: false ingress: enabled: false + +platformConfig: + auth: + enabled: true + policy_decision_point_provider: embedded + policy_data_refresh_interval: 2 + bundle_cache_seconds: 2 + admin_email: "admin@example.com" + inference_gateway: + mock_provider_prefix: igw-mock- + models: + controller: + backends: + nim_operator: + enabled: false diff --git a/e2e/k8s/values/minikube-auth.yaml b/e2e/k8s/values/minikube-auth.yaml index 5aa3b36251..a3f7eab3aa 100644 --- a/e2e/k8s/values/minikube-auth.yaml +++ b/e2e/k8s/values/minikube-auth.yaml @@ -1,17 +1,37 @@ -# Auth overlay for minikube — use on top of minikube.yaml. +# CPU-only minikube values for local auth E2E verification. # # Usage: -# helm upgrade -i nemo-platform k8s/helm \ -# -f e2e/k8s/values/minikube.yaml \ -# -f e2e/k8s/values/minikube-auth.yaml \ -# --set api.image.tag= ... +# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube-auth.yaml # -# Disables the NIM operator and enables embedded auth with short -# cache/refresh intervals for fast test feedback. +# This values file is intentionally local-friendly: +# - disables the NIM operator dependency +# - enables platform auth without requiring external OIDC +# - relies on the chart default that keeps split pods on the API service URL +# while the API pod itself loops back to localhost +# - keeps ingress enabled for browser/curl access through minikube ingress k8s-nim-operator: enabled: false +postgresql: + persistence: + storageClass: standard + +core: + storage: + storageClass: standard + volumePermissionsImage: busybox + +ingress: + enabled: true + className: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + platformConfig: auth: enabled: true @@ -19,6 +39,8 @@ platformConfig: policy_data_refresh_interval: 2 bundle_cache_seconds: 2 admin_email: "admin@example.com" + inference_gateway: + mock_provider_prefix: igw-mock- models: controller: backends: From 5f006a4b197c5c5319efc9b32cbe9a3d4ebb6e11 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 19:47:06 -0700 Subject: [PATCH 5/7] make vendor Signed-off-by: Matthew Grossman --- .../nemo-platform/.nmpcontext/openapi.yaml | 93 ++- .../nemo-platform/.nmpcontext/stainless.yaml | 154 ++--- .../beta/evaluator/agent_eval/dashboard.py | 143 +++++ .../beta/evaluator/agent_eval/evaluator.py | 563 ++++++++++++++++++ .../beta/evaluator/agent_eval/persistence.py | 61 ++ .../agent_eval/runtimes/callable_runtime.py | 105 ++++ .../agent_eval/runtimes/codex/runtime.py | 414 +++++++++++++ .../agent_eval/runtimes/docker_sandbox.py | 346 +++++++++++ .../beta/evaluator/agent_eval/tasks.py | 2 + .../beta/evaluator/execution/samples.py | 3 +- .../src/nemo_platform/cli/app.py | 2 +- .../src/nemo_platform/cli/commands/auth.py | 16 +- .../src/nemo_platform/config/models.py | 31 +- .../resources/experiments/api.md | 2 + .../resources/experiments/experiments.py | 210 ++++++- .../src/nemo_platform/resources/files/api.md | 2 +- .../nemo_platform/resources/files/filesets.py | 1 - .../src/nemo_platform/types/__init__.py | 1 + .../experiments/experiment_filter_param.py | 6 + .../experiments/experiment_list_params.py | 5 +- .../types/experiments/experiment_response.py | 6 + .../src/nemo_platform/types/files/__init__.py | 2 - .../src/nemo_platform/types/files/fileset.py | 2 +- .../types/files/fileset_create_params.py | 1 - .../types/files/fileset_metadata_param.py | 47 -- .../nemo_platform/types/shared/__init__.py | 1 + .../{files => shared}/fileset_metadata.py | 4 +- .../shared_params/fileset_metadata_param.py | 4 +- .../tests/api_resources/test_experiments.py | 210 +++++++ .../cli/commands/test_auth.py | 24 + .../cli/commands/test_setup.py | 2 +- .../nemo_platform_ext/config/test_config.py | 55 ++ sdk/stainless.yaml | 154 ++--- 33 files changed, 2432 insertions(+), 240 deletions(-) create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata_param.py rename sdk/python/nemo-platform/src/nemo_platform/types/{files => shared}/fileset_metadata.py (91%) diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index bd1e211721..5251f16cbe 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -3719,6 +3719,8 @@ paths: - updated_at - -name - name + - -pinned_at + - pinned_at type: string description: Sort field; prefix with '-' for descending. default: -created_at @@ -3733,7 +3735,8 @@ paths: $ref: '#/components/schemas/ExperimentFilter' description: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true - to return only soft-deleted experiments; omit to see only live ones. + to return only soft-deleted experiments; omit to see only live ones. Pass + is_pinned=true (or false) to filter by pinned state; omit to return both. responses: '200': description: Successful Response @@ -3851,6 +3854,82 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin: + post: + tags: + - Experiments + summary: Pin Experiment + description: 'Pin an experiment to the top of the list (workspace-shared). + + + Re-pinning an already-pinned experiment refreshes ``pinned_at`` to the current + timestamp, + + which is intentional (most-recently-pinned sorts first).' + operationId: pin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Experiments + summary: Unpin Experiment + description: 'Unpin an experiment. Idempotent: unpinning an already-unpinned + experiment is a no-op.' + operationId: unpin_experiment_apis_intake_v2_workspaces__workspace__experiments__name__pin_delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentResponse' + '404': + description: Experiment not found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/intake/v2/workspaces/{workspace}/experiments/{name}/sessions: get: tags: @@ -10234,6 +10313,11 @@ components: false) to see only live experiments. title: Is Deleted type: boolean + is_pinned: + description: When true, returns only pinned experiments. When false, returns + only unpinned experiments. Omit to return both. + title: Is Pinned + type: boolean title: ExperimentFilter type: object ExperimentGroupFilter: @@ -10416,6 +10500,13 @@ components: title: Updated At type: string format: date-time + pinned_at: + title: Pinned At + description: Timestamp at which the experiment was pinned, or null if unpinned. + Managed via POST/DELETE /experiments/{name}/pin. + nullable: true + type: string + format: date-time evaluator_names: items: type: string diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index fcbb5c2a58..2f0a999c90 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -81,86 +81,86 @@ client_settings: # `pagination` defines [pagination schemes] which provides a template to match # endpoints and generate next-page and auto-pagination helpers in the SDKs. pagination: - - name: default_pagination - type: page_number - request: - page: - type: integer - x-stainless-pagination-property: - purpose: page_number_param - page_size: - type: integer - response: - data: - type: array - x-stainless-pagination-property: - purpose: items - items: - type: object - additionalProperties: true - pagination: +- name: default_pagination + type: page_number + request: + page: + type: integer + x-stainless-pagination-property: + purpose: page_number_param + page_size: + type: integer + response: + data: + type: array + x-stainless-pagination-property: + purpose: items + items: type: object - properties: - page: - type: integer - title: Page - description: The current page number. - x-stainless-pagination-property: - purpose: current_page_number_field - page_size: - type: integer - title: Page Size - description: The page size used for the query. - current_page_size: - type: integer - title: Current Page Size - description: The size for the current page. - total_pages: - type: integer - title: Total Pages - description: The total number of pages. - x-stainless-pagination-property: - purpose: total_page_count_field - total_results: - type: integer - title: Total Results - description: The total number of results. - required: - - page - - page_size - - total_pages - - total_results - - current_page_size - - name: logs_pagination - type: cursor - request: - limit: - type: integer - page_cursor: - type: string - x-stainless-pagination-property: - purpose: next_cursor_param - response: - data: - type: array - x-stainless-pagination-property: - purpose: items - items: - type: object - additionalProperties: true - next_page: - type: string - x-stainless-pagination-property: - purpose: next_cursor_field + additionalProperties: true + pagination: + type: object + properties: + page: + type: integer + title: Page + description: The current page number. + x-stainless-pagination-property: + purpose: current_page_number_field + page_size: + type: integer + title: Page Size + description: The page size used for the query. + current_page_size: + type: integer + title: Current Page Size + description: The size for the current page. + total_pages: + type: integer + title: Total Pages + description: The total number of pages. + x-stainless-pagination-property: + purpose: total_page_count_field + total_results: + type: integer + title: Total Results + description: The total number of results. + required: + - page + - page_size + - total_pages + - total_results + - current_page_size +- name: logs_pagination + type: cursor + request: + limit: + type: integer + page_cursor: + type: string + x-stainless-pagination-property: + purpose: next_cursor_param + response: + data: + type: array + x-stainless-pagination-property: + purpose: items + items: + type: object + additionalProperties: true + next_page: + type: string + x-stainless-pagination-property: + purpose: next_cursor_field streaming: on_event: - - data_starts_with: "[DONE]" - handle: done - - event_type: error - handle: error - - event_type: - handle: yield + - data_starts_with: "[DONE]" + handle: done + - event_type: error + handle: error + - event_type: + handle: yield readme: example_requests: @@ -919,6 +919,8 @@ resources: retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} + pin: post /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin + unpin: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin subresources: sessions: models: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py new file mode 100644 index 0000000000..c537ce2274 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small HTML dashboard for standalone agent-eval result bundles.""" + +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult +from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalTaskScore +from pydantic import BaseModel + + +def write_dashboard(result: AgentEvalResult, output_path: str | Path) -> Path: + """Write an HTML dashboard and return its path.""" + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_dashboard(result), encoding="utf-8") + return path + + +def render_dashboard(result: AgentEvalResult) -> str: + """Render a compact generic report for metric outputs.""" + return f""" + + + + + Agent Eval Report + + + +
+

Agent Eval Report

+
Run {_e(result.run_id)} · {_e(result.summary.task_count)} tasks · {_e(result.summary.trial_count)} trials
+
+
+
+
Tasks{_e(result.summary.task_count)}
+
Trials{_e(result.summary.trial_count)}
+
Metric Scores{_e(result.summary.score_count)}
+
+

Metric Rollups

+ {_metric_rollups(result)} +

Scores

+ {_score_table(result.scores)} +
+ + +""" + + +def _metric_rollups(result: AgentEvalResult) -> str: + aggregated = result.summary.scores.scores + if not aggregated: + return '

No numeric metric outputs to summarize.

' + rows: list[str] = [] + for score in sorted(aggregated, key=lambda item: item.name): + rows.append( + "" + f"{_e(score.name)}" + f"{_format_score(score.mean)}" + f"{_e(score.count)}" + f"{_e(score.nan_count)}" + "" + ) + return ( + "" + + "".join(rows) + + "
NameMeanCountNaN
" + ) + + +def _score_table(scores: list[AgentEvalTaskScore]) -> str: + if not scores: + return '

No metric scores.

' + rows = [ + "" + f"{_e(score.task_id)}" + f"{_e(score.trial_id)}" + f"{_e(score.metric_type)}" + f"{_outputs(score)}" + "" + for score in scores + ] + return ( + "" + + "".join(rows) + + "
TaskTrialMetricOutputs
" + ) + + +def _outputs(score: AgentEvalTaskScore) -> str: + chunks = [] + for output in score.outputs: + chunks.append( + f'
{_e(output.name)}
{_e(_jsonish(output.value))}
' + ) + return '
' + "".join(chunks) + "
" + + +def _jsonish(value: Any) -> str: + if isinstance(value, BaseModel): + value = value.model_dump(mode="json") + try: + return json.dumps(value, indent=2, sort_keys=True) + except (TypeError, ValueError): + # ValueError covers circular references; fall back to a plain string rather than crash rendering. + return str(value) + + +def _format_score(value: float | None) -> str: + if value is None: + return "n/a" + return f"{value:.3f}" + + +def _e(value: object) -> str: + return html.escape(str(value), quote=True) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py new file mode 100644 index 0000000000..c0c5099f1a --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -0,0 +1,563 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Standalone agent evaluation orchestration.""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from collections.abc import Awaitable, Callable, Sequence +from datetime import UTC, datetime +from logging import getLogger +from pathlib import Path +from typing import Any, cast +from urllib.parse import urlparse + +import httpx +import nemo_platform.beta.evaluator.inference as inference +from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard +from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run +from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_platform.beta.evaluator.agent_eval.scores import ( + AgentEvalDiagnostic, + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, +) +from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_platform.beta.evaluator.agent_eval.trials import ( + AgentEvalTarget, + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + AgentTaskRunner, +) +from nemo_platform.beta.evaluator.agent_inference import ( + AgentInferenceFn, + make_agent_inference_request, + new_agent_inference_client, +) +from nemo_platform.beta.evaluator.execution.metric_execution import generate_online_sample, run_sync +from nemo_platform.beta.evaluator.execution.samples import build_metric_input +from nemo_platform.beta.evaluator.inference import InferenceFn +from nemo_platform.beta.evaluator.metrics.protocol import Metric, validate_metric_result +from nemo_platform.beta.evaluator.metrics.utils import metric_type_name +from nemo_platform.beta.evaluator.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel +from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor +from openai import AsyncOpenAI + +log = getLogger(__name__) + + +class AgentEvaluator: + """Run stored-trial or live-target agent evaluations. + + The online inference seam (an optional ``inference_fn``, transport ``client``, and + ``default_headers``) is injected on the evaluator instance rather than the run config, + because these are runtime transport concerns rather than declarative run settings. A + single ``inference_fn``/``client`` pair serves both model and agent targets; leave them + unset to let the evaluator build a default client for the resolved target type. + """ + + def __init__( + self, + *, + inference_fn: InferenceFn | AgentInferenceFn | None = None, + client: AsyncOpenAI | httpx.AsyncClient | None = None, + default_headers: dict[str, str] | None = None, + ) -> None: + self.inference_fn = inference_fn + self.client = client + self.default_headers = default_headers + + async def run( + self, + *, + tasks: Sequence[AgentEvalTask], + trials: Sequence[AgentEvalTrial] | None = None, + target: AgentEvalTarget | None = None, + config: AgentEvalRunConfig | None = None, + ) -> AgentEvalResult: + """Evaluate imported trials or generate live trials before scoring. + + Exactly one of ``trials`` or ``target`` must be provided. + """ + resolved_config = config or AgentEvalRunConfig() + task_list = list(tasks) + if not task_list: + raise ValueError("at least one task is required") + + run_id = resolved_config.run_id or _new_run_id() + runtime_config = resolved_config.model_copy(update={"run_id": run_id}) + + # Branch on which seam was supplied so the type checker can narrow ``target`` to a + # concrete ``AgentEvalTarget`` without a cast. + if trials is not None: + if target is not None: + raise ValueError("provide exactly one of trials or target") + trial_list = list(trials) + elif target is not None: + trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config) + else: + raise ValueError("provide exactly one of trials or target") + scores = await self._score_trials( + tasks=task_list, + trials=trial_list, + config=runtime_config, + run_id=run_id, + ) + benchmark = {**_benchmark_metadata(task_list), **runtime_config.benchmark} + result = AgentEvalResult( + run_id=run_id, + tasks=task_list, + trials=trial_list, + scores=scores, + summary=AgentEvalSummary.from_scores(scores, tasks=task_list), + benchmark=benchmark, + ) + + if runtime_config.output_dir is not None: + result = _persist_with_optional_dashboard(result, runtime_config.output_dir, runtime_config.write_dashboard) + return result + + def run_sync( + self, + *, + tasks: Sequence[AgentEvalTask], + trials: Sequence[AgentEvalTrial] | None = None, + target: AgentEvalTarget | None = None, + config: AgentEvalRunConfig | None = None, + ) -> AgentEvalResult: + """Synchronous bridge for :meth:`run`.""" + return run_sync(lambda: self.run(tasks=tasks, trials=trials, target=target, config=config)) + + async def _score_trials( + self, + *, + tasks: list[AgentEvalTask], + trials: list[AgentEvalTrial], + config: AgentEvalRunConfig, + run_id: str, + ) -> list[AgentEvalTaskScore]: + tasks_by_id = {task.id: task for task in tasks} + task_index_by_id = {task.id: index for index, task in enumerate(tasks)} + trials_by_task: dict[str, list[AgentEvalTrial]] = defaultdict(list) + for trial in trials: + if trial.task_id not in tasks_by_id: + raise ValueError(f"trial {trial.id!r} references unknown task {trial.task_id!r}") + trials_by_task[trial.task_id].append(trial) + + # Fail loudly when a task produced no trial. Imported trials or an AgentTaskRunner may omit a + # task entirely; without this an incomplete run would look successful aside from lower summary + # counts. (A richer alternative is to emit a "missing trial" failed score per metric.) + tasks_without_trials = [task.id for task in tasks if not trials_by_task.get(task.id)] + if tasks_without_trials: + raise ValueError(f"no trials produced for tasks: {sorted(tasks_without_trials)}") + + for task in tasks: + if not task.metrics: + raise ValueError(f"task {task.id!r} does not declare any metrics") + + semaphore = asyncio.Semaphore(config.parallelism) + + async def guarded_score(task: AgentEvalTask, trial: AgentEvalTrial, metric: Metric) -> AgentEvalTaskScore: + async with semaphore: + row_index = task_index_by_id[task.id] + if trial.status == AgentEvalTrialStatus.FAILED: + return _failed_metric_score( + run_id=run_id, + task=task, + trial=trial, + metric=metric, + row_index=row_index, + diagnostic=AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message=f"trial {trial.id!r} is failed", + source=metric_type_name(metric), + details={"trial_status": trial.status.value}, + ), + ) + try: + return await _score_metric( + run_id=run_id, + task=task, + trial=trial, + metric=metric, + row_index=row_index, + ) + except Exception as exc: + if config.fail_fast: + raise + log.warning( + "metric %s failed for trial %r (task %r): %s", + metric_type_name(metric), + trial.id, + task.id, + exc, + ) + return _failed_metric_score( + run_id=run_id, + task=task, + trial=trial, + metric=metric, + row_index=row_index, + diagnostic=_exception_diagnostic(exc, metric_type_name(metric)), + ) + + return await asyncio.gather( + *[ + guarded_score(task, trial, metric) + for task in tasks + for trial in trials_by_task.get(task.id, []) + for metric in task.metrics + ] + ) + + async def _generate_trials( + self, + *, + tasks: list[AgentEvalTask], + target: AgentEvalTarget, + config: AgentEvalRunConfig, + ) -> list[AgentEvalTrial]: + if isinstance(target, AgentTaskRunner): + return list(await target.run_tasks(tasks, config=config)) + if not isinstance(target, (Model, Agent)): + raise NotImplementedError(f"unsupported agent-eval target type: {type(target).__name__}") + + params = _resolve_live_params(config, target) + prompt_template = config.prompt_template or _default_prompt_template(target) + semaphore = asyncio.Semaphore(params.parallelism) + + # Use the injected transport client when provided; otherwise build a default for the + # resolved target type and close it when generation finishes. + client = self.client + close_client: Callable[[], Awaitable[Any]] | None = None + if client is None and self.inference_fn is None: + if isinstance(target, Model): + client = inference.new_inference_client(target) + close_client = client.close + else: + client = new_agent_inference_client() + close_client = client.aclose + + try: + # When config.params.ignore_request_failure is set, convert a failed generation request + # into a FAILED trial (which the scorer turns into failed metric scores) instead of + # aborting the whole run. This matches the existing online-evaluator contract. + async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + try: + sample = await _generate_sample( + target=target, + row=_task_row(task), + index=index, + prompt_template=prompt_template, + params=params, + inference_fn=self.inference_fn, + client=client, + default_headers=self.default_headers, + ) + except Exception as exc: + if params.ignore_request_failure: + return _failed_generation_trial(task, target, exc) + raise + return _trial_from_sample(task, target, sample) + + return await asyncio.gather(*(generate_one(index, task) for index, task in enumerate(tasks))) + finally: + if close_client is not None: + await close_client() + + +async def _generate_sample( + *, + target: Model | Agent, + row: dict[str, Any], + index: int, + prompt_template: str | dict[str, Any], + params: RunConfigOnline | RunConfigOnlineModel, + inference_fn: InferenceFn | AgentInferenceFn | None, + client: AsyncOpenAI | httpx.AsyncClient | None, + default_headers: dict[str, str] | None, +) -> dict[str, Any]: + # InferenceFn and AgentInferenceFn are callable protocols, so isinstance cannot discriminate + # the injected fn; narrow it per target type with a cast (matching execution/benchmark_execution). + # The transport client is a real class union, so isinstance narrowing is enough there. + if isinstance(target, Model): + model_params = cast(RunConfigOnlineModel, params) + preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params, model_format=target.format) + model_inference_fn = ( + cast(InferenceFn, inference_fn) if inference_fn is not None else inference.make_inference_request + ) + return await generate_online_sample( + target=target, + row=row, + index=index, + prompt_template=prompt_template, + params=model_params, + inference_fn=model_inference_fn, + client=client if isinstance(client, AsyncOpenAI) else None, + preprocess_hooks=preprocess_hooks, + postprocess_hooks=postprocess_hooks, + default_headers=default_headers, + ) + + agent_inference_fn = ( + cast(AgentInferenceFn, inference_fn) if inference_fn is not None else make_agent_inference_request + ) + return await generate_online_sample( + target=target, + row=row, + index=index, + prompt_template=prompt_template, + params=params, + inference_fn=agent_inference_fn, + client=client if isinstance(client, httpx.AsyncClient) else None, + default_headers=default_headers, + ) + + +def _trial_from_sample(task: AgentEvalTask, target: Model | Agent, sample: dict[str, Any]) -> AgentEvalTrial: + output_text = sample.get("output_text") + if not (isinstance(output_text, str) and output_text.strip()): + # Reasoning models that exhaust the token budget can return only + # `reasoning_content` with empty `content`. Fall back to that text so the + # trial stays scorable instead of being dropped as empty output. + output_text = _reasoning_content_fallback(sample.get("response")) + if "trajectory" in sample: + trace = EvidenceDescriptor(kind="trace", format="json", data=sample["trajectory"]) + else: + trace = EvidenceDescriptor(kind="sdk_online_generation", data={"task_id": task.id, "target": target.name}) + + return AgentEvalTrial( + id=f"{task.id}:{target.name}", + task_id=task.id, + status=AgentEvalTrialStatus.COMPLETED, + output=AgentOutput( + output_text=output_text if isinstance(output_text, str) else None, + response=sample.get("response"), + metadata={ + key: value for key, value in sample.items() if key not in {"output_text", "response", "trajectory"} + }, + ), + evidence=CandidateEvidence(descriptors={"trace": trace}), + metadata={ + "model_id": target.name, + "target_name": target.name, + "generated": True, + }, + ) + + +def _reasoning_content_fallback(response: Any) -> str | None: + if not isinstance(response, dict): + return None + choices = response.get("choices") + if not isinstance(choices, list): + return None + for choice in choices: + message = choice.get("message") if isinstance(choice, dict) else None + if not isinstance(message, dict): + continue + reasoning = message.get("reasoning_content") + if isinstance(reasoning, str) and reasoning.strip(): + return reasoning + return None + + +def _failed_generation_trial(task: AgentEvalTask, target: Model | Agent, exc: Exception) -> AgentEvalTrial: + return AgentEvalTrial( + id=f"{task.id}:{target.name}", + task_id=task.id, + status=AgentEvalTrialStatus.FAILED, + output=None, + evidence=CandidateEvidence( + descriptors={ + "error": EvidenceDescriptor( + kind="error", + data={"error_type": exc.__class__.__name__, "error": str(exc)}, + ) + } + ), + metadata={ + "model_id": target.name, + "target_name": target.name, + "generated": True, + "error_type": exc.__class__.__name__, + "error": str(exc), + }, + ) + + +async def _score_metric( + *, + run_id: str, + task: AgentEvalTask, + trial: AgentEvalTrial, + metric: Metric, + row_index: int, +) -> AgentEvalTaskScore: + output_spec = metric.output_spec() + metric_result = validate_metric_result( + await metric.compute_scores(build_metric_input(_metric_row(task, trial), _trial_sample(trial), row_index)), + output_spec, + ) + metric_type = metric_type_name(metric) + return AgentEvalTaskScore( + id=_score_id(run_id, task.id, trial.id, metric_type), + run_id=run_id, + task_id=task.id, + trial_id=trial.id, + metric_type=metric_type, + status=AgentEvalScoreStatus.COMPLETED, + outputs=metric_result.outputs, + metadata={ + "row_index": row_index, + "trial_metadata": trial.metadata, + }, + ) + + +def _failed_metric_score( + *, + run_id: str, + task: AgentEvalTask, + trial: AgentEvalTrial, + metric: Metric, + row_index: int, + diagnostic: AgentEvalDiagnostic, +) -> AgentEvalTaskScore: + metric_type = metric_type_name(metric) + return AgentEvalTaskScore( + id=_score_id(run_id, task.id, trial.id, metric_type), + run_id=run_id, + task_id=task.id, + trial_id=trial.id, + metric_type=metric_type, + status=AgentEvalScoreStatus.FAILED, + outputs=[], + diagnostics=[diagnostic], + metadata={ + "row_index": row_index, + "trial_metadata": trial.metadata, + }, + ) + + +def _exception_diagnostic(exc: Exception, metric_type: str) -> AgentEvalDiagnostic: + return AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message=str(exc) or exc.__class__.__name__, + source=metric_type, + details={"exception_type": exc.__class__.__name__}, + ) + + +def _score_id(run_id: str, task_id: str, trial_id: str, metric_type: str) -> str: + return f"{run_id}:{task_id}:{trial_id}:{metric_type}" + + +def _trial_sample(trial: AgentEvalTrial) -> dict[str, Any]: + if trial.output is None: + return {} + sample: dict[str, Any] = { + **trial.metadata, + **trial.output.metadata, + } + if trial.output.output_text is not None: + sample["output_text"] = trial.output.output_text + if trial.output.response is not None: + sample["response"] = trial.output.response + if trial.evidence is not None: + sample["evidence"] = trial.evidence + return sample + + +def _resolve_live_params( + config: AgentEvalRunConfig, + target: Model | Agent, +) -> RunConfigOnline | RunConfigOnlineModel: + params = config.params + if isinstance(target, Model): + if params is None: + return RunConfigOnlineModel(parallelism=config.parallelism) + if isinstance(params, RunConfigOnlineModel): + return params + if isinstance(params, RunConfigOnline): + return RunConfigOnlineModel(**params.model_dump(mode="python")) + if isinstance(params, RunConfig): + return RunConfigOnlineModel(**params.model_dump(mode="python")) + + if params is None: + return RunConfigOnline(parallelism=config.parallelism) + if isinstance(params, RunConfigOnlineModel): + return RunConfigOnline( + **params.model_dump( + mode="python", + exclude={"inference", "system_prompt", "reasoning", "structured_output"}, + ) + ) + if isinstance(params, RunConfigOnline): + return params + return RunConfigOnline(**params.model_dump(mode="python")) + + +def _default_prompt_template(target: Model | Agent) -> dict[str, Any]: + if isinstance(target, Model) and _is_completions_endpoint(target.url): + return {"prompt": "{{item.prompt}}"} + return {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} + + +def _task_row(task: AgentEvalTask) -> dict[str, Any]: + return { + **task.inputs, + "task_id": task.id, + "prompt": task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent, + } + + +def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]: + return { + "task": { + "id": task.id, + "intent": task.intent, + "metadata": task.metadata, + }, + "inputs": task.inputs, + "trial": { + "id": trial.id, + "task_id": trial.task_id, + "status": trial.status.value, + "metadata": trial.metadata, + }, + } + + +def _is_completions_endpoint(url: str) -> bool: + path = urlparse(url).path.rstrip("/") + return path.endswith("/completions") and not path.endswith("/chat/completions") + + +def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]: + benchmarks = sorted({str(task.metadata.get("benchmark")) for task in tasks if task.metadata.get("benchmark")}) + if not benchmarks: + return {} + return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks} + + +def _persist_with_optional_dashboard( + result: AgentEvalResult, + output_dir: Path, + write_html: bool, +) -> AgentEvalResult: + path = Path(output_dir) + dashboard_path = None + if write_html: + dashboard_path = write_dashboard(result.model_copy(update={"output_dir": path}), path / "report.html") + return persist_run(result.model_copy(update={"output_dir": path, "dashboard_path": dashboard_path}), path) + + +def _new_run_id() -> str: + return f"agent-eval-{datetime.now(UTC).strftime('%Y%m%d%H%M%S')}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py new file mode 100644 index 0000000000..fbabd00670 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Persistence helpers for standalone agent-eval result bundles.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult +from pydantic import BaseModel + + +def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalResult: + """Persist a completed run bundle to ``output_dir``.""" + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + + _write_json(path / "benchmark.json", result.benchmark) + _write_jsonl(path / "tasks.jsonl", result.tasks) + _write_jsonl(path / "trials.jsonl", result.trials) + _write_jsonl(path / "scores.jsonl", result.scores) + _write_json(path / "summary.json", result.summary) + + updated = result.model_copy(update={"output_dir": path}) + _write_json(path / "run.json", _run_manifest(updated)) + return updated + + +def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: + return { + "run_id": result.run_id, + "output_dir": str(result.output_dir) if result.output_dir is not None else None, + "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, + "artifacts": { + "benchmark": "benchmark.json", + "tasks": "tasks.jsonl", + "trials": "trials.jsonl", + "scores": "scores.jsonl", + "summary": "summary.json", + }, + } + + +def _write_json(path: Path, value: BaseModel | dict[str, Any]) -> None: + if isinstance(value, BaseModel): + payload = value.model_dump(mode="json") + else: + payload = value + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _write_jsonl(path: Path, rows: Sequence[BaseModel]) -> None: + # Stream row-by-row instead of joining the whole payload in memory first. + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row.model_dump(mode="json"), sort_keys=True)) + handle.write("\n") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py new file mode 100644 index 0000000000..1a36ac4868 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal, dependency-light AgentTaskRunner backed by a user-supplied callable.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence + + +@dataclass(slots=True) +class TrialDraft: + """What an agent callable returns for one task: final output plus optional evidence. + + The runtime wraps this into a completed :class:`AgentEvalTrial`. Returning a + :class:`AgentOutput` or a plain string is also accepted as shorthand. + """ + + output: AgentOutput + evidence: CandidateEvidence | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +AgentTaskFn = Callable[[AgentEvalTask], Awaitable[TrialDraft | AgentOutput | str]] + + +class CallableAgentTaskRunner: + """Smallest possible :class:`AgentTaskRunner`: delegate each task to an async callable. + + The callable receives an :class:`AgentEvalTask` and returns the agent's final output as + a :class:`TrialDraft`, an :class:`AgentOutput`, or a plain string. This runtime adds only + what the ``AgentTaskRunner`` contract needs: bounded concurrency, stable trial ids, and + failure capture (an exception becomes a ``FAILED`` trial instead of aborting the batch). + It requires no Docker or external agent SDK, so it doubles as a reference for richer + runtimes and as the seam an ``AgentEvaluator`` drives via ``run(target=runner)``. + """ + + def __init__( + self, + agent_fn: AgentTaskFn, + *, + parallelism: int | None = None, + trial_id_suffix: str = "trial", + ) -> None: + self._agent_fn = agent_fn + self._parallelism = parallelism + self._trial_id_suffix = trial_id_suffix + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> list[AgentEvalTrial]: + """Run every task through the callable and return one trial per task, in order.""" + parallelism = self._parallelism if self._parallelism is not None else (config.parallelism if config else 4) + semaphore = asyncio.Semaphore(max(1, parallelism)) + + async def run_one(task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + try: + result = await self._agent_fn(task) + except Exception as exc: # noqa: BLE001 - surfaced as a FAILED trial, not a crash + return self._failed_trial(task, exc) + return self._completed_trial(task, result) + + return list(await asyncio.gather(*(run_one(task) for task in tasks))) + + def _trial_id(self, task: AgentEvalTask) -> str: + return f"{task.id}:{self._trial_id_suffix}" + + def _completed_trial(self, task: AgentEvalTask, result: TrialDraft | AgentOutput | str) -> AgentEvalTrial: + draft = _as_trial_draft(result) + return AgentEvalTrial( + id=self._trial_id(task), + task_id=task.id, + status=AgentEvalTrialStatus.COMPLETED, + output=draft.output, + evidence=draft.evidence, + metadata=draft.metadata, + ) + + def _failed_trial(self, task: AgentEvalTask, exc: Exception) -> AgentEvalTrial: + return AgentEvalTrial( + id=self._trial_id(task), + task_id=task.id, + status=AgentEvalTrialStatus.FAILED, + metadata={"error": f"{type(exc).__name__}: {exc}"}, + ) + + +def _as_trial_draft(result: TrialDraft | AgentOutput | str) -> TrialDraft: + if isinstance(result, TrialDraft): + return result + if isinstance(result, AgentOutput): + return TrialDraft(output=result) + if isinstance(result, str): + return TrialDraft(output=AgentOutput(output_text=result)) + raise TypeError(f"agent callable must return TrialDraft, AgentOutput, or str; got {type(result).__name__}") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py new file mode 100644 index 0000000000..b9e929d3da --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py @@ -0,0 +1,414 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Codex-backed agent-eval runtimes.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import shlex +import shutil +import subprocess +from collections.abc import Awaitable, Callable, Mapping, Sequence +from enum import StrEnum +from pathlib import Path +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime +from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor + +DEFAULT_CODEX_TIMEOUT_S = 600 +DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" +DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" +DEFAULT_CODEX_DOCKER_CLI_PACKAGE = "@openai/codex@0.137.0" +ProcessFactory = Callable[..., Awaitable[Any]] + + +class RuntimeChoice(StrEnum): + DOCKER = "docker" + LOCAL = "local" + + +class EffectiveCodexRuntime(StrEnum): + DOCKER_SANDBOX = "docker_sandbox" + DOCKER_CLI = "docker_cli" + LOCAL_CLI = "local_cli" + + +class CodexCliAgentRuntime: + """AgentTaskRunner that uses the locally installed Codex CLI credentials.""" + + def __init__( + self, + *, + model: str | None = None, + work_root: str | Path | None = None, + codex_bin: str = "codex", + timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, + process_factory: ProcessFactory | None = None, + runtime_name: str = "codex_cli", + ) -> None: + self._model = model + self._work_root = Path(work_root).expanduser() if work_root is not None else None + self._codex_bin = codex_bin + self._timeout_s = timeout_s + self._process_factory = process_factory or asyncio.create_subprocess_exec + self._runtime_name = runtime_name + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + if shutil.which(self._codex_bin) is None: + raise RuntimeError(f"Codex CLI executable {self._codex_bin!r} was not found on PATH") + + resolved_config = config or AgentEvalRunConfig() + semaphore = asyncio.Semaphore(resolved_config.parallelism) + + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task(index, task, resolved_config) + + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + + async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> AgentEvalTrial: + evidence_dir = self._evidence_dir(index, task, config) + workspace_dir = evidence_dir / "workspace" + evidence_dir.mkdir(parents=True, exist_ok=True) + workspace_dir.mkdir(parents=True, exist_ok=True) + + prompt = _codex_prompt(task) + prompt_path = evidence_dir / "prompt.txt" + task_path = evidence_dir / "task.json" + stdout_path = evidence_dir / "stdout.jsonl" + stderr_path = evidence_dir / "stderr.txt" + final_output_path = evidence_dir / "final_output.txt" + + prompt_path.write_text(prompt, encoding="utf-8") + task_path.write_text(task.model_dump_json(indent=2), encoding="utf-8") + + command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) + process: Any | None = None + try: + process = await self._process_factory( + *command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for( + process.communicate(prompt.encode("utf-8")), + timeout=self._timeout_s, + ) + except TimeoutError as exc: + await _terminate_process(process) + return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) + except Exception as exc: + return _failed_codex_trial(task, evidence_dir, exc, runtime_name=self._runtime_name) + + stdout_text = _decode_process_output(stdout) + stderr_text = _decode_process_output(stderr) + stdout_path.write_text(stdout_text, encoding="utf-8") + stderr_path.write_text(stderr_text, encoding="utf-8") + + if process.returncode != 0: + return _failed_codex_trial( + task, + evidence_dir, + RuntimeError(f"codex exec exited with status {process.returncode}: {stderr_text.strip()}"), + runtime_name=self._runtime_name, + ) + + if final_output_path.exists(): + output_text = final_output_path.read_text(encoding="utf-8") + else: + output_text = stdout_text + final_output_path.write_text(output_text, encoding="utf-8") + return AgentEvalTrial( + id=f"{task.id}:codex", + task_id=task.id, + status=AgentEvalTrialStatus.COMPLETED, + output=AgentOutput( + output_text=output_text, + metadata={ + "runtime": self._runtime_name, + "agent": "codex", + "agent_model": self._model, + "evidence_dir": str(evidence_dir), + }, + ), + evidence=CandidateEvidence( + descriptors={ + "workspace": EvidenceDescriptor(kind="filesystem", ref=str(workspace_dir)), + "prompt": EvidenceDescriptor(kind="text", format="txt", ref=str(prompt_path)), + "task": EvidenceDescriptor(kind="json", format="json", ref=str(task_path)), + "stdout": EvidenceDescriptor(kind="codex_stdout", format="jsonl", ref=str(stdout_path)), + "stderr": EvidenceDescriptor(kind="text", format="txt", ref=str(stderr_path)), + "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), + }, + metadata={"runtime": self._runtime_name, "agent": "codex"}, + ), + metadata={ + "runtime": self._runtime_name, + "agent": "codex", + "agent_model": self._model, + "generated": True, + }, + ) + + def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: + command = [ + self._codex_bin, + "exec", + "--skip-git-repo-check", + "--ephemeral", + "--ignore-user-config", + "--sandbox", + "workspace-write", + "--cd", + str(workspace_dir), + "--output-last-message", + str(final_output_path), + "--json", + ] + if self._model is not None: + command.extend(["--model", self._model]) + command.append("-") + return command + + def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: + root = self._work_root + if root is None: + root = (config.output_dir or Path.cwd()) / "evidence" / "codex" + safe_task_id = _safe_path_name(task.id) + task_dir = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" + return Path(root) / task_dir + + +class CodexDockerCliAgentRuntime(CodexCliAgentRuntime): + """AgentTaskRunner that runs Codex CLI inside a Docker container.""" + + def __init__( + self, + *, + model: str | None = None, + work_root: str | Path | None = None, + docker_bin: str = "docker", + image: str = DEFAULT_CODEX_DOCKER_CLI_IMAGE, + codex_package: str = DEFAULT_CODEX_DOCKER_CLI_PACKAGE, + auth_path: str | Path | None = None, + timeout_s: int = DEFAULT_CODEX_TIMEOUT_S, + process_factory: ProcessFactory | None = None, + ) -> None: + super().__init__( + model=model, + work_root=work_root, + timeout_s=timeout_s, + process_factory=process_factory, + runtime_name="codex_docker_cli", + ) + self._docker_bin = docker_bin + self._image = image + self._codex_package = codex_package + self._auth_path = ( + Path(auth_path).expanduser() if auth_path is not None else Path.home() / ".codex" / "auth.json" + ) + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + if shutil.which(self._docker_bin) is None: + raise RuntimeError(f"Docker executable {self._docker_bin!r} was not found on PATH") + if not self._auth_path.exists(): + raise RuntimeError( + f"Codex auth file was not found at {self._auth_path}. Run `codex login` or use OPENAI_API_KEY " + "so --runtime docker can use DockerSandboxAgentRuntime." + ) + + resolved_config = config or AgentEvalRunConfig() + semaphore = asyncio.Semaphore(resolved_config.parallelism) + + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task(index, task, resolved_config) + + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + + def _command(self, *, workspace_dir: Path, final_output_path: Path) -> list[str]: + evidence_dir = final_output_path.parent + inner_command = [ + "npx", + "-y", + self._codex_package, + "exec", + "--skip-git-repo-check", + "--ephemeral", + "--sandbox", + "danger-full-access", + "--cd", + "/workspace", + "--output-last-message", + "/evidence/final_output.txt", + "--json", + ] + if self._model is not None: + inner_command.extend(["--model", self._model]) + inner_command.append("-") + return [ + self._docker_bin, + "run", + "--rm", + "-i", + "-v", + f"{self._auth_path.resolve()}:/root/.codex/auth.json:ro", + "-v", + f"{workspace_dir.resolve()}:/workspace", + "-v", + f"{evidence_dir.resolve()}:/evidence", + self._image, + "sh", + "-lc", + shlex.join(inner_command), + ] + + +def resolve_codex_target( + *, + runtime: RuntimeChoice, + model: str | None, + output_dir: Path, + env: Mapping[str, str] = os.environ, +) -> tuple[CodexCliAgentRuntime | CodexDockerCliAgentRuntime | DockerSandboxAgentRuntime, str, EffectiveCodexRuntime]: + """Resolve a Codex-backed agent-eval target for ProfBench-style candidate runs.""" + effective_runtime = _resolve_codex_runtime(runtime, env) + if effective_runtime == EffectiveCodexRuntime.LOCAL_CLI: + return ( + CodexCliAgentRuntime(model=model, work_root=output_dir / "evidence" / "codex"), + "codex_cli_candidate_and_live_judge", + effective_runtime, + ) + if effective_runtime == EffectiveCodexRuntime.DOCKER_CLI: + return ( + CodexDockerCliAgentRuntime(model=model, work_root=output_dir / "evidence" / "codex-docker"), + "codex_docker_cli_candidate_and_live_judge", + effective_runtime, + ) + if effective_runtime == EffectiveCodexRuntime.DOCKER_SANDBOX: + return ( + DockerSandboxAgentRuntime(model=model or DEFAULT_CODEX_DOCKER_MODEL), + "docker_sandbox_candidate_and_live_judge", + effective_runtime, + ) + raise ValueError(f"unsupported Codex runtime {runtime!r}") + + +def _resolve_codex_runtime(runtime: RuntimeChoice, env: Mapping[str, str] = os.environ) -> EffectiveCodexRuntime: + if runtime == RuntimeChoice.LOCAL: + return EffectiveCodexRuntime.LOCAL_CLI + if runtime == RuntimeChoice.DOCKER: + if _openai_sdk_secret_key_is_set(env): + return EffectiveCodexRuntime.DOCKER_SANDBOX + return EffectiveCodexRuntime.DOCKER_CLI + raise ValueError(f"unsupported Codex runtime {runtime!r}") + + +def list_codex_agent_models(*, codex_bin: str = "codex") -> list[dict[str, Any]]: + """Return visible Codex model descriptors from the local Codex CLI.""" + if shutil.which(codex_bin) is None: + raise RuntimeError(f"Codex CLI executable {codex_bin!r} was not found on PATH") + result = subprocess.run( + [codex_bin, "debug", "models"], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + models = payload.get("models") + if not isinstance(models, list): + raise RuntimeError("Codex model catalog did not contain a models list") + visible = [model for model in models if isinstance(model, dict) and model.get("visibility") == "list"] + return sorted(visible, key=lambda model: int(model.get("priority") or 0), reverse=True) + + +def print_codex_agent_models(*, codex_bin: str = "codex") -> None: + """Print local Codex model slugs and display names.""" + for model in list_codex_agent_models(codex_bin=codex_bin): + slug = model.get("slug") + if not isinstance(slug, str): + continue + display_name = model.get("display_name") + if isinstance(display_name, str) and display_name != slug: + print(f"{slug}\t{display_name}") + else: + print(slug) + + +def _codex_prompt(task: AgentEvalTask) -> str: + return ( + "Answer the ProfBench task below. Return only the final answer text; do not include " + "analysis, markdown fences, tool logs, or commentary.\n\n" + f"Task id: {task.id}\n" + f"Intent: {task.intent}\n" + f"Inputs: {task.inputs}\n" + ) + + +def _failed_codex_trial( + task: AgentEvalTask, + evidence_dir: Path, + exc: Exception, + *, + runtime_name: str = "codex_cli", +) -> AgentEvalTrial: + error_path = evidence_dir / "error.json" + error_path.write_text( + json.dumps({"error_type": exc.__class__.__name__, "error": str(exc)}) + "\n", encoding="utf-8" + ) + return AgentEvalTrial( + id=f"{task.id}:codex", + task_id=task.id, + status=AgentEvalTrialStatus.FAILED, + output=None, + evidence=CandidateEvidence( + descriptors={"error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path))}, + metadata={"runtime": runtime_name, "agent": "codex"}, + ), + metadata={ + "runtime": runtime_name, + "agent": "codex", + "error_type": exc.__class__.__name__, + "error": str(exc), + }, + ) + + +async def _terminate_process(process: Any | None) -> None: + if process is None or process.returncode is not None: + return + process.kill() + with contextlib.suppress(Exception): + await process.wait() + + +def _decode_process_output(value: bytes | str | None) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + return value.decode("utf-8", errors="replace") + + +def _safe_path_name(value: str) -> str: + return "".join(char if char.isalnum() or char in "._-" else "-" for char in value).strip(".-")[:120] + + +def _openai_sdk_secret_key_is_set(env: Mapping[str, str] = os.environ) -> bool: + return env.get("OPENAI_API_KEY", "").strip().startswith("sk-") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py new file mode 100644 index 0000000000..f1f6ddd4f3 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Docker-backed sandbox runtime for agent-eval trials.""" + +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import json +import re +import shutil +import tarfile +import tempfile +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor +from pydantic_core import to_jsonable_python + +DEFAULT_INSTRUCTIONS = ( + "Complete the task inside the sandbox workspace. Inspect the provided task files, " + "write any durable artifacts under output/, and return a concise final answer." +) +_RUNTIME_NAME = "docker_sandbox" +_SAFE_NAME_PATTERN = re.compile(r"[^A-Za-z0-9_.-]+") + + +@dataclass(frozen=True) +class SandboxSDK: + """Loaded OpenAI Agents SDK symbols used by the runtime.""" + + Runner: Any + RunConfig: Any + SandboxRunConfig: Any + Manifest: Any + SandboxAgent: Any + DockerSandboxClient: Any + DockerSandboxClientOptions: Any + File: Any + Dir: Any + LocalDir: Any + DEFAULT_PYTHON_SANDBOX_IMAGE: str + docker_from_env: Callable[[], Any] + + +def _load_agents_sdk() -> SandboxSDK: + try: + # The OpenAI Agents SDK ships under the `nemo-evaluator-sdk[agent-runtimes]` extra and is + # imported only when this Docker runtime is actually used, so it is absent from the default + # type-checking environment. + from agents import Runner # ty: ignore[unresolved-import] + from agents.run import RunConfig # ty: ignore[unresolved-import] + from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig # ty: ignore[unresolved-import] + from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE # ty: ignore[unresolved-import] + from agents.sandbox.entries import Dir, File, LocalDir # ty: ignore[unresolved-import] + from agents.sandbox.sandboxes.docker import ( # ty: ignore[unresolved-import] + DockerSandboxClient, + DockerSandboxClientOptions, + ) + + from docker import from_env as docker_from_env + except ImportError as exc: + raise RuntimeError("DockerSandboxAgentRuntime requires `nemo-evaluator-sdk[agent-runtimes]`") from exc + + return SandboxSDK( + Runner=Runner, + RunConfig=RunConfig, + SandboxRunConfig=SandboxRunConfig, + Manifest=Manifest, + SandboxAgent=SandboxAgent, + DockerSandboxClient=DockerSandboxClient, + DockerSandboxClientOptions=DockerSandboxClientOptions, + File=File, + Dir=Dir, + LocalDir=LocalDir, + DEFAULT_PYTHON_SANDBOX_IMAGE=DEFAULT_PYTHON_SANDBOX_IMAGE, + docker_from_env=docker_from_env, + ) + + +class DockerSandboxAgentRuntime: + """Generate agent-eval trials by running a SandboxAgent in Docker per task.""" + + def __init__( + self, + *, + model: str | None = None, + instructions: str | None = None, + image: str | None = None, + work_root: Path | None = None, + timeout_s: float | None = None, + agent_factory: Callable[..., Any] | None = None, + sandbox_client_factory: Callable[[], Any] | None = None, + runner: Any | None = None, + ) -> None: + self._model = model + self._instructions = instructions or DEFAULT_INSTRUCTIONS + self._image = image + self._work_root = work_root + self._timeout_s = timeout_s + self._agent_factory = agent_factory + self._sandbox_client_factory = sandbox_client_factory + self._runner = runner + + async def run_tasks( + self, + tasks: Sequence[AgentEvalTask], + config: AgentEvalRunConfig | None = None, + ) -> Sequence[AgentEvalTrial]: + resolved_config = config or AgentEvalRunConfig() + if resolved_config.run_id is None: + resolved_config = resolved_config.model_copy(update={"run_id": _new_runtime_run_id()}) + sdk = _load_agents_sdk() + semaphore = asyncio.Semaphore(resolved_config.parallelism) + + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task(index, task, resolved_config, sdk) + + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + + async def _run_task( + self, + index: int, + task: AgentEvalTask, + config: AgentEvalRunConfig, + sdk: SandboxSDK, + ) -> AgentEvalTrial: + evidence_dir = self._evidence_dir(index, task, config) + evidence_dir.mkdir(parents=True, exist_ok=True) + prompt = _task_prompt(task) + manifest = self._build_manifest(task, sdk) + agent = self._build_agent(manifest, sdk) + client = self._build_client(sdk) + sandbox = None + + try: + sandbox = await client.create( + manifest=manifest, + options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE), + ) + async with sandbox: + result = await self._run_agent(agent, prompt, sandbox, sdk) + return await self._completed_trial(task, result, sandbox, evidence_dir) + except Exception as exc: + return self._failed_trial(task, exc, evidence_dir) + finally: + if sandbox is not None: + with contextlib.suppress(Exception): + await client.delete(sandbox) + + def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: + entries: dict[str, Any] = { + "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), + "task.json": sdk.File(content=task.model_dump_json().encode("utf-8")), + "output": sdk.Dir(), + } + workspace_dir = task.inputs.get("workspace_dir") + if workspace_dir is not None: + entries["workspace"] = sdk.LocalDir(src=_validated_workspace_dir(workspace_dir)) + return sdk.Manifest(root="/workspace", entries=entries) + + def _build_agent(self, manifest: Any, sdk: SandboxSDK) -> Any: + agent_factory = self._agent_factory or sdk.SandboxAgent + kwargs = { + "name": "NeMo Agent Eval Docker Sandbox Runtime", + "instructions": self._instructions, + "default_manifest": manifest, + } + if self._model is not None: + kwargs["model"] = self._model + return agent_factory(**kwargs) + + def _build_client(self, sdk: SandboxSDK) -> Any: + if self._sandbox_client_factory is not None: + return self._sandbox_client_factory() + return sdk.DockerSandboxClient(sdk.docker_from_env()) + + async def _run_agent(self, agent: Any, prompt: str, sandbox: Any, sdk: SandboxSDK) -> Any: + runner = self._runner or sdk.Runner + run = runner.run( + agent, + prompt, + run_config=sdk.RunConfig(sandbox=sdk.SandboxRunConfig(session=sandbox)), + ) + if self._timeout_s is not None: + return await asyncio.wait_for(_maybe_await(run), timeout=self._timeout_s) + return await _maybe_await(run) + + async def _completed_trial( + self, + task: AgentEvalTask, + result: Any, + sandbox: Any, + evidence_dir: Path, + ) -> AgentEvalTrial: + final_output = getattr(result, "final_output", None) + final_output_text = "" if final_output is None else str(final_output) + + final_output_path = evidence_dir / "final_output.txt" + run_items_path = evidence_dir / "run_items.json" + raw_responses_path = evidence_dir / "raw_responses.json" + workspace_tar_path = evidence_dir / "workspace.tar" + final_state_dir = evidence_dir / "final_state" + + final_output_path.write_text(final_output_text, encoding="utf-8") + _write_json(run_items_path, _jsonable(getattr(result, "new_items", []))) + _write_json(raw_responses_path, _jsonable(getattr(result, "raw_responses", []))) + await _persist_workspace(sandbox, workspace_tar_path, final_state_dir) + + return AgentEvalTrial( + id=f"{task.id}:docker-sandbox", + task_id=task.id, + status=AgentEvalTrialStatus.COMPLETED, + output=AgentOutput( + output_text=final_output_text, + response={"final_output": final_output_text}, + metadata={ + "runtime": _RUNTIME_NAME, + "evidence_dir": str(evidence_dir), + }, + ), + evidence=CandidateEvidence( + descriptors={ + "final_state": EvidenceDescriptor(kind="filesystem", ref=str(final_state_dir)), + "workspace_archive": EvidenceDescriptor(kind="archive", format="tar", ref=str(workspace_tar_path)), + "run_items": EvidenceDescriptor(kind="run_items", format="json", ref=str(run_items_path)), + "raw_responses": EvidenceDescriptor( + kind="raw_responses", format="json", ref=str(raw_responses_path) + ), + "final_output": EvidenceDescriptor(kind="text", format="txt", ref=str(final_output_path)), + }, + metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, + ), + metadata={"runtime": _RUNTIME_NAME, "generated": True}, + ) + + def _failed_trial(self, task: AgentEvalTask, exc: Exception, evidence_dir: Path) -> AgentEvalTrial: + error_path = evidence_dir / "error.json" + _write_json( + error_path, + { + "error_type": exc.__class__.__name__, + "error": str(exc), + }, + ) + return AgentEvalTrial( + id=f"{task.id}:docker-sandbox", + task_id=task.id, + status=AgentEvalTrialStatus.FAILED, + output=None, + evidence=CandidateEvidence( + descriptors={ + "error": EvidenceDescriptor(kind="error", format="json", ref=str(error_path)), + }, + metadata={"runtime": _RUNTIME_NAME, "sandbox_backend": "docker"}, + ), + metadata={ + "runtime": _RUNTIME_NAME, + "error_type": exc.__class__.__name__, + "error": str(exc), + }, + ) + + def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: + root = config.output_dir if config.output_dir is not None else self._work_root + if root is None: + root = Path(tempfile.gettempdir()) / "nemo-evaluator-agent-runtime" + run_id = config.run_id or _new_runtime_run_id() + safe_task_id = _safe_path_name(task.id) + task_name = f"{index:06d}-{safe_task_id}" if safe_task_id else f"task-{index:06d}" + return Path(root) / "agent-runtime" / run_id / task_name + + +def _validated_workspace_dir(workspace_dir: Any) -> Path: + if not isinstance(workspace_dir, (str, Path)): + raise ValueError(f"workspace_dir must be a path, got {type(workspace_dir).__name__}") + path = Path(workspace_dir).expanduser() + if not path.is_absolute(): + raise ValueError(f"workspace_dir must be an absolute path; got {workspace_dir!r}") + resolved = path.resolve() + if not resolved.is_dir(): + raise ValueError(f"workspace_dir does not exist or is not a directory: {resolved}") + return resolved + + +def _task_prompt(task: AgentEvalTask) -> str: + return str(task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent) + + +async def _maybe_await(value: Awaitable[Any] | Any) -> Any: + if inspect.isawaitable(value): + return await value + return value + + +async def _persist_workspace(sandbox: Any, workspace_tar_path: Path, final_state_dir: Path) -> None: + archive = await sandbox.persist_workspace() + try: + with workspace_tar_path.open("wb") as out: + shutil.copyfileobj(archive, out) + finally: + close = getattr(archive, "close", None) + if close is not None: + close() + + _extract_tar_safely(workspace_tar_path, final_state_dir) + + +def _extract_tar_safely(archive_path: Path, destination_root: Path) -> None: + if destination_root.exists(): + shutil.rmtree(destination_root) + destination_root.mkdir(parents=True, exist_ok=True) + + # The stdlib `data` filter (Python 3.12+) rejects absolute paths, parent-directory + # traversal, links, and special files, so we do not hand-roll those guards. + with tarfile.open(archive_path, "r:*") as archive: + archive.extractall(destination_root, filter="data") + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _jsonable(value: Any) -> Any: + # Normalize pydantic models, dataclasses, Paths, sets, etc. into JSON-safe values; + # `repr` is the last-resort fallback for anything still not serializable. + return to_jsonable_python(value, fallback=repr) + + +def _safe_path_name(value: str) -> str: + sanitized = _SAFE_NAME_PATTERN.sub("-", value).strip(".-") + return sanitized[:120] + + +def _new_runtime_run_id() -> str: + timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") + return f"agent-runtime-{timestamp}-{uuid4().hex[:8]}" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index f4ca34350e..0755c9cd1f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -62,6 +62,8 @@ class SemanticView(BaseModel): class AgentEvalTask(BaseModel): """Standalone agent-eval task: the unit of work being evaluated.""" + # TODO: Tasks may need to define a set of required_capabilities or something that allow the + # runtime to skip trying to complete a task that isn't possible. model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") id: str = Field(description="Stable task identifier, unique within the supplied task collection.") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py index 49f9f34da0..0d3619cf50 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/samples.py @@ -7,7 +7,7 @@ from nemo_platform.beta.evaluator.metrics.protocol import CandidateOutput, DatasetRow, MetricInput -_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory"}) +_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory", "evidence"}) def build_offline_sample(row: dict[str, Any]) -> dict[str, Any]: @@ -37,6 +37,7 @@ def build_metric_input(row: dict[str, Any], sample: dict[str, Any], index: int | output_text=output_text if isinstance(output_text, str) else None, response=sample.get("response"), trajectory=sample.get("trajectory"), + evidence=sample.get("evidence"), metadata=metadata, ), ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py b/sdk/python/nemo-platform/src/nemo_platform/cli/app.py index 05f8a26dfb..73ac115b89 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/app.py @@ -224,7 +224,7 @@ def main( """ Command-line interface for NeMo Platform. - :books: Documentation: https://nvidia-nemo.github.io/nemo-platform/main/ + :books: Documentation: https://docs.nvidia.com/nemo-platform [green]Getting started:[/] - Browse documentation with [cyan]`nemo docs --list`[/] diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py index e44cf4bfc5..a3498e967f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py @@ -450,7 +450,7 @@ def login( if oidc_config.device_authorization_endpoint is None: raise AuthError("This cluster does not support device flow authentication.") client_id = cast(str, oidc_config.client_id) - device_authorization_endpoint = cast(str, oidc_config.device_authorization_endpoint) + device_authorization_endpoint = oidc_config.device_authorization_endpoint try: token_response = asyncio.run( authenticate_with_device_flow( @@ -468,9 +468,12 @@ def login( claims = decode_jwt_claims(token) user_email = claims.get("upn") or claims.get("email") or claims.get("preferred_username") - granted_scopes = claims.get("scp") or claims.get("scope") or "" - if isinstance(granted_scopes, str): - granted_scopes = granted_scopes.split() + raw_granted_scopes = claims.get("scp") or claims.get("scope") + granted_scopes: list[str] = [] + if isinstance(raw_granted_scopes, str): + granted_scopes = raw_granted_scopes.split() + elif isinstance(raw_granted_scopes, list): + granted_scopes = [scope for scope in raw_granted_scopes if isinstance(scope, str)] validate_requested_scopes_granted(effective_scope, granted_scopes, scope_prefix) @@ -522,10 +525,7 @@ def logout(ctx: typer.Context) -> None: console.print("[yellow]Authentication is disabled on this cluster — nothing to log out from.[/]") return - logout_params = cast( - ConfigParams, - {"access_token": None, "refresh_token": None}, - ) + logout_params: ConfigParams = {"access_token": None, "refresh_token": None} Config.write(logout_params, context_name=context.context_name) console.print("[green]Logged out successfully.[/]") diff --git a/sdk/python/nemo-platform/src/nemo_platform/config/models.py b/sdk/python/nemo-platform/src/nemo_platform/config/models.py index 4c26f27291..3a13e80916 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/config/models.py +++ b/sdk/python/nemo-platform/src/nemo_platform/config/models.py @@ -169,8 +169,8 @@ class ConfigParams(TypedDict, total=False): base_url: str # OAuth fields (for OAuthUser) - access_token: str - refresh_token: str + access_token: str | None + refresh_token: str | None workspace: str default_model: str @@ -257,6 +257,8 @@ def ensure_context( # Find existing or create user user: User = next((u for u in self.users if u.name == user_name), None) # type: ignore[assignment] + access_token_provided = "access_token" in params + refresh_token_provided = "refresh_token" in params access_token = params.get("access_token") refresh_token = params.get("refresh_token") @@ -271,25 +273,28 @@ def ensure_context( else: user = NoAuthUser(name=user_name) self.users.append(user) - elif access_token: - # Replace existing user with OAuthUser + elif access_token_provided: + # Replace existing user with OAuthUser, or clear auth when the + # caller explicitly passes access_token=None. idx = next(i for i, u in enumerate(self.users) if u.name == user_name) - user = OAuthUser( - name=user_name, - token=SecretStr(access_token), - refresh_token=SecretStr(refresh_token) if refresh_token else None, - ) + if access_token: + user = OAuthUser( + name=user_name, + token=SecretStr(access_token), + refresh_token=SecretStr(refresh_token) if refresh_token else None, + ) + else: + user = NoAuthUser(name=user_name) self.users[idx] = user - elif isinstance(user, OAuthUser) and refresh_token: - # Update existing OAuthUser with new refresh token info + elif isinstance(user, OAuthUser) and refresh_token_provided: + # Allow callers to explicitly clear or rotate just the refresh token. idx = next(i for i, u in enumerate(self.users) if u.name == user_name) user = OAuthUser( name=user_name, token=user.token, - refresh_token=SecretStr(refresh_token) if refresh_token else user.refresh_token, + refresh_token=SecretStr(refresh_token) if refresh_token else None, ) self.users[idx] = user - # Find existing or create context context = existing_context if context is None: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md index 57f4bd2b37..d6275f4e35 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/api.md @@ -19,6 +19,8 @@ Methods: - client.experiments.update(path_name, \*, workspace, \*\*params) -> ExperimentResponse - client.experiments.list(\*, workspace, \*\*params) -> SyncDefaultPagination[ExperimentResponse] - client.experiments.delete(name, \*, workspace) -> None +- client.experiments.pin(name, \*, workspace) -> ExperimentResponse +- client.experiments.unpin(name, \*, workspace) -> ExperimentResponse ## Sessions diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py index 7d6a674f4b..8931b898c8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/experiments/experiments.py @@ -286,7 +286,10 @@ def list( filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, - sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + sort: Literal[ + "-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at" + ] + | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -300,7 +303,8 @@ def list( Args: filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. + soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + false) to filter by pinned state; omit to return both. page: Page number. @@ -380,6 +384,91 @@ def delete( cast_to=NoneType, ) + def pin( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Pin an experiment to the top of the list (workspace-shared). + + Re-pinning an already-pinned experiment refreshes `pinned_at` to the current + timestamp, which is intentional (most-recently-pinned sorts first). + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._post( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + def unpin( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """Unpin an experiment. + + Idempotent: unpinning an already-unpinned experiment is a + no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + class AsyncExperimentsResource(AsyncAPIResource): @cached_property @@ -613,7 +702,10 @@ def list( filter: ExperimentFilterParam | Omit = omit, page: int | Omit = omit, page_size: int | Omit = omit, - sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] | Omit = omit, + sort: Literal[ + "-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at" + ] + | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -627,7 +719,8 @@ def list( Args: filter: Filter experiments by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. + soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + false) to filter by pinned state; omit to return both. page: Page number. @@ -707,6 +800,91 @@ async def delete( cast_to=NoneType, ) + async def pin( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """ + Pin an experiment to the top of the list (workspace-shared). + + Re-pinning an already-pinned experiment refreshes `pinned_at` to the current + timestamp, which is intentional (most-recently-pinned sorts first). + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._post( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + + async def unpin( + self, + name: str, + *, + workspace: str | None = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> ExperimentResponse: + """Unpin an experiment. + + Idempotent: unpinning an already-unpinned experiment is a + no-op. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not name: + raise ValueError(f"Expected a non-empty value for `name` but received {name!r}") + return await self._delete( + path_template( + "/apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin", workspace=workspace, name=name + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ExperimentResponse, + ) + class ExperimentsResourceWithRawResponse: def __init__(self, experiments: ExperimentsResource) -> None: @@ -727,6 +905,12 @@ def __init__(self, experiments: ExperimentsResource) -> None: self.delete = to_raw_response_wrapper( experiments.delete, ) + self.pin = to_raw_response_wrapper( + experiments.pin, + ) + self.unpin = to_raw_response_wrapper( + experiments.unpin, + ) @cached_property def sessions(self) -> SessionsResourceWithRawResponse: @@ -752,6 +936,12 @@ def __init__(self, experiments: AsyncExperimentsResource) -> None: self.delete = async_to_raw_response_wrapper( experiments.delete, ) + self.pin = async_to_raw_response_wrapper( + experiments.pin, + ) + self.unpin = async_to_raw_response_wrapper( + experiments.unpin, + ) @cached_property def sessions(self) -> AsyncSessionsResourceWithRawResponse: @@ -777,6 +967,12 @@ def __init__(self, experiments: ExperimentsResource) -> None: self.delete = to_streamed_response_wrapper( experiments.delete, ) + self.pin = to_streamed_response_wrapper( + experiments.pin, + ) + self.unpin = to_streamed_response_wrapper( + experiments.unpin, + ) @cached_property def sessions(self) -> SessionsResourceWithStreamingResponse: @@ -802,6 +998,12 @@ def __init__(self, experiments: AsyncExperimentsResource) -> None: self.delete = async_to_streamed_response_wrapper( experiments.delete, ) + self.pin = async_to_streamed_response_wrapper( + experiments.pin, + ) + self.unpin = async_to_streamed_response_wrapper( + experiments.unpin, + ) @cached_property def sessions(self) -> AsyncSessionsResourceWithStreamingResponse: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/files/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/files/api.md index 882f649add..72e7b5ca66 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/files/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/files/api.md @@ -33,7 +33,7 @@ Methods: Types: ```python -from nemo_platform.types.files import FilesetFilter, FilesetMetadata, FilesetMetadataParam +from nemo_platform.types.files import FilesetFilter ``` Methods: diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py b/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py index 2fbd9935dc..9b7afcb95f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/files/filesets.py @@ -34,7 +34,6 @@ from ...pagination import SyncDefaultPagination, AsyncDefaultPagination from ...types.files import ( FilesetPurpose, - FilesetMetadataParam, fileset_list_params, fileset_create_params, fileset_update_params, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py index 571b87927b..eb3af5c4f7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/__init__.py @@ -32,6 +32,7 @@ PlatformJobLog as PlatformJobLog, ToolCallConfig as ToolCallConfig, APIEndpointData as APIEndpointData, + FilesetMetadata as FilesetMetadata, FileStorageType as FileStorageType, InferenceParams as InferenceParams, LinearLayerSpec as LinearLayerSpec, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py index 27477bf2c1..93e43c31a4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_filter_param.py @@ -50,6 +50,12 @@ class ExperimentFilterParam(TypedDict, total=False): Omit (or false) to see only live experiments. """ + is_pinned: bool + """When true, returns only pinned experiments. + + When false, returns only unpinned experiments. Omit to return both. + """ + name: str """Filter experiments by name.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py index 8c2adab1ce..84183f259e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_list_params.py @@ -31,7 +31,8 @@ class ExperimentListParams(TypedDict, total=False): """ Filter experiments by name, experiment_group_id, dataset_name, dataset_version, created_by, created_at, or updated_at. Pass is_deleted=true to return only - soft-deleted experiments; omit to see only live ones. + soft-deleted experiments; omit to see only live ones. Pass is_pinned=true (or + false) to filter by pinned state; omit to return both. """ page: int @@ -40,5 +41,5 @@ class ExperimentListParams(TypedDict, total=False): page_size: int """Page size.""" - sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"] + sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name", "-pinned_at", "pinned_at"] """Sort field; prefix with '-' for descending.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py index 9967b18544..51be0b692c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/experiments/experiment_response.py @@ -66,6 +66,12 @@ class ExperimentResponse(BaseModel): model_names: Optional[List[str]] = None """Distinct model names observed across ingested sessions for this experiment.""" + pinned_at: Optional[datetime] = None + """Timestamp at which the experiment was pinned, or null if unpinned. + + Managed via POST/DELETE /experiments/{name}/pin. + """ + run_count: Optional[int] = None """ Number of distinct ingested experiment sessions; one session is treated as one diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/__init__.py index b76dd4a694..3833c1d785 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/files/__init__.py @@ -22,7 +22,6 @@ from .cache_status import CacheStatus as CacheStatus from .fileset_file import FilesetFile as FilesetFile from .fileset_purpose import FilesetPurpose as FilesetPurpose -from .fileset_metadata import FilesetMetadata as FilesetMetadata from .s3_storage_config import S3StorageConfig as S3StorageConfig from .ngc_storage_config import NGCStorageConfig as NGCStorageConfig from .fileset_list_params import FilesetListParams as FilesetListParams @@ -33,7 +32,6 @@ from .fileset_create_params import FilesetCreateParams as FilesetCreateParams from .fileset_update_params import FilesetUpdateParams as FilesetUpdateParams from .file_list_files_params import FileListFilesParams as FileListFilesParams -from .fileset_metadata_param import FilesetMetadataParam as FilesetMetadataParam from .file_upload_file_params import FileUploadFileParams as FileUploadFileParams from .s3_storage_config_param import S3StorageConfigParam as S3StorageConfigParam from .ngc_storage_config_param import NGCStorageConfigParam as NGCStorageConfigParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset.py index e6d9642b7a..810d5ce990 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset.py @@ -20,10 +20,10 @@ from ..._models import BaseModel from .fileset_purpose import FilesetPurpose -from .fileset_metadata import FilesetMetadata from .s3_storage_config import S3StorageConfig from .ngc_storage_config import NGCStorageConfig from .local_storage_config import LocalStorageConfig +from ..shared.fileset_metadata import FilesetMetadata from .huggingface_storage_config import HuggingfaceStorageConfig __all__ = ["Fileset", "Storage"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py index ea3cb763f7..9836fcb477 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_create_params.py @@ -21,7 +21,6 @@ from typing_extensions import Required, TypeAlias, TypedDict from .fileset_purpose import FilesetPurpose -from .fileset_metadata_param import FilesetMetadataParam from .s3_storage_config_param import S3StorageConfigParam from .ngc_storage_config_param import NGCStorageConfigParam from .local_storage_config_param import LocalStorageConfigParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata_param.py deleted file mode 100644 index 66f37de921..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata_param.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from __future__ import annotations - -from typing_extensions import TypedDict - -from ..shared_params.model_metadata_content import ModelMetadataContent -from ..shared_params.dataset_metadata_content import DatasetMetadataContent - -__all__ = ["FilesetMetadataParam"] - - -class FilesetMetadataParam(TypedDict, total=False): - """Tagged metadata container - the key indicates the type. - - Example: - metadata = FilesetMetadata( - dataset=DatasetMetadataContent( - schema={"columns": ["id", "name"]}, - ) - ) - """ - - dataset: DatasetMetadataContent - """Content for dataset-type filesets.""" - - model: ModelMetadataContent - """Content for model-type filesets. - - Contains tool calling configuration that is merged into the ModelSpec during - checkpoint analysis. - """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py index 70ea9bdc92..d16fead87f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared/__init__.py @@ -26,6 +26,7 @@ from .delete_response import DeleteResponse as DeleteResponse from .finetuning_type import FinetuningType as FinetuningType from .pagination_data import PaginationData as PaginationData +from .fileset_metadata import FilesetMetadata as FilesetMetadata from .inference_params import InferenceParams as InferenceParams from .platform_job_log import PlatformJobLog as PlatformJobLog from .tool_call_config import ToolCallConfig as ToolCallConfig diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata.py similarity index 91% rename from sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata.py rename to sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata.py index 36573bd374..b35b6d8ecc 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/files/fileset_metadata.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared/fileset_metadata.py @@ -18,8 +18,8 @@ from typing import Optional from ..._models import BaseModel -from ..shared.model_metadata_content import ModelMetadataContent -from ..shared.dataset_metadata_content import DatasetMetadataContent +from .model_metadata_content import ModelMetadataContent +from .dataset_metadata_content import DatasetMetadataContent __all__ = ["FilesetMetadata"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py index 66f37de921..e3f510ca6e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/shared_params/fileset_metadata_param.py @@ -19,8 +19,8 @@ from typing_extensions import TypedDict -from ..shared_params.model_metadata_content import ModelMetadataContent -from ..shared_params.dataset_metadata_content import DatasetMetadataContent +from .model_metadata_content import ModelMetadataContent +from .dataset_metadata_content import DatasetMetadataContent __all__ = ["FilesetMetadataParam"] diff --git a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py index 2aba797f84..68f5457688 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_experiments.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_experiments.py @@ -265,6 +265,7 @@ def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: "dataset_version": "dataset_version", "experiment_group_id": "experiment_group_id", "is_deleted": True, + "is_pinned": True, "name": "name", "updated_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -363,6 +364,110 @@ def test_path_params_delete(self, client: NeMoPlatform) -> None: workspace="workspace", ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_pin(self, client: NeMoPlatform) -> None: + experiment = client.experiments.pin( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_pin(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.pin( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_pin(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.pin( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_pin(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.pin( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiments.with_raw_response.pin( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_unpin(self, client: NeMoPlatform) -> None: + experiment = client.experiments.unpin( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_unpin(self, client: NeMoPlatform) -> None: + response = client.experiments.with_raw_response.unpin( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_unpin(self, client: NeMoPlatform) -> None: + with client.experiments.with_streaming_response.unpin( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_unpin(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.experiments.with_raw_response.unpin( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + client.experiments.with_raw_response.unpin( + name="", + workspace="workspace", + ) + class TestAsyncExperiments: parametrize = pytest.mark.parametrize( @@ -598,6 +703,7 @@ async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform "dataset_version": "dataset_version", "experiment_group_id": "experiment_group_id", "is_deleted": True, + "is_pinned": True, "name": "name", "updated_at": { "gte": parse_datetime("2019-12-27T18:11:19.117Z"), @@ -695,3 +801,107 @@ async def test_path_params_delete(self, async_client: AsyncNeMoPlatform) -> None name="", workspace="workspace", ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_pin(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.pin( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_pin(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.pin( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_pin(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.pin( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_pin(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.pin( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiments.with_raw_response.pin( + name="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_unpin(self, async_client: AsyncNeMoPlatform) -> None: + experiment = await async_client.experiments.unpin( + name="name", + workspace="workspace", + ) + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_unpin(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.experiments.with_raw_response.unpin( + name="name", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_unpin(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.experiments.with_streaming_response.unpin( + name="name", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + experiment = await response.parse() + assert_matches_type(ExperimentResponse, experiment, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_unpin(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.experiments.with_raw_response.unpin( + name="name", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `name` but received ''"): + await async_client.experiments.with_raw_response.unpin( + name="", + workspace="workspace", + ) diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py index dc3e07269b..bc8fd32ca2 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py @@ -124,6 +124,30 @@ def test_auth_logout_writes_to_selected_context(oauth_config_file: Path, monkeyp assert mock_write.call_args.kwargs["context_name"] == "foo" +def test_auth_logout_clears_selected_context_credentials( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("nemo_platform.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + + result = runner.invoke(app, ["--context", "foo", "auth", "logout"]) + + assert_exit_code(result, 0) + assert "Logged out successfully" in result.output + + with open(oauth_config_file) as f: + data = yaml.safe_load(f) + + default_user = next(user for user in data["users"] if user["name"] == "default") + foo_user = next(user for user in data["users"] if user["name"] == "foo") + + assert default_user["type"] == "oauth" + assert default_user["token"] == "default-token" + assert default_user["refresh_token"] == "default-refresh" + assert foo_user["type"] == "no-auth" + assert "token" not in foo_user + assert "refresh_token" not in foo_user + + # --------------------------------------------------------------------------- # refresh # --------------------------------------------------------------------------- diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py index 65267bdce3..4a39347d66 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py @@ -2379,7 +2379,7 @@ def test_explore_card_contains_skill_prompt_and_docs(self): panel = mock_console.print.call_args_list[0].args[0] content = panel.renderable assert "What can I do with NeMo Platform?" in content - assert "nvidia-nemo.github.io/nemo-platform" in content + assert "docs.nvidia.com/nemo-platform" in content def test_unknown_value_is_silently_skipped(self): with patch(f"{SETUP_MOD}.console") as mock_console: diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py index 7d04d030b7..3a7ebb4b4f 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py @@ -587,6 +587,61 @@ def test_write_with_access_token(self, tmp_path: Path): assert isinstance(user, OAuthUser) assert user.token.get_secret_value() == "test-token-123" + def test_write_with_access_token_none_clears_oauth_user(self, tmp_path: Path): + """Test that write() clears OAuth credentials when access_token is explicitly None.""" + config_path = tmp_path / "config.yaml" + Config.write( + { + "base_url": "http://test.example.com", + "access_token": "test-token-123", + "refresh_token": "test-refresh-123", + }, + context_name="default", + config_path=config_path, + ) + + config = Config.write( + {"access_token": None, "refresh_token": None}, + context_name="default", + config_path=config_path, + ) + + user = config.get_config_file().users[0] + assert isinstance(user, NoAuthUser) + + with open(config_path) as f: + data = yaml.safe_load(f) + + stored_user = data["users"][0] + assert stored_user["type"] == "no-auth" + assert "token" not in stored_user + assert "refresh_token" not in stored_user + + def test_write_without_access_token_preserves_oauth_user(self, tmp_path: Path): + """Test that unrelated config writes preserve existing OAuth credentials.""" + config_path = tmp_path / "config.yaml" + Config.write( + { + "base_url": "http://test.example.com", + "access_token": "test-token-123", + "refresh_token": "test-refresh-123", + }, + context_name="default", + config_path=config_path, + ) + + config = Config.write( + {"workspace": "updated-workspace"}, + context_name="default", + config_path=config_path, + ) + + user = config.get_config_file().users[0] + assert isinstance(user, OAuthUser) + assert user.token.get_secret_value() == "test-token-123" + assert user.refresh_token is not None + assert user.refresh_token.get_secret_value() == "test-refresh-123" + def test_write_without_access_token_creates_noauth_user(self, tmp_path: Path): """Test that write() creates NoAuthUser when no access_token provided.""" config_path = tmp_path / "config.yaml" diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index fcbb5c2a58..2f0a999c90 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -81,86 +81,86 @@ client_settings: # `pagination` defines [pagination schemes] which provides a template to match # endpoints and generate next-page and auto-pagination helpers in the SDKs. pagination: - - name: default_pagination - type: page_number - request: - page: - type: integer - x-stainless-pagination-property: - purpose: page_number_param - page_size: - type: integer - response: - data: - type: array - x-stainless-pagination-property: - purpose: items - items: - type: object - additionalProperties: true - pagination: +- name: default_pagination + type: page_number + request: + page: + type: integer + x-stainless-pagination-property: + purpose: page_number_param + page_size: + type: integer + response: + data: + type: array + x-stainless-pagination-property: + purpose: items + items: type: object - properties: - page: - type: integer - title: Page - description: The current page number. - x-stainless-pagination-property: - purpose: current_page_number_field - page_size: - type: integer - title: Page Size - description: The page size used for the query. - current_page_size: - type: integer - title: Current Page Size - description: The size for the current page. - total_pages: - type: integer - title: Total Pages - description: The total number of pages. - x-stainless-pagination-property: - purpose: total_page_count_field - total_results: - type: integer - title: Total Results - description: The total number of results. - required: - - page - - page_size - - total_pages - - total_results - - current_page_size - - name: logs_pagination - type: cursor - request: - limit: - type: integer - page_cursor: - type: string - x-stainless-pagination-property: - purpose: next_cursor_param - response: - data: - type: array - x-stainless-pagination-property: - purpose: items - items: - type: object - additionalProperties: true - next_page: - type: string - x-stainless-pagination-property: - purpose: next_cursor_field + additionalProperties: true + pagination: + type: object + properties: + page: + type: integer + title: Page + description: The current page number. + x-stainless-pagination-property: + purpose: current_page_number_field + page_size: + type: integer + title: Page Size + description: The page size used for the query. + current_page_size: + type: integer + title: Current Page Size + description: The size for the current page. + total_pages: + type: integer + title: Total Pages + description: The total number of pages. + x-stainless-pagination-property: + purpose: total_page_count_field + total_results: + type: integer + title: Total Results + description: The total number of results. + required: + - page + - page_size + - total_pages + - total_results + - current_page_size +- name: logs_pagination + type: cursor + request: + limit: + type: integer + page_cursor: + type: string + x-stainless-pagination-property: + purpose: next_cursor_param + response: + data: + type: array + x-stainless-pagination-property: + purpose: items + items: + type: object + additionalProperties: true + next_page: + type: string + x-stainless-pagination-property: + purpose: next_cursor_field streaming: on_event: - - data_starts_with: "[DONE]" - handle: done - - event_type: error - handle: error - - event_type: - handle: yield + - data_starts_with: "[DONE]" + handle: done + - event_type: error + handle: error + - event_type: + handle: yield readme: example_requests: @@ -919,6 +919,8 @@ resources: retrieve: get /apis/intake/v2/workspaces/{workspace}/experiments/{name} update: put /apis/intake/v2/workspaces/{workspace}/experiments/{name} delete: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name} + pin: post /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin + unpin: delete /apis/intake/v2/workspaces/{workspace}/experiments/{name}/pin subresources: sessions: models: From 0d57e9d46453c140a89eaf52303496c976745e9f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 21:56:18 -0700 Subject: [PATCH 6/7] remove unnecessary comment Signed-off-by: Matthew Grossman --- e2e/conftest.py | 2 +- e2e/k8s/values/minikube.yaml | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/e2e/conftest.py b/e2e/conftest.py index 8c5aecb96a..feb0a7efcd 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -13,7 +13,7 @@ When ``NMP_BASE_URL`` is set the harness skips service startup/shutdown and connects to the given URL. Otherwise it spawns ``nemo services run`` as a -child process on a free port, polls ``/health/ready`` until ready, and +child process on a free port, polls ``/status`` until ready, and terminates the process after the session. """ diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml index 147345f426..2570a9b716 100644 --- a/e2e/k8s/values/minikube.yaml +++ b/e2e/k8s/values/minikube.yaml @@ -1,23 +1,7 @@ # Minikube values for local development # This is a standalone values file - use it in place of ./default.yaml # -# Usage (local build): -# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \ -# --set api.image.repository=docker.io/my-registry/nmp-api \ -# --set api.image.tag=local --set api.image.pullPolicy=Never \ -# --set core.image.repository=docker.io/my-registry/nmp-api \ -# --set core.image.tag=local --set core.image.pullPolicy=Never \ -# --set platformConfig.platform.image_registry=docker.io/my-registry \ -# --set platformConfig.platform.image_tag=local -# -# Usage (GHCR): -# helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml \ -# --set api.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \ -# --set api.image.tag=latest \ -# --set core.image.repository=ghcr.io/nvidia-nemo/platform/nmp-api \ -# --set core.image.tag=latest \ -# --set platformConfig.platform.image_registry=ghcr.io/nvidia-nemo/platform \ -# --set platformConfig.platform.image_tag=latest +# Usage: helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml # Enable NIM operator for local GPU testing k8s-nim-operator: From 556df03682152a4486bfba8e16d31f2898c05ba9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 22 Jun 2026 22:07:12 -0700 Subject: [PATCH 7/7] lint Signed-off-by: Matthew Grossman --- docs/cli/reference.mdx | 2 +- .../tests/cli/commands/test_services_lifecycle.py | 4 ++-- .../nemo_platform_ext/cli/commands/test_services_lifecycle.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/cli/reference.mdx b/docs/cli/reference.mdx index 42011f745e..d25445b563 100644 --- a/docs/cli/reference.mdx +++ b/docs/cli/reference.mdx @@ -145,7 +145,7 @@ nemo services run [OPTIONS] Start platform services in the background. -Detaches the process, polls /health/ready, then returns. +Detaches the process, polls /status, then returns. **Examples:** diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py b/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py index da9953770b..fd45922a1c 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py @@ -413,10 +413,10 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: with open(desc_path, "w") as f: json.dump(desc, f, indent=2) -# HTTP server for /health/ready +# HTTP server for /status class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): - if self.path == "/health/ready": + if self.path == "/status": self.send_response(200) self.end_headers() self.wfile.write(b"ok") diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py index 24efecdd32..2d035e1355 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py @@ -413,10 +413,10 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: with open(desc_path, "w") as f: json.dump(desc, f, indent=2) -# HTTP server for /health/ready +# HTTP server for /status class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): - if self.path == "/health/ready": + if self.path == "/status": self.send_response(200) self.end_headers() self.wfile.write(b"ok")