diff --git a/.cursor/skills/nemoclaw-k8s-hpa/SKILL.md b/.cursor/skills/nemoclaw-k8s-hpa/SKILL.md new file mode 100644 index 00000000000..3cf189bdd9e --- /dev/null +++ b/.cursor/skills/nemoclaw-k8s-hpa/SKILL.md @@ -0,0 +1,178 @@ +--- +name: nemoclaw-k8s-hpa +description: >- + Deploy and validate NemoClaw CPU agent tier on Kubernetes with Helm, CPU-based + HPA, metrics-server, and load tests (Inference Hub / Nemotron). Use when the user + asks about nemoclaw-cpu Helm chart, Kubernetes autoscaling, HPA scale-up/down, + agent pods, load testing for HPA, or K8s packaging separate from VM NemoClaw/Telegram. +--- + +# NemoClaw Kubernetes HPA (nemoclaw-cpu) + +## Architecture (read first) + +```text +Telegram / OpenShell sandbox (VM) ≠ K8s agent pods (nemoclaw-cpu chart) + │ │ + └─ Inference Hub (Nemotron Ultra) ◄────────────┘ (optional same API) +``` + +| Layer | Location | Scales with HPA? | +|-------|----------|------------------| +| Full NemoClaw (Telegram, OpenShell) | VM `nemoclaw onboard` | **No** | +| CPU agent pods | `deploy/helm/nemoclaw-cpu/` | **Yes** | + +HPA watches **CPU % of pod CPU requests** on agent pods, not VM processes. + +**Load balancing:** `ClusterIP` Service `nemoclaw-nemoclaw-cpu-agent` is the in-cluster LB. No cloud ALB required. Traffic only reaches **Ready** pods (`kubectl get endpoints`). + +--- + +## Repo map + +| Path | Purpose | +|------|---------| +| `deploy/helm/nemoclaw-cpu/` | Helm chart (agent Deployment, Service, HPA, ConfigMap) | +| `deploy/helm/nemoclaw-cpu/scripts/install-hpa.sh` | **One-command** CPU HPA (metrics-server) | +| `deploy/helm/nemoclaw-cpu/scripts/install-performance-hpa.sh` | Optional Prometheus + performance HPA | +| `deploy/helm/nemoclaw-cpu/scripts/hpa-load-test.sh` | End-to-end HPA load test | +| `deploy/helm/nemoclaw-cpu/files/load-generator.mjs` | In-cluster load Job | +| `deploy/helm/nemoclaw-cpu/files/agent-server.mjs` | `/healthz`, `/readyz`, `/bench`, `/v1/chat/completions` | +| `deploy/helm/nemoclaw-cpu/values-step2-hpa.yaml` | Enable CPU HPA | +| `deploy/helm/nemoclaw-cpu/values-step2-hpa-saturate.yaml` | HPA + **400m CPU request/pod** (fit 7 on 8 vCPU node) | +| `deploy/helm/nemoclaw-cpu/scripts/hpa-reset.sh` | Clean Jobs/HPA/pods before a new test | + +--- + +## Prerequisites + +- Kubernetes (MicroK8s OK), `helm` 3, `kubectl` +- `NVIDIA_INFERENCE_HUB_API_KEY` in `~/.nemoclaw/secrets.env` (`sk-*`) +- **metrics-server** for CPU HPA: `microk8s enable metrics-server` +- Verify: `kubectl get apiservice v1beta1.metrics.k8s.io` → `AVAILABLE True` + +--- + +## One-command install (recommended) + +CPU HPA + metrics-server (no Prometheus): + +```bash +cd deploy/helm/nemoclaw-cpu +source ~/.nemoclaw/secrets.env +./scripts/install-hpa.sh +kubectl get hpa,pods -n nemoclaw -w +``` + +Optional Prometheus path: `./scripts/install-performance-hpa.sh` + +**Do not** use placeholder `sk-YOUR-INFERENCE-HUB-KEY` — `/readyz` stays 503 and pods never become Ready. + +--- + +## Alternate — CPU HPA only (no Prometheus) + +```bash +helm upgrade --install nemoclaw . -n nemoclaw --create-namespace \ + --set namespace.create=false \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" + +helm upgrade nemoclaw . -n nemoclaw --reuse-values \ + -f values-step2-hpa-saturate.yaml \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" +``` + +| Value | Typical | +|-------|---------| +| `autoscaling.minReplicas` | 1 | +| `autoscaling.maxReplicas` | 7 (8 vCPU node) | +| `autoscaling.targetCPUUtilizationPercentage` | 50–65 (saturate); 65 (1 CPU/pod) | +| `autoscaling.behavior.scaleDown.stabilizationWindowSeconds` | 90–120 (not 300) | +| `cpuScaling.perPodRequest` | **400m** (fit 7 pods); **1** only fits ~4–6 | + +When `autoscaling.enabled=true`, Helm sets `spec.replicas` to `minReplicas` (≥1); HPA scales between min and max. + +--- + +## Step 3 — Run HPA load test + +```bash +./scripts/hpa-reset.sh # optional: clean Jobs/HPA/pods first +./scripts/hpa-load-test.sh +``` + +Defaults: `TARGET_PODS=7`, `CONCURRENCY_PER_POD=40`, `/bench` with worker-thread CPU spin, **one** load Job pod (`JOB_PARALLELISM=1`). + +**Watch (two terminals — `kubectl get -w` accepts only one resource type):** + +```bash +watch -n 5 'kubectl get hpa,pods -n nemoclaw; kubectl top pods -n nemoclaw 2>/dev/null | grep agent' +kubectl get endpoints nemoclaw-nemoclaw-cpu-agent -n nemoclaw +``` + +**Success:** `REPLICAS` climbs toward 7; multiple endpoint IPs; agent pods `1/1 Ready`; `kubectl top` shows **300m+** CPU per pod under load. + +**After test:** + +```bash +helm upgrade nemoclaw . -n nemoclaw --reuse-values --set loadTest.cpuSpinMs=0 +``` + +--- + +## Manual scale (no HPA) + +```bash +helm upgrade nemoclaw . -n nemoclaw --reuse-values --set autoscaling.enabled=false --set cpuScaling.count=4 +``` + +`cpuScaling.count=N` → N pods, each `perPodRequest` CPU (default 1). + +--- + +## Troubleshooting (symptom → cause) + +| Symptom | Likely cause | +|---------|----------------| +| Helm `namespace already exists` | Chart + `--create-namespace` both create NS → `namespace.create=false` | +| Pod `0/1`, readiness 503 | Bad Inference Hub key | +| HPA `cpu: ` | metrics-server missing or pod unready | +| Stuck at 3–4 replicas, `cpu: 99%` | **Node CPU requests full** (1 CPU/pod); use `values-step2-hpa-saturate.yaml` (400m) | +| One endpoint IP only | Only one pod Ready — fix probes / worker-thread `/bench` | +| Load Job `fetch failed` | Pods unready or overloaded; check endpoints | +| `7/8` manual scale, pod Pending | Insufficient allocatable CPU | +| HPA scale-down slow | 5 min stabilization window (by design) | + +**Not caused by missing external load balancer** — Service + Endpoints handle distribution. + +--- + +## Agent HTTP API (for load & health) + +| Path | Role | +|------|------| +| `GET /healthz` | Liveness | +| `GET /readyz` | Readiness (cached Inference Hub check) | +| `POST /bench?ms=450&threads=2` | CPU load for HPA tests (worker threads) | +| `POST /v1/chat/completions` | Proxy to Inference Hub (same model as NemoClaw) | +| `GET /metrics` | Prometheus metrics | + +--- + +## Agent building checklist + +When implementing or extending a **Kubernetes HPA agent** for NemoClaw: + +- [ ] Distinguish VM NemoClaw vs K8s agent tier in docs and tests +- [ ] Confirm metrics-server before HPA +- [ ] Use saturate values on single-node 8 vCPU clusters +- [ ] Keep `/healthz` responsive under load (CPU spin off main thread) +- [ ] Verify `endpoints` has multiple IPs during load test +- [ ] Size `maxReplicas` to allocatable CPU on node +- [ ] Store secrets in `~/.nemoclaw/secrets.env`, never commit keys + +--- + +## Additional resources + +- Deep dive and env vars: [reference.md](reference.md) diff --git a/.cursor/skills/nemoclaw-k8s-hpa/reference.md b/.cursor/skills/nemoclaw-k8s-hpa/reference.md new file mode 100644 index 00000000000..c603a1a8525 --- /dev/null +++ b/.cursor/skills/nemoclaw-k8s-hpa/reference.md @@ -0,0 +1,98 @@ +# NemoClaw K8s HPA — Reference + +## HPA math (CPU resource metric) + +- Target: `averageUtilization` = % of **CPU request** (not limit). +- Example: request `1` CPU, target `35%` → HPA wants ~350m average per pod. +- With `perPodRequest: 400m` and target `30%` → ~120m average triggers scale-up. + +Desired replicas (simplified): + +```text +desired = ceil(currentReplicas × (currentCPU% / targetCPU%)) +``` + +Scale-up stabilization: **0s**. Scale-down: **120s** (chart default). + +## 8 vCPU node capacity (example) + +| Config | 7 pods schedulable? | +|--------|---------------------| +| `perPodRequest: 1` | Usually **no** (~7 + system > 8) | +| `perPodRequest: 400m` | **Often yes** (~2.8 + ~1.5 system) | + +Check: `kubectl describe node | grep -A6 "Allocated resources"` + +## Load generator env vars + +Used by `scripts/hpa-load-test.sh` → Job `nemoclaw-hpa-load-test`: + +| Variable | Default | Meaning | +|----------|---------|---------| +| `TARGET_PODS` | 7 | HPA max / load target | +| `CONCURRENCY_PER_POD` | 40 | In-flight requests per pod (via Service) | +| `BENCH_MS` | 450 | CPU spin per `/bench` | +| `BENCH_THREADS` | 2 | Worker threads per bench (agent) | +| `JOB_PARALLELISM` | 1 | Load Job pods (default: one generator) | +| `DURATION_SEC` | 720 | Test duration | +| `RAMP_SEC` | 90 | Ramp to full concurrency | +| `HPA_TARGET_CPU` | 30 | Helm HPA target % | +| `SCALE_UP_TARGET` | 7 | Script success threshold | + +Heavier run: + +```bash +CONCURRENCY_PER_POD=55 BENCH_MS=500 JOB_PARALLELISM=3 HPA_TARGET_CPU=25 \ + ./scripts/hpa-load-test.sh +``` + +## Helm values quick reference + +```yaml +# Manual scale +cpuScaling: + oneReplicaPerCpu: true + count: 4 + perPodRequest: "1" + +# HPA +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 7 + targetCPUUtilizationPercentage: 30 + +# Load test CPU on agent +loadTest: + cpuSpinMs: 450 +``` + +## Inference: VM vs K8s + +| | VM NemoClaw | K8s agent pod | +|--|-------------|---------------| +| Telegram | Yes | No | +| Config | `~/.nemoclaw/`, openclaw.json | Helm values + Secret | +| Model | Nemotron Ultra via Inference Hub | Same via proxy | +| Scales with HPA | No | Yes | + +## Performance HPA (Step 2b, optional) + +Requires Prometheus + prometheus-adapter. Custom metric: `nemoclaw_http_inflight_requests` from `/metrics`. See `values-step2-hpa-performance.yaml` and `deploy/helm/nemoclaw-cpu/observability.md`. + +## Reset + load test scripts + +| Script | Purpose | +|--------|---------| +| `scripts/hpa-reset.sh` | Delete Jobs/HPA/stuck pods; reinstall baseline (min 1 replica) | +| `scripts/hpa-load-test.sh` | Run load Job; wait for scale-up/down | +| `scripts/hpa-common.sh` | Shared helpers (never scale to 0) | + +## Git / fork notes + +- Fork branch work: Inference Hub, Tavily, Telegram fixes live on VM sandbox path. +- `deploy/helm/nemoclaw-cpu/` may be uncommitted — commit when stabilizing HPA agent. + +## Riva HPA pattern (prior art) + +NVIDIA blog: autoscaling Riva with K8s HPA + Grafana — same pattern (metrics → HPA → dashboards). CPU HPA is Step 2a; queue/latency metrics are Step 2b. diff --git a/.gitignore b/.gitignore index ddbb67731c2..1f30b7f6d00 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ vdr-notes/ .npmrc .pypirc credentials.json +secrets.env DRAFT-*.md key.json secrets.json diff --git a/Dockerfile b/Dockerfile index 1bd9054ea15..f73b6ebc77f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -409,9 +409,10 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ # Build args for config that varies per deployment. # nemoclaw onboard passes these at image build time. -ARG NEMOCLAW_MODEL=nvidia/nemotron-3-super-120b-a12b +# Nemotron Ultra — inference-api.nvidia.com (see src/lib/inference/config.ts) +ARG NEMOCLAW_MODEL=nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1 ARG NEMOCLAW_PROVIDER_KEY=inference -ARG NEMOCLAW_PRIMARY_MODEL_REF=inference/nvidia/nemotron-3-super-120b-a12b +ARG NEMOCLAW_PRIMARY_MODEL_REF=inference/nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1 # Default dashboard port 18789 — override at runtime via NEMOCLAW_DASHBOARD_PORT. ARG CHAT_UI_URL=http://127.0.0.1:18789 ARG NEMOCLAW_INFERENCE_BASE_URL=https://inference.local/v1 @@ -488,6 +489,7 @@ ARG NEMOCLAW_PROXY_PORT=3128 # The actual API key is injected at runtime via openshell:resolve:env, never # baked into the image. ARG NEMOCLAW_WEB_SEARCH_ENABLED=0 +ARG NEMOCLAW_WEB_SEARCH_PROVIDER=brave # SECURITY: Promote build-args to env vars so the Python script reads them # via os.environ, never via string interpolation into Python source code. @@ -515,7 +517,8 @@ ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \ NEMOCLAW_DISABLE_DEVICE_AUTH=${NEMOCLAW_DISABLE_DEVICE_AUTH} \ NEMOCLAW_PROXY_HOST=${NEMOCLAW_PROXY_HOST} \ NEMOCLAW_PROXY_PORT=${NEMOCLAW_PROXY_PORT} \ - NEMOCLAW_WEB_SEARCH_ENABLED=${NEMOCLAW_WEB_SEARCH_ENABLED} + NEMOCLAW_WEB_SEARCH_ENABLED=${NEMOCLAW_WEB_SEARCH_ENABLED} \ + NEMOCLAW_WEB_SEARCH_PROVIDER=${NEMOCLAW_WEB_SEARCH_PROVIDER} WORKDIR /sandbox USER sandbox diff --git a/README.fork.md b/README.fork.md new file mode 100644 index 00000000000..694ec24005f --- /dev/null +++ b/README.fork.md @@ -0,0 +1,278 @@ +# NemoClaw (maggiezha fork) + +Personal fork of [NVIDIA/NemoClaw](https://github.com/NVIDIA/NemoClaw) with **Tavily Web Search**, **Nemotron Ultra via NVIDIA Inference Hub**, **Telegram** messaging, and **split API credentials** (`nvapi-*` vs `sk-*`). + +- **Fork:** https://github.com/maggiezha/NemoClaw +- **Branch:** `2026-05-27-0hlj` +- **Upstream docs:** https://docs.nvidia.com/nemoclaw/latest/ + +> This document describes **this fork only**. For the standard NemoClaw install, prerequisites, and architecture, see **[README.md](README.md)** in this repo (same as [NVIDIA/NemoClaw](https://github.com/NVIDIA/NemoClaw)) and the [NVIDIA documentation](https://docs.nvidia.com/nemoclaw/latest/). + +--- + +## What this fork adds + +| Feature | Summary | +|--------|---------| +| **Tavily Web Search** | Choose Tavily during onboard; `tavily` network preset; OpenClaw `plugins.entries.tavily` + `tools.web.search.provider=tavily` | +| **Nemotron Ultra** | First-class cloud model on **Inference Hub** +| **Telegram plugin fix** | Enables `plugins.entries.telegram` when Telegram is in messaging channels (OpenClaw 2026.5.x) | + +--- + +## Quick start (this fork) + +### 1. Clone and build + +```bash +git clone https://github.com/maggiezha/NemoClaw.git +cd NemoClaw +git checkout 2026-05-27-0hlj +npm install --ignore-scripts +npm run build:cli +``` + +Use the repo CLI (not an old global install): + +```bash +export PATH="$PWD:$PATH" # or: alias nemoclaw='node bin/nemoclaw.js' +node bin/nemoclaw.js --version +``` + +### 2. API keys (`~/.nemoclaw/secrets.env`) + +```bash +node bin/nemoclaw.js credentials init-secrets +# Edit ~/.nemoclaw/secrets.env — never commit real keys +``` + +| Variable | Format | Used for | +|----------|--------|----------| +| `NVIDIA_INFERENCE_HUB_API_KEY` | `sk-...` | **Nemotron Ultra** | +| `NVIDIA_API_KEY` | `nvapi-...` | Nemotron Super / Build models → `https://integrate.api.nvidia.com/v1` | +| `TAVILY_API_KEY` | `tvly-...` | Tavily web search | +| `TELEGRAM_BOT_TOKEN` | from @BotFather | Telegram bot | + +Get keys: + +- Inference Hub +- NVIDIA Build: https://build.nvidia.com/settings/api-keys +- Tavily: https://tavily.com +- Telegram: [@BotFather](https://t.me/BotFather) → `/newbot` + +### 3. Onboard + +```bash +set -a && source ~/.nemoclaw/secrets.env && set +a +node bin/nemoclaw.js onboard +``` + +During onboard: + +1. **Inference** → NVIDIA Endpoints → **Nemotron Ultra 253B** (uses Inference Hub key). +2. **Web search** → **Yes** → **2) Tavily Search** → paste `TAVILY_API_KEY`. +3. **Messaging** → enable **Telegram** (or add later with `channels add telegram`). + +Non-interactive resume (example): + +```bash +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_SANDBOX_NAME=my-assistant +export TELEGRAM_ALLOWED_IDS= +export TELEGRAM_REQUIRE_MENTION=1 +node bin/nemoclaw.js onboard --resume +``` + +### 4. Telegram bot token (if not done in onboard) + +On your Mac/phone (not on the VM): create bot with @BotFather, copy token. + +On the VM: + +```bash +node bin/nemoclaw.js my-assistant channels add telegram +# Paste TELEGRAM_BOT_TOKEN when prompted; confirm rebuild +``` + +--- + +## Steps taken to add Tavily Web Search + +1. **Policy** — Added `nemoclaw-blueprint/policies/presets/tavily.yaml` (egress to `api.tavily.com`). +2. **Onboard flow** — Extended `src/lib/onboard/web-search-flow.ts`: prompt “Brave vs Tavily”, validate Tavily key, save `TAVILY_API_KEY`, set `NEMOCLAW_WEB_SEARCH_PROVIDER=tavily`. +3. **Types / env** — `src/lib/inference/web-search.ts`: `WebSearchProvider`, `TAVILY_API_KEY_ENV`, `resolveWebSearchProvider()`. +4. **Sandbox build** — `src/lib/onboard/dockerfile-patch.ts` patches `NEMOCLAW_WEB_SEARCH_ENABLED` and `NEMOCLAW_WEB_SEARCH_PROVIDER`. +5. **OpenClaw config** — `scripts/generate-openclaw-config.py`: + - `tools.web.search` with `provider: tavily` + - `plugins.entries.tavily.config.webSearch.apiKey` (OpenClaw 2026.5+; legacy `tools.web.search.tavily.*` stripped) +6. **Credentials in sandbox** — `src/lib/onboard.ts`: register `*-tavily-search` OpenShell provider; fail fast if key missing before recreate (#3626). +7. **Resume / rebuild** — `src/lib/onboard/machine/handlers/sandbox.ts`: revalidate Tavily (not Brave) when provider is Tavily. +8. **Verification** — `src/lib/onboard/web-search-verify.ts` + post-onboard probe in finalization. +9. **Agent hint** — Workspace instruction: reply **“Tavily Web Search is used”** when search runs. +10. **Helpers** — `scripts/setup-tavily-search.sh`, `scripts/test-tavily-flow.sh` for pre-commit smoke checks. + +**Enable Tavily on an existing sandbox:** + +```bash +./scripts/setup-tavily-search.sh my-assistant +``` +Screenshot 2026-05-27 at 2 53 37 PM + + +--- + +## Steps taken to add Nemotron Ultra (Inference Hub) + +1. **Routing** — `src/lib/inference/config.ts`: + - `resolveNvidiaCloudModelRoute()` sends Ultra to `inference-api.nvidia.com` with `NVIDIA_INFERENCE_HUB_API_KEY` + - Super / other catalog models stay on `integrate.api.nvidia.com` with `NVIDIA_API_KEY` + - Ultra uses OpenAI-compatible provider type (`openai`), not legacy `nvidia` → integrate +2. **Cloud model list** — `nemoclaw/src/index.ts`: Nemotron Ultra 253B under **NVIDIA Endpoints → Cloud models**. +3. **Onboard** — Pick model first → set `endpointUrl` + `credentialEnv` → `ensureNvidiaEndpointCredential()`. +4. **Validation** — `src/lib/validation.ts`: strict key shapes (`sk-*` only on Inference Hub env, `nvapi-*` on Build env). +5. **Credentials** — `src/lib/credentials/store.ts` + `secrets-env.ts`: load `~/.nemoclaw/secrets.env`; `nemoclaw credentials init-secrets`. +6. **Resume repair** — `provider-inference.ts` fixes `credentialEnv` / `endpointUrl` for `nvidia-prod` from selected model. +7. **Reference client** — `scripts/examples/nemotron-ultra-inference.py` (uses `NVIDIA_INFERENCE_HUB_API_KEY`). + +Screenshot 2026-05-27 at 2 53 05 PM + + + +--- + +## Steps taken for Telegram + +1. Create bot via [@BotFather](https://t.me/BotFather) (`/newbot`) on Mac/phone — copy token. +2. `node bin/nemoclaw.js channels add telegram` — paste token; rebuild sandbox. +3. Store `TELEGRAM_BOT_TOKEN` in `~/.nemoclaw/secrets.env` for non-interactive runs. +4. Network policy preset `telegram` → `api.telegram.org`. +5. DM allowlist: `TELEGRAM_ALLOWED_IDS` (your numeric user ID). +6. Groups: `TELEGRAM_REQUIRE_MENTION=1` — bot only answers when @mentioned. +7. **Plugin fix** — `generate-openclaw-config.py` sets `plugins.entries.telegram.enabled: true` when Telegram is configured (otherwise gateway has config but no polling). + +**Get your Telegram user ID:** message [@userinfobot](https://t.me/userinfobot) or inspect gateway logs after you DM the bot (`Inbound message telegram:`). + +--- + +## Telegram test questions + +Use **DM** first (simplest). In a **group**, **@mention** the bot on every message. + +### 1. Basic reply (Nemotron Ultra only) + +Confirms inference works end-to-end. + +- `Hello — are you there? Reply in one sentence.` +- `What is 17 × 23? Show the answer only.` +- `In one paragraph, what is NVIDIA Nemotron?` + + +### 2. Tavily / web search + +Screenshot 2026-05-27 at 5 41 04 PM + + +Ask for **current** or **live** info so the agent should call Tavily. + +- `Use web search: what is today's date and one major tech headline from today?` +- `Search the web for the latest news about NVIDIA Nemotron Ultra and summarize in 3 bullets.` +- `Use Tavily to find the current price of Bitcoin and cite the source URL.` +- `Web search: who won the most recent Super Bowl? Include the year.` + +### 3. Search + reasoning + +- `Search the web for NVIDIA's latest earnings or product news, then explain in 2 sentences why it matters for AI developers.` +- `Find today's weather in San Francisco using web search, then suggest what to wear.` + +### 4. Follow-up (same chat thread) + +Send after a successful answer: + +- `Summarize what you just told me in one sentence.` +- `What source did you use for that?` (after a web-search answer) + +### 5. Negative / edge cases + +- `Do not use web search: what is the capital of France?` (should answer without browsing) +- `Search the web for xyznonexistenttopic12345zzz` (weak or empty results) + +### 6. Group chat + +Format: + +- `@YourBotName Use web search: what happened in AI news this week?` + +Also set BotFather **Group Privacy** to **Disabled** if the bot must see all group messages (otherwise only commands/@mentions reach it). + +### What “good” looks like + +| Test | Good sign | +|------|-----------| +| Basic | Fast text reply | +| Web search | Recent info, links/snippets, or **“Tavily Web Search is used”** | +| No search | Answers without claiming it browsed the web | +| Follow-up | Stays on topic in the same thread | + +**Recommended order:** run **§1** then **§2** (`Use web search: what is today's date...`) to confirm Ultra + Tavily quickly. + +### If something fails + +| Symptom | What to check | +|---------|----------------| +| No reply in DM | `node bin/nemoclaw.js my-assistant logs --follow` | +| No reply in group | @mention the bot; BotFather privacy **Disabled** | +| Reply but no web | Say: `You must use web search for this.` | +| Silent bot, logs show `HTTP 404` / `model_not_found` | Ultra must use **Inference Hub** (`sk-*`, `inference-api.nvidia.com`), not integrate.api — re-onboard or fix `nvidia-prod` provider | +| Rebuild wiped Telegram | Confirm `plugins.entries.telegram.enabled` in sandbox `openclaw.json` | + +**Avoid** `channels stop telegram` if it triggers a full rebuild you did not intend — prefer `recover` or gateway restart when possible. + +--- + +## Useful commands + +```bash +# Status +node bin/nemoclaw.js my-assistant status + +# Logs (while testing Telegram) +node bin/nemoclaw.js my-assistant logs --follow + +# Tavily smoke (no full sandbox) +npm run build:cli && ./scripts/test-tavily-flow.sh + +# Non-interactive resume +NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_SANDBOX_NAME=my-assistant \ + node bin/nemoclaw.js onboard --resume +``` + +--- + +## Git workflow (fork only) + +Work on a feature branch — **do not** put custom commits on `main`: + +```bash +git checkout 2026-05-27-0hlj +git push fork 2026-05-27-0hlj +``` + +`main` on this fork tracks [NVIDIA/NemoClaw main](https://github.com/NVIDIA/NemoClaw); upstream is unchanged unless you open a PR there. + +--- + +## Key files (fork diff) + +| Area | Files | +|------|--------| +| Inference Hub / Ultra | `src/lib/inference/config.ts`, `nemoclaw/src/index.ts`, `src/lib/credentials/*`, `src/lib/validation.ts` | +| Tavily | `src/lib/onboard/web-search-flow.ts`, `scripts/generate-openclaw-config.py`, `nemoclaw-blueprint/policies/presets/tavily.yaml` | +| Telegram plugin | `scripts/generate-openclaw-config.py` | +| Secrets | `secrets.env.example`, `src/lib/credentials/secrets-env.ts` | +| Tests | `test/generate-openclaw-config.test.ts`, `src/lib/onboard/web-search-verify.test.ts`, `src/lib/inference/config.test.ts` | + +--- + +## License + +Same as upstream: **Apache-2.0**. See [LICENSE](LICENSE). diff --git a/README.md b/README.md index 223bdbd4bcd..1a122cf35a3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ SPDX-License-Identifier: Apache-2.0 --> +> **Fork ([maggiezha/NemoClaw](https://github.com/maggiezha/NemoClaw), branch `2026-05-27-0hlj`):** Tavily web search, Nemotron Ultra (Inference Hub), and Telegram setup notes are in **[README.fork.md](README.fork.md)**. + # 🦞 NVIDIA NemoClaw: Reference Stack for Running OpenClaw in OpenShell diff --git a/deploy/README-cpu.md b/deploy/README-cpu.md new file mode 100644 index 00000000000..d25ae596d8c --- /dev/null +++ b/deploy/README-cpu.md @@ -0,0 +1,313 @@ +# NemoClaw Kubernetes deployment — CPU (HPA, optional) + +**Optional pre-GPU chart** — use only if you do not have cluster GPUs yet and want to test agents/HPA against **NVIDIA Inference Hub** (Nemotron Ultra). For the real deployment, use the **[GPU chart](README-gpu.md)**. + +Helm chart and scripts for **CPU agent pods** with **HPA**. Each pod uses **1 CPU** and proxies remote inference (no `nvidia.com/gpu`). + +**Index:** [deploy/README.md](README.md) · **Models (CPU vs GPU):** [helm/README.md](helm/README.md) + +This document is the **CPU deployment guide** — install, operate, and load-test the optional CPU chart. GPU deployment: [README-gpu.md](README-gpu.md). + +--- + +## Install (CPU only) + +You do **not** install the chart, then HPA, then load test as three separate steps. One script installs **metrics-server + Helm chart + CPU HPA** together. + +```bash +cd ~/NemoClaw/deploy/helm/nemoclaw-cpu +source ~/.nemoclaw/secrets.env # NVIDIA_INFERENCE_HUB_API_KEY=sk-... + +./scripts/install-hpa.sh +``` + +Confirm idle state (no workload required): + +```bash +kubectl get hpa -n nemoclaw +kubectl get pods -n nemoclaw +``` + +Expect **1 Running pod** and HPA **REPLICAS 1** (CPU well below the ~65% target). **Autoscaling only happens under load** — see [Idle vs load test](#idle-vs-load-test). + +--- + +## Port-forward + +CPU agent listens on **8080** (Service, pod, and local port-forward): + +```bash +kubectl port-forward -n nemoclaw svc/nemoclaw-nemoclaw-cpu-agent 8080:8080 +``` + +**Verify** Inference Hub (needed for chat, not for rollout): + +```bash +curl -s http://127.0.0.1:8080/healthz # always ok if pod is up +curl -s http://127.0.0.1:8080/readyz # 200 = Hub key OK; 503 = fix secrets +``` + +Override host port: `LOCAL_PORT=8080 ./scripts/install-hpa.sh` (default is 8080). + +Optional demo of scale up/down: + +```bash +./scripts/hpa-load-test.sh +./scripts/hpa-reset.sh # return to idle baseline after the test +``` + + +Screenshot 2026-05-28 at 12 49 11 PM + + + + +Screenshot 2026-05-28 at 12 42 34 PM + + + + +If install keeps failing with **rollout failed** or Deployment **`0 up-to-date`**: + +```bash +./scripts/cluster-recover.sh +``` + +Chart details: [helm/nemoclaw-cpu/README.md](helm/nemoclaw-cpu/README.md) · [CPU vs GPU comparison](helm/README.md) + +--- + +## Scripts + +All scripts live under `deploy/helm/nemoclaw-cpu/scripts/`. Run from the chart directory after sourcing `~/.nemoclaw/secrets.env`. + +| Script | What it does | When to use | +|--------|----------------|-------------| +| **`install-hpa.sh`** | Enables **metrics-server**; `helm upgrade --install` with **CPU HPA** (`values-step2-hpa.yaml`, min **1** / max **7**); readiness on **`/healthz`** so rollout succeeds without Hub traffic. | **First install** or healthy cluster refresh. | +| **`hpa-reset.sh`** | Deletes load-test Jobs; force-deletes **pods** and stale ReplicaSets; **keeps HPA + Deployment** by default; `helm upgrade` to idle baseline. | After load test, stuck load-test pods, return to idle. | +| **`cluster-recover.sh`** | Removes stray resources, **`helm uninstall`**, wipes namespace leftovers, **restarts MicroK8s**, runs **`install-hpa.sh`**. | Repeated **rollout failed**, Deployment **`0 updated \| 0 total`**, ghost HPA. | +| **`hpa-load-test.sh`** | Switches to **saturate** values (400m CPU/pod); runs in-cluster load Job (~12 min); watches scale up then down. | Prove HPA only — not required for setup. | +| **`install-performance-hpa.sh`** | Optional Prometheus + adapter + inflight-metric HPA. Heavy; may timeout on small VMs. | Advanced only. | +| **`hpa-common.sh`** | Shared helpers (not run directly). | — | + +### Workflow + +```text +First time: install-hpa.sh +After load test / tidy: hpa-reset.sh +Rollout keeps failing: cluster-recover.sh +Prove autoscaling: hpa-load-test.sh → watch → hpa-reset.sh +``` + +Do **not** run `hpa-reset.sh && install-hpa.sh` unless you used `SKIP_HELM=1` on reset. Reset already runs `helm upgrade`. + +### Environment variables + +| Variable | Scripts | Effect | +|----------|---------|--------| +| `DELETE_DEPLOYMENT=1` | `hpa-reset.sh` | Delete Deployment before reinstall (stuck rollout) | +| `DELETE_HPA=1` | `hpa-reset.sh` | Delete HPA; two-phase reinstall (`desiredReplicas=0`) | +| `SKIP_HELM=1` | `hpa-reset.sh` | kubectl cleanup only; then run `install-hpa.sh` | +| `RUN_LOAD_TEST=1` | `hpa-reset.sh` | Run `hpa-load-test.sh` after reset | +| `RESTART_MICROK8S=0` | `cluster-recover.sh` | Skip MicroK8s restart (cleanup only) | +| `RUN_INSTALL=0` | `cluster-recover.sh` | Cleanup without reinstall | +| `MIN_REPLICAS` / `MAX_REPLICAS` | install, reset | Override HPA bounds (default 1 / 7) | +| `ROLLOUT_TIMEOUT` | install, reset | Seconds to wait for rollout (default 300) | +| `LOCAL_PORT` | install | Host port for port-forward hints (default **8080**) | + +--- + +## Architecture + +```text +┌─────────────────────────────────────────────────────────────────────────┐ +│ Your VM (Brev) │ +│ ┌──────────────────────┐ ┌──────────────────────────────────┐ │ +│ │ NemoClaw (fork) │ │ MicroK8s — namespace: nemoclaw │ │ +│ │ Telegram / OpenShell │ │ agent pods: 1 CPU each (idle) │ │ +│ │ (not in this chart) │ │ HPA min 1 … max 7 (CPU metrics) │ │ +│ └──────────┬───────────┘ └──────────────┬───────────────────┘ │ +│ │ same API key │ metrics-server │ +└─────────────┼───────────────────────────────────┼────────────────────────┘ + │ HTTPS │ + ▼ ▼ + ┌────────────────────────────────────────────────┐ + │ NVIDIA Inference Hub (Nemotron Ultra) │ + └────────────────────────────────────────────────┘ +``` + +HPA scales on **CPU of agent pods** in `nemoclaw`, not on VM Telegram/OpenShell traffic. + +--- + +## Load balancer + +This chart uses a **ClusterIP** Service (`nemoclaw-nemoclaw-cpu-agent`) — **no AWS ALB/NLB** and no Ingress by default. + +| Resource | Type | Role | +|----------|------|------| +| Agent Service | **ClusterIP** | In-cluster VIP; kube-proxy balances **new connections** across Ready CPU agent pods | +| `kubectl port-forward` | Local tunnel | Debug on **8080**; not production routing | +| Load-test Job | In-cluster client | Hits Service DNS → all replicas as HPA scales | + +HPA changes replica count; the Service picks up new pods automatically. That is **not** the same as an external load balancer or GPU-aware routing. + +**Next step (not implemented):** add **NGINX Ingress** (or AWS Load Balancer Controller on EKS) for external clients and smarter HTTP routing (least-connections, timeouts, rate limits). See [README-gpu.md — Load balancer](README-gpu.md#load-balancer) for the same pattern on GPU (including example **`g6e.12xlarge`** / 4× L40S reference node). + +--- + +## Prerequisites + +| Item | Notes | +|------|--------| +| MicroK8s or K8s 1.25+ | `microk8s status --wait-ready` | +| `helm` 3, `kubectl` | | +| Inference Hub key | `~/.nemoclaw/secrets.env` | + +```bash +# ~/.nemoclaw/secrets.env +export NVIDIA_INFERENCE_HUB_API_KEY='sk-...' +``` + +`install-hpa.sh` enables metrics-server on MicroK8s when present. Confirm: + +```bash +kubectl get apiservice v1beta1.metrics.k8s.io +# AVAILABLE should be True +``` + +--- + +## Readiness and Inference Hub + +| Probe | Path | Used for | +|-------|------|----------| +| Liveness | `/healthz` | Pod restart if process dead | +| Readiness (default) | `/healthz` | Rollout success (`probes.readinessChecksInferenceHub: false`) | +| Manual check | `/readyz` | Inference Hub reachable (chat / load test) | + +Install succeeds when the agent process is up. **`/readyz` can still return 503** if the API key is wrong — fix secrets and `helm upgrade`, or use: + +```bash +helm upgrade nemoclaw . -n nemoclaw -f values-step2-hpa.yaml \ + --reuse-values --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" +``` + +To require Hub for readiness (stricter, install may fail without valid key): + +```bash +helm upgrade nemoclaw . -n nemoclaw --reuse-values \ + --set probes.readinessChecksInferenceHub=true +``` + +--- + +## Values overlays + +| File | Used by | Purpose | +|------|---------|---------| +| `values-step2-hpa.yaml` | `install-hpa.sh`, `hpa-reset.sh` | Idle baseline: 1 CPU/pod, CPU % HPA, `/healthz` readiness | +| `values-step2-hpa-saturate.yaml` | `hpa-load-test.sh` | 400m CPU request/pod so up to 7 replicas fit on 8 vCPU | +| `values-step2-hpa-performance.yaml` | `install-performance-hpa.sh` | Scale on `nemoclaw_http_inflight_requests` (Prometheus) | + +--- + +## Idle vs load test + +### Idle (no traffic) + +| Resource | Expected | +|----------|----------| +| Pods | **1** agent, `Running`, `READY 1/1` | +| HPA | `MINPODS 1`, `MAXPODS 7`, **`REPLICAS 1`** | +| TARGETS | `cpu: %/65%` (may show `` briefly after install) | + +**No scale-up or scale-down** without workload — HPA stays at min replicas. + +### Under load (`hpa-load-test.sh`) + +Replicas rise toward **7** while CPU is high; after the Job ends, count drifts back to **1** over ~2–8 minutes (scale-down stabilization ~120s). + +Watch (use separate commands — some kubectl versions reject `hpa,pods` together): + +```bash +kubectl get hpa -n nemoclaw -w +kubectl get pods -n nemoclaw -w +kubectl top pods -n nemoclaw +less /tmp/nemoclaw-hpa-watch.log +``` + +--- + +## Troubleshooting + +| Symptom | What to try | +|---------|-------------| +| `install-hpa.sh` → **rollout failed** | `./scripts/cluster-recover.sh` | +| Deployment **`0 up-to-date`**, no ReplicaSet | `./scripts/cluster-recover.sh` (restarts MicroK8s) | +| HPA **`REPLICAS 0`** / `desiredReplicas=0` | `DELETE_HPA=1 ./scripts/hpa-reset.sh` — never `kubectl scale … --replicas=0` | +| `/readyz` **503** after install | Fix `NVIDIA_INFERENCE_HUB_API_KEY`; pod can still be Running | +| Port-forward fails | Check nothing else is bound to local port **8080** | +| Load-test pods stuck `Terminating` | `./scripts/hpa-reset.sh` | +| HPA slow to scale down | Normal after load stops | +| Prometheus install timeout | Use CPU path only (`install-hpa.sh`) | + +--- + +## Optional: Prometheus performance HPA + +Scales on `nemoclaw_http_inflight_requests` instead of CPU. Requires kube-prometheus-stack + prometheus-adapter; **heavy** on small VMs. + +```bash +PROM_HELM_TIMEOUT=35m ./scripts/install-performance-hpa.sh +``` + +See [helm/nemoclaw-cpu/observability.md](helm/nemoclaw-cpu/observability.md). + +--- + +## Directory layout + +```text +deploy/ +├── README-cpu.md ← this file +├── README-gpu.md ← GPU deployment +└── helm/ + ├── README.md ← CPU vs GPU comparison (models, endpoints) + ├── nemoclaw-cpu/ + │ ├── README.md + │ ├── values.yaml + │ ├── values-step2-hpa.yaml + │ ├── values-step2-hpa-saturate.yaml + │ ├── values-step2-hpa-performance.yaml + │ ├── scripts/ + │ │ ├── install-hpa.sh ← start here + │ │ ├── cluster-recover.sh ← rollout / controller stuck + │ │ ├── hpa-reset.sh + │ │ ├── hpa-load-test.sh + │ │ ├── install-performance-hpa.sh + │ │ └── hpa-common.sh + │ └── files/ + └── nemoclaw-gpu/ +``` + +Cursor skill: `.cursor/skills/nemoclaw-k8s-hpa/` + +--- + +## Uninstall + +```bash +helm uninstall nemoclaw -n nemoclaw +helm uninstall prometheus-adapter kube-prometheus -n monitoring 2>/dev/null || true +kubectl delete namespace nemoclaw monitoring --ignore-not-found +``` + +--- + +## Further reading + +- [README-gpu.md](README-gpu.md) — GPU + Ollama deployment +- [helm/nemoclaw-cpu/README.md](helm/nemoclaw-cpu/README.md) — chart values, manual helm, load-test tuning +- [NVIDIA Inference Hub](https://inference-api.nvidia.com) +- [Kubernetes HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) diff --git a/deploy/README-gpu.md b/deploy/README-gpu.md new file mode 100644 index 00000000000..b71729fe78d --- /dev/null +++ b/deploy/README-gpu.md @@ -0,0 +1,525 @@ + +# NemoClaw Kubernetes deployment — GPU (HPA) + +**Main deployment path** for NemoClaw on Kubernetes: **GPU agent pods** with **Horizontal Pod Autoscaler (HPA)**. Each pod uses **one GPU** and runs **local Ollama inference** (same pattern as NemoClaw GPU onboard / E2E). + +**Index:** [deploy/README.md](README.md) · **Models (CPU vs GPU):** [helm/README.md](helm/README.md) + +This document is the **GPU deployment guide** — install, operate, load-test, architecture, HPA metrics, ports, and troubleshooting. Optional CPU chart: [README-cpu.md](README-cpu.md). + +**Reference hardware:** **4× L40S** on single-node **MicroK8s** — one agent pod per GPU, HPA `MAX_REPLICAS=4`. Other GPU counts work; set `MAX_REPLICAS` to your allocatable `nvidia.com/gpu`. + +**Not the VM NemoClaw sandbox** (Telegram, OpenShell). The VM bot and cluster agents are separate deployments. + +**Default GPU HPA autoscaling metric name:** `gpu_utilization_percent` +(Prometheus source: `DCGM_FI_DEV_GPU_UTIL` from nvidia-dcgm-exporter → prometheus-adapter → custom.metrics.k8s.io) + +## Contents + +- [Reference environment (4× L40S)](#reference-environment-4-l40s) +- [New users (one command)](#new-users-one-command) +- [Port-forward](#port-forward) +- [Comparison (CPU vs GPU)](#comparison-cpu-vs-gpu) +- [Scripts](#scripts) +- [Architecture](#architecture) +- [Pod resources (`gpuScaling`)](#pod-resources-gpuscaling--why-ollama-has-cpu-on-a-gpu-pod) +- [HPA metrics (GPU utilization)](#hpa-metrics-gpu-utilization) +- [Load balancer](#load-balancer) +- [Prerequisites](#prerequisites) +- [Readiness](#readiness) +- [Troubleshooting](#troubleshooting) +- [Directory layout](#directory-layout) +- [Uninstall](#uninstall) + +--- + +## Reference environment (4× L40S) + +This guide is written and load-tested on **4× NVIDIA L40S**. Default install and HPA bounds assume **one agent pod per GPU** (`MAX_REPLICAS=4`). + + +Screenshot 2026-06-09 at 7 01 28 PM + + +| Property | Value | +|----------|--------| +| **GPUs** | **4× L40S** | +| **Kubernetes** | Single-node **MicroK8s** | +| **HPA** | `MIN_REPLICAS=1`, `MAX_REPLICAS=4` | + +If you have fewer or more GPUs, set `MAX_REPLICAS` to match allocatable `nvidia.com/gpu`. + +--- + +## New users (one command) + +```bash +cd ~/NemoClaw/deploy/helm/nemoclaw-gpu + +# Prerequisites (MicroK8s example) +microk8s enable gpu metrics-server +microk8s status --wait-ready + +./scripts/install-hpa.sh +``` + +Set `MAX_REPLICAS` to the number of allocatable GPUs on your node (**4** on the reference **4× L40S** node): + +```bash +MAX_REPLICAS=4 ./scripts/install-hpa.sh # 4× L40S +``` + +First startup pulls the Ollama model — allow **5–15 minutes**: + +```bash +ROLLOUT_TIMEOUT=1200 INFERENCE_MODEL=llama3.2:3b ./scripts/install-hpa.sh +``` + +Confirm idle state: + +```bash +kubectl get hpa -n nemoclaw-gpu +# HPA metric: gpu_utilization_percent +# TARGETS column: current/target (may show milli-units, e.g. 33500m/40 = 33.5%/40) + +kubectl get pods -n nemoclaw-gpu -l component=gpu-agent +``` + +Expect **1 Running pod** (`2/2` containers: Ollama + agent) and HPA **REPLICAS 1**. + +**Watch HPA live** (recommended): + +```bash +kubectl get hpa -n nemoclaw-gpu -w +``` + +Optional second terminal — per-pod GPU % (not the HPA average): + +```bash +./scripts/get-agent-pods.sh -n nemoclaw-gpu -w +``` + +--- + +## Port-forward + +GPU agent listens on **8081** (Service, pod, and local port-forward): + +```bash +kubectl port-forward -n nemoclaw-gpu svc/nemoclaw-gpu-agent 8081:8081 +``` + +Ollama runs on **11434** inside the pod only (not exposed on the Service). + +**Verify** (run port-forward in one terminal): + +```bash +curl -s http://127.0.0.1:8081/healthz +curl -s http://127.0.0.1:8081/readyz +curl -s http://127.0.0.1:8081/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Hi"}],"max_tokens":16}' +``` + +Override host port: `LOCAL_PORT=8082 ./scripts/install-hpa.sh` + +Optional autoscaling demo: + + +Screenshot 2026-06-09 at 7 02 00 PM + +Screenshot 2026-06-09 at 7 39 36 PM + + +```bash +./scripts/hpa-load-test.sh + +./scripts/hpa-reset.sh +``` + + + + + +If install keeps failing: + +```bash +./scripts/cluster-recover.sh +``` + +Chart: [helm/nemoclaw-gpu/](helm/nemoclaw-gpu/) · **Models (CPU vs GPU):** [helm/README.md](helm/README.md) + + + +--- + +## Comparison (CPU vs GPU) + +See [deploy/README.md](README.md) and [helm/README.md](helm/README.md). This guide covers **GPU only** — install with `./scripts/install-hpa.sh` in this chart directory. + +--- + +## Scripts + +All scripts live under `deploy/helm/nemoclaw-gpu/scripts/`. Run from the chart directory. + +| Script | What it does | When to use | +|--------|----------------|-------------| +| **`install-hpa.sh`** | Enables **GPU plugin + DCGM**; installs **Prometheus + prometheus-adapter**; chart with **GPU utilization HPA** (`DCGM_FI_DEV_GPU_UTIL`, default target **70%**). | **First install** or refresh. | +| **`hpa-reset.sh`** | Deletes load-test Jobs; force-deletes pods; **helm upgrade** to idle baseline. | After load test or stuck pods. | +| **`cluster-recover.sh`** | Full uninstall, namespace cleanup, optional MicroK8s restart, reinstall. | Repeated rollout failures. | +| **`hpa-load-test.sh`** | Chat load against local Ollama; watches HPA scale up/down. | Prove autoscaling only. | +| **`get-agent-pods.sh`** | Agent pods with per-pod **GPU UTIL %** (optional `-w` refresh). | Second terminal while `kubectl get hpa -w`. | +| **`get-hpa.sh`** | One-shot HPA with readable **GPU UTIL %** column. | Quick check without milli-units. | +| **`hpa-watch.sh`** | Alias for `kubectl get hpa -n nemoclaw-gpu -w`. | Same as kubectl watch. | +| **`hpa-common.sh`** | Shared helpers (not run directly). | — | + +### Workflow + +```text +First time: install-hpa.sh +After load test / tidy: hpa-reset.sh +Rollout keeps failing: cluster-recover.sh +Prove autoscaling: hpa-load-test.sh +Watch HPA: kubectl get hpa -n nemoclaw-gpu -w +Per-pod GPU (optional): ./scripts/get-agent-pods.sh -w +After load test: hpa-reset.sh +``` + +### Load test tuning (4× L40S) + +`hpa-load-test.sh` auto-detects allocatable GPUs and defaults for the reference **4× L40S** node: + +| Setting | Default | Purpose | +|---------|---------|---------| +| `TARGET_PODS` | allocatable GPUs (**4** on L40S node) | HPA max + load target | +| `CONCURRENCY_PER_POD` | **64** | Base concurrency tuning knob | +| `JOB_PARALLELISM` | **8** | Eight load-generator pods | +| `LOAD_MULTIPLIER` | **4** (4× L40S) | **4096** in-flight requests per agent GPU at peak | +| `INFLIGHT_PER_GPU` | **1024** | Base per-GPU in-flight before multiplier | +| `MAX_TOKENS` | **512** | Longer generations = more GPU compute per request | +| `HPA_TARGET_GPU` | **50** | Scale when average GPU util > 50% (load test only; production default is 70%) | +| Load model | **direct to each Running pod IP** + **HPA compensation** | When HPA=2 but only 1 pod ready, 2× load on ready pod keeps avg above 50% | + +**Why 84% → 42% at 2 replicas:** HPA averages GPU % across all replicas. A new pod at **0%** while starting pulls the average to ~half. Compensation + hitting Running pods (not only Service endpoints) fixes this. + +```bash +./scripts/hpa-load-test.sh +``` + +Watch while it runs: + +```bash +kubectl get hpa -n nemoclaw-gpu -w +``` + +Optional second terminal: + +```bash +./scripts/get-agent-pods.sh -w +``` + +### Environment variables + +| Variable | Scripts | Effect | +|----------|---------|--------| +| `MIN_REPLICAS` / `MAX_REPLICAS` | install, reset, load-test | HPA bounds (default 1 / 4) | +| `CONCURRENCY_PER_POD` / `MAX_TOKENS` / `JOB_PARALLELISM` | load-test | GPU load intensity (defaults tuned for **4× L40S**) | +| `HPA_TARGET_GPU` | load-test | GPU util target during test (default **50**) | +| `ROLLOUT_TIMEOUT` | install, reset | Seconds to wait for rollout (default 900) | +| `INFERENCE_MODEL` | install | Ollama model tag (default `llama3.2:3b`) | +| `GPU_TARGET` | install, reset | HPA target GPU util % (default **70**) | +| `PROM_HELM_TIMEOUT` | install | Prometheus helm wait (default 25m) | +| `LOCAL_PORT` | install | Host port for port-forward hints (default **8081**) | +| `DELETE_DEPLOYMENT=1` | `hpa-reset.sh` | Delete Deployment before reinstall | +| `DELETE_HPA=1` | `hpa-reset.sh` | Delete HPA before reinstall | +| `SKIP_HELM=1` | `hpa-reset.sh` | kubectl cleanup only | +| `RESTART_MICROK8S=0` | `cluster-recover.sh` | Skip MicroK8s restart | + +--- + +## Architecture + +```text +┌─────────────────────────────────────────────────────────────┐ +│ GPU node — 4× L40S — nemoclaw-gpu │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Pod (1× nvidia.com/gpu = 1× L40S) │ │ +│ │ ┌──────────────┐ ┌─────────────────────────────┐ │ │ +│ │ │ Ollama │◄───│ agent (Node.js) :8081 │ │ │ +│ │ │ :11434 GPU │ │ /healthz /readyz /metrics │ │ │ +│ │ └──────────────┘ └─────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ HPA scales 1 … 4 pods (one pod per L40S) │ +└─────────────────────────────────────────────────────────────┘ + ▲ + │ DCGM_FI_DEV_GPU_UTIL (Prometheus ← nvidia-dcgm-exporter) + │ same signal family as nvidia-smi GPU utilization % +``` + +## Pod resources (`gpuScaling`) — why Ollama has CPU on a GPU pod + +`values-step2-hpa.yaml` (and `values.yaml`) define two sections that are easy to confuse: + +| Section | Purpose | +|---------|---------| +| **`autoscaling:`** | HPA — *when* to scale (GPU util %, min/max replicas) | +| **`gpuScaling:`** | Pod sizing — *how big* each pod is (CPU, memory, GPU per container) | + +The **`perPodCpuRequest`** / **`perPodCpuLimit`** fields under `gpuScaling` are **not** the HPA scale signal. They do not drive scale-up when `autoscaling.mode=gpu` (HPA uses DCGM GPU utilization instead). + +Each agent pod runs **two containers**: + +```text +┌─────────────────────────────────────────────┐ +│ pod: nemoclaw-gpu-agent-… │ +│ ┌──────────────────┐ ┌─────────────────┐ │ +│ │ ollama (1 GPU) │ │ agent (no GPU) │ │ +│ │ perPodCpu* │ │ agentCpu* │ │ +│ │ perPodMemory* │ │ agentMemory* │ │ +│ │ nvidia.com/gpu:1 │ │ │ │ +│ └──────────────────┘ └─────────────────┘ │ +└─────────────────────────────────────────────┘ +``` + +| Field | Applies to | Role | +|-------|------------|------| +| `perPodCpuRequest` / `perPodCpuLimit` | **Ollama** | Host CPU reservation and cap for the inference container | +| `perPodMemory` / `perPodMemoryLimit` | **Ollama** | Host RAM (model cache, buffers — not VRAM) | +| `perPodGpu` | **Ollama** | GPUs per pod (default **1**) | +| `agentCpuRequest` / `agentCpuLimit` | **Node.js agent** | Small proxy sidecar (health, `/metrics`, chat API) | + +### Why Ollama still needs CPU + +GPU inference does not run with zero host CPU. Ollama uses CPU for: + +- HTTP/API handling and request parsing +- Tokenization and preprocessing before GPU work +- Loading and unpacking model weights into GPU memory +- Runtime overhead around the CUDA kernels + +Kubernetes also **requires** CPU and memory **requests** on every container so the scheduler can place pods and enforce QoS. Without them, the cluster cannot reserve node capacity reliably. + +Default sizing (e.g. **2** CPU request / **4** limit for Ollama on **4× L40S**) is a **scheduling guardrail**, not “scale when CPU hits 65%.” Four pods × 2 CPU request ≈ 8 vCPUs for Ollama, plus ~1 vCPU for agents, leaving headroom for kube-system, DCGM, Prometheus, and load-test Jobs. + +`targetCPUUtilizationPercentage` in `values.yaml` is only used when `autoscaling.mode=resource` (CPU-based HPA via metrics-server). It is **ignored** in the default GPU HPA mode. + +Example from `values-step2-hpa.yaml`: + +```yaml +gpuScaling: + perPodCpuRequest: "2" + perPodCpuLimit: "4" + perPodMemory: "16Gi" + agentCpuRequest: "250m" + agentCpuLimit: "1" + +autoscaling: + mode: gpu + targetGPUUtilizationPercentage: 70 # HPA signal — not perPodCpu* +``` + +## HPA metrics (GPU utilization) + +**HPA autoscaling metric name:** `gpu_utilization_percent` + +| Layer | What | +|-------|------| +| **HPA reads** | **`gpu_utilization_percent`** (custom.metrics.k8s.io, per pod) | +| **Prometheus source** | `DCGM_FI_DEV_GPU_UTIL` (nvidia-dcgm-exporter) | +| **Adapter alias** | `gpu_utilization_percent` (prometheus-adapter rule) | +| **Source** | `nvidia-dcgm-exporter` (GPU operator / `microk8s enable gpu`) | +| **Pipeline** | DCGM → Prometheus → prometheus-adapter → HPA custom metrics API | +| **Target** | Default **70%** average GPU util per pod (`GPU_TARGET=70`) | + +**Watch HPA** (primary): + +```bash +kubectl get hpa -n nemoclaw-gpu -w +``` + +The **TARGETS** column shows current/target. Kubernetes may display milli-units (`33500m/70` = 33.5%/70%) — same numbers, different format. Scale-up happens when **current** stays above **target** (e.g. average `84%/50%` across pods → add replicas). + +**Per-pod GPU %** (HPA averages across all agent pods — use this to see each pod): + +```bash +./scripts/get-agent-pods.sh -n nemoclaw-gpu -w +``` + +**One-shot readable HPA** (optional): + +```bash +./scripts/get-hpa.sh -n nemoclaw-gpu +# prints GPU UTIL % as 33.5%/70% instead of milli-units +``` + +(Prometheus source: `DCGM_FI_DEV_GPU_UTIL` from nvidia-dcgm-exporter — same family as **nvidia-smi** GPU util %.) + +```bash +kubectl describe hpa -n nemoclaw-gpu | grep nemoclaw.ai/hpa-metric +``` + +(not `cpu: …/65%`). Metrics may show `` for 1–2 minutes after install until Prometheus scrapes DCGM. + +**Fallback modes** (not default): `autoscaling.mode=performance` (inflight HTTP), `latency` (LLM p95 ms), or `resource` (CPU %) via helm `--set`. + +### Agent `/metrics` (LLM response time) + +Each agent pod exposes Prometheus metrics at `GET /metrics`. LLM latency is measured **inside the chat/completions proxy** (time from outbound request to Ollama until the response body is received). + +| Metric | Type | Use | +|--------|------|-----| +| `nemoclaw_llm_request_duration_seconds` | histogram | Full latency distribution (Prometheus/Grafana) | +| `nemoclaw_llm_latency_p95_milliseconds` | gauge | Rolling **p95** over recent requests (default window: 128); **HPA-friendly** | +| `nemoclaw_llm_latency_p50_milliseconds` | gauge | Rolling median latency | +| `nemoclaw_llm_latency_avg_milliseconds` | gauge | Rolling average latency | +| `nemoclaw_llm_requests_total{result=…}` | counter | Success/error counts | +| `nemoclaw_http_inflight_requests` | gauge | Concurrent requests (queue/backpressure) | + +After port-forward, inspect locally: + +```bash +kubectl port-forward -n nemoclaw-gpu svc/nemoclaw-gpu-agent 8081:8081 +curl -s http://127.0.0.1:8081/metrics | grep nemoclaw_llm +``` + +**Latency-based HPA** (optional — needs ServiceMonitor + adapter rules from `install-hpa.sh`): + +```bash +helm upgrade nemoclaw-gpu . -n nemoclaw-gpu \ + -f values-step2-hpa-latency.yaml \ + --set autoscaling.performance.latencyP95Milliseconds=2000 +``` + +HPA scales up when average pod p95 exceeds the target (e.g. `2500/2000` ms). Tune window size with env `LLM_LATENCY_WINDOW_SIZE` on the agent container if needed. + +--- + +## Load balancer + +### AWS instance + +See [Reference environment (4× L40S)](#reference-environment-4-l40s). This chart targets **one agent pod per L40S** on the node, not multi-node autoscaling groups (yet). + +### What we use today (in-cluster only) + +There is **no AWS Application/Network Load Balancer** in this chart. Traffic uses a **ClusterIP** Service: + +| Resource | Type | Role | +|----------|------|------| +| `nemoclaw-gpu-agent` | **ClusterIP** | In-cluster VIP; kube-proxy spreads **new TCP connections** across Ready pods | +| `kubectl port-forward` | Local tunnel | Debug only (`8081` → Service); not production routing | +| Load-test Job | In-cluster HTTP client | Hits `http://nemoclaw-gpu-agent:8081` → Service → pods | + +```text +In-cluster client (load-test Job, another pod) + → Service nemoclaw-gpu-agent:8081 (ClusterIP) + → kube-proxy (iptables / IPVS) + → agent pod 1 … agent pod N (one GPU each) + +Your laptop + → kubectl port-forward :8081 (debug path only) +``` + +**HPA** adds or removes pods; the Service **automatically** picks up new endpoints when pods become Ready. No separate LB configuration is required for that. + +### Even distribution — limits of ClusterIP + +Kubernetes Service balancing is **connection-level** (rough round-robin), not GPU-aware: + +- Long **Ollama chat** requests can leave one pod hot while others are idle. +- HPA scales **replica count** from **GPU utilization**; it does not reshuffle an existing queue. +- For dev/demo and `./scripts/hpa-load-test.sh`, ClusterIP is enough to fan out load as replicas grow. + +### Next step (not implemented yet): smarter ingress routing + +Planned follow-up — **not in this chart today**: + +- **NGINX Ingress Controller** (or similar) in front of the Service for HTTP routing, timeouts, and optional rate limits. +- **AWS Load Balancer Controller** / `type: LoadBalancer` for external clients (NLB/ALB) when running on EKS with a cloud controller. +- **Queue- or latency-aware** routing (e.g. least-connections, custom metrics) so inference spreads more evenly than default kube-proxy. + +Until then, use **in-cluster Service DNS** for load tests and **port-forward** only for manual checks. Track ingress/LB work as a separate install step when you need external traffic or fairer GPU scheduling. + +--- + +## Prerequisites + +| Item | Notes | +|------|--------| +| **Reference GPUs** | **4× L40S** — see [Reference environment](#reference-environment-4-l40s) | +| MicroK8s or K8s 1.25+ | `microk8s status --wait-ready` | +| NVIDIA device plugin | `microk8s enable gpu` (DCGM for L40S util metrics) | +| metrics-server | `microk8s enable metrics-server` | +| `helm` 3, `kubectl` | | +| Allocatable GPUs | `kubectl describe node \| grep nvidia.com/gpu` — expect **4** on reference node | + +--- + +## Readiness + +| Probe | Path | Used for | +|-------|------|----------| +| Liveness | `/healthz` | Agent process up | +| Readiness | `/readyz` | Ollama model pulled and ready | +| Startup | `/readyz` | Long window for first `ollama pull` | + +**`/readyz` may return 503 for several minutes** on first install while the model downloads. + +--- + +## Troubleshooting + +| Symptom | What to try | +|---------|-------------| +| `Insufficient nvidia.com/gpu` | Lower `MAX_REPLICAS`; check allocatable GPUs | +| Service not found | Use `nemoclaw-gpu-agent`, not `nemoclaw-gpu-nemoclaw-gpu-agent` | +| Port-forward fails | Check nothing else is bound to local port **8081** | +| `/readyz` 503 for minutes | Normal during model pull; `kubectl logs -c ollama` | +| HPA stays at 1 on 1-GPU node | Expected — need multiple GPUs for scale-up (reference node has **4× L40S**) | +| HPA TARGETS `/70` | Metrics pipeline broken — re-run `MAX_REPLICAS=4 GPU_TARGET=70 ./scripts/install-hpa.sh` on **4× L40S** node; verify adapter points at Prometheus (not Grafana) and custom metric exists: `kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/nemoclaw-gpu/pods/*/gpu_utilization_percent"` | +| HPA `ScalingActive: False` | Same as above — HPA cannot scale until `gpu_utilization_percent` is readable | +| Rollout failed | `ROLLOUT_TIMEOUT=1200 ./scripts/install-hpa.sh` or `cluster-recover.sh` | + +--- + +## Directory layout + +```text +deploy/ +├── README.md ← deploy index (links here) +├── README-gpu.md ← this file (GPU how-to) +├── README-cpu.md ← optional CPU how-to +└── helm/ + ├── README.md ← model comparison only + ├── nemoclaw-gpu/ + │ ├── values-step2-hpa.yaml + │ ├── values-step2-hpa-performance.yaml + │ ├── values-step2-hpa-latency.yaml + │ └── scripts/ + │ ├── install-hpa.sh + │ ├── hpa-reset.sh + │ ├── cluster-recover.sh + │ ├── hpa-load-test.sh + │ ├── get-agent-pods.sh + │ ├── get-hpa.sh + │ └── hpa-watch.sh # → kubectl get hpa -w + └── nemoclaw-cpu/ +``` + +--- + +## Uninstall + +```bash +helm uninstall nemoclaw-gpu -n nemoclaw-gpu +kubectl delete namespace nemoclaw-gpu --ignore-not-found +``` + +--- + +## Further reading + +- [helm/README.md](helm/README.md) — model comparison (CPU vs GPU) +- [README-cpu.md](README-cpu.md) — optional pre-GPU / Inference Hub deployment +- [Kubernetes HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000000..c7bf0888a24 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,39 @@ +# NemoClaw Kubernetes deployment + +**How to run the GPU deployment:** **[README-gpu.md](README-gpu.md)** — install, port-forward, HPA, load test, architecture, metrics, ports, and troubleshooting. + +**Model comparison (CPU vs GPU):** table below and [helm/README.md](helm/README.md). + +--- + +## Quick links + +| | GPU (primary) | CPU (optional) | +|--|---------------|----------------| +| **How to run** | **[README-gpu.md](README-gpu.md)** | **[README-cpu.md](README-cpu.md)** | +| **Chart** | [helm/nemoclaw-gpu/](helm/nemoclaw-gpu/) | [helm/nemoclaw-cpu/](helm/nemoclaw-cpu/) | +| **Namespace** | `nemoclaw-gpu` | `nemoclaw` | +| **Port** | 8081 | 8080 | +| **Default model** | **Llama 3.2 3B** — `llama3.2:3b` | **Nemotron Ultra 253B** — `nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1` | +| **Inference backend** | Local **Ollama** on pod GPU (`127.0.0.1:11434`) | Remote **NVIDIA Inference Hub** (`inference-api.nvidia.com`) | +| **Where weights run** | In cluster (1× GPU per pod) | In NVIDIA cloud (no cluster GPU) | +| **HPA signal** | GPU util % (DCGM) | CPU % | + +Full model comparison (sizes, auth, overrides): [helm/README.md](helm/README.md). + +Charts are fully independent (separate install scripts, Services, HPAs). Do **not** use deprecated `deploy/scripts/*-both.sh` helpers. + +--- + +## Layout + +```text +deploy/ +├── README.md ← this index +├── README-gpu.md ← GPU install & ops (main guide) +├── README-cpu.md ← optional pre-GPU testing +└── helm/ + ├── README.md ← model comparison (CPU vs GPU) + ├── nemoclaw-gpu/ ← primary chart + └── nemoclaw-cpu/ ← optional chart +``` diff --git a/deploy/helm/README.md b/deploy/helm/README.md new file mode 100644 index 00000000000..1a28c9f22df --- /dev/null +++ b/deploy/helm/README.md @@ -0,0 +1,83 @@ +# Models — CPU vs GPU charts + +How the two NemoClaw Kubernetes charts differ in **which model** they use and **where inference runs**. + +| | **GPU chart** ([nemoclaw-gpu](nemoclaw-gpu/)) | **CPU chart** ([nemoclaw-cpu](nemoclaw-cpu/)) | +|--|-----------------------------------------------|-----------------------------------------------| +| **Role** | **Primary** — local inference on cluster GPU | **Optional** — pre-GPU testing, no cluster GPU | +| **Default model** | `llama3.2:3b` | `nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1` | +| **Display name** | Llama 3.2 **3B** (Ollama) | **Nemotron Ultra 253B** (NVIDIA Inference Hub) | +| **Approx. size** | ~3B parameters | ~253B parameters | +| **Where weights run** | On **your pod GPU** (pulled into cluster) | In **NVIDIA cloud** (not in your cluster) | +| **Backend** | [Ollama](https://ollama.com) sidecar in pod | [NVIDIA Inference Hub](https://inference-api.nvidia.com) HTTPS API | +| **API endpoint** | `http://127.0.0.1:11434/v1` (in-pod) | `https://inference-api.nvidia.com/v1` | +| **Model ID format** | Ollama **tag** (e.g. `llama3.2:3b`, `qwen2.5:7b`) | Hub **model path** (e.g. `nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1`) | +| **Auth** | None (default) | `NVIDIA_INFERENCE_HUB_API_KEY` (`sk-*`) | +| **First-start delay** | **5–15 min** — `ollama pull` on each new pod/node | Seconds — no local pull | +| **GPU memory** | Uses **1× `nvidia.com/gpu` / pod** | None | +| **Quality / use case** | Fast, small local model; good for HPA/load demos on L40S | Large Nemotron; matches VM NemoClaw + Inference Hub setup | +| **Set in** | `nemoclaw-gpu/values.yaml` → `inference.model` | `nemoclaw-cpu/values.yaml` → `inference.model` | + +Defaults are defined in each chart’s `values.yaml`: + +```yaml +# nemoclaw-gpu/values.yaml +inference: + baseUrl: "http://127.0.0.1:11434/v1" + model: "llama3.2:3b" + +# nemoclaw-cpu/values.yaml +inference: + baseUrl: "https://inference-api.nvidia.com/v1" + model: "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" +``` + +--- + +## Request path + +**GPU (local Ollama)** + +```text +Client → agent :8081 → Ollama :11434 → GPU +``` + +**CPU (remote Hub)** + +```text +Client → agent :8080 → HTTPS Inference Hub → Nemotron Ultra (cloud) +``` + +Same agent container pattern (health, metrics, OpenAI-compatible `/v1/chat/completions`); only the **inference backend and model** change. + +--- + +## Changing the model + +Model overrides are done at install/upgrade time in each chart’s ops guide — not in this file. + +| Chart | How to run & change model | +|-------|---------------------------| +| **GPU** | [../README-gpu.md](../README-gpu.md) — `INFERENCE_MODEL=… ./scripts/install-hpa.sh` or `helm upgrade … --set inference.model=…` | +| **CPU** | [../README-cpu.md](../README-cpu.md) — install, port-forward, load test, and Hub model overrides | + +Hub model ids and Ollama tags are **not interchangeable** — use the chart that matches your backend. + +## Why the defaults differ + +| Chart | Default model choice | +|-------|----------------------| +| **GPU** | Small Ollama model (`llama3.2:3b`) so pods start quickly on a single GPU, HPA load tests stay predictable, and VRAM use is modest on demo hardware. | +| **CPU** | Nemotron Ultra via Hub so pre-GPU testing matches **NemoClaw on a VM** when configured for Inference Hub — no local weights, no GPU. | + +--- + +## Related docs + +| Doc | Content | +|-----|---------| +| [../README-gpu.md](../README-gpu.md) | **How to run GPU** — install, HPA, load test, model override | +| [../README-cpu.md](../README-cpu.md) | **How to run CPU** — install, port-forward, load test, Hub model override | +| [../README.md](../README.md) | Deploy index (GPU-first) | +| [nemoclaw-gpu/](nemoclaw-gpu/) | GPU Helm chart | +| [nemoclaw-cpu/](nemoclaw-cpu/) | CPU Helm chart | diff --git a/deploy/helm/nemoclaw-cpu/.helmignore b/deploy/helm/nemoclaw-cpu/.helmignore new file mode 100644 index 00000000000..7da0ef05be4 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +*.swp +*.bak +*.tmp +.git/ diff --git a/deploy/helm/nemoclaw-cpu/Chart.yaml b/deploy/helm/nemoclaw-cpu/Chart.yaml new file mode 100644 index 00000000000..5c4441c0c9d --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/Chart.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v2 +name: nemoclaw-cpu +description: CPU-only NemoClaw agent replicas on Kubernetes (remote Nemotron Ultra via Inference Hub) +type: application +version: 0.1.0 +appVersion: "2026.05.27" +keywords: + - nemoclaw + - inference-hub + - cpu +maintainers: + - name: maggiezha diff --git a/deploy/helm/nemoclaw-cpu/README.md b/deploy/helm/nemoclaw-cpu/README.md new file mode 100644 index 00000000000..c40de219cb1 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/README.md @@ -0,0 +1,16 @@ +# nemoclaw-cpu Helm chart + +**How to run the CPU deployment:** **[../../README-cpu.md](../../README-cpu.md)** — install, port-forward, HPA, load test, and troubleshooting. + +Optional pre-GPU chart: CPU agent pods proxy **[NVIDIA Inference Hub](https://inference-api.nvidia.com)** (Nemotron Ultra by default). No cluster GPU required. + +| | | +|--|--| +| **Ops guide** | [../../README-cpu.md](../../README-cpu.md) | +| **Models (vs GPU)** | [../README.md](../README.md) | +| **Primary deployment** | [nemoclaw-gpu](../nemoclaw-gpu/) · [../../README-gpu.md](../../README-gpu.md) | +| **Chart path** | `deploy/helm/nemoclaw-cpu/` | +| **Install script** | `./scripts/install-hpa.sh` | +| **Default model** | `inference.model` in `values.yaml` → Nemotron Ultra 253B on Inference Hub | + +For Helm values, templates, and advanced overlays, see sections in [README-cpu.md](../../README-cpu.md) and files under this directory (`values.yaml`, `values-step2-hpa.yaml`, `templates/`). diff --git a/deploy/helm/nemoclaw-cpu/files/agent-metrics.mjs b/deploy/helm/nemoclaw-cpu/files/agent-metrics.mjs new file mode 100644 index 00000000000..f8c703eb9de --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/files/agent-metrics.mjs @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Shared Prometheus helpers for agent /metrics (LLM latency, HTTP counters). + +const LLM_LATENCY_WINDOW = Number(process.env.LLM_LATENCY_WINDOW_SIZE || 128); +const llmDurationsMs = []; +let llmDurationSumSec = 0; +let llmDurationCount = 0; +let llmRequestsOk = 0; +let llmRequestsError = 0; +const llmHistogramBucketsSec = [0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300]; +const llmHistogramCounts = Array.from({ length: llmHistogramBucketsSec.length + 1 }, () => 0); + +export function recordLlmLatency(durationMs, ok) { + const sec = Math.max(0, durationMs) / 1000; + llmDurationSumSec += sec; + llmDurationCount += 1; + if (ok) llmRequestsOk += 1; + else llmRequestsError += 1; + + llmDurationsMs.push(durationMs); + if (llmDurationsMs.length > LLM_LATENCY_WINDOW) llmDurationsMs.shift(); + + let bucketIdx = llmHistogramBucketsSec.findIndex((bound) => sec <= bound); + if (bucketIdx === -1) bucketIdx = llmHistogramBucketsSec.length; + for (let i = bucketIdx; i < llmHistogramCounts.length; i += 1) { + llmHistogramCounts[i] += 1; + } +} + +function percentileMs(sorted, p) { + if (!sorted.length) return 0; + const idx = Math.ceil(sorted.length * p) - 1; + return sorted[Math.max(0, idx)]; +} + +function llmLatencySnapshotMs() { + if (!llmDurationsMs.length) { + return { p50: 0, p95: 0, avg: 0 }; + } + const sorted = [...llmDurationsMs].sort((a, b) => a - b); + const sum = sorted.reduce((acc, v) => acc + v, 0); + return { + p50: percentileMs(sorted, 0.5), + p95: percentileMs(sorted, 0.95), + avg: sum / sorted.length, + }; +} + +export function llmMetricsLines() { + const { p50, p95, avg } = llmLatencySnapshotMs(); + const lines = [ + "# HELP nemoclaw_llm_requests_total Chat/completions proxied to inference backend", + "# TYPE nemoclaw_llm_requests_total counter", + `nemoclaw_llm_requests_total{result="success"} ${llmRequestsOk}`, + `nemoclaw_llm_requests_total{result="error"} ${llmRequestsError}`, + "# HELP nemoclaw_llm_request_duration_seconds LLM chat/completions end-to-end proxy latency", + "# TYPE nemoclaw_llm_request_duration_seconds histogram", + ]; + + for (let i = 0; i < llmHistogramBucketsSec.length; i += 1) { + lines.push( + `nemoclaw_llm_request_duration_seconds_bucket{le="${llmHistogramBucketsSec[i]}"} ${llmHistogramCounts[i]}`, + ); + } + lines.push( + `nemoclaw_llm_request_duration_seconds_bucket{le="+Inf"} ${llmHistogramCounts[llmHistogramCounts.length - 1]}`, + `nemoclaw_llm_request_duration_seconds_sum ${llmDurationSumSec}`, + `nemoclaw_llm_request_duration_seconds_count ${llmDurationCount}`, + "# HELP nemoclaw_llm_latency_p50_milliseconds Rolling p50 LLM latency (recent window)", + "# TYPE nemoclaw_llm_latency_p50_milliseconds gauge", + `nemoclaw_llm_latency_p50_milliseconds ${Math.round(p50)}`, + "# HELP nemoclaw_llm_latency_p95_milliseconds Rolling p95 LLM latency (recent window; HPA-friendly)", + "# TYPE nemoclaw_llm_latency_p95_milliseconds gauge", + `nemoclaw_llm_latency_p95_milliseconds ${Math.round(p95)}`, + "# HELP nemoclaw_llm_latency_avg_milliseconds Rolling average LLM latency (recent window)", + "# TYPE nemoclaw_llm_latency_avg_milliseconds gauge", + `nemoclaw_llm_latency_avg_milliseconds ${Math.round(avg)}`, + ); + return lines; +} diff --git a/deploy/helm/nemoclaw-cpu/files/agent-server.mjs b/deploy/helm/nemoclaw-cpu/files/agent-server.mjs new file mode 100644 index 00000000000..b194edfa314 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/files/agent-server.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Minimal CPU agent pod: health + Prometheus metrics for HPA tuning. +// Inference runs on NVIDIA Inference Hub (no local GPU). + +import http from "node:http"; +import { Worker } from "node:worker_threads"; +import { llmMetricsLines, recordLlmLatency } from "./agent-metrics.mjs"; + +const PORT = Number(process.env.PORT || 8080); +const BASE_URL = (process.env.INFERENCE_BASE_URL || "").replace(/\/$/, ""); +const MODEL = process.env.INFERENCE_MODEL || ""; +const API_KEY = process.env.NVIDIA_INFERENCE_HUB_API_KEY || ""; +/** Optional per-request CPU spin (ms) so HPA sees pod CPU rise under load. */ +const LOAD_TEST_CPU_SPIN_MS = Number(process.env.LOAD_TEST_CPU_SPIN_MS || 0); + +let inflight = 0; +let totalRequests = 0; +let inferenceReachable = 0; +/** Cache /readyz Hub check so probes do not compete with load-test chat traffic. */ +let inferenceCache = { ok: false, at: 0 }; +const INFERENCE_CACHE_MS = Number(process.env.INFERENCE_READY_CACHE_MS || 30_000); + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +/** Spin in worker threads so /healthz and /readyz stay responsive under load. */ +function cpuSpinWorkers(ms, threads = 1) { + if (!ms || ms <= 0) return Promise.resolve(); + const n = Math.max(1, Math.min(Number(threads) || 1, 2)); + const eachMs = Math.ceil(ms / n); + return Promise.all( + Array.from({ length: n }, () => + new Promise((resolve, reject) => { + const w = new Worker(new URL("./cpu-spin-worker.mjs", import.meta.url), { + workerData: { ms: eachMs }, + }); + w.once("message", () => { + w.terminate().catch(() => {}); + resolve(); + }); + w.once("error", reject); + w.once("exit", (code) => { + if (code !== 0) reject(new Error(`cpu-spin-worker exited ${code}`)); + }); + }), + ), + ); +} + +async function proxyChatCompletions(req, res) { + if (!BASE_URL || !API_KEY) { + res.writeHead(503, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "inference not configured" })); + return; + } + let raw; + try { + raw = await readBody(req); + } catch { + res.writeHead(400, { "content-type": "text/plain" }); + res.end("bad request\n"); + return; + } + let body; + try { + body = raw ? JSON.parse(raw) : {}; + } catch { + res.writeHead(400, { "content-type": "text/plain" }); + res.end("invalid json\n"); + return; + } + if (!body.model) body.model = MODEL; + const spinHeader = Number(req.headers["x-nemoclaw-load-spin-ms"]); + const spinMs = Number.isFinite(spinHeader) && spinHeader > 0 ? spinHeader : LOAD_TEST_CPU_SPIN_MS; + await cpuSpinWorkers(spinMs, 1); + const llmStart = performance.now(); + let llmOk = false; + try { + const hubRes = await fetch(`${BASE_URL}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(180_000), + }); + llmOk = hubRes.ok; + const text = await hubRes.text(); + res.writeHead(hubRes.status, { "content-type": "application/json" }); + res.end(text); + } catch (err) { + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: String(err) })); + } finally { + recordLlmLatency(performance.now() - llmStart, llmOk); + } +} + +async function checkInference() { + if (!BASE_URL || !API_KEY) return false; + const now = Date.now(); + if (now - inferenceCache.at < INFERENCE_CACHE_MS) return inferenceCache.ok; + try { + const res = await fetch(`${BASE_URL}/models`, { + headers: { Authorization: `Bearer ${API_KEY}` }, + signal: AbortSignal.timeout(10_000), + }); + inferenceCache = { ok: res.ok, at: now }; + return res.ok; + } catch { + inferenceCache = { ok: false, at: now }; + return false; + } +} + +function metricsText() { + return [ + "# HELP nemoclaw_http_requests_total Total HTTP requests to agent pod", + "# TYPE nemoclaw_http_requests_total counter", + `nemoclaw_http_requests_total ${totalRequests}`, + "# HELP nemoclaw_http_inflight_requests In-flight HTTP requests", + "# TYPE nemoclaw_http_inflight_requests gauge", + `nemoclaw_http_inflight_requests ${inflight}`, + "# HELP nemoclaw_inference_hub_reachable 1 if Inference Hub /models OK", + "# TYPE nemoclaw_inference_hub_reachable gauge", + `nemoclaw_inference_hub_reachable ${inferenceReachable}`, + ...llmMetricsLines(), + "", + ].join("\n"); +} + +const server = http.createServer(async (req, res) => { + totalRequests += 1; + inflight += 1; + try { + if (req.url === "/healthz" || req.url === "/health") { + res.writeHead(200, { "content-type": "text/plain" }); + res.end("ok\n"); + return; + } + if (req.url === "/readyz" || req.url === "/ready") { + const ok = await checkInference(); + inferenceReachable = ok ? 1 : 0; + res.writeHead(ok ? 200 : 503, { "content-type": "text/plain" }); + res.end(ok ? "ready\n" : "inference not reachable\n"); + return; + } + if (req.url === "/metrics") { + res.writeHead(200, { "content-type": "text/plain; version=0.0.4" }); + res.end(metricsText()); + return; + } + const pathOnly = (req.url || "").split("?")[0]; + if (pathOnly === "/bench" && (req.method === "POST" || req.method === "GET")) { + const parsed = new URL(req.url || "/bench", "http://127.0.0.1"); + const headerSpin = Number(req.headers["x-nemoclaw-load-spin-ms"]); + const qMs = Number(parsed.searchParams.get("ms")); + const qThreads = Number(parsed.searchParams.get("threads")); + const spinMs = + (Number.isFinite(qMs) && qMs > 0 ? qMs : 0) || + (Number.isFinite(headerSpin) && headerSpin > 0 ? headerSpin : 0) || + LOAD_TEST_CPU_SPIN_MS || + 100; + const threads = + Number.isFinite(qThreads) && qThreads > 0 + ? qThreads + : Number(process.env.LOAD_TEST_BENCH_THREADS || 2); + await cpuSpinWorkers(spinMs, threads); + res.writeHead(200, { "content-type": "text/plain" }); + res.end(`ok spin=${spinMs} threads=${threads}\n`); + return; + } + if ( + (pathOnly === "/v1/chat/completions" || pathOnly === "/chat/completions") && + req.method === "POST" + ) { + await proxyChatCompletions(req, res); + return; + } + if (req.url === "/" && req.method === "GET") { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + service: "nemoclaw-cpu-agent", + model: MODEL, + inferenceBaseUrl: BASE_URL, + loadTestCpuSpinMs: LOAD_TEST_CPU_SPIN_MS, + endpoints: ["/healthz", "/readyz", "/metrics", "POST /bench", "POST /v1/chat/completions"], + note: "Remote Nemotron Ultra via Inference Hub; scale replicas with kubectl or HPA", + }), + ); + return; + } + res.writeHead(404); + res.end("not found\n"); + } finally { + inflight -= 1; + } +}); + +server.listen(PORT, () => { + console.log(`nemoclaw-cpu-agent listening on :${PORT} model=${MODEL}`); +}); diff --git a/deploy/helm/nemoclaw-cpu/files/cpu-spin-worker.mjs b/deploy/helm/nemoclaw-cpu/files/cpu-spin-worker.mjs new file mode 100644 index 00000000000..c13cc2af40a --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/files/cpu-spin-worker.mjs @@ -0,0 +1,11 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import { parentPort, workerData } from "node:worker_threads"; + +const ms = Number(workerData?.ms) || 100; +const end = performance.now() + ms; +while (performance.now() < end) { + /* intentional CPU load for HPA testing */ +} +parentPort?.postMessage("ok"); diff --git a/deploy/helm/nemoclaw-cpu/files/load-generator.mjs b/deploy/helm/nemoclaw-cpu/files/load-generator.mjs new file mode 100644 index 00000000000..0ef9f7bce96 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/files/load-generator.mjs @@ -0,0 +1,147 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Saturate agent pods for CPU HPA: per-target-pod concurrency, parallel /bench + worker threads. + +import fs from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; + +const TARGET = (process.env.TARGET_URL || "http://nemoclaw-nemoclaw-cpu-agent:8080").replace( + /\/$/, + "", +); +const DURATION_SEC = Number(process.env.DURATION_SEC || 720); +/** HPA max replicas we want to drive (8-vCPU node ≈ 7 with 400m request/pod). */ +const TARGET_PODS = Number(process.env.TARGET_PODS || 7); +/** Steady-state in-flight requests per agent pod (via Service LB). */ +const CONCURRENCY_PER_POD = Number(process.env.CONCURRENCY_PER_POD || 40); +const BENCH_MS = Number(process.env.BENCH_MS || process.env.SPIN_MS || 450); +const BENCH_THREADS = Number(process.env.BENCH_THREADS || 2); +const BENCH_RATIO = Number(process.env.BENCH_RATIO || 1); +const RAMP_SEC = Number(process.env.RAMP_SEC || 90); +const REQUEST_TIMEOUT_MS = Number(process.env.REQUEST_TIMEOUT_MS || 90_000); + +const PEAK_INFLIGHT = TARGET_PODS * CONCURRENCY_PER_POD; +const WORKER_COUNT = Math.max(8, Math.min(64, Math.ceil(PEAK_INFLIGHT / 12))); + +function loadQuestions() { + try { + const lines = fs + .readFileSync(process.env.QUESTIONS_FILE || "/questions/questions.txt", "utf8") + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + if (lines.length) return lines; + } catch { + /* use fallback */ + } + return ["Briefly explain Kubernetes HPA."]; +} + +async function bench() { + const url = `${TARGET}/bench?ms=${BENCH_MS}&threads=${BENCH_THREADS}`; + const res = await fetch(url, { + method: "POST", + headers: { "X-NemoClaw-Load-Spin-Ms": String(BENCH_MS) }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`bench HTTP ${res.status}`); + await res.text(); +} + +async function ask(questions) { + const q = questions[Math.floor(Math.random() * questions.length)]; + const res = await fetch(`${TARGET}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", content: q }], + max_tokens: 24, + }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`chat HTTP ${res.status}`); + await res.json(); +} + +async function oneShot(questions, stats) { + if (Math.random() < BENCH_RATIO) { + await bench(); + stats.bench += 1; + } else { + await ask(questions); + stats.chat += 1; + } +} + +/** Keep exactly `limit` requests in flight (token bucket style). */ +async function saturationLoop(limit, questions, endAt, stats, wid) { + const tasks = new Set(); + while (Date.now() < endAt) { + while (tasks.size < limit && Date.now() < endAt) { + const p = oneShot(questions, stats) + .catch((err) => { + stats.fail += 1; + if (stats.fail <= 10 || stats.fail % 200 === 0) { + console.error(`[sat-${wid}] ${err.message}`); + } + }) + .finally(() => tasks.delete(p)); + tasks.add(p); + } + if (tasks.size > 0) await Promise.race(tasks); + else await sleep(20); + } + await Promise.all(tasks); +} + +async function main() { + const questions = loadQuestions(); + const endAt = Date.now() + DURATION_SEC * 1000; + const stats = { bench: 0, chat: 0, fail: 0 }; + + console.log( + JSON.stringify({ + target: TARGET, + targetPods: TARGET_PODS, + concurrencyPerPod: CONCURRENCY_PER_POD, + peakInflight: PEAK_INFLIGHT, + workerGoroutines: WORKER_COUNT, + benchMs: BENCH_MS, + benchThreads: BENCH_THREADS, + rampSec: RAMP_SEC, + durationSec: DURATION_SEC, + }), + ); + + const loops = []; + for (let w = 0; w < WORKER_COUNT; w += 1) { + const share = Math.ceil(PEAK_INFLIGHT / WORKER_COUNT); + loops.push( + (async () => { + const rampEnd = Date.now() + RAMP_SEC * 1000; + let limit = Math.max(2, Math.ceil(share / 4)); + while (Date.now() < endAt) { + if (Date.now() < rampEnd) { + const progress = (Date.now() - (rampEnd - RAMP_SEC * 1000)) / (RAMP_SEC * 1000); + limit = Math.max(2, Math.ceil(share * (0.25 + 0.75 * progress))); + } else { + limit = share; + } + await saturationLoop(limit, questions, Math.min(endAt, Date.now() + 5000), stats, w); + } + })(), + ); + } + + await Promise.all(loops); + const ok = stats.bench + stats.chat; + console.log(`done bench=${stats.bench} chat=${stats.chat} fail=${stats.fail}`); + process.exit(stats.fail > ok * 3 ? 1 : 0); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/deploy/helm/nemoclaw-cpu/files/questions-sample.txt b/deploy/helm/nemoclaw-cpu/files/questions-sample.txt new file mode 100644 index 00000000000..24ebe2b944d --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/files/questions-sample.txt @@ -0,0 +1,10 @@ +What is Kubernetes HPA and when should you use it? +Explain the difference between CPU requests and limits on a pod. +How does NVIDIA Inference Hub route requests to Nemotron Ultra? +Write a short Python function to compute Fibonacci numbers recursively. +What are the tradeoffs of manual replica scaling vs autoscaling? +Describe how a readiness probe differs from a liveness probe. +What is the CAP theorem in distributed systems? +How would you debug a pod stuck in Pending state? +Summarize how Prometheus metrics feed into Kubernetes HPA. +What is the purpose of a HorizontalPodAutoscaler behavior stabilization window? diff --git a/deploy/helm/nemoclaw-cpu/monitoring/kube-prometheus-microk8s.yaml b/deploy/helm/nemoclaw-cpu/monitoring/kube-prometheus-microk8s.yaml new file mode 100644 index 00000000000..b4d6b84c74f --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/monitoring/kube-prometheus-microk8s.yaml @@ -0,0 +1,50 @@ +# Slim kube-prometheus-stack for single-node MicroK8s (~8 vCPU / 8 GiB). +# Enough for ServiceMonitor scraping + HPA via prometheus-adapter. + +alertmanager: + enabled: false + +grafana: + enabled: true + defaultDashboardsEnabled: false + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false +kubeEtcd: + enabled: false +coreDns: + enabled: false + +prometheusOperator: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +prometheus: + prometheusSpec: + retention: 6h + scrapeInterval: 30s + evaluationInterval: 30s + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 300m + memory: 512Mi + limits: + cpu: "1" + memory: 1536Mi diff --git a/deploy/helm/nemoclaw-cpu/monitoring/prometheus-adapter-values.yaml b/deploy/helm/nemoclaw-cpu/monitoring/prometheus-adapter-values.yaml new file mode 100644 index 00000000000..380d25a921c --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/monitoring/prometheus-adapter-values.yaml @@ -0,0 +1,32 @@ +# prometheus-adapter: expose nemoclaw_http_inflight_requests to the HPA custom metrics API. +# Install with scripts/install-performance-hpa.sh (sets prometheus.url automatically). + +prometheus: + url: http://REPLACE_PROMETHEUS_SERVICE.monitoring.svc + port: 9090 + +rules: + default: false + custom: + - seriesQuery: 'nemoclaw_http_inflight_requests{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)$" + as: "nemoclaw_http_inflight_requests" + metricsQuery: 'avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' + - seriesQuery: 'nemoclaw_llm_latency_p95_milliseconds{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)$" + as: "nemoclaw_llm_latency_p95_milliseconds" + metricsQuery: 'avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' diff --git a/deploy/helm/nemoclaw-cpu/observability.md b/deploy/helm/nemoclaw-cpu/observability.md new file mode 100644 index 00000000000..c10c725638a --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/observability.md @@ -0,0 +1,43 @@ +# Observability (performance HPA) + +CPU-based HPA only needs **metrics-server**. For **performance HPA** (inflight requests, latency), use **Prometheus** + **prometheus-adapter**. + +| Component | CPU HPA | Performance HPA | +|-----------|---------|-----------------| +| metrics-server | Yes | Optional | +| Prometheus | No | Yes | +| prometheus-adapter | No | Yes | +| Grafana | Optional | Recommended for tuning | + +## Agent metrics (`GET /metrics`) + +| Metric | Use | +|--------|-----| +| `nemoclaw_http_inflight_requests` | Backpressure / queue proxy | +| `nemoclaw_http_requests_total` | Throughput | +| `nemoclaw_llm_request_duration_seconds` | LLM chat/completions latency histogram | +| `nemoclaw_llm_latency_p95_milliseconds` | Rolling p95 LLM latency (ms); use with `autoscaling.mode=latency` | +| `nemoclaw_llm_latency_p50_milliseconds` | Rolling median LLM latency (ms) | +| `nemoclaw_llm_latency_avg_milliseconds` | Rolling average LLM latency (ms) | +| `nemoclaw_llm_requests_total` | LLM proxy success/error counts | +| `nemoclaw_inference_hub_reachable` | Hub health | + +Enable scraping: set `metrics.serviceMonitor.enabled: true` in Helm values (requires Prometheus Operator in cluster). + +## kube-prometheus-stack (example) + +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +kubectl create namespace monitoring +helm install kube-prometheus prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +``` + +Then upgrade the chart with `metrics.serviceMonitor.enabled: true` and install [prometheus-adapter](https://github.com/kubernetes-sigs/prometheus-adapter) with a rule for `nemoclaw_http_inflight_requests` (see `values-step2-hpa-performance.yaml` and chart README). + +## Further reading + +- [Kubernetes HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) +- [NVIDIA Riva autoscaling with HPA + Grafana](https://developer.nvidia.com/blog/autoscaling-nvidia-riva-deployment-with-kubernetes-for-speech-ai-in-production/) (same metrics → HPA pattern) diff --git a/deploy/helm/nemoclaw-cpu/scripts/cluster-recover.sh b/deploy/helm/nemoclaw-cpu/scripts/cluster-recover.sh new file mode 100755 index 00000000000..a9714e8c6d8 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/cluster-recover.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw}" +RELEASE="${RELEASE:-nemoclaw}" +RESTART_MICROK8S="${RESTART_MICROK8S:-1}" +RUN_INSTALL="${RUN_INSTALL:-1}" + +require_cmd kubectl +require_cmd helm + +if [[ -f "${HOME}/.nemoclaw/secrets.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.nemoclaw/secrets.env" + set +a +fi + +kubectl delete deploy,svc,hpa -n "${NAMESPACE}" -l 'app.kubernetes.io/name=nemoclaw-cpu' --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete deploy,svc t-nemoclaw-cpu-agent -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete hpa -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete job -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +helm uninstall "${RELEASE}" -n "${NAMESPACE}" 2>/dev/null || true +sleep 3 + +kubectl delete deploy,rs,hpa,job -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +if [[ "${RESTART_MICROK8S}" == "1" ]] && command -v microk8s >/dev/null 2>&1; then + microk8s stop + microk8s start + microk8s status --wait-ready +fi + +if [[ "${RUN_INSTALL}" == "1" ]]; then + exec "${SCRIPT_DIR}/install-hpa.sh" +fi diff --git a/deploy/helm/nemoclaw-cpu/scripts/hpa-common.sh b/deploy/helm/nemoclaw-cpu/scripts/hpa-common.sh new file mode 100755 index 00000000000..1e538eae156 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/hpa-common.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Shared helpers for hpa-reset.sh and hpa-load-test.sh + +hpa_common_log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + +hpa_common_print_hpa() { + local ns="${1:?namespace}" + kubectl get hpa -n "${ns}" 2>/dev/null || true +} + +hpa_common_log_hpa_if_changed() { + local ns="${1:?namespace}" + local last_var="${2:?lastLineVar}" + local line last + line="$(kubectl get hpa -n "${ns}" --no-headers 2>/dev/null | head -1 || true)" + [[ -z "${line}" ]] && return 0 + last="${!last_var}" + if [[ "${line}" != "${last}" ]]; then + hpa_common_log "${line}" + printf -v "${last_var}" '%s' "${line}" + fi +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing command: $1" >&2 + exit 1 + } +} + +# shellcheck disable=SC2034 +hpa_common_agent_deployment() { + echo "${RELEASE:-nemoclaw}-nemoclaw-cpu-agent" +} + +hpa_common_clear_stuck_pods() { + local ns="${1:?namespace}" + local pod + for pod in $(kubectl get pods -n "${ns}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + [[ -z "${pod}" ]] && continue + kubectl patch pod "${pod}" -n "${ns}" -p '{"metadata":{"finalizers":null}}' --type=merge \ + >/dev/null 2>&1 || true + done + kubectl delete pods -n "${ns}" --all --force --grace-period=0 >/dev/null 2>&1 || true +} + +# Bring up one Ready agent before HPA (avoids desiredReplicas=0 / unknown metrics deadlock). +hpa_common_ensure_agent_ready() { + local ns="${1:?namespace}" + local release="${2:?release}" + local chart_dir="${3:?chartDir}" + local api_key="${4:?apiKey}" + local values_file="${5:-}" + local rollout_timeout="${6:-300}" + local deploy + deploy="$(RELEASE="${release}" hpa_common_agent_deployment)" + + local helm_args=( + upgrade "${release}" "${chart_dir}" -n "${ns}" + --reuse-values + --set "namespace.create=false" + --set "inference.apiKey=${api_key}" + --set "autoscaling.enabled=false" + --set "cpuScaling.count=1" + --set "loadTest.cpuSpinMs=0" + ) + if [[ -n "${values_file}" && -f "${values_file}" ]]; then + helm_args+=(-f "${values_file}") + fi + helm "${helm_args[@]}" >/dev/null + + hpa_common_kick_deployment "${ns}" "${deploy}" || helm "${helm_args[@]}" >/dev/null + + if ! kubectl rollout status "deployment/${deploy}" -n "${ns}" --timeout="${rollout_timeout}s" >/dev/null; then + hpa_common_diagnose_rollout "${ns}" "${deploy}" + return 1 + fi + + local ready + ready="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo 0)" + if [[ "${ready}" != "1" ]]; then + hpa_common_diagnose_rollout "${ns}" "${deploy}" + return 1 + fi + return 0 +} + +hpa_common_wait_rollout() { + local deploy="${1:?deploy}" + local ns="${2:?namespace}" + local timeout="${3:-300}" + kubectl rollout status "deployment/${deploy}" -n "${ns}" --timeout="${timeout}s" >/dev/null +} + +# If Deployment has no ReplicaSet (stuck controller), nudge or delete so helm can recreate. +hpa_common_kick_deployment() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local rs + rs="$(kubectl get rs -n "${ns}" -l "app.kubernetes.io/name=nemoclaw-cpu,component=cpu-agent" \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)" + if [[ -n "${rs}" ]]; then + return 0 + fi + kubectl rollout restart "deployment/${deploy}" -n "${ns}" >/dev/null 2>&1 || true + sleep 8 + rs="$(kubectl get rs -n "${ns}" -l "app.kubernetes.io/name=nemoclaw-cpu,component=cpu-agent" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + [[ -n "${rs}" ]] && return 0 + kubectl delete "deployment/${deploy}" -n "${ns}" --ignore-not-found --wait=false 2>/dev/null || true + sleep 3 + return 1 +} + +hpa_common_diagnose_rollout() { + local ns="${1:?namespace}" + hpa_common_print_hpa "${ns}" + kubectl describe hpa -n "${ns}" 2>/dev/null | tail -20 || true +} + +# Never leave Deployment below minReplicas (HPA min is 1 — no scale-to-zero). +hpa_common_enforce_replica_floor() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local min="${3:-1}" + local spec + spec="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")" + if [[ -z "${spec}" ]] || [[ "${spec}" -lt "${min}" ]]; then + kubectl patch "deployment/${deploy}" -n "${ns}" \ + --type=merge -p "{\"spec\":{\"replicas\":${min}}}" + fi +} + +# After HPA is applied, ensure spec/status honor min..max (fix desiredReplicas=0 deadlock). +hpa_common_verify_hpa_bounds() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local hpa_name="${3:-${deploy}}" + local min="${4:-1}" + local max="${5:-7}" + + if ! kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" >/dev/null 2>&1; then + echo "HPA ${hpa_name} not found" >&2 + return 1 + fi + + local spec_min spec_max desired deploy_spec + spec_min="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.spec.minReplicas}' 2>/dev/null || echo 0)" + spec_max="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.spec.maxReplicas}' 2>/dev/null || echo 0)" + desired="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.status.desiredReplicas}' 2>/dev/null || echo "")" + deploy_spec="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")" + + if [[ "${spec_min}" -lt "${min}" ]]; then + echo "HPA spec.minReplicas=${spec_min} invalid" >&2 + return 1 + fi + + hpa_common_enforce_replica_floor "${ns}" "${deploy}" "${min}" + + if [[ -n "${desired}" && "${desired}" =~ ^[0-9]+$ && "${desired}" -lt "${min}" ]]; then + kubectl patch "deployment/${deploy}" -n "${ns}" \ + --type=merge -p "{\"spec\":{\"replicas\":${min}}}" + sleep 5 + fi + + return 0 +} diff --git a/deploy/helm/nemoclaw-cpu/scripts/hpa-load-test.sh b/deploy/helm/nemoclaw-cpu/scripts/hpa-load-test.sh new file mode 100755 index 00000000000..cf018c90780 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/hpa-load-test.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# HPA scale-up / scale-down test. Prints HPA status only. +# +# Usage: +# cd deploy/helm/nemoclaw-cpu +# source ~/.nemoclaw/secrets.env +# ./scripts/hpa-load-test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" +NAMESPACE="${NAMESPACE:-nemoclaw}" +RELEASE="${RELEASE:-nemoclaw}" +JOB_NAME="${JOB_NAME:-nemoclaw-hpa-load-test}" +TARGET_PODS="${TARGET_PODS:-7}" +CONCURRENCY_PER_POD="${CONCURRENCY_PER_POD:-40}" +BENCH_THREADS="${BENCH_THREADS:-2}" +DURATION_SEC="${DURATION_SEC:-720}" +RAMP_SEC="${RAMP_SEC:-90}" +BENCH_MS="${BENCH_MS:-${SPIN_MS:-450}}" +BENCH_RATIO="${BENCH_RATIO:-1}" +JOB_PARALLELISM="${JOB_PARALLELISM:-1}" +SCALE_UP_TARGET="${SCALE_UP_TARGET:-7}" +SCALE_UP_WAIT_LOOPS="${SCALE_UP_WAIT_LOOPS:-64}" +LOAD_TEST_CPU_SPIN_MS="${LOAD_TEST_CPU_SPIN_MS:-$BENCH_MS}" +HPA_TARGET_CPU="${HPA_TARGET_CPU:-50}" +SATURATE_VALUES="${SATURATE_VALUES:-${CHART_DIR}/values-step2-hpa-saturate.yaml}" +SCALE_DOWN_WAIT_LOOPS="${SCALE_DOWN_WAIT_LOOPS:-32}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300}" +DEPLOYMENT="${DEPLOYMENT:-${RELEASE}-nemoclaw-cpu-agent}" +SERVICE="${SERVICE:-${DEPLOYMENT}}" +SERVICE_PORT="${SERVICE_PORT:-8080}" +LAST_HPA_LINE="" + +require_cmd kubectl +require_cmd helm + +if [[ -z "${NVIDIA_INFERENCE_HUB_API_KEY:-}" ]]; then + if [[ -f "${HOME}/.nemoclaw/secrets.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.nemoclaw/secrets.env" + set +a + fi +fi + +if [[ -z "${NVIDIA_INFERENCE_HUB_API_KEY:-}" ]]; then + echo "Set NVIDIA_INFERENCE_HUB_API_KEY or add it to ~/.nemoclaw/secrets.env" >&2 + exit 1 +fi + +if ! kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True; then + echo "metrics-server not ready — CPU HPA unavailable" >&2 + exit 1 +fi + +if ! hpa_common_ensure_agent_ready "${NAMESPACE}" "${RELEASE}" "${CHART_DIR}" \ + "${NVIDIA_INFERENCE_HUB_API_KEY}" "${SATURATE_VALUES}" "${ROLLOUT_TIMEOUT}"; then + echo "Baseline pod not ready — HPA test cannot start" >&2 + exit 1 +fi + +helm upgrade "${RELEASE}" "${CHART_DIR}" -n "${NAMESPACE}" \ + --reuse-values \ + -f "${SATURATE_VALUES}" \ + --set namespace.create=false \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas=1 \ + --set autoscaling.maxReplicas="${TARGET_PODS}" \ + --set autoscaling.targetCPUUtilizationPercentage="${HPA_TARGET_CPU}" \ + --set loadTest.cpuSpinMs="${LOAD_TEST_CPU_SPIN_MS}" \ + >/dev/null + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${DEPLOYMENT}" 1 "${TARGET_PODS}" || true + +if ! hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}"; then + hpa_common_diagnose_rollout "${NAMESPACE}" "${DEPLOYMENT}" + exit 1 +fi + +hpa_common_print_hpa "${NAMESPACE}" + +cleanup() { + kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true +} +trap cleanup EXIT + +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true + +kubectl create configmap "${JOB_NAME}-scripts" -n "${NAMESPACE}" \ + --from-file=load-generator.mjs="${CHART_DIR}/files/load-generator.mjs" \ + --from-file=questions.txt="${CHART_DIR}/files/questions-sample.txt" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +cat </dev/null +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: ${NAMESPACE} +spec: + backoffLimit: 0 + parallelism: ${JOB_PARALLELISM} + completions: ${JOB_PARALLELISM} + ttlSecondsAfterFinished: 600 + template: + spec: + restartPolicy: Never + containers: + - name: load-generator + image: node:22-bookworm-slim + command: ["node", "/scripts/load-generator.mjs"] + env: + - name: TARGET_URL + value: "http://${SERVICE}:${SERVICE_PORT}" + - name: TARGET_PODS + value: "${TARGET_PODS}" + - name: CONCURRENCY_PER_POD + value: "${CONCURRENCY_PER_POD}" + - name: BENCH_THREADS + value: "${BENCH_THREADS}" + - name: RAMP_SEC + value: "${RAMP_SEC}" + - name: DURATION_SEC + value: "${DURATION_SEC}" + - name: BENCH_MS + value: "${BENCH_MS}" + - name: BENCH_RATIO + value: "${BENCH_RATIO}" + - name: QUESTIONS_FILE + value: "/questions/questions.txt" + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: questions + mountPath: /questions + readOnly: true + volumes: + - name: scripts + configMap: + name: ${JOB_NAME}-scripts + items: + - key: load-generator.mjs + path: load-generator.mjs + - name: questions + configMap: + name: ${JOB_NAME}-scripts + items: + - key: questions.txt + path: questions.txt +EOF + +SCALE_UP_OK=0 +for _ in $(seq 1 "${SCALE_UP_WAIT_LOOPS}"); do + hpa_common_log_hpa_if_changed "${NAMESPACE}" LAST_HPA_LINE + REPLICAS="$(kubectl get hpa -n "${NAMESPACE}" -o jsonpath='{.items[0].status.currentReplicas}' 2>/dev/null || echo 0)" + if [[ "${REPLICAS}" -ge "${SCALE_UP_TARGET}" ]]; then + SCALE_UP_OK=1 + break + fi + sleep 15 +done + +if [[ "${SCALE_UP_OK}" -ne 1 ]]; then + echo "HPA did not scale to ${SCALE_UP_TARGET} replicas" >&2 +fi + +kubectl wait --for=condition=complete "job/${JOB_NAME}" -n "${NAMESPACE}" --timeout="$((DURATION_SEC + 120))s" >/dev/null 2>&1 || true +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true + +for _ in $(seq 1 "${SCALE_DOWN_WAIT_LOOPS}"); do + hpa_common_log_hpa_if_changed "${NAMESPACE}" LAST_HPA_LINE + REPLICAS="$(kubectl get hpa -n "${NAMESPACE}" -o jsonpath='{.items[0].status.currentReplicas}' 2>/dev/null || echo 0)" + [[ "${REPLICAS}" -le 1 ]] && break + sleep 15 +done + +hpa_common_print_hpa "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-cpu/scripts/hpa-reset.sh b/deploy/helm/nemoclaw-cpu/scripts/hpa-reset.sh new file mode 100755 index 00000000000..1a50cf97036 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/hpa-reset.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Tear down load-test Jobs and agent pods, then helm upgrade idle baseline. +# Keeps HPA and Deployment by default (only deletes pods + stale ReplicaSets). +# +# Usage: +# cd deploy/helm/nemoclaw-cpu +# source ~/.nemoclaw/secrets.env +# ./scripts/hpa-reset.sh +# +# Env: +# NAMESPACE=nemoclaw RELEASE=nemoclaw JOB_NAME=nemoclaw-hpa-load-test +# REINSTALL_HELM=1 # helm upgrade after cleanup (default 1) +# HPA_VALUES=values-step2-hpa.yaml # idle baseline; load-test script applies saturate overlay +# SKIP_HELM=1 # only kubectl cleanup, no helm +# DELETE_DEPLOYMENT=0 # set 1 to delete Deployment before helm reinstall (stuck clusters) +# DELETE_HPA=0 # set 1 to delete HPA before reinstall (desiredReplicas=0 deadlock) +# RUN_LOAD_TEST=0 # set 1 to run ./scripts/hpa-load-test.sh after reset + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" +NAMESPACE="${NAMESPACE:-nemoclaw}" +RELEASE="${RELEASE:-nemoclaw}" +JOB_NAME="${JOB_NAME:-nemoclaw-hpa-load-test}" +DEPLOYMENT="${DEPLOYMENT:-${RELEASE}-nemoclaw-cpu-agent}" +HPA_NAME="${HPA_NAME:-${DEPLOYMENT}}" +REINSTALL_HELM="${REINSTALL_HELM:-1}" +SKIP_HELM="${SKIP_HELM:-0}" +DELETE_DEPLOYMENT="${DELETE_DEPLOYMENT:-0}" +DELETE_HPA="${DELETE_HPA:-0}" +RUN_LOAD_TEST="${RUN_LOAD_TEST:-0}" +HPA_VALUES="${HPA_VALUES:-${CHART_DIR}/values-step2-hpa.yaml}" +WAIT_ROLLOUT="${WAIT_ROLLOUT:-1}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300}" + +require_cmd kubectl + +if [[ "${SKIP_HELM}" != "1" ]] || [[ "${RUN_LOAD_TEST}" == "1" ]]; then + require_cmd helm +fi + +if [[ -f "${HOME}/.nemoclaw/secrets.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.nemoclaw/secrets.env" + set +a +fi + +if [[ "${SKIP_HELM}" != "1" ]] && [[ -z "${NVIDIA_INFERENCE_HUB_API_KEY:-}" ]]; then + echo "Set NVIDIA_INFERENCE_HUB_API_KEY or add it to ~/.nemoclaw/secrets.env" >&2 + exit 1 +fi + +namespace_exists() { + kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 +} + +clear_pod_finalizers() { + local pod + for pod in $(kubectl get pods -n "${NAMESPACE}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + [[ -z "${pod}" ]] && continue + kubectl patch pod "${pod}" -n "${NAMESPACE}" -p '{"metadata":{"finalizers":null}}' --type=merge \ + >/dev/null 2>&1 || true + done +} + +if ! namespace_exists; then + exit 0 +fi + +kubectl delete job -n "${NAMESPACE}" -l "job-name=${JOB_NAME}" --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete configmap "${JOB_NAME}-scripts" -n "${NAMESPACE}" --ignore-not-found 2>/dev/null || true + +if [[ "${DELETE_HPA}" == "1" ]]; then + kubectl delete hpa "${HPA_NAME}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true + kubectl delete hpa -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-cpu --ignore-not-found --wait=false 2>/dev/null || true +fi + +if [[ "${DELETE_DEPLOYMENT}" == "1" ]]; then + kubectl delete deployment "${DEPLOYMENT}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true +fi + +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true +sleep 2 +clear_pod_finalizers +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true + +kubectl delete rs -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-cpu --ignore-not-found --wait=false 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +if [[ "${SKIP_HELM}" == "1" ]]; then + hpa_common_print_hpa "${NAMESPACE}" + exit 0 +fi + +MIN_REPLICAS="${MIN_REPLICAS:-1}" +MAX_REPLICAS="${MAX_REPLICAS:-7}" + +if [[ "${DELETE_HPA}" == "1" ]]; then + if ! hpa_common_ensure_agent_ready "${NAMESPACE}" "${RELEASE}" "${CHART_DIR}" \ + "${NVIDIA_INFERENCE_HUB_API_KEY}" "${HPA_VALUES}" "${ROLLOUT_TIMEOUT}"; then + echo "HPA reset failed — baseline pod not ready" >&2 + exit 1 + fi +fi + +helm upgrade "${RELEASE}" "${CHART_DIR}" -n "${NAMESPACE}" \ + --reuse-values \ + -f "${HPA_VALUES}" \ + --set namespace.create=false \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" \ + --set loadTest.cpuSpinMs=0 \ + --set probes.readinessChecksInferenceHub=false \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas="${MIN_REPLICAS}" \ + --set autoscaling.maxReplicas="${MAX_REPLICAS}" \ + >/dev/null + +hpa_common_kick_deployment "${NAMESPACE}" "${DEPLOYMENT}" || helm upgrade "${RELEASE}" "${CHART_DIR}" -n "${NAMESPACE}" \ + --reuse-values -f "${HPA_VALUES}" \ + --set namespace.create=false \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" \ + --set loadTest.cpuSpinMs=0 \ + --set probes.readinessChecksInferenceHub=false \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas="${MIN_REPLICAS}" \ + --set autoscaling.maxReplicas="${MAX_REPLICAS}" \ + >/dev/null + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${HPA_NAME}" "${MIN_REPLICAS}" "${MAX_REPLICAS}" || true + +if [[ "${WAIT_ROLLOUT}" == "1" ]]; then + if ! hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}"; then + hpa_common_diagnose_rollout "${NAMESPACE}" "${DEPLOYMENT}" + fi +fi + +hpa_common_print_hpa "${NAMESPACE}" + +if [[ "${RUN_LOAD_TEST}" == "1" ]]; then + exec "${SCRIPT_DIR}/hpa-load-test.sh" +fi diff --git a/deploy/helm/nemoclaw-cpu/scripts/install-hpa.sh b/deploy/helm/nemoclaw-cpu/scripts/install-hpa.sh new file mode 100755 index 00000000000..c236c85b7eb --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/install-hpa.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Install CPU HPA (metrics-server). Output is HPA-focused only. +# +# Usage: +# cd deploy/helm/nemoclaw-cpu +# source ~/.nemoclaw/secrets.env +# ./scripts/install-hpa.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw}" +RELEASE="${RELEASE:-nemoclaw}" +DEPLOYMENT="${DEPLOYMENT:-${RELEASE}-nemoclaw-cpu-agent}" +HPA_NAME="${HPA_NAME:-${DEPLOYMENT}}" +HPA_VALUES="${HPA_VALUES:-${CHART_DIR}/values-step2-hpa.yaml}" +MIN_REPLICAS="${MIN_REPLICAS:-1}" +MAX_REPLICAS="${MAX_REPLICAS:-7}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300}" + +require_cmd kubectl +require_cmd helm + +if [[ -f "${HOME}/.nemoclaw/secrets.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.nemoclaw/secrets.env" + set +a +fi + +if [[ -z "${NVIDIA_INFERENCE_HUB_API_KEY:-}" ]]; then + echo "Set NVIDIA_INFERENCE_HUB_API_KEY in ~/.nemoclaw/secrets.env" >&2 + exit 1 +fi + +helm_install() { + helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set namespace.create=false \ + -f "${HPA_VALUES}" \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" \ + --set loadTest.cpuSpinMs=0 \ + --set probes.readinessChecksInferenceHub=false \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas="${MIN_REPLICAS}" \ + --set autoscaling.maxReplicas="${MAX_REPLICAS}" \ + >/dev/null +} + +if command -v microk8s >/dev/null 2>&1; then + microk8s enable metrics-server 2>/dev/null || true +fi +if ! kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True; then + for _ in $(seq 1 24); do + kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True && break + sleep 5 + done +fi +kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True || { + echo "metrics-server not ready — CPU HPA unavailable" >&2 + exit 1 +} + +helm_install +hpa_common_kick_deployment "${NAMESPACE}" "${DEPLOYMENT}" && helm_install || true + +if ! hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}"; then + hpa_common_diagnose_rollout "${NAMESPACE}" "${DEPLOYMENT}" + exit 1 +fi + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${HPA_NAME}" "${MIN_REPLICAS}" "${MAX_REPLICAS}" || true +hpa_common_print_hpa "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-cpu/scripts/install-performance-hpa.sh b/deploy/helm/nemoclaw-cpu/scripts/install-performance-hpa.sh new file mode 100755 index 00000000000..c55815d16d4 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/scripts/install-performance-hpa.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# OPTIONAL: performance HPA (inflight requests). Output is HPA-focused only. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw}" +RELEASE="${RELEASE:-nemoclaw}" +MONITORING_NS="${MONITORING_NS:-monitoring}" +PROM_RELEASE="${PROM_RELEASE:-kube-prometheus}" +ADAPTER_RELEASE="${ADAPTER_RELEASE:-prometheus-adapter}" +MAX_REPLICAS="${MAX_REPLICAS:-7}" +MIN_REPLICAS="${MIN_REPLICAS:-1}" +INFLIGHT_TARGET="${INFLIGHT_TARGET:-10}" +PROM_HELM_TIMEOUT="${PROM_HELM_TIMEOUT:-25m}" +PROM_VALUES="${PROM_VALUES:-${CHART_DIR}/monitoring/kube-prometheus-microk8s.yaml}" + +helm_release_failed() { + local rel="${1:?}" ns="${2:?}" + local st + st="$(helm status "${rel}" -n "${ns}" -o json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('info',{}).get('status',''))" 2>/dev/null || true)" + [[ "${st}" == "failed" ]] +} + +require_cmd kubectl +require_cmd helm + +if [[ -f "${HOME}/.nemoclaw/secrets.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.nemoclaw/secrets.env" + set +a +fi + +if [[ -z "${NVIDIA_INFERENCE_HUB_API_KEY:-}" ]]; then + echo "Set NVIDIA_INFERENCE_HUB_API_KEY in ~/.nemoclaw/secrets.env" >&2 + exit 1 +fi + +if command -v microk8s >/dev/null 2>&1; then + microk8s enable metrics-server 2>/dev/null || true +fi + +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null 2>&1 || true +helm repo update prometheus-community >/dev/null 2>&1 || helm repo update >/dev/null 2>&1 + +kubectl create namespace "${MONITORING_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +if helm_release_failed "${PROM_RELEASE}" "${MONITORING_NS}"; then + helm uninstall "${PROM_RELEASE}" -n "${MONITORING_NS}" --wait --timeout 5m 2>/dev/null || true +fi + +PROM_HELM_ARGS=( + upgrade --install "${PROM_RELEASE}" prometheus-community/kube-prometheus-stack + --namespace "${MONITORING_NS}" + --create-namespace + -f "${PROM_VALUES}" + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false + --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false + --set prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues=false + --timeout "${PROM_HELM_TIMEOUT}" +) + +helm "${PROM_HELM_ARGS[@]}" --wait >/dev/null 2>&1 || { + if ! kubectl get pods -n "${MONITORING_NS}" -l app.kubernetes.io/name=prometheus-operator 2>/dev/null | grep -q Running; then + echo "Prometheus stack did not come up — performance HPA metrics unavailable" >&2 + exit 1 + fi +} + +kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/name=prometheus-operator \ + -n "${MONITORING_NS}" \ + --timeout=600s >/dev/null 2>&1 || true + +kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/name=prometheus \ + -n "${MONITORING_NS}" \ + --timeout=600s >/dev/null 2>&1 || true + +PROM_SVC="$(kubectl get svc -n "${MONITORING_NS}" \ + -l app.kubernetes.io/name=prometheus,app.kubernetes.io/component=server \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -z "${PROM_SVC}" ]]; then + PROM_SVC="$(kubectl get svc -n "${MONITORING_NS}" -o name | grep prometheus | head -1 | sed 's|service/||')" +fi +if [[ -z "${PROM_SVC}" ]]; then + echo "Prometheus not found — performance HPA metrics unavailable" >&2 + exit 1 +fi +PROM_URL="http://${PROM_SVC}.${MONITORING_NS}.svc" + +helm upgrade --install "${ADAPTER_RELEASE}" prometheus-community/prometheus-adapter \ + --namespace "${MONITORING_NS}" \ + -f "${CHART_DIR}/monitoring/prometheus-adapter-values.yaml" \ + --set "prometheus.url=${PROM_URL}" \ + --set prometheus.port=9090 \ + --wait --timeout 10m >/dev/null + +for _ in $(seq 1 30); do + kubectl get apiservice v1beta1.custom.metrics.k8s.io 2>/dev/null | grep -q True && break + sleep 5 +done +kubectl get apiservice v1beta1.custom.metrics.k8s.io >/dev/null 2>&1 || { + echo "custom.metrics.k8s.io not ready — performance HPA unavailable" >&2 + exit 1 +} + +hpa_common_ensure_agent_ready "${NAMESPACE}" "${RELEASE}" "${CHART_DIR}" \ + "${NVIDIA_INFERENCE_HUB_API_KEY}" "${CHART_DIR}/values-step2-hpa-performance.yaml" 300 + +helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set namespace.create=false \ + -f "${CHART_DIR}/values-step2-hpa-performance.yaml" \ + --set inference.apiKey="${NVIDIA_INFERENCE_HUB_API_KEY}" \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas="${MIN_REPLICAS}" \ + --set autoscaling.maxReplicas="${MAX_REPLICAS}" \ + --set "autoscaling.performance.inflightRequestsPerPod=${INFLIGHT_TARGET}" \ + --set metrics.serviceMonitor.enabled=true \ + --set "metrics.serviceMonitor.labels.release=${PROM_RELEASE}" \ + >/dev/null + +DEPLOYMENT="$(RELEASE="${RELEASE}" hpa_common_agent_deployment)" +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${DEPLOYMENT}" "${MIN_REPLICAS}" "${MAX_REPLICAS}" || true + +hpa_common_print_hpa "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-cpu/templates/NOTES.txt b/deploy/helm/nemoclaw-cpu/templates/NOTES.txt new file mode 100644 index 00000000000..d708e99fec9 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/NOTES.txt @@ -0,0 +1 @@ +HPA: kubectl get hpa -n {{ include "nemoclaw-cpu.namespace" . }} diff --git a/deploy/helm/nemoclaw-cpu/templates/_helpers.tpl b/deploy/helm/nemoclaw-cpu/templates/_helpers.tpl new file mode 100644 index 00000000000..88dd3c861e2 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/_helpers.tpl @@ -0,0 +1,99 @@ +{{/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */}} +{{/* SPDX-License-Identifier: Apache-2.0 */}} +{{- define "nemoclaw-cpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nemoclaw-cpu.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "nemoclaw-cpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nemoclaw-cpu.labels" -}} +helm.sh/chart: {{ include "nemoclaw-cpu.chart" . }} +{{ include "nemoclaw-cpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "nemoclaw-cpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nemoclaw-cpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +component: cpu-agent +nemoclaw.ai/workload-type: cpu +{{- end }} + +{{- define "nemoclaw-cpu.namespace" -}} +{{- .Values.namespace.name }} +{{- end }} + +{{- define "nemoclaw-cpu.secretName" -}} +{{- if .Values.inference.existingSecret }} +{{- .Values.inference.existingSecret }} +{{- else }} +{{- .Values.inference.secretName }} +{{- end }} +{{- end }} + +{{/* + One replica per CPU: desired cluster CPUs == replica count; each pod requests 1 CPU. +*/}} +{{- define "nemoclaw-cpu.replicas" -}} +{{- if .Values.cpuScaling.oneReplicaPerCpu -}} +{{- .Values.cpuScaling.count | int }} +{{- else -}} +{{- .Values.replicaCount | int }} +{{- end -}} +{{- end }} + +{{- define "nemoclaw-cpu.agentResources" -}} +{{- if .Values.cpuScaling.oneReplicaPerCpu }} +requests: + cpu: {{ .Values.cpuScaling.perPodRequest | quote }} + memory: {{ .Values.cpuScaling.perPodMemory | quote }} +limits: + cpu: {{ .Values.cpuScaling.perPodLimit | quote }} + memory: {{ .Values.cpuScaling.perPodMemoryLimit | quote }} +{{- else }} +{{- toYaml .Values.resources }} +{{- end }} +{{- end }} + +{{/* + HPA max replicas: explicit maxReplicas, else maxCpus when one pod per CPU. +*/}} +{{- define "nemoclaw-cpu.hpaMaxReplicas" -}} +{{- if gt (int .Values.autoscaling.maxReplicas) 0 -}} +{{- int .Values.autoscaling.maxReplicas -}} +{{- else if .Values.cpuScaling.oneReplicaPerCpu -}} +{{- int .Values.autoscaling.maxCpus -}} +{{- else -}} +{{- 10 -}} +{{- end -}} +{{- end }} + +{{/* + HPA min replicas: always at least 1 when autoscaling is on (never scale to zero). +*/}} +{{- define "nemoclaw-cpu.hpaMinReplicas" -}} +{{- $min := int .Values.autoscaling.minReplicas -}} +{{- if lt $min 1 -}} +{{- 1 -}} +{{- else -}} +{{- $min -}} +{{- end -}} +{{- end }} diff --git a/deploy/helm/nemoclaw-cpu/templates/configmap.yaml b/deploy/helm/nemoclaw-cpu/templates/configmap.yaml new file mode 100644 index 00000000000..54af87792d6 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} +data: + agent-server.mjs: | +{{ .Files.Get "files/agent-server.mjs" | indent 4 }} + agent-metrics.mjs: | +{{ .Files.Get "files/agent-metrics.mjs" | indent 4 }} + cpu-spin-worker.mjs: | +{{ .Files.Get "files/cpu-spin-worker.mjs" | indent 4 }} + INFERENCE_BASE_URL: {{ .Values.inference.baseUrl | quote }} + INFERENCE_MODEL: {{ .Values.inference.model | quote }} diff --git a/deploy/helm/nemoclaw-cpu/templates/deployment.yaml b/deploy/helm/nemoclaw-cpu/templates/deployment.yaml new file mode 100644 index 00000000000..b1520d4b124 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/deployment.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} +spec: + {{- if .Values.autoscaling.enabled }} + # Floor for HPA (min 1). HPA scales between minReplicas and maxReplicas under load. + replicas: {{ include "nemoclaw-cpu.hpaMinReplicas" . }} + {{- else }} + replicas: {{ include "nemoclaw-cpu.replicas" . }} + {{- end }} + selector: + matchLabels: + {{- include "nemoclaw-cpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "nemoclaw-cpu.selectorLabels" . | nindent 8 }} + annotations: + checksum/agent-config: {{ printf "%s%s%s" (.Files.Get "files/agent-server.mjs") (.Files.Get "files/agent-metrics.mjs") (.Files.Get "files/cpu-spin-worker.mjs") | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: agent + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["node", "/app/agent-server.mjs"] + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + env: + - name: PORT + value: {{ .Values.service.port | quote }} + - name: INFERENCE_BASE_URL + valueFrom: + configMapKeyRef: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + key: INFERENCE_BASE_URL + - name: INFERENCE_MODEL + valueFrom: + configMapKeyRef: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + key: INFERENCE_MODEL + - name: NVIDIA_INFERENCE_HUB_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "nemoclaw-cpu.secretName" . }} + key: NVIDIA_INFERENCE_HUB_API_KEY + - name: LOAD_TEST_CPU_SPIN_MS + value: {{ .Values.loadTest.cpuSpinMs | quote }} + volumeMounts: + - name: app + mountPath: /app + readOnly: true + resources: + {{- include "nemoclaw-cpu.agentResources" . | nindent 12 }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 5 + readinessProbe: + httpGet: + path: {{ if .Values.probes.readinessChecksInferenceHub }}/readyz{{ else }}/healthz{{ end }} + port: http + initialDelaySeconds: {{ if .Values.probes.readinessChecksInferenceHub }}5{{ else }}3{{ end }} + periodSeconds: 20 + timeoutSeconds: {{ if .Values.probes.readinessChecksInferenceHub }}5{{ else }}3{{ end }} + failureThreshold: 5 + volumes: + - name: app + configMap: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + items: + - key: agent-server.mjs + path: agent-server.mjs + - key: agent-metrics.mjs + path: agent-metrics.mjs + - key: cpu-spin-worker.mjs + path: cpu-spin-worker.mjs diff --git a/deploy/helm/nemoclaw-cpu/templates/hpa.yaml b/deploy/helm/nemoclaw-cpu/templates/hpa.yaml new file mode 100644 index 00000000000..798d188fba1 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/hpa.yaml @@ -0,0 +1,74 @@ +{{- if .Values.autoscaling.enabled }} +{{- $hpaMin := include "nemoclaw-cpu.hpaMinReplicas" . | int }} +{{- $hpaMax := include "nemoclaw-cpu.hpaMaxReplicas" . | int }} +{{- if lt $hpaMax $hpaMin }} +{{- fail (printf "autoscaling.maxReplicas (%d) must be >= minReplicas (%d)" $hpaMax $hpaMin) }} +{{- end }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} + annotations: + nemoclaw.ai/hpa-policy: "min-{{ $hpaMin }}-max-{{ $hpaMax }}-never-zero" + nemoclaw.ai/hpa-mode: {{ .Values.autoscaling.mode | quote }} + {{- if eq .Values.autoscaling.mode "performance" }} + nemoclaw.ai/hpa-metric: "nemoclaw_http_inflight_requests" + nemoclaw.ai/hpa-metric-display: "In-flight HTTP requests per pod" + nemoclaw.ai/hpa-targets-format: "nemoclaw_http_inflight_requests: /" + {{- else if eq .Values.autoscaling.mode "latency" }} + nemoclaw.ai/hpa-metric: "nemoclaw_llm_latency_p95_milliseconds" + nemoclaw.ai/hpa-metric-display: "LLM response time p95 (ms per pod)" + nemoclaw.ai/hpa-targets-format: "nemoclaw_llm_latency_p95_milliseconds: / (ms)" + {{- else }} + nemoclaw.ai/hpa-metric: "cpu" + nemoclaw.ai/hpa-metric-display: "CPU utilization % of pod request" + nemoclaw.ai/hpa-targets-format: "cpu: %/%" + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + minReplicas: {{ $hpaMin }} + maxReplicas: {{ $hpaMax }} + metrics: + {{- if eq .Values.autoscaling.mode "performance" }} + - type: Pods + pods: + metric: + name: nemoclaw_http_inflight_requests + target: + type: AverageValue + averageValue: {{ .Values.autoscaling.performance.inflightRequestsPerPod | quote }} + {{- else if eq .Values.autoscaling.mode "latency" }} + - type: Pods + pods: + metric: + name: nemoclaw_llm_latency_p95_milliseconds + target: + type: AverageValue + averageValue: {{ .Values.autoscaling.performance.latencyP95Milliseconds | quote }} + {{- else }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/nemoclaw-cpu/templates/namespace.yaml b/deploy/helm/nemoclaw-cpu/templates/namespace.yaml new file mode 100644 index 00000000000..ae68118ad4e --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/namespace.yaml @@ -0,0 +1,8 @@ +{{- if .Values.namespace.create }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} +{{- end }} diff --git a/deploy/helm/nemoclaw-cpu/templates/secret.yaml b/deploy/helm/nemoclaw-cpu/templates/secret.yaml new file mode 100644 index 00000000000..d582a0450da --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/secret.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.inference.existingSecret }} +{{- if .Values.inference.apiKey }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "nemoclaw-cpu.secretName" . }} + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} +type: Opaque +stringData: + NVIDIA_INFERENCE_HUB_API_KEY: {{ .Values.inference.apiKey | quote }} +{{- end }} +{{- end }} diff --git a/deploy/helm/nemoclaw-cpu/templates/service.yaml b/deploy/helm/nemoclaw-cpu/templates/service.yaml new file mode 100644 index 00000000000..8609ca6eac3 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} + annotations: + nemoclaw.ai/agent-port: {{ .Values.service.port | quote }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "nemoclaw-cpu.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/nemoclaw-cpu/templates/servicemonitor.yaml b/deploy/helm/nemoclaw-cpu/templates/servicemonitor.yaml new file mode 100644 index 00000000000..d032c4b5b71 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nemoclaw-cpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-cpu.namespace" . }} + labels: + {{- include "nemoclaw-cpu.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "nemoclaw-cpu.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: {{ .Values.metrics.path }} + interval: {{ .Values.metrics.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/nemoclaw-cpu/values-step2-hpa-performance.yaml b/deploy/helm/nemoclaw-cpu/values-step2-hpa-performance.yaml new file mode 100644 index 00000000000..9a0742d9f0e --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/values-step2-hpa-performance.yaml @@ -0,0 +1,34 @@ +# Performance HPA: Prometheus scrapes /metrics → prometheus-adapter → HPA on inflight requests. +# Requires: kube-prometheus-stack + prometheus-adapter (see scripts/install-performance-hpa.sh) +# +# One Nemoclaw agent pod per CPU; min 1, max 7 on ~8 vCPU node. + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 7 + maxCpus: 8 + mode: performance + # Scale out when average in-flight HTTP requests per pod exceeds this value + performance: + inflightRequestsPerPod: "10" + +cpuScaling: + oneReplicaPerCpu: true + count: 1 + perPodRequest: "1" + perPodLimit: "1" + perPodMemory: "1Gi" + perPodMemoryLimit: "2Gi" + +loadTest: + cpuSpinMs: 0 + +metrics: + enabled: true + path: /metrics + serviceMonitor: + enabled: true + interval: 15s + labels: + release: kube-prometheus diff --git a/deploy/helm/nemoclaw-cpu/values-step2-hpa-saturate.yaml b/deploy/helm/nemoclaw-cpu/values-step2-hpa-saturate.yaml new file mode 100644 index 00000000000..04139f6bcf3 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/values-step2-hpa-saturate.yaml @@ -0,0 +1,42 @@ +# Load-test overlay on a small node: lower CPU *request* so HPA can schedule more pods +# for the same physical CPUs. Each pod still caps at 1 CPU (limit). +# For production-style "1 request = 1 CPU", use values-step2-hpa.yaml instead. + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 7 + maxCpus: 8 + targetCPUUtilizationPercentage: 50 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + - type: Pods + value: 4 + periodSeconds: 30 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 90 + policies: + - type: Percent + value: 50 + periodSeconds: 45 + - type: Pods + value: 2 + periodSeconds: 45 + selectPolicy: Max + +cpuScaling: + oneReplicaPerCpu: true + count: 1 + perPodRequest: "400m" + perPodLimit: "1" + perPodMemory: "512Mi" + perPodMemoryLimit: "2Gi" + +loadTest: + cpuSpinMs: 450 diff --git a/deploy/helm/nemoclaw-cpu/values-step2-hpa.yaml b/deploy/helm/nemoclaw-cpu/values-step2-hpa.yaml new file mode 100644 index 00000000000..b4c5534111d --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/values-step2-hpa.yaml @@ -0,0 +1,29 @@ +# CPU HPA — one agent pod per CPU. +# Requires metrics-server: microk8s enable metrics-server +# +# helm upgrade nemoclaw . -n nemoclaw -f values-step2-hpa.yaml --reuse-values \ +# --set inference.apiKey="$NVIDIA_INFERENCE_HUB_API_KEY" +# +# On an 8-vCPU node, maxReplicas: 7 leaves ~1 CPU for kube/system. + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 7 + maxCpus: 8 + mode: resource + targetCPUUtilizationPercentage: 65 + +cpuScaling: + oneReplicaPerCpu: true + count: 1 + perPodRequest: "1" + perPodLimit: "1" + perPodMemory: "1Gi" + perPodMemoryLimit: "2Gi" + +loadTest: + cpuSpinMs: 0 + +probes: + readinessChecksInferenceHub: false diff --git a/deploy/helm/nemoclaw-cpu/values.yaml b/deploy/helm/nemoclaw-cpu/values.yaml new file mode 100644 index 00000000000..3c73bfae399 --- /dev/null +++ b/deploy/helm/nemoclaw-cpu/values.yaml @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Manual scaling: one pod per CPU (optional pre-GPU chart). +# helm install nemoclaw . -n nemoclaw --create-namespace --set cpuScaling.count=4 +# helm upgrade nemoclaw . -n nemoclaw --set cpuScaling.count=2 # 2 CPUs → 2 pods +# kubectl scale deployment -n nemoclaw -l app.kubernetes.io/name=nemoclaw-cpu --replicas=4 +cpuScaling: + oneReplicaPerCpu: true + count: 1 # desired CPUs in cluster == number of replicas + perPodRequest: "1" # each pod requests exactly 1 CPU + perPodLimit: "1" # limit 1 CPU per pod (set higher for burst, e.g. "2") + perPodMemory: "1Gi" + perPodMemoryLimit: "2Gi" + +# Used only when cpuScaling.oneReplicaPerCpu is false +replicaCount: 1 + +image: + repository: node + tag: "22-bookworm-slim" + pullPolicy: IfNotPresent + +nameOverride: "" +fullnameOverride: "" + +namespace: + # Prefer: helm install --create-namespace (avoids "namespace already exists" errors) + create: false + name: nemoclaw + +service: + type: ClusterIP + # Agent HTTP port (GPU chart uses 8081 — keep charts on different ports) + port: 8080 + +# Remote inference (no GPU in cluster) — same model path as NemoClaw on your VM +inference: + baseUrl: "https://inference-api.nvidia.com/v1" + model: "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" + # Set via --set-file or external Secret in production + existingSecret: "" + secretName: nemoclaw-inference-hub + apiKey: "" # NVIDIA_INFERENCE_HUB_API_KEY (sk-*); leave empty if using existingSecret + +# Used only when cpuScaling.oneReplicaPerCpu is false +resources: + requests: + cpu: "1" + memory: "1Gi" + limits: + cpu: "4" + memory: "4Gi" + +podAnnotations: {} + +# Readiness: /healthz (process up) vs /readyz (Inference Hub reachable). +# Use false for install/rollout; verify Hub with: curl .../readyz after install. +probes: + readinessChecksInferenceHub: false + +nodeSelector: {} +tolerations: [] +# Optional: prefer nodes without GPUs when your cluster has CPU-only workers. +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 80 + preference: + matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + +# HPA (enabled by install-hpa.sh). +# Each agent pod requests 1 CPU (cpuScaling); HPA adds/removes pods under load. +# Enable: helm upgrade ... -f values-step2-hpa.yaml +autoscaling: + enabled: false + minReplicas: 1 # never 0 — at least one Nemoclaw agent pod always runs + maxReplicas: 7 # set to 0 to use maxCpus instead; 7 fits 8-vCPU node (1 CPU/pod + system) + maxCpus: 8 # fallback max when maxReplicas is 0 and oneReplicaPerCpu + # resource | performance + mode: resource + targetCPUUtilizationPercentage: 65 + targetMemoryUtilizationPercentage: null + # Scale-up is aggressive; scale-down is slower but much faster than default 5m window. + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + - type: Pods + value: 4 + periodSeconds: 30 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 120 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + - type: Pods + value: 2 + periodSeconds: 60 + selectPolicy: Max + # Performance mode (needs Prometheus + prometheus-adapter) + # See observability.md and NVIDIA Riva autoscaling blog: + # https://developer.nvidia.com/blog/autoscaling-nvidia-riva-deployment-with-kubernetes-for-speech-ai-in-production/ + performance: + requestsPerSecondPerPod: "20" + inflightRequestsPerPod: "10" + latencyP95Milliseconds: "500" + +# HPA load test: POST /v1/chat/completions (Inference Hub) + optional CPU spin per request +loadTest: + cpuSpinMs: 0 # hpa-load-test.sh sets ~350; POST /bench also spins CPU + targetUtilization: 35 # optional: lower HPA target during test (easier scale-up to 7 pods) + +metrics: + enabled: true + path: /metrics + serviceMonitor: + enabled: false + interval: 30s + labels: {} diff --git a/deploy/helm/nemoclaw-gpu/.helmignore b/deploy/helm/nemoclaw-gpu/.helmignore new file mode 100644 index 00000000000..7da0ef05be4 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +*.swp +*.bak +*.tmp +.git/ diff --git a/deploy/helm/nemoclaw-gpu/Chart.yaml b/deploy/helm/nemoclaw-gpu/Chart.yaml new file mode 100644 index 00000000000..1d474ab7be7 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/Chart.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v2 +name: nemoclaw-gpu +description: GPU NemoClaw agent pods on Kubernetes with local Ollama inference and HPA +type: application +version: 0.1.0 +appVersion: "2026.05.27" +keywords: + - nemoclaw + - ollama + - gpu +maintainers: + - name: maggiezha diff --git a/deploy/helm/nemoclaw-gpu/README.md b/deploy/helm/nemoclaw-gpu/README.md new file mode 100644 index 00000000000..2dba54f9640 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/README.md @@ -0,0 +1,19 @@ +# nemoclaw-gpu Helm chart + +**How to run the GPU deployment:** **[../../README-gpu.md](../../README-gpu.md)** — install, port-forward, HPA, load test, architecture, metrics, and troubleshooting. + +Primary NemoClaw Kubernetes chart: **Ollama on one GPU per pod** + Node.js agent (health, metrics, OpenAI-compatible API). Namespace `nemoclaw-gpu`, agent port **8081**. + +| | | +|--|--| +| **Ops guide** | [../../README-gpu.md](../../README-gpu.md) | +| **Models (vs CPU)** | [../README.md](../README.md) | +| **Deploy index** | [../../README.md](../../README.md) | +| **Optional CPU chart** | [nemoclaw-cpu](../nemoclaw-cpu/) · [../../README-cpu.md](../../README-cpu.md) | +| **Chart path** | `deploy/helm/nemoclaw-gpu/` | +| **Install script** | `./scripts/install-hpa.sh` | +| **Watch HPA** | `kubectl get hpa -n nemoclaw-gpu -w` | +| **Per-pod GPU %** | `./scripts/get-agent-pods.sh -n nemoclaw-gpu -w` | +| **Default model** | `inference.model` in `values.yaml` → `llama3.2:3b` (Ollama) | + +For Helm values, overlays (`values-step2-hpa.yaml`, latency/performance variants), and templates, see [README-gpu.md](../../README-gpu.md) and files under this directory. diff --git a/deploy/helm/nemoclaw-gpu/files/agent-metrics.mjs b/deploy/helm/nemoclaw-gpu/files/agent-metrics.mjs new file mode 100644 index 00000000000..f8c703eb9de --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/files/agent-metrics.mjs @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Shared Prometheus helpers for agent /metrics (LLM latency, HTTP counters). + +const LLM_LATENCY_WINDOW = Number(process.env.LLM_LATENCY_WINDOW_SIZE || 128); +const llmDurationsMs = []; +let llmDurationSumSec = 0; +let llmDurationCount = 0; +let llmRequestsOk = 0; +let llmRequestsError = 0; +const llmHistogramBucketsSec = [0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300]; +const llmHistogramCounts = Array.from({ length: llmHistogramBucketsSec.length + 1 }, () => 0); + +export function recordLlmLatency(durationMs, ok) { + const sec = Math.max(0, durationMs) / 1000; + llmDurationSumSec += sec; + llmDurationCount += 1; + if (ok) llmRequestsOk += 1; + else llmRequestsError += 1; + + llmDurationsMs.push(durationMs); + if (llmDurationsMs.length > LLM_LATENCY_WINDOW) llmDurationsMs.shift(); + + let bucketIdx = llmHistogramBucketsSec.findIndex((bound) => sec <= bound); + if (bucketIdx === -1) bucketIdx = llmHistogramBucketsSec.length; + for (let i = bucketIdx; i < llmHistogramCounts.length; i += 1) { + llmHistogramCounts[i] += 1; + } +} + +function percentileMs(sorted, p) { + if (!sorted.length) return 0; + const idx = Math.ceil(sorted.length * p) - 1; + return sorted[Math.max(0, idx)]; +} + +function llmLatencySnapshotMs() { + if (!llmDurationsMs.length) { + return { p50: 0, p95: 0, avg: 0 }; + } + const sorted = [...llmDurationsMs].sort((a, b) => a - b); + const sum = sorted.reduce((acc, v) => acc + v, 0); + return { + p50: percentileMs(sorted, 0.5), + p95: percentileMs(sorted, 0.95), + avg: sum / sorted.length, + }; +} + +export function llmMetricsLines() { + const { p50, p95, avg } = llmLatencySnapshotMs(); + const lines = [ + "# HELP nemoclaw_llm_requests_total Chat/completions proxied to inference backend", + "# TYPE nemoclaw_llm_requests_total counter", + `nemoclaw_llm_requests_total{result="success"} ${llmRequestsOk}`, + `nemoclaw_llm_requests_total{result="error"} ${llmRequestsError}`, + "# HELP nemoclaw_llm_request_duration_seconds LLM chat/completions end-to-end proxy latency", + "# TYPE nemoclaw_llm_request_duration_seconds histogram", + ]; + + for (let i = 0; i < llmHistogramBucketsSec.length; i += 1) { + lines.push( + `nemoclaw_llm_request_duration_seconds_bucket{le="${llmHistogramBucketsSec[i]}"} ${llmHistogramCounts[i]}`, + ); + } + lines.push( + `nemoclaw_llm_request_duration_seconds_bucket{le="+Inf"} ${llmHistogramCounts[llmHistogramCounts.length - 1]}`, + `nemoclaw_llm_request_duration_seconds_sum ${llmDurationSumSec}`, + `nemoclaw_llm_request_duration_seconds_count ${llmDurationCount}`, + "# HELP nemoclaw_llm_latency_p50_milliseconds Rolling p50 LLM latency (recent window)", + "# TYPE nemoclaw_llm_latency_p50_milliseconds gauge", + `nemoclaw_llm_latency_p50_milliseconds ${Math.round(p50)}`, + "# HELP nemoclaw_llm_latency_p95_milliseconds Rolling p95 LLM latency (recent window; HPA-friendly)", + "# TYPE nemoclaw_llm_latency_p95_milliseconds gauge", + `nemoclaw_llm_latency_p95_milliseconds ${Math.round(p95)}`, + "# HELP nemoclaw_llm_latency_avg_milliseconds Rolling average LLM latency (recent window)", + "# TYPE nemoclaw_llm_latency_avg_milliseconds gauge", + `nemoclaw_llm_latency_avg_milliseconds ${Math.round(avg)}`, + ); + return lines; +} diff --git a/deploy/helm/nemoclaw-gpu/files/agent-server.mjs b/deploy/helm/nemoclaw-gpu/files/agent-server.mjs new file mode 100644 index 00000000000..bf616e4b898 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/files/agent-server.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// GPU agent pod: health + Prometheus metrics + OpenAI-compatible proxy to local Ollama. + +import http from "node:http"; +import { llmMetricsLines, recordLlmLatency } from "./agent-metrics.mjs"; + +const PORT = Number(process.env.PORT || 8081); +const BASE_URL = (process.env.INFERENCE_BASE_URL || "http://127.0.0.1:11434/v1").replace(/\/$/, ""); +const OLLAMA_BASE = (process.env.OLLAMA_BASE_URL || "http://127.0.0.1:11434").replace(/\/$/, ""); +const MODEL = process.env.INFERENCE_MODEL || ""; + +let inflight = 0; +let totalRequests = 0; +let inferenceReachable = 0; +let inferenceCache = { ok: false, at: 0 }; +const INFERENCE_CACHE_MS = Number(process.env.INFERENCE_READY_CACHE_MS || 15_000); +let inferenceReadyEver = false; +let inferenceFailStreak = 0; +const INFERENCE_FAIL_MAX = Number(process.env.INFERENCE_FAIL_MAX || 8); + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +async function proxyChatCompletions(req, res) { + let raw; + try { + raw = await readBody(req); + } catch { + res.writeHead(400, { "content-type": "text/plain" }); + res.end("bad request\n"); + return; + } + let body; + try { + body = raw ? JSON.parse(raw) : {}; + } catch { + res.writeHead(400, { "content-type": "text/plain" }); + res.end("invalid json\n"); + return; + } + if (!body.model) body.model = MODEL; + const llmStart = performance.now(); + let llmOk = false; + try { + const hubRes = await fetch(`${BASE_URL}/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(300_000), + }); + llmOk = hubRes.ok; + const text = await hubRes.text(); + res.writeHead(hubRes.status, { "content-type": "application/json" }); + res.end(text); + } catch (err) { + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: String(err) })); + } finally { + recordLlmLatency(performance.now() - llmStart, llmOk); + } +} + +async function checkInference() { + const now = Date.now(); + if (now - inferenceCache.at < INFERENCE_CACHE_MS) return inferenceCache.ok; + try { + const res = await fetch(`${OLLAMA_BASE}/api/tags`, { + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) { + inferenceCache = { ok: false, at: now }; + return false; + } + const data = await res.json(); + const names = (data.models || []).map((m) => m.name || m.model || ""); + const want = MODEL.split(":")[0]; + const ok = + names.some((n) => n === MODEL || n.startsWith(`${want}:`) || n.includes(MODEL)) || + names.length > 0; + if (ok) { + inferenceReadyEver = true; + inferenceFailStreak = 0; + inferenceCache = { ok: true, at: now }; + return true; + } + inferenceFailStreak += 1; + if (inferenceReadyEver && (inflight > 0 || inferenceFailStreak < INFERENCE_FAIL_MAX)) { + inferenceCache = { ok: true, at: now }; + return true; + } + inferenceCache = { ok: false, at: now }; + return false; + } catch { + inferenceFailStreak += 1; + if (inferenceReadyEver && (inflight > 0 || inferenceFailStreak < INFERENCE_FAIL_MAX)) { + inferenceCache = { ok: true, at: now }; + return true; + } + inferenceCache = { ok: false, at: now }; + return false; + } +} + +function metricsText() { + return [ + "# HELP nemoclaw_http_requests_total Total HTTP requests to agent pod", + "# TYPE nemoclaw_http_requests_total counter", + `nemoclaw_http_requests_total ${totalRequests}`, + "# HELP nemoclaw_http_inflight_requests In-flight HTTP requests", + "# TYPE nemoclaw_http_inflight_requests gauge", + `nemoclaw_http_inflight_requests ${inflight}`, + "# HELP nemoclaw_inference_reachable 1 if local Ollama model is ready", + "# TYPE nemoclaw_inference_reachable gauge", + `nemoclaw_inference_reachable ${inferenceReachable}`, + ...llmMetricsLines(), + "", + ].join("\n"); +} + +const server = http.createServer(async (req, res) => { + totalRequests += 1; + inflight += 1; + try { + if (req.url === "/healthz" || req.url === "/health") { + res.writeHead(200, { "content-type": "text/plain" }); + res.end("ok\n"); + return; + } + if (req.url === "/readyz" || req.url === "/ready") { + const ok = await checkInference(); + inferenceReachable = ok ? 1 : 0; + res.writeHead(ok ? 200 : 503, { "content-type": "text/plain" }); + res.end(ok ? "ready\n" : "ollama model not ready\n"); + return; + } + if (req.url === "/metrics") { + res.writeHead(200, { "content-type": "text/plain; version=0.0.4" }); + res.end(metricsText()); + return; + } + const pathOnly = (req.url || "").split("?")[0]; + if ( + (pathOnly === "/v1/chat/completions" || pathOnly === "/chat/completions") && + req.method === "POST" + ) { + await proxyChatCompletions(req, res); + return; + } + if (req.url === "/" && req.method === "GET") { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + service: "nemoclaw-gpu-agent", + model: MODEL, + inferenceBaseUrl: BASE_URL, + ollamaBaseUrl: OLLAMA_BASE, + endpoints: ["/healthz", "/readyz", "/metrics", "POST /v1/chat/completions"], + note: "Local Ollama on GPU; scale replicas with kubectl or HPA (one pod per GPU)", + }), + ); + return; + } + res.writeHead(404); + res.end("not found\n"); + } finally { + inflight -= 1; + } +}); + +server.listen(PORT, () => { + console.log(`nemoclaw-gpu-agent listening on :${PORT} model=${MODEL}`); +}); diff --git a/deploy/helm/nemoclaw-gpu/files/load-generator.mjs b/deploy/helm/nemoclaw-gpu/files/load-generator.mjs new file mode 100644 index 00000000000..7664df6ac3e --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/files/load-generator.mjs @@ -0,0 +1,625 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Drive GPU utilization for HPA: chat completions directly to each agent pod IP. +// Each Running agent pod gets PER_POD_PEAK × compensation concurrent requests. +// Compensation = HPA currentReplicas / loadTargetCount so cold pods at 0% GPU +// do not drag the average to ~42% while a new replica is starting. + +import fs from "node:fs"; +import https from "node:https"; +import { setTimeout as sleep } from "node:timers/promises"; + +const SERVICE_FALLBACK = (process.env.TARGET_URL || "http://nemoclaw-gpu-agent:8081").replace( + /\/$/, + "", +); +const DURATION_SEC = Number(process.env.DURATION_SEC || 720); +const TARGET_PODS = Number(process.env.TARGET_PODS || 4); +const HPA_TARGET_GPU = Number(process.env.HPA_TARGET_GPU || 40); +const JOB_PARALLELISM = Number(process.env.JOB_PARALLELISM || 1); +const AGENT_PORT = Number(process.env.AGENT_PORT || 8081); +const INFLIGHT_PER_GPU = Number(process.env.INFLIGHT_PER_GPU || 384); +const LOAD_MULTIPLIER = Number(process.env.LOAD_MULTIPLIER || 1); +const PER_POD_PEAK = INFLIGHT_PER_GPU * LOAD_MULTIPLIER; +const RAMP_SEC = Number(process.env.RAMP_SEC || 60); +const REQUEST_TIMEOUT_MS = Number(process.env.REQUEST_TIMEOUT_MS || 300_000); +const MAX_TOKENS = Number(process.env.MAX_TOKENS || 512); +const LOG_EVERY_SEC = Number(process.env.LOG_EVERY_SEC || 15); +const TARGET_POLL_SEC = Number(process.env.TARGET_POLL_SEC || 1); +const K8S_NAMESPACE = process.env.K8S_NAMESPACE || "nemoclaw-gpu"; +const AGENT_SERVICE = process.env.AGENT_SERVICE || "nemoclaw-gpu-agent"; +const HPA_NAME = process.env.HPA_NAME || AGENT_SERVICE; +const AGENT_LABEL_SELECTOR = + process.env.AGENT_LABEL_SELECTOR || "app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent"; +const ESCALATE_INTERVAL_SEC = Number(process.env.ESCALATE_INTERVAL_SEC || 10); +const ESCALATE_FACTOR = Number(process.env.ESCALATE_FACTOR || 0.5); +const ESCALATE_MAX_MULT = Number(process.env.ESCALATE_MAX_MULT || 3); +const REQUEST_RETRIES = Number(process.env.REQUEST_RETRIES || 2); +const LOAD_COMPENSATION_SAFETY = Number(process.env.LOAD_COMPENSATION_SAFETY || 3); +const MAX_COMPENSATION = Number(process.env.MAX_COMPENSATION || 16); +const NEW_POD_RAMP_SEC = Number(process.env.NEW_POD_RAMP_SEC || 0); +const MAX_INFLIGHT_PER_POD = Number(process.env.MAX_INFLIGHT_PER_POD || 6144); +const WARMUP_SEC = Number(process.env.WARMUP_SEC || 45); +const ERROR_BACKOFF_FACTOR = Number(process.env.ERROR_BACKOFF_FACTOR || 0.92); +const ERROR_BACKOFF_MIN = Number(process.env.ERROR_BACKOFF_MIN || 0.4); +const ERROR_BACKOFF_RECOVERY = Number(process.env.ERROR_BACKOFF_RECOVERY || 1.15); +const BOOTSTRAP_INFLIGHT = Number(process.env.BOOTSTRAP_INFLIGHT || 4); +const NEW_POD_WARMUP_PARALLEL = Number(process.env.NEW_POD_WARMUP_PARALLEL || 8); +const NEW_POD_WARMUP_MAX_SEC = Number(process.env.NEW_POD_WARMUP_MAX_SEC || 120); +const CIRCUIT_BREAKER_BACKOFF = Number(process.env.CIRCUIT_BREAKER_BACKOFF || 0.15); +const MIN_INFLIGHT_FLOOR = Number(process.env.MIN_INFLIGHT_FLOOR || 8); +const MIN_RECOVERY_INFLIGHT = Number(process.env.MIN_RECOVERY_INFLIGHT || 4); +const READYZ_GRACE_SEC = Number(process.env.READYZ_GRACE_SEC || 45); +const REQUIRE_CHAT_PROBE = + process.env.REQUIRE_CHAT_PROBE === "1" || process.env.REQUIRE_CHAT_PROBE === "true"; +const PROBE_CHAT_TIMEOUT_MS = Number(process.env.PROBE_CHAT_TIMEOUT_MS || 30_000); + +let podTargets = []; +let podCandidates = []; +let hpaReplicas = 1; +let hpaDesired = 1; +let loadCompensation = 1; +let lastTargetPoll = 0; +const podFirstSeen = new Map(); +const targetBackoff = new Map(); +const targetChatOk = new Set(); +const warmInFlight = new Set(); +const readyzLastOk = new Map(); + +function loadQuestions() { + try { + const lines = fs + .readFileSync(process.env.QUESTIONS_FILE || "/questions/questions.txt", "utf8") + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + if (lines.length) return lines; + } catch { + /* use fallback */ + } + return [ + "Explain Kubernetes HPA and GPU autoscaling in detail with examples.", + "Write a long summary of transformer inference on NVIDIA GPUs.", + "Describe how Ollama serves models and batches concurrent chat requests.", + ]; +} + +function k8sGet(path) { + const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + if (!fs.existsSync(tokenPath) || !process.env.KUBERNETES_SERVICE_HOST) { + return Promise.resolve(null); + } + const token = fs.readFileSync(tokenPath, "utf8"); + const ca = fs.readFileSync(caPath); + return new Promise((resolve) => { + const req = https.request( + { + hostname: process.env.KUBERNETES_SERVICE_HOST, + port: process.env.KUBERNETES_SERVICE_PORT || 443, + path, + method: "GET", + headers: { Authorization: `Bearer ${token}` }, + ca, + rejectUnauthorized: true, + }, + (res) => { + let data = ""; + res.on("data", (chunk) => { + data += chunk; + }); + res.on("end", () => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + try { + resolve(JSON.parse(data)); + } catch { + resolve(null); + } + } else { + resolve(null); + } + }); + }, + ); + req.on("error", () => resolve(null)); + req.setTimeout(5000, () => { + req.destroy(); + resolve(null); + }); + req.end(); + }); +} + +function ipsFromEndpointSliceList(list) { + const ips = new Set(); + for (const slice of list?.items || []) { + for (const ep of slice.endpoints || []) { + if (ep.conditions?.ready === false) continue; + for (const addr of ep.addresses || []) { + if (addr) ips.add(addr); + } + } + } + return ips; +} + +function ipsFromRunningPods(list) { + const ips = new Set(); + for (const pod of list?.items || []) { + if (pod.status?.phase !== "Running") continue; + const ip = pod.status?.podIP; + if (ip) ips.add(ip); + } + return ips; +} + +async function pollHpaReplicas() { + const hpa = await k8sGet( + `/apis/autoscaling/v2/namespaces/${K8S_NAMESPACE}/horizontalpodautoscalers/${HPA_NAME}`, + ); + if (hpa?.status?.currentReplicas >= 1) { + hpaReplicas = hpa.status.currentReplicas; + } + if (hpa?.status?.desiredReplicas >= 1) { + hpaDesired = hpa.status.desiredReplicas; + } + return hpaReplicas; +} + +async function probeInferenceReady(target) { + const ip = podIpFromTarget(target); + try { + const res = await fetch(`${target}/readyz`, { + signal: AbortSignal.timeout(10_000), + }); + if (res.ok) { + readyzLastOk.set(ip, Date.now()); + return true; + } + } catch { + /* grace below */ + } + const last = readyzLastOk.get(ip); + if (last && (Date.now() - last) / 1000 < READYZ_GRACE_SEC) return true; + return false; +} + +async function pollAgentPodTargets() { + const now = Date.now(); + if (now - lastTargetPoll < TARGET_POLL_SEC * 1000) { + return podTargets; + } + lastTargetPoll = now; + + await pollHpaReplicas(); + + const ips = new Set(); + + const sliceList = await k8sGet( + `/apis/discovery.k8s.io/v1/namespaces/${K8S_NAMESPACE}/endpointslices?labelSelector=${encodeURIComponent(`kubernetes.io/service-name=${AGENT_SERVICE}`)}`, + ); + for (const ip of ipsFromEndpointSliceList(sliceList)) ips.add(ip); + + const podList = await k8sGet( + `/api/v1/namespaces/${K8S_NAMESPACE}/pods?labelSelector=${encodeURIComponent(AGENT_LABEL_SELECTOR)}`, + ); + for (const ip of ipsFromRunningPods(podList)) ips.add(ip); + + if (ips.size) { + const nowMs = Date.now(); + for (const ip of ips) { + if (!podFirstSeen.has(ip)) { + podFirstSeen.set(ip, nowMs); + console.log(JSON.stringify({ event: "newPodDiscovered", ip })); + } + } + const candidates = [...ips].map((ip) => `http://${ip}:${AGENT_PORT}`); + podCandidates = candidates; + const ready = []; + await Promise.all( + candidates.map(async (target) => { + if (await probeInferenceReady(target)) ready.push(target); + }), + ); + podTargets = ready; + for (const target of podCandidates) scheduleWarmTarget(target); + } else { + podCandidates = []; + podTargets = []; + } + + // HPA may show N replicas while pods warm — compensate only until every replica has a candidate IP. + const readyCount = Math.max(1, podCandidates.length); + const hpaCount = Math.max(hpaReplicas, hpaDesired, readyCount, 1); + if (readyCount >= TARGET_PODS) { + loadCompensation = 1; + } else if (readyCount >= hpaCount) { + // Fair share: spread load so each GPU pod targets ~TARGET_PODS/readyCount of peak. + loadCompensation = Math.min(MAX_COMPENSATION, TARGET_PODS / readyCount); + } else { + loadCompensation = Math.min( + MAX_COMPENSATION, + Math.max(1, (TARGET_PODS / readyCount) * LOAD_COMPENSATION_SAFETY), + ); + } + + return podTargets; +} + +function podIpFromTarget(target) { + return target.match(/^http:\/\/([^:/]+)/)?.[1] || target; +} + +function newPodRampMultiplier(ip) { + if (NEW_POD_RAMP_SEC <= 0) return 1; + const seenAt = podFirstSeen.get(ip); + if (!seenAt) return 1; + const ageSec = (Date.now() - seenAt) / 1000; + if (ageSec >= NEW_POD_RAMP_SEC) return 1; + return 0.75 + 0.25 * (ageSec / NEW_POD_RAMP_SEC); +} + +function getTargetBackoff(ip) { + return targetBackoff.get(ip) ?? 1; +} + +function noteTargetResult(ip, ok) { + const cur = targetBackoff.get(ip) ?? 1; + if (!ok) { + targetBackoff.set(ip, Math.max(ERROR_BACKOFF_MIN, cur * ERROR_BACKOFF_FACTOR)); + } else if (cur < 1) { + targetBackoff.set(ip, Math.min(1, cur * ERROR_BACKOFF_RECOVERY)); + } +} + +function activeReplicaCount() { + return Math.max(hpaReplicas, hpaDesired, podCandidates.length, podTargets.length, 1); +} + +function strugglingTargetCount() { + let n = 0; + for (const ip of podFirstSeen.keys()) { + if (getTargetBackoff(ip) <= CIRCUIT_BREAKER_BACKOFF) n += 1; + } + return n; +} + +function healthyBoostMultiplier(ip) { + const n = activeReplicaCount(); + if (n < 2) return 1; + const backoff = getTargetBackoff(ip); + if (backoff <= CIRCUIT_BREAKER_BACKOFF) return 1; + const struggling = strugglingTargetCount(); + if (struggling <= 0) return 1; + return Math.min(2, 1 + struggling / Math.max(1, targetChatOk.size)); +} + +async function probeChatWorks(target) { + try { + const res = await fetch(`${target}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", content: "Say OK." }], + max_tokens: 8, + stream: false, + }), + signal: AbortSignal.timeout(PROBE_CHAT_TIMEOUT_MS), + }); + if (!res.ok) return false; + await res.json(); + return true; + } catch { + return false; + } +} + +async function requirePodTargets(deadlineMs) { + while (Date.now() < deadlineMs) { + await pollAgentPodTargets(); + const targets = podCandidates.length ? podCandidates : podTargets; + if (targets.length === 0) { + await sleep(1000); + continue; + } + if (!REQUIRE_CHAT_PROBE) { + console.log(JSON.stringify({ event: "targetsReady", loadTargets: targets, readyzOk: podTargets.length })); + return targets; + } + for (const target of podTargets.length ? podTargets : targets) { + if (await probeChatWorks(target)) { + targetChatOk.add(target); + console.log(JSON.stringify({ event: "chatProbeOk", target })); + console.log(JSON.stringify({ event: "targetsReady", loadTargets: podTargets })); + return podTargets; + } + } + console.log(JSON.stringify({ event: "waitingForChatProbe", loadTargets: podTargets })); + await sleep(3000); + } + console.error( + REQUIRE_CHAT_PROBE + ? "FATAL: agent pod(s) found but chat probe never succeeded — wait for Ollama model pull" + : "FATAL: no ready agent pod IPs — check RBAC (pods/endpointslices) and agent pods", + ); + process.exit(1); +} + +function inflightFloor(multiReplica) { + if (multiReplica) return Math.max(MIN_INFLIGHT_FLOOR, 12); + return Math.max(MIN_INFLIGHT_FLOOR, MIN_RECOVERY_INFLIGHT); +} + +function escalationCap() { + let cap = ESCALATE_MAX_MULT; + const hpaN = Math.max(hpaReplicas, hpaDesired); + if (hpaN >= 2 && targetChatOk.size < hpaN) { + cap = Math.min(cap, 1.25); + } + return cap; +} + +function rampMultiplier(elapsedSec) { + const cap = escalationCap(); + if (RAMP_SEC <= 0) { + const steps = Math.floor(elapsedSec / ESCALATE_INTERVAL_SEC); + return Math.min(cap, 1 + steps * ESCALATE_FACTOR); + } + if (elapsedSec < RAMP_SEC) { + const progress = elapsedSec / RAMP_SEC; + return Math.min(cap, 0.75 + 0.25 * progress); + } + const steps = Math.floor((elapsedSec - RAMP_SEC) / ESCALATE_INTERVAL_SEC); + return Math.min(cap, 1 + steps * ESCALATE_FACTOR); +} + +function warmupCompensationScale(startedAt) { + const elapsed = (Date.now() - startedAt) / 1000; + if (WARMUP_SEC <= 0 || elapsed >= WARMUP_SEC) return 1; + return 0.35 + 0.65 * (elapsed / WARMUP_SEC); +} + +function effectiveCompensation(startedAt) { + const warm = warmupCompensationScale(startedAt); + return 1 + (loadCompensation - 1) * warm; +} + +function baseInflightPerPodPerGenerator(startedAt, stats) { + const mult = rampMultiplier((Date.now() - startedAt) / 1000); + const comp = effectiveCompensation(startedAt); + let raw = Math.ceil((PER_POD_PEAK * mult * comp) / JOB_PARALLELISM); + const total = (stats?.chat ?? 0) + (stats?.fail ?? 0); + if (total >= 40) { + const failRate = (stats?.fail ?? 0) / total; + if (failRate > 0.6) raw = Math.ceil(raw * 0.25); + else if (failRate > 0.3) raw = Math.ceil(raw * 0.5); + else if (failRate > 0.15) raw = Math.ceil(raw * 0.75); + } + const cap = Math.max(2, Math.ceil(MAX_INFLIGHT_PER_POD / JOB_PARALLELISM)); + const floor = inflightFloor(activeReplicaCount() >= 2); + return Math.max(floor, Math.min(raw, cap)); +} + +function inflightForTarget(target, startedAt, inferenceReady = true, stats = null) { + const ip = podIpFromTarget(target); + const backoff = getTargetBackoff(ip); + const multiReplica = activeReplicaCount() >= 2; + const floor = inflightFloor(multiReplica || targetChatOk.has(target)); + + if (backoff <= CIRCUIT_BREAKER_BACKOFF) { + return Math.max(floor, MIN_RECOVERY_INFLIGHT); + } + if (!inferenceReady) { + return Math.max(floor, BOOTSTRAP_INFLIGHT); + } + if (!targetChatOk.has(target)) { + const ageSec = (Date.now() - (podFirstSeen.get(ip) || Date.now())) / 1000; + if (ageSec > NEW_POD_WARMUP_MAX_SEC) { + return Math.max( + floor, + BOOTSTRAP_INFLIGHT, + Math.ceil(baseInflightPerPodPerGenerator(startedAt, stats) * 0.25), + ); + } + return Math.max(floor, BOOTSTRAP_INFLIGHT); + } + const base = baseInflightPerPodPerGenerator(startedAt, stats); + let limit = Math.ceil(base * newPodRampMultiplier(ip) * backoff * healthyBoostMultiplier(ip)); + return Math.max(floor, limit); +} + +async function warmTarget(target) { + if (targetChatOk.has(target)) return true; + const probes = []; + for (let i = 0; i < NEW_POD_WARMUP_PARALLEL; i++) { + probes.push(probeChatWorks(target)); + } + const ok = (await Promise.all(probes)).some(Boolean); + if (ok) { + targetChatOk.add(target); + console.log(JSON.stringify({ event: "podWarmed", target })); + } + return ok; +} + +function scheduleWarmTarget(target) { + if (targetChatOk.has(target) || warmInFlight.has(target)) return; + warmInFlight.add(target); + warmTarget(target) + .catch(() => false) + .finally(() => warmInFlight.delete(target)); +} + +async function ask(target, questions, stats) { + const ip = podIpFromTarget(target); + const q = questions[Math.floor(Math.random() * questions.length)]; + let lastErr; + for (let attempt = 0; attempt <= REQUEST_RETRIES; attempt++) { + try { + const res = await fetch(`${target}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", content: q }], + max_tokens: MAX_TOKENS, + stream: false, + }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`chat HTTP ${res.status}`); + await res.json(); + stats.chat += 1; + targetChatOk.add(target); + noteTargetResult(ip, true); + return; + } catch (err) { + lastErr = err; + if (attempt < REQUEST_RETRIES) await sleep(100 * (attempt + 1)); + } + } + noteTargetResult(ip, false); + throw lastErr; +} + +const targetWorkers = new Map(); +const workerPromises = new Map(); + +async function runTargetWorker(target, questions, endAt, stats) { + const state = { limit: 2, tasks: new Set() }; + targetWorkers.set(target, state); + + while (Date.now() < endAt) { + while (state.tasks.size < state.limit && Date.now() < endAt) { + const p = ask(target, questions, stats) + .catch((err) => { + stats.fail += 1; + if (stats.fail <= 10 || stats.fail % 100 === 0) { + console.error(`[gpu-load] ${target} ${err.message}`); + } + }) + .finally(() => state.tasks.delete(p)); + state.tasks.add(p); + } + if (state.tasks.size > 0) await Promise.race(state.tasks); + else await sleep(20); + } + await Promise.all(state.tasks); + targetWorkers.delete(target); +} + +function syncTargetWorkers(targets, startedAt, stats) { + const readySet = new Set(podTargets); + const active = new Set(targets); + for (const target of active) { + scheduleWarmTarget(target); + const inferenceReady = readySet.has(target); + const limit = inflightForTarget(target, startedAt, inferenceReady, stats); + const worker = targetWorkers.get(target); + if (worker) worker.limit = limit; + } + for (const target of [...targetWorkers.keys()]) { + if (!active.has(target)) { + const worker = targetWorkers.get(target); + if (worker) worker.limit = 0; + } + } +} + +async function main() { + const questions = loadQuestions(); + const startedAt = Date.now(); + const endAt = startedAt + DURATION_SEC * 1000; + const stats = { chat: 0, fail: 0 }; + let lastLog = startedAt; + let lastUnevenLog = 0; + + console.log( + JSON.stringify({ + targetPods: TARGET_PODS, + hpaTargetGpu: HPA_TARGET_GPU, + jobParallelism: JOB_PARALLELISM, + perPodPeak: PER_POD_PEAK, + loadMultiplier: LOAD_MULTIPLIER, + loadCompensationSafety: LOAD_COMPENSATION_SAFETY, + maxTokens: MAX_TOKENS, + rampSec: RAMP_SEC, + durationSec: DURATION_SEC, + loadModel: "direct-pod-IP saturation (multi-replica floor, per-target backoff, no full idle)", + maxInflightPerPod: MAX_INFLIGHT_PER_POD, + escalateMaxMult: ESCALATE_MAX_MULT, + }), + ); + + await requirePodTargets(startedAt + 90_000); + + while (Date.now() < endAt) { + await pollAgentPodTargets(); + const targets = podCandidates.length ? podCandidates : podTargets; + if (!targets.length) { + await sleep(1000); + continue; + } + + if (hpaReplicas > podTargets.length && Date.now() - lastUnevenLog >= 30_000) { + console.log( + JSON.stringify({ + event: "unevenReplicas", + hpaReplicas, + hpaDesired, + readyLoadTargets: podTargets.length, + candidatePods: targets.length, + warmedPods: targetChatOk.size, + message: "HPA has more replicas than warmed GPUs — bootstrapping new pods", + }), + ); + lastUnevenLog = Date.now(); + } + + for (const target of targets) { + if (!workerPromises.has(target)) { + workerPromises.set(target, runTargetWorker(target, questions, endAt, stats)); + } + } + syncTargetWorkers(targets, startedAt, stats); + + if (Date.now() - lastLog >= LOG_EVERY_SEC * 1000) { + const sampleTarget = podTargets[0] || targets[0]; + const sampleLimit = sampleTarget + ? inflightForTarget(sampleTarget, startedAt, podTargets.includes(sampleTarget), stats) + : 0; + console.log( + JSON.stringify({ + event: "progress", + hpaReplicas, + hpaDesired, + loadTargets: podTargets.length, + candidatePods: targets.length, + warmedPods: targetChatOk.size, + loadCompensation, + effectiveCompensation: effectiveCompensation(startedAt), + inflightPerPodPerGenerator: sampleLimit, + perPodClusterInflight: sampleLimit * JOB_PARALLELISM, + sampleTarget, + chat: stats.chat, + fail: stats.fail, + elapsedSec: Math.round((Date.now() - startedAt) / 1000), + }), + ); + lastLog = Date.now(); + } + + await sleep(1000); + } + + await Promise.all([...workerPromises.values()]); + + console.log(`done chat=${stats.chat} fail=${stats.fail} lastPodCount=${podTargets.length}`); + process.exit(0); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/deploy/helm/nemoclaw-gpu/files/ollama-start.sh b/deploy/helm/nemoclaw-gpu/files/ollama-start.sh new file mode 100644 index 00000000000..84cd3dedd59 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/files/ollama-start.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -eu + +MODEL="${OLLAMA_MODEL:?OLLAMA_MODEL required}" +HOST="${OLLAMA_HOST:-0.0.0.0:11434}" + +ollama serve & +SERVE_PID=$! + +cleanup() { + kill "${SERVE_PID}" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +echo "Waiting for Ollama API..." +for _ in $(seq 1 120); do + if ollama list >/dev/null 2>&1; then + break + fi + sleep 2 +done + +# Pull only when the model is not already in /root/.ollama (PVC, hostPath, or emptyDir). +if ollama show "${MODEL}" >/dev/null 2>&1; then + echo "Model ${MODEL} already present — skipping pull" +else + echo "Pulling model ${MODEL} (first time on this volume; may take several minutes)..." + ollama pull "${MODEL}" +fi + +echo "Ollama ready with model ${MODEL}" +wait "${SERVE_PID}" diff --git a/deploy/helm/nemoclaw-gpu/files/questions-sample.txt b/deploy/helm/nemoclaw-gpu/files/questions-sample.txt new file mode 100644 index 00000000000..24ebe2b944d --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/files/questions-sample.txt @@ -0,0 +1,10 @@ +What is Kubernetes HPA and when should you use it? +Explain the difference between CPU requests and limits on a pod. +How does NVIDIA Inference Hub route requests to Nemotron Ultra? +Write a short Python function to compute Fibonacci numbers recursively. +What are the tradeoffs of manual replica scaling vs autoscaling? +Describe how a readiness probe differs from a liveness probe. +What is the CAP theorem in distributed systems? +How would you debug a pod stuck in Pending state? +Summarize how Prometheus metrics feed into Kubernetes HPA. +What is the purpose of a HorizontalPodAutoscaler behavior stabilization window? diff --git a/deploy/helm/nemoclaw-gpu/monitoring/dcgm-servicemonitor.yaml b/deploy/helm/nemoclaw-gpu/monitoring/dcgm-servicemonitor.yaml new file mode 100644 index 00000000000..e1ea50b82c2 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/monitoring/dcgm-servicemonitor.yaml @@ -0,0 +1,18 @@ +# Scrape NVIDIA DCGM exporter (already installed by microk8s enable gpu / GPU operator). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: nvidia-dcgm-exporter + namespace: gpu-operator-resources + labels: + release: kube-prometheus +spec: + namespaceSelector: + matchNames: + - gpu-operator-resources + selector: + matchLabels: + app: nvidia-dcgm-exporter + endpoints: + - port: gpu-metrics + interval: 15s diff --git a/deploy/helm/nemoclaw-gpu/monitoring/kube-prometheus-microk8s.yaml b/deploy/helm/nemoclaw-gpu/monitoring/kube-prometheus-microk8s.yaml new file mode 100644 index 00000000000..2cbcab52d07 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/monitoring/kube-prometheus-microk8s.yaml @@ -0,0 +1,49 @@ +# Slim kube-prometheus-stack for single-node MicroK8s (GPU HPA via DCGM metrics). + +alertmanager: + enabled: false + +grafana: + enabled: true + defaultDashboardsEnabled: false + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false +kubeEtcd: + enabled: false +coreDns: + enabled: false + +prometheusOperator: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +prometheus: + prometheusSpec: + retention: 6h + scrapeInterval: 30s + evaluationInterval: 30s + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + resources: + requests: + cpu: 300m + memory: 512Mi + limits: + cpu: "1" + memory: 1536Mi diff --git a/deploy/helm/nemoclaw-gpu/monitoring/prometheus-adapter-gpu-values.yaml b/deploy/helm/nemoclaw-gpu/monitoring/prometheus-adapter-gpu-values.yaml new file mode 100644 index 00000000000..261bc8d4812 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/monitoring/prometheus-adapter-gpu-values.yaml @@ -0,0 +1,45 @@ +# prometheus-adapter: expose DCGM GPU utilization and agent LLM metrics to HPA. +# DCGM_FI_DEV_GPU_UTIL is scraped from nvidia-dcgm-exporter (GPU operator). +# Agent /metrics (inflight, LLM latency) require ServiceMonitor on nemoclaw-gpu-agent. + +prometheus: + url: http://REPLACE_PROMETHEUS_SERVICE.monitoring.svc + port: 9090 + +rules: + default: false + custom: + # DCGM exporter attributes GPU util to workloads via exported_* labels (not pod/namespace). + - seriesQuery: 'DCGM_FI_DEV_GPU_UTIL{exported_namespace!="",exported_pod!=""}' + resources: + overrides: + exported_namespace: + resource: namespace + exported_pod: + resource: pod + name: + matches: "^(.*)$" + as: "gpu_utilization_percent" + metricsQuery: 'avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' + - seriesQuery: 'nemoclaw_http_inflight_requests{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)$" + as: "nemoclaw_http_inflight_requests" + metricsQuery: 'avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' + - seriesQuery: 'nemoclaw_llm_latency_p95_milliseconds{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^(.*)$" + as: "nemoclaw_llm_latency_p95_milliseconds" + metricsQuery: 'avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' diff --git a/deploy/helm/nemoclaw-gpu/scripts/cluster-recover.sh b/deploy/helm/nemoclaw-gpu/scripts/cluster-recover.sh new file mode 100755 index 00000000000..1a236317cb9 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/cluster-recover.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +RELEASE="${RELEASE:-nemoclaw-gpu}" +RESTART_MICROK8S="${RESTART_MICROK8S:-1}" +RUN_INSTALL="${RUN_INSTALL:-1}" + +require_cmd kubectl +require_cmd helm + +kubectl delete deploy,svc,hpa -n "${NAMESPACE}" -l 'app.kubernetes.io/name=nemoclaw-gpu' --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete hpa -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete job -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +helm uninstall "${RELEASE}" -n "${NAMESPACE}" 2>/dev/null || true +sleep 3 + +kubectl delete deploy,rs,hpa,job -n "${NAMESPACE}" --all --ignore-not-found --wait=false 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +if [[ "${RESTART_MICROK8S}" == "1" ]] && command -v microk8s >/dev/null 2>&1; then + microk8s stop + microk8s start + microk8s status --wait-ready + microk8s enable gpu 2>/dev/null || true +fi + +if [[ "${RUN_INSTALL}" == "1" ]]; then + exec "${SCRIPT_DIR}/install-hpa.sh" +fi diff --git a/deploy/helm/nemoclaw-gpu/scripts/get-agent-pods.sh b/deploy/helm/nemoclaw-gpu/scripts/get-agent-pods.sh new file mode 100755 index 00000000000..592a79787e8 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/get-agent-pods.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Per-pod GPU agent status (READY, GPU UTIL %, load-test generators). +# Use alongside: kubectl get hpa -n nemoclaw-gpu -w +# +# Usage: +# ./scripts/get-agent-pods.sh -n nemoclaw-gpu +# ./scripts/get-agent-pods.sh -n nemoclaw-gpu -w + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +INTERVAL="${INTERVAL:-5}" +WATCH=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + -n | --namespace) + NAMESPACE="${2:?namespace required after -n}" + shift 2 + ;; + -w | --watch) + WATCH=1 + shift + ;; + -h | --help) + cat <&2 + exit 1 + ;; + esac +done + +require_cmd kubectl + +if [[ "${WATCH}" -eq 1 ]]; then + while true; do + clear 2>/dev/null || true + hpa_common_print_agent_pods "${NAMESPACE}" + sleep "${INTERVAL}" + done +fi + +hpa_common_print_agent_pods "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-gpu/scripts/get-hpa.sh b/deploy/helm/nemoclaw-gpu/scripts/get-hpa.sh new file mode 100755 index 00000000000..067f7884aca --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/get-hpa.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# One-shot HPA with readable GPU % (30.25%/40%, not 30250m/40). +# For live updates prefer: kubectl get hpa -n nemoclaw-gpu -w +# +# Usage: +# ./scripts/get-hpa.sh -n nemoclaw-gpu +# ./scripts/get-hpa.sh -n nemoclaw-gpu -w # same as kubectl get hpa -w + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +WATCH=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + -n | --namespace) + NAMESPACE="${2:?namespace required after -n}" + shift 2 + ;; + -w | --watch) + WATCH=1 + shift + ;; + -h | --help) + cat <&2 + exit 1 + ;; + esac +done + +require_cmd kubectl + +if [[ "${WATCH}" -eq 1 ]]; then + exec kubectl get hpa -n "${NAMESPACE}" -w +fi + +hpa_common_print_hpa "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-gpu/scripts/hpa-common.sh b/deploy/helm/nemoclaw-gpu/scripts/hpa-common.sh new file mode 100755 index 00000000000..0c3bc2c16a6 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/hpa-common.sh @@ -0,0 +1,559 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Shared helpers for nemoclaw-gpu HPA scripts + +hpa_common_log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } + +# Kubernetes custom metrics use Quantity milli-units (33500m = 33.5). Format as plain % for scripts. +# Style: script (GPU UTIL % column + subtitle) | kubectl (matches kubectl get hpa TARGETS column). +hpa_common_format_hpa() { + local ns="${1:?namespace}" + local headers="${2:-1}" + local style="${3:-script}" + python3 - "${ns}" "${headers}" "${style}" <<'PY' +import json, subprocess, sys +from datetime import datetime, timezone + +ns, headers = sys.argv[1], sys.argv[2] == "1" +style = sys.argv[3] if len(sys.argv) > 3 else "script" + +def qty(raw): + if raw is None: + return None + s = str(raw).strip() + if not s or s == "": + return None + if s.endswith("m"): + return float(s[:-1]) / 1000.0 + return float(s) + +def fmt_pct(n): + if n is None: + return "" + if abs(n - round(n)) < 1e-6: + return f"{int(round(n))}%" + s = f"{n:.2f}".rstrip("0").rstrip(".") + return f"{s}%" + +def age(ts): + if not ts: + return "?" + created = datetime.fromisoformat(ts.replace("Z", "+00:00")) + secs = int((datetime.now(timezone.utc) - created).total_seconds()) + if secs < 60: + return f"{secs}s" + if secs < 3600: + return f"{secs // 60}m" + if secs < 86400: + return f"{secs // 3600}h" + return f"{secs // 86400}d" + +def targets(h): + spec_metrics = h.get("spec", {}).get("metrics") or [] + current = h.get("status", {}).get("currentMetrics") or [] + parts = [] + for i, sm in enumerate(spec_metrics): + mtype = sm.get("type") + cm = current[i] if i < len(current) else {} + if mtype == "Pods": + name = sm["pods"]["metric"]["name"] + target = sm["pods"]["target"] + tgt_raw = target.get("averageValue") or target.get("value") + cur_raw = (cm.get("pods") or {}).get("current", {}) + cur_raw = cur_raw.get("averageValue") or cur_raw.get("value") + if name == "gpu_utilization_percent": + parts.append(f"{fmt_pct(qty(cur_raw))}/{fmt_pct(qty(tgt_raw))}") + else: + cur = cur_raw if cur_raw not in (None, "") else "" + parts.append(f"{cur}/{tgt_raw}") + elif mtype == "Resource": + res = sm["resource"]["name"] + target = sm["resource"]["target"] + cur_res = (cm.get("resource") or {}).get("current", {}) + if target.get("type") == "Utilization": + cur = cur_res.get("averageUtilization") + tgt = target.get("averageUtilization") + cur_s = f"{cur}%" if cur is not None else "" + parts.append(f"{res}: {cur_s}/{tgt}%") + else: + cur = cur_res.get("averageValue") or cur_res.get("value") + tgt = target.get("averageValue") or target.get("value") + parts.append(f"{res}: {cur or ''}/{tgt}") + return " ".join(parts) if parts else "" + +def print_row(h): + meta = h["metadata"] + spec = h["spec"] + status = h.get("status") or {} + ref = spec["scaleTargetRef"] + ref_str = f"{ref['kind']}/{ref['name']}" + tgt = targets(h) + if style == "kubectl": + print( + f"{meta['name']:<20} " + f"{ref_str:<31} " + f"{tgt:<11} " + f"{spec.get('minReplicas', ''):<9} " + f"{spec.get('maxReplicas', ''):<9} " + f"{status.get('currentReplicas', ''):<10} " + f"{age(meta.get('creationTimestamp'))}" + ) + else: + print( + f"{meta['name']:<22} " + f"{ref_str:<31} " + f"{tgt:<18} " + f"{spec.get('minReplicas', ''):<8} " + f"{spec.get('maxReplicas', ''):<8} " + f"{status.get('currentReplicas', ''):<10} " + f"{age(meta.get('creationTimestamp'))}" + ) + +try: + raw = subprocess.check_output( + ["kubectl", "get", "hpa", "-n", ns, "-o", "json"], + stderr=subprocess.DEVNULL, + text=True, + ) + items = json.loads(raw).get("items") or [] +except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError): + sys.exit(1) + +if not items: + sys.exit(1) + +if headers: + if style == "kubectl": + print( + f"{'NAME':<20} {'REFERENCE':<31} {'TARGETS':<11} " + f"{'MINPODS':<9} {'MAXPODS':<9} {'REPLICAS':<10} AGE" + ) + else: + print("GPU utilization rate (avg per pod): current / target") + print( + f"{'NAME':<22} {'REFERENCE':<31} {'GPU UTIL %':<18} " + f"{'MINPODS':<8} {'MAXPODS':<8} {'REPLICAS':<10} AGE" + ) + +for h in items: + print_row(h) +PY +} + +# Autoscaling-only stdout: GPU utilization as 30.25%/40% (not kubectl milli-units). +hpa_common_print_hpa() { + local ns="${1:?namespace}" + if ! hpa_common_format_hpa "${ns}" 1 "script"; then + kubectl get hpa -n "${ns}" 2>/dev/null || true + fi +} + +# Agent pods + per-pod GPU % (same namespace as HPA). +hpa_common_print_agent_pods() { + local ns="${1:?namespace}" + python3 - "${ns}" <<'PY' +import json, subprocess, sys + +ns = sys.argv[1] + +def qty(raw): + if raw is None: + return None + s = str(raw).strip() + if not s or s == "": + return None + if s.endswith("m"): + return float(s[:-1]) / 1000.0 + return float(s) + +def fmt_pct(n): + if n is None: + return "" + if abs(n - round(n)) < 1e-6: + return f"{int(round(n))}%" + s = f"{n:.2f}".rstrip("0").rstrip(".") + return f"{s}%" + +def age(ts): + if not ts: + return "?" + from datetime import datetime, timezone + created = datetime.fromisoformat(ts.replace("Z", "+00:00")) + secs = int((datetime.now(timezone.utc) - created).total_seconds()) + if secs < 60: + return f"{secs}s" + if secs < 3600: + return f"{secs // 60}m" + if secs < 86400: + return f"{secs // 3600}h" + return f"{secs // 86400}d" + +gpu = {} +try: + raw = subprocess.check_output( + [ + "kubectl", "get", "--raw", + f"/apis/custom.metrics.k8s.io/v1beta1/namespaces/{ns}/pods/*/gpu_utilization_percent", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + for item in json.loads(raw).get("items") or []: + pod = item.get("describedObject", {}).get("name", "") + gpu[pod] = fmt_pct(qty(item.get("value"))) +except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError): + pass + +try: + raw = subprocess.check_output( + [ + "kubectl", "get", "pods", "-n", ns, + "-l", "app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent", + "-o", "json", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + items = json.loads(raw).get("items") or [] +except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError): + items = [] + +print() +print("Agent pods (avg GPU util per pod):") +if not items: + print(" (no gpu-agent pods)") +else: + print( + f"{'NAME':<42} {'READY':<7} {'STATUS':<11} {'RESTARTS':<9} " + f"{'GPU UTIL':<10} AGE" + ) + for pod in sorted(items, key=lambda p: p["metadata"]["name"]): + meta = pod["metadata"] + status = pod.get("status") or {} + name = meta["name"] + ready = sum( + 1 for c in (status.get("containerStatuses") or []) + if c.get("ready") + ) + total = len(status.get("containerStatuses") or []) + ready_s = f"{ready}/{total}" if total else "?" + phase = status.get("phase") or "?" + restarts = sum( + (c.get("restartCount") or 0) for c in (status.get("containerStatuses") or []) + ) + print( + f"{name:<42} {ready_s:<7} {phase:<11} {restarts:<9} " + f"{gpu.get(name, ''):<10} {age(meta.get('creationTimestamp'))}" + ) + +# Load-test job pods (if running) +try: + raw = subprocess.check_output( + [ + "kubectl", "get", "pods", "-n", ns, + "-l", "job-name=nemoclaw-gpu-hpa-load-test", + "-o", "json", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + load_items = json.loads(raw).get("items") or [] +except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError): + load_items = [] + +if load_items: + print() + print("Load-test generators:") + print(f"{'NAME':<42} {'READY':<7} {'STATUS':<11} {'RESTARTS':<9} AGE") + for pod in sorted(load_items, key=lambda p: p["metadata"]["name"]): + meta = pod["metadata"] + status = pod.get("status") or {} + name = meta["name"] + ready = sum( + 1 for c in (status.get("containerStatuses") or []) + if c.get("ready") + ) + total = len(status.get("containerStatuses") or []) + ready_s = f"{ready}/{total}" if total else "?" + phase = status.get("phase") or "?" + restarts = sum( + (c.get("restartCount") or 0) for c in (status.get("containerStatuses") or []) + ) + print( + f"{name:<42} {ready_s:<7} {phase:<11} {restarts:<9} " + f"{age(meta.get('creationTimestamp'))}" + ) +PY +} + +# Log one HPA row when TARGETS or REPLICAS change (load-test loops). +# Usage: hpa_common_log_hpa_if_changed +hpa_common_log_hpa_if_changed() { + local ns="${1:?namespace}" + local last_var="${2:?lastLineVar}" + local line last + line="$(hpa_common_format_hpa "${ns}" 0 "script" 2>/dev/null | head -1 || true)" + [[ -z "${line}" ]] && return 0 + last="${!last_var}" + if [[ "${line}" != "${last}" ]]; then + hpa_common_log "${line}" + printf -v "${last_var}" '%s' "${line}" + fi +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing command: $1" >&2 + exit 1 + } +} + +# Match Helm fullname in templates/_helpers.tpl (release name contains chart name → use release only). +hpa_common_release_fullname() { + local release="${1:-${RELEASE:-nemoclaw-gpu}}" + local chart="${2:-${CHART_NAME:-nemoclaw-gpu}}" + if [[ "${release}" == *"${chart}"* ]]; then + echo "${release}" + else + echo "${release}-${chart}" + fi +} + +hpa_common_agent_deployment() { + echo "$(hpa_common_release_fullname)-agent" +} + +hpa_common_agent_service() { + echo "$(hpa_common_release_fullname)-agent" +} + +# Old releases used component=agent; chart now uses gpu-agent + workload-type (immutable selector). +hpa_common_gpu_stale_workload() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local comp + comp="$(kubectl get "deployment/${deploy}" -n "${ns}" \ + -o jsonpath='{.spec.selector.matchLabels.component}' 2>/dev/null || true)" + [[ "${comp}" == "agent" ]] +} + +hpa_common_gpu_recreate_stale_workload() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local svc="${3:-${deploy}}" + if hpa_common_gpu_stale_workload "${ns}" "${deploy}"; then + kubectl delete "deployment/${deploy}" "service/${svc}" -n "${ns}" \ + --ignore-not-found --wait=false 2>/dev/null || true + sleep 2 + fi +} + +# Idle GPU HPA baseline (no --reuse-values — avoids Service port merge bugs). +hpa_common_gpu_helm_upgrade() { + local release="${1:?release}" + local chart_dir="${2:?chartDir}" + local ns="${3:?namespace}" + local hpa_values="${4:?valuesFile}" + local min="${5:-1}" + local max="${6:-4}" + local gpu_target="${7:-40}" + local inference_model="${8:-llama3.2:3b}" + + helm upgrade --install "${release}" "${chart_dir}" \ + --namespace "${ns}" \ + --create-namespace \ + --set namespace.create=false \ + -f "${hpa_values}" \ + --set inference.model="${inference_model}" \ + --set probes.readinessChecksInference=true \ + --set autoscaling.enabled=true \ + --set autoscaling.mode=gpu \ + --set autoscaling.minReplicas="${min}" \ + --set autoscaling.maxReplicas="${max}" \ + --set "autoscaling.targetGPUUtilizationPercentage=${gpu_target}" \ + >/dev/null +} + +hpa_common_clear_stuck_pods() { + local ns="${1:?namespace}" + local pod + for pod in $(kubectl get pods -n "${ns}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + [[ -z "${pod}" ]] && continue + kubectl patch pod "${pod}" -n "${ns}" -p '{"metadata":{"finalizers":null}}' --type=merge \ + >/dev/null 2>&1 || true + done + kubectl delete pods -n "${ns}" --all --force --grace-period=0 >/dev/null 2>&1 || true +} + +hpa_common_ensure_agent_ready() { + local ns="${1:?namespace}" + local release="${2:?release}" + local chart_dir="${3:?chartDir}" + local values_file="${4:-}" + local rollout_timeout="${5:-600}" + local deploy + deploy="$(RELEASE="${release}" hpa_common_agent_deployment)" + + local helm_args=( + upgrade --install "${release}" "${chart_dir}" -n "${ns}" + --set "namespace.create=false" + --set "autoscaling.enabled=false" + --set "gpuScaling.count=1" + ) + if [[ -n "${values_file}" && -f "${values_file}" ]]; then + helm_args+=(-f "${values_file}") + fi + helm "${helm_args[@]}" >/dev/null + + hpa_common_kick_deployment "${ns}" "${deploy}" || helm "${helm_args[@]}" >/dev/null + + if ! kubectl rollout status "deployment/${deploy}" -n "${ns}" --timeout="${rollout_timeout}s" >/dev/null; then + hpa_common_diagnose_rollout "${ns}" "${deploy}" + return 1 + fi + + local ready + ready="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo 0)" + if [[ "${ready}" != "1" ]]; then + hpa_common_diagnose_rollout "${ns}" "${deploy}" + return 1 + fi + return 0 +} + +hpa_common_wait_rollout() { + local deploy="${1:?deploy}" + local ns="${2:?namespace}" + local timeout="${3:-600}" + kubectl rollout status "deployment/${deploy}" -n "${ns}" --timeout="${timeout}s" >/dev/null +} + +hpa_common_kick_deployment() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local rs + rs="$(kubectl get rs -n "${ns}" -l "app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent" \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)" + if [[ -n "${rs}" ]]; then + return 0 + fi + kubectl rollout restart "deployment/${deploy}" -n "${ns}" >/dev/null 2>&1 || true + sleep 8 + rs="$(kubectl get rs -n "${ns}" -l "app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + [[ -n "${rs}" ]] && return 0 + kubectl delete "deployment/${deploy}" -n "${ns}" --ignore-not-found --wait=false 2>/dev/null || true + sleep 3 + return 1 +} + +hpa_common_diagnose_rollout() { + local ns="${1:?namespace}" + hpa_common_print_hpa "${ns}" + kubectl describe hpa -n "${ns}" 2>/dev/null | tail -20 || true +} + +hpa_common_enforce_replica_floor() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local min="${3:-1}" + local spec + spec="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")" + if [[ -z "${spec}" ]] || [[ "${spec}" -lt "${min}" ]]; then + kubectl patch "deployment/${deploy}" -n "${ns}" \ + --type=merge -p "{\"spec\":{\"replicas\":${min}}}" + fi +} + +hpa_common_verify_hpa_bounds() { + local ns="${1:?namespace}" + local deploy="${2:?deploy}" + local hpa_name="${3:-${deploy}}" + local min="${4:-1}" + local max="${5:-4}" + + if ! kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" >/dev/null 2>&1; then + echo "HPA ${hpa_name} not found" >&2 + return 1 + fi + + local spec_min spec_max desired deploy_spec + spec_min="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.spec.minReplicas}' 2>/dev/null || echo 0)" + spec_max="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.spec.maxReplicas}' 2>/dev/null || echo 0)" + desired="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" -o jsonpath='{.status.desiredReplicas}' 2>/dev/null || echo "")" + deploy_spec="$(kubectl get "deployment/${deploy}" -n "${ns}" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")" + + hpa_common_enforce_replica_floor "${ns}" "${deploy}" "${min}" + + if [[ -n "${desired}" && "${desired}" =~ ^[0-9]+$ && "${desired}" -lt "${min}" ]]; then + kubectl patch "deployment/${deploy}" -n "${ns}" \ + --type=merge -p "{\"spec\":{\"replicas\":${min}}}" + sleep 5 + fi + + return 0 +} + +hpa_common_verify_gpu_nodes() { + local gpu_count + gpu_count="$(hpa_common_allocatable_gpus)" + if [[ "${gpu_count}" -lt 1 ]]; then + echo "No allocatable nvidia.com/gpu — HPA cannot scale GPU pods" >&2 + return 1 + fi + return 0 +} + +hpa_common_allocatable_gpus() { + kubectl get nodes -o jsonpath='{range .items[*]}{.status.allocatable.nvidia\.com/gpu}{"\n"}{end}' 2>/dev/null \ + | awk 'NF && $1+0>0 {s+=$1} END {print s+0}' +} + +hpa_common_verify_gpu_hpa_metric() { + local ns="${1:-${NAMESPACE:-nemoclaw-gpu}}" + if kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/${ns}/pods/*/gpu_utilization_percent" 2>/dev/null \ + | grep -q '"metricName":"gpu_utilization_percent"'; then + return 0 + fi + echo "gpu_utilization_percent not available — HPA cannot scale on GPU util" >&2 + return 1 +} + +# Human-readable HPA metric (optional; VERBOSE=1 for full legend). +hpa_common_hpa_metric_display() { + local ns="${1:?namespace}" + local hpa_name="${2:-}" + if [[ "${VERBOSE:-0}" != "1" ]]; then + return 0 + fi + if [[ -z "${hpa_name}" ]]; then + hpa_name="$(kubectl get hpa -n "${ns}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + fi + [[ -n "${hpa_name}" ]] || return 0 + + local metric spec_target spec_type + spec_type="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" \ + -o jsonpath='{.spec.metrics[0].type}' 2>/dev/null || true)" + if [[ "${spec_type}" == "Pods" ]]; then + metric="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" \ + -o jsonpath='{.spec.metrics[0].pods.metric.name}' 2>/dev/null || true)" + spec_target="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" \ + -o jsonpath='{.spec.metrics[0].pods.target.averageValue}' 2>/dev/null || true)" + elif [[ "${spec_type}" == "Resource" ]]; then + metric="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" \ + -o jsonpath='{.spec.metrics[0].resource.name}' 2>/dev/null || true)" + spec_target="$(kubectl get "horizontalpodautoscaler/${hpa_name}" -n "${ns}" \ + -o jsonpath='{.spec.metrics[0].resource.target.averageUtilization}' 2>/dev/null || true)" + fi + echo "HPA metric: ${metric:-unknown} target=${spec_target:-?}" +} + +# Default GPU HPA custom metric (prometheus-adapter → custom.metrics.k8s.io). +hpa_common_gpu_hpa_metric_name() { + echo "gpu_utilization_percent" +} + +hpa_common_print_hpa_status() { + hpa_common_print_hpa "${1:?namespace}" +} diff --git a/deploy/helm/nemoclaw-gpu/scripts/hpa-load-test.sh b/deploy/helm/nemoclaw-gpu/scripts/hpa-load-test.sh new file mode 100755 index 00000000000..569046910f3 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/hpa-load-test.sh @@ -0,0 +1,387 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# HPA scale-up / scale-down test driven by GPU utilization (DCGM → HPA). +# Goal: raise average GPU util above HPA target so replicas grow to TARGET_PODS. +# +# Usage: +# cd deploy/helm/nemoclaw-gpu +# ./scripts/hpa-load-test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +RELEASE="${RELEASE:-nemoclaw-gpu}" +JOB_NAME="${JOB_NAME:-nemoclaw-gpu-hpa-load-test}" +ALLOC_GPUS="$(hpa_common_allocatable_gpus 2>/dev/null || echo 4)" +TARGET_PODS="${TARGET_PODS:-${ALLOC_GPUS}}" + +# Backoff / floor — never drive all GPUs to 0% when HPA has 2+ replicas (circuit breaker keeps probe load). +ERROR_BACKOFF_FACTOR="${ERROR_BACKOFF_FACTOR:-0.92}" +ERROR_BACKOFF_MIN="${ERROR_BACKOFF_MIN:-0.4}" +ERROR_BACKOFF_RECOVERY="${ERROR_BACKOFF_RECOVERY:-1.15}" +CIRCUIT_BREAKER_BACKOFF="${CIRCUIT_BREAKER_BACKOFF:-0.15}" +MIN_INFLIGHT_FLOOR="${MIN_INFLIGHT_FLOOR:-12}" +MIN_RECOVERY_INFLIGHT="${MIN_RECOVERY_INFLIGHT:-4}" +READYZ_GRACE_SEC="${READYZ_GRACE_SEC:-45}" + +# Steady GPU saturation — scales to all allocatable GPUs (TARGET_PODS defaults to GPU count). +# Defaults avoid overload → 502/503 → 0% GPU → retry spikes (see README-gpu.md). +# Override any knob via env, e.g. INFLIGHT_PER_GPU=512 ./scripts/hpa-load-test.sh +# HPA target GPU util % (default 40 — easier scale-up vs 50 while load spreads across GPUs). +if [[ "${TARGET_PODS}" -ge 4 ]]; then + JOB_PARALLELISM="${JOB_PARALLELISM:-4}" + MAX_TOKENS="${MAX_TOKENS:-128}" + HPA_TARGET_GPU="${HPA_TARGET_GPU:-40}" + INFLIGHT_PER_GPU="${INFLIGHT_PER_GPU:-64}" + LOAD_MULTIPLIER="${LOAD_MULTIPLIER:-2}" + LOAD_COMPENSATION_SAFETY="${LOAD_COMPENSATION_SAFETY:-2}" + MAX_COMPENSATION="${MAX_COMPENSATION:-4}" + MAX_INFLIGHT_PER_POD="${MAX_INFLIGHT_PER_POD:-512}" + WARMUP_SEC="${WARMUP_SEC:-90}" + NEW_POD_RAMP_SEC="${NEW_POD_RAMP_SEC:-0}" + BOOTSTRAP_INFLIGHT="${BOOTSTRAP_INFLIGHT:-8}" + NEW_POD_WARMUP_PARALLEL="${NEW_POD_WARMUP_PARALLEL:-8}" + RAMP_SEC="${RAMP_SEC:-45}" + ESCALATE_INTERVAL_SEC="${ESCALATE_INTERVAL_SEC:-15}" + ESCALATE_FACTOR="${ESCALATE_FACTOR:-0.35}" + ESCALATE_MAX_MULT="${ESCALATE_MAX_MULT:-1.5}" + TARGET_POLL_SEC="${TARGET_POLL_SEC:-1}" + SCALE_UP_POLL_SEC="${SCALE_UP_POLL_SEC:-10}" +else + JOB_PARALLELISM="${JOB_PARALLELISM:-2}" + LOAD_MULTIPLIER="${LOAD_MULTIPLIER:-2}" + MAX_TOKENS="${MAX_TOKENS:-128}" + HPA_TARGET_GPU="${HPA_TARGET_GPU:-40}" + INFLIGHT_PER_GPU="${INFLIGHT_PER_GPU:-64}" + LOAD_COMPENSATION_SAFETY="${LOAD_COMPENSATION_SAFETY:-3}" + MAX_COMPENSATION="${MAX_COMPENSATION:-8}" + MAX_INFLIGHT_PER_POD="${MAX_INFLIGHT_PER_POD:-512}" + WARMUP_SEC="${WARMUP_SEC:-90}" + BOOTSTRAP_INFLIGHT="${BOOTSTRAP_INFLIGHT:-8}" + NEW_POD_WARMUP_PARALLEL="${NEW_POD_WARMUP_PARALLEL:-8}" + RAMP_SEC="${RAMP_SEC:-20}" + ESCALATE_INTERVAL_SEC="${ESCALATE_INTERVAL_SEC:-15}" + ESCALATE_FACTOR="${ESCALATE_FACTOR:-0.4}" + ESCALATE_MAX_MULT="${ESCALATE_MAX_MULT:-2}" + SCALE_UP_POLL_SEC="${SCALE_UP_POLL_SEC:-10}" +fi + +DURATION_SEC="${DURATION_SEC:-720}" +SCALE_UP_TARGET="${SCALE_UP_TARGET:-${TARGET_PODS}}" +SCALE_UP_WAIT_LOOPS="${SCALE_UP_WAIT_LOOPS:-60}" +HPA_VALUES="${HPA_VALUES:-${CHART_DIR}/values-step2-hpa.yaml}" +LOAD_TEST_HPA_VALUES="${LOAD_TEST_HPA_VALUES:-${CHART_DIR}/values-load-test-hpa.yaml}" +SCALE_DOWN_WAIT_LOOPS="${SCALE_DOWN_WAIT_LOOPS:-40}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-900}" +DEPLOYMENT="${DEPLOYMENT:-$(RELEASE="${RELEASE}" CHART_NAME=nemoclaw-gpu hpa_common_agent_deployment)}" +SERVICE="${SERVICE:-$(RELEASE="${RELEASE}" CHART_NAME=nemoclaw-gpu hpa_common_agent_service)}" +SERVICE_PORT="${SERVICE_PORT:-8081}" +LAST_HPA_LINE="" + +require_cmd kubectl +require_cmd helm + +kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True || { + echo "metrics-server not ready" >&2 + exit 1 +} +hpa_common_verify_gpu_nodes || exit 1 +hpa_common_verify_gpu_hpa_metric "${NAMESPACE}" || exit 1 + +if [[ "${TARGET_PODS}" -gt "${ALLOC_GPUS}" ]]; then + TARGET_PODS="${ALLOC_GPUS}" + SCALE_UP_TARGET="${ALLOC_GPUS}" +fi + +if ! hpa_common_ensure_agent_ready "${NAMESPACE}" "${RELEASE}" "${CHART_DIR}" \ + "${HPA_VALUES}" "${ROLLOUT_TIMEOUT}"; then + echo "Baseline pod not ready — HPA test cannot start" >&2 + exit 1 +fi + +hpa_common_gpu_recreate_stale_workload "${NAMESPACE}" "${DEPLOYMENT}" "${SERVICE}" + +INFERENCE_MODEL="${INFERENCE_MODEL:-llama3.2:3b}" +helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + --set namespace.create=false \ + -f "${HPA_VALUES}" \ + -f "${LOAD_TEST_HPA_VALUES}" \ + --set inference.model="${INFERENCE_MODEL}" \ + --set probes.readinessChecksInference=true \ + --set autoscaling.enabled=true \ + --set autoscaling.mode=gpu \ + --set autoscaling.minReplicas=1 \ + --set autoscaling.maxReplicas="${TARGET_PODS}" \ + --set "autoscaling.targetGPUUtilizationPercentage=${HPA_TARGET_GPU}" \ + >/dev/null + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${DEPLOYMENT}" 1 "${TARGET_PODS}" || true +hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}" +hpa_common_print_hpa "${NAMESPACE}" + +# Ensure agent pods are Ready (Ollama loaded) before load starts. +kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent \ + -n "${NAMESPACE}" --timeout=600s >/dev/null 2>&1 || { + echo "Agent pods not Ready — run ./scripts/hpa-reset.sh then retry" >&2 + exit 1 +} + +# Wait for inference ready (Ollama model loaded) before starting load Job. +hpa_common_log "Waiting for agent /readyz (model loaded)..." +READY_OK=0 +for _ in $(seq 1 60); do + AGENT_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [[ -n "${AGENT_POD}" ]] && kubectl exec -n "${NAMESPACE}" "${AGENT_POD}" -c agent -- \ + node -e "fetch('http://127.0.0.1:${SERVICE_PORT}/readyz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" \ + >/dev/null 2>&1; then + READY_OK=1 + break + fi + sleep 3 +done +if [[ "${READY_OK}" -ne 1 ]]; then + echo "Agent /readyz not stable — Ollama may still be pulling the model. Run ./scripts/hpa-reset.sh then retry" >&2 + exit 1 +fi + +# Smoke-test one chat completion before load Job starts. +hpa_common_log "Smoke test: chat completion on agent pod..." +SMOKE_OK=0 +for _ in $(seq 1 60); do + AGENT_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-gpu,component=gpu-agent \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [[ -n "${AGENT_POD}" ]] && kubectl exec -n "${NAMESPACE}" "${AGENT_POD}" -c agent -- \ + node -e "fetch('http://127.0.0.1:${SERVICE_PORT}/v1/chat/completions',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({messages:[{role:'user',content:'Say OK.'}],max_tokens:8,stream:false})}).then(r=>r.ok?process.exit(0):process.exit(1)).catch(()=>process.exit(1));" \ + >/dev/null 2>&1; then + SMOKE_OK=1 + break + fi + sleep 5 +done +if [[ "${SMOKE_OK}" -ne 1 ]]; then + echo "Chat smoke test failed — inference not serving yet" >&2 + exit 1 +fi +hpa_common_log "Smoke test OK — starting load generators" + +cleanup() { + kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true +} +trap cleanup EXIT + +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true + +LOAD_SA="${JOB_NAME}-sa" +kubectl apply -f - >/dev/null </dev/null 2>&1 || true +kubectl create configmap "${JOB_NAME}-scripts" -n "${NAMESPACE}" \ + --from-file=load-generator.mjs="${CHART_DIR}/files/load-generator.mjs" \ + --from-file=questions.txt="${CHART_DIR}/files/questions-sample.txt" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +cat </dev/null +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: ${NAMESPACE} +spec: + backoffLimit: 0 + parallelism: ${JOB_PARALLELISM} + completions: ${JOB_PARALLELISM} + ttlSecondsAfterFinished: 600 + template: + spec: + serviceAccountName: ${LOAD_SA} + restartPolicy: Never + containers: + - name: load-generator + image: node:22-bookworm-slim + command: ["node", "/scripts/load-generator.mjs"] + env: + - name: TARGET_URL + value: "http://${SERVICE}:${SERVICE_PORT}" + - name: TARGET_PODS + value: "${TARGET_PODS}" + - name: HPA_TARGET_GPU + value: "${HPA_TARGET_GPU}" + - name: JOB_PARALLELISM + value: "${JOB_PARALLELISM}" + - name: INFLIGHT_PER_GPU + value: "${INFLIGHT_PER_GPU}" + - name: LOAD_MULTIPLIER + value: "${LOAD_MULTIPLIER}" + - name: LOAD_COMPENSATION_SAFETY + value: "${LOAD_COMPENSATION_SAFETY}" + - name: MAX_COMPENSATION + value: "${MAX_COMPENSATION}" + - name: NEW_POD_RAMP_SEC + value: "${NEW_POD_RAMP_SEC}" + - name: MAX_INFLIGHT_PER_POD + value: "${MAX_INFLIGHT_PER_POD}" + - name: WARMUP_SEC + value: "${WARMUP_SEC}" + - name: BOOTSTRAP_INFLIGHT + value: "${BOOTSTRAP_INFLIGHT}" + - name: NEW_POD_WARMUP_PARALLEL + value: "${NEW_POD_WARMUP_PARALLEL}" + - name: ERROR_BACKOFF_FACTOR + value: "${ERROR_BACKOFF_FACTOR}" + - name: ERROR_BACKOFF_MIN + value: "${ERROR_BACKOFF_MIN}" + - name: ERROR_BACKOFF_RECOVERY + value: "${ERROR_BACKOFF_RECOVERY}" + - name: CIRCUIT_BREAKER_BACKOFF + value: "${CIRCUIT_BREAKER_BACKOFF}" + - name: MIN_INFLIGHT_FLOOR + value: "${MIN_INFLIGHT_FLOOR}" + - name: MIN_RECOVERY_INFLIGHT + value: "${MIN_RECOVERY_INFLIGHT}" + - name: READYZ_GRACE_SEC + value: "${READYZ_GRACE_SEC}" + - name: REQUIRE_CHAT_PROBE + value: "false" + - name: TARGET_POLL_SEC + value: "${TARGET_POLL_SEC:-1}" + - name: K8S_NAMESPACE + value: "${NAMESPACE}" + - name: AGENT_SERVICE + value: "${SERVICE}" + - name: HPA_NAME + value: "${DEPLOYMENT}" + - name: AGENT_PORT + value: "${SERVICE_PORT}" + - name: RAMP_SEC + value: "${RAMP_SEC}" + - name: DURATION_SEC + value: "${DURATION_SEC}" + - name: MAX_TOKENS + value: "${MAX_TOKENS}" + - name: ESCALATE_INTERVAL_SEC + value: "${ESCALATE_INTERVAL_SEC}" + - name: ESCALATE_FACTOR + value: "${ESCALATE_FACTOR}" + - name: ESCALATE_MAX_MULT + value: "${ESCALATE_MAX_MULT}" + - name: QUESTIONS_FILE + value: "/questions/questions.txt" + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: questions + mountPath: /questions + readOnly: true + volumes: + - name: scripts + configMap: + name: ${JOB_NAME}-scripts + items: + - key: load-generator.mjs + path: load-generator.mjs + - name: questions + configMap: + name: ${JOB_NAME}-scripts + items: + - key: questions.txt + path: questions.txt +EOF + +PER_POD_PEAK=$((INFLIGHT_PER_GPU * LOAD_MULTIPLIER)) +hpa_common_log "GPU load: ${JOB_PARALLELISM} generators × ${MAX_TOKENS} tokens → each Ready agent pod; base ~${PER_POD_PEAK} in-flight/pod (${LOAD_MULTIPLIER}×), cap ${MAX_INFLIGHT_PER_POD}/pod, warmup ${WARMUP_SEC}s, bootstrap ${BOOTSTRAP_INFLIGHT}; HPA target ${HPA_TARGET_GPU}% → max ${TARGET_PODS} replicas" + +kubectl wait --for=condition=ready pod -l "job-name=${JOB_NAME}" -n "${NAMESPACE}" --timeout=120s >/dev/null 2>&1 || { + echo "Load-generator pods not ready — check: kubectl get pods -n ${NAMESPACE} -l job-name=${JOB_NAME}" >&2 +} + +if ! kubectl logs -n "${NAMESPACE}" -l "job-name=${JOB_NAME}" --tail=200 2>/dev/null \ + | grep -q 'targetsReady'; then + hpa_common_log "Waiting for load generators to discover agent pods..." + for _ in $(seq 1 15); do + kubectl logs -n "${NAMESPACE}" -l "job-name=${JOB_NAME}" --tail=200 2>/dev/null \ + | grep -q 'targetsReady' && break + sleep 1 + done +fi + +SCALE_UP_OK=0 +SCALE_UP_POLL_SEC="${SCALE_UP_POLL_SEC:-10}" +for _ in $(seq 1 "${SCALE_UP_WAIT_LOOPS}"); do + hpa_common_log_hpa_if_changed "${NAMESPACE}" LAST_HPA_LINE + REPLICAS="$(kubectl get hpa -n "${NAMESPACE}" -o jsonpath='{.items[0].status.currentReplicas}' 2>/dev/null || echo 0)" + if [[ "${REPLICAS}" -ge "${SCALE_UP_TARGET}" ]]; then + SCALE_UP_OK=1 + hpa_common_log "Scale-up OK: ${REPLICAS}/${SCALE_UP_TARGET} replicas" + break + fi + sleep "${SCALE_UP_POLL_SEC}" +done + +if [[ "${SCALE_UP_OK}" -ne 1 ]]; then + echo "HPA did not scale to ${SCALE_UP_TARGET} replicas" >&2 +fi + +kubectl wait --for=condition=complete "job/${JOB_NAME}" -n "${NAMESPACE}" --timeout="$((DURATION_SEC + 180))s" >/dev/null 2>&1 || true +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found=true >/dev/null 2>&1 || true + +for _ in $(seq 1 "${SCALE_DOWN_WAIT_LOOPS}"); do + hpa_common_log_hpa_if_changed "${NAMESPACE}" LAST_HPA_LINE + REPLICAS="$(kubectl get hpa -n "${NAMESPACE}" -o jsonpath='{.items[0].status.currentReplicas}' 2>/dev/null || echo 0)" + [[ "${REPLICAS}" -le 1 ]] && break + sleep 15 +done + +hpa_common_print_hpa "${NAMESPACE}" + +trap - EXIT +if [[ "${SCALE_UP_OK}" -eq 1 ]]; then + hpa_common_log "Load test complete: scaled to ${SCALE_UP_TARGET}/${TARGET_PODS} GPU replicas" +else + echo "HPA load test incomplete: did not reach ${SCALE_UP_TARGET} replicas" >&2 +fi diff --git a/deploy/helm/nemoclaw-gpu/scripts/hpa-reset.sh b/deploy/helm/nemoclaw-gpu/scripts/hpa-reset.sh new file mode 100755 index 00000000000..49a6cccc586 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/hpa-reset.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Tear down load-test Jobs and GPU agent pods, then helm upgrade idle baseline. +# +# Usage: +# cd deploy/helm/nemoclaw-gpu +# ./scripts/hpa-reset.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +RELEASE="${RELEASE:-nemoclaw-gpu}" +JOB_NAME="${JOB_NAME:-nemoclaw-gpu-hpa-load-test}" +DEPLOYMENT="${DEPLOYMENT:-$(RELEASE="${RELEASE}" CHART_NAME=nemoclaw-gpu hpa_common_agent_deployment)}" +HPA_NAME="${HPA_NAME:-${DEPLOYMENT}}" +REINSTALL_HELM="${REINSTALL_HELM:-1}" +SKIP_HELM="${SKIP_HELM:-0}" +DELETE_DEPLOYMENT="${DELETE_DEPLOYMENT:-0}" +DELETE_HPA="${DELETE_HPA:-0}" +RUN_LOAD_TEST="${RUN_LOAD_TEST:-0}" +HPA_VALUES="${HPA_VALUES:-${CHART_DIR}/values-step2-hpa.yaml}" +WAIT_ROLLOUT="${WAIT_ROLLOUT:-1}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-900}" +MIN_REPLICAS="${MIN_REPLICAS:-1}" +MAX_REPLICAS="${MAX_REPLICAS:-4}" +GPU_TARGET="${GPU_TARGET:-40}" +INFERENCE_MODEL="${INFERENCE_MODEL:-llama3.2:3b}" +SERVICE="${SERVICE:-$(RELEASE="${RELEASE}" CHART_NAME=nemoclaw-gpu hpa_common_agent_service)}" + +require_cmd kubectl + +if [[ "${SKIP_HELM}" != "1" ]] || [[ "${RUN_LOAD_TEST}" == "1" ]]; then + require_cmd helm +fi + +namespace_exists() { + kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 +} + +clear_pod_finalizers() { + local pod + for pod in $(kubectl get pods -n "${NAMESPACE}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + [[ -z "${pod}" ]] && continue + kubectl patch pod "${pod}" -n "${NAMESPACE}" -p '{"metadata":{"finalizers":null}}' --type=merge \ + >/dev/null 2>&1 || true + done +} + +if ! namespace_exists; then + exit 0 +fi + +kubectl delete job "${JOB_NAME}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true +kubectl delete configmap "${JOB_NAME}-scripts" -n "${NAMESPACE}" --ignore-not-found 2>/dev/null || true + +if [[ "${DELETE_HPA}" == "1" ]]; then + kubectl delete hpa "${HPA_NAME}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true + kubectl delete hpa -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-gpu --ignore-not-found --wait=false 2>/dev/null || true +fi + +if [[ "${DELETE_DEPLOYMENT}" == "1" ]]; then + kubectl delete deployment "${DEPLOYMENT}" -n "${NAMESPACE}" --ignore-not-found --wait=false 2>/dev/null || true +fi + +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true +sleep 2 +clear_pod_finalizers +kubectl delete pods -n "${NAMESPACE}" --all --force --grace-period=0 2>/dev/null || true + +kubectl delete rs -n "${NAMESPACE}" -l app.kubernetes.io/name=nemoclaw-gpu --ignore-not-found --wait=false 2>/dev/null || true +hpa_common_clear_stuck_pods "${NAMESPACE}" + +if [[ "${SKIP_HELM}" == "1" ]]; then + hpa_common_print_hpa "${NAMESPACE}" + exit 0 +fi + +if [[ "${DELETE_HPA}" == "1" ]]; then + if ! hpa_common_ensure_agent_ready "${NAMESPACE}" "${RELEASE}" "${CHART_DIR}" \ + "${HPA_VALUES}" "${ROLLOUT_TIMEOUT}"; then + echo "HPA reset failed — baseline pod not ready" >&2 + exit 1 + fi +fi + +hpa_common_gpu_recreate_stale_workload "${NAMESPACE}" "${DEPLOYMENT}" "${SERVICE}" + +hpa_common_gpu_helm_upgrade "${RELEASE}" "${CHART_DIR}" "${NAMESPACE}" "${HPA_VALUES}" \ + "${MIN_REPLICAS}" "${MAX_REPLICAS}" "${GPU_TARGET}" "${INFERENCE_MODEL}" + +hpa_common_kick_deployment "${NAMESPACE}" "${DEPLOYMENT}" || hpa_common_gpu_helm_upgrade "${RELEASE}" "${CHART_DIR}" "${NAMESPACE}" "${HPA_VALUES}" \ + "${MIN_REPLICAS}" "${MAX_REPLICAS}" "${GPU_TARGET}" "${INFERENCE_MODEL}" + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${HPA_NAME}" "${MIN_REPLICAS}" "${MAX_REPLICAS}" || true + +if [[ "${WAIT_ROLLOUT}" == "1" ]]; then + hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}" || \ + hpa_common_diagnose_rollout "${NAMESPACE}" "${DEPLOYMENT}" +fi + +hpa_common_print_hpa "${NAMESPACE}" + +if [[ "${RUN_LOAD_TEST}" == "1" ]]; then + exec "${SCRIPT_DIR}/hpa-load-test.sh" +fi diff --git a/deploy/helm/nemoclaw-gpu/scripts/hpa-watch.sh b/deploy/helm/nemoclaw-gpu/scripts/hpa-watch.sh new file mode 100755 index 00000000000..7901d00d57c --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/hpa-watch.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Live HPA watch (kubectl native streaming). +# Same as: kubectl get hpa -n nemoclaw-gpu -w + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/get-hpa.sh" -w "$@" diff --git a/deploy/helm/nemoclaw-gpu/scripts/install-hpa.sh b/deploy/helm/nemoclaw-gpu/scripts/install-hpa.sh new file mode 100755 index 00000000000..89937c76fd1 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/scripts/install-hpa.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Install GPU HPA (DCGM → prometheus-adapter → gpu_utilization_percent). +# Script output is HPA-focused only; see README-gpu.md for full ops. +# +# Usage: +# cd deploy/helm/nemoclaw-gpu +# ./scripts/install-hpa.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=hpa-common.sh +source "${SCRIPT_DIR}/hpa-common.sh" + +NAMESPACE="${NAMESPACE:-nemoclaw-gpu}" +RELEASE="${RELEASE:-nemoclaw-gpu}" +MONITORING_NS="${MONITORING_NS:-monitoring}" +PROM_RELEASE="${PROM_RELEASE:-kube-prometheus}" +ADAPTER_RELEASE="${ADAPTER_RELEASE:-prometheus-adapter}" +DEPLOYMENT="${DEPLOYMENT:-$(RELEASE="${RELEASE}" CHART_NAME=nemoclaw-gpu hpa_common_agent_deployment)}" +HPA_NAME="${HPA_NAME:-${DEPLOYMENT}}" +HPA_VALUES="${HPA_VALUES:-${CHART_DIR}/values-step2-hpa.yaml}" +MIN_REPLICAS="${MIN_REPLICAS:-1}" +MAX_REPLICAS="${MAX_REPLICAS:-4}" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-900}" +INFERENCE_MODEL="${INFERENCE_MODEL:-llama3.2:3b}" +GPU_TARGET="${GPU_TARGET:-40}" +PROM_HELM_TIMEOUT="${PROM_HELM_TIMEOUT:-25m}" +PROM_VALUES="${PROM_VALUES:-${CHART_DIR}/monitoring/kube-prometheus-microk8s.yaml}" +ADAPTER_VALUES="${ADAPTER_VALUES:-${CHART_DIR}/monitoring/prometheus-adapter-gpu-values.yaml}" + +require_cmd kubectl +require_cmd helm + +custom_metrics_ready() { + kubectl get apiservice v1beta1.custom.metrics.k8s.io 2>/dev/null | grep -q True +} + +prometheus_service_name() { + local svc="" + svc="$(kubectl get svc -n "${MONITORING_NS}" \ + -l 'app=kube-prometheus-stack-prometheus' \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [[ -z "${svc}" ]]; then + svc="$(kubectl get svc -n "${MONITORING_NS}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | grep -E 'kube-prome-prometheus$' | head -1 || true)" + fi + [[ -n "${svc}" ]] || return 1 + printf '%s' "${svc}" +} + +ensure_prometheus_stack() { + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null 2>&1 || true + helm repo update prometheus-community >/dev/null 2>&1 || helm repo update >/dev/null 2>&1 + + kubectl create namespace "${MONITORING_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + + if ! helm status "${PROM_RELEASE}" -n "${MONITORING_NS}" >/dev/null 2>&1; then + helm upgrade --install "${PROM_RELEASE}" prometheus-community/kube-prometheus-stack \ + --namespace "${MONITORING_NS}" \ + --create-namespace \ + -f "${PROM_VALUES}" \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \ + --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ + --set prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues=false \ + --timeout "${PROM_HELM_TIMEOUT}" \ + --wait >/dev/null 2>&1 || true + fi + + kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/name=prometheus \ + -n "${MONITORING_NS}" \ + --timeout=600s >/dev/null 2>&1 || true + + kubectl apply -f "${CHART_DIR}/monitoring/dcgm-servicemonitor.yaml" >/dev/null + + PROM_SVC="$(prometheus_service_name)" || { + echo "Prometheus not found — GPU HPA metric pipeline unavailable" >&2 + exit 1 + } + PROM_URL="http://${PROM_SVC}.${MONITORING_NS}.svc" + + helm upgrade --install "${ADAPTER_RELEASE}" prometheus-community/prometheus-adapter \ + --namespace "${MONITORING_NS}" \ + -f "${ADAPTER_VALUES}" \ + --set "prometheus.url=${PROM_URL}" \ + --set prometheus.port=9090 \ + --wait --timeout 10m >/dev/null + + for _ in $(seq 1 36); do + custom_metrics_ready && break + sleep 5 + done + custom_metrics_ready || { + echo "custom.metrics.k8s.io not ready — HPA cannot use gpu_utilization_percent" >&2 + exit 1 + } +} + +INFERENCE_MODEL="${INFERENCE_MODEL:-llama3.2:3b}" + +helm_install() { + hpa_common_gpu_helm_upgrade "${RELEASE}" "${CHART_DIR}" "${NAMESPACE}" "${HPA_VALUES}" \ + "${MIN_REPLICAS}" "${MAX_REPLICAS}" "${GPU_TARGET}" "${INFERENCE_MODEL}" +} + +if command -v microk8s >/dev/null 2>&1; then + microk8s enable gpu 2>/dev/null || true + microk8s enable metrics-server 2>/dev/null || true +fi +kubectl get apiservice v1beta1.metrics.k8s.io 2>/dev/null | grep -q True || { + echo "metrics-server not ready — CPU/memory HPA APIs unavailable" >&2 + exit 1 +} +hpa_common_verify_gpu_nodes || exit 1 +kubectl get pods -n gpu-operator-resources -l app=nvidia-dcgm-exporter 2>/dev/null | grep -q Running || { + echo "nvidia-dcgm-exporter not running — GPU HPA metric unavailable" >&2 + exit 1 +} + +ensure_prometheus_stack + +hpa_common_gpu_recreate_stale_workload "${NAMESPACE}" "${DEPLOYMENT}" "${DEPLOYMENT}" + +helm_install +hpa_common_kick_deployment "${NAMESPACE}" "${DEPLOYMENT}" && helm_install || true + +if ! hpa_common_wait_rollout "${DEPLOYMENT}" "${NAMESPACE}" "${ROLLOUT_TIMEOUT}"; then + hpa_common_diagnose_rollout "${NAMESPACE}" "${DEPLOYMENT}" + exit 1 +fi + +hpa_common_verify_hpa_bounds "${NAMESPACE}" "${DEPLOYMENT}" "${HPA_NAME}" "${MIN_REPLICAS}" "${MAX_REPLICAS}" || true +hpa_common_print_hpa "${NAMESPACE}" diff --git a/deploy/helm/nemoclaw-gpu/templates/NOTES.txt b/deploy/helm/nemoclaw-gpu/templates/NOTES.txt new file mode 100644 index 00000000000..f00e41d5304 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/NOTES.txt @@ -0,0 +1,2 @@ +HPA: kubectl get hpa -n {{ include "nemoclaw-gpu.namespace" . }} -w +Pods: ./scripts/get-agent-pods.sh -n {{ include "nemoclaw-gpu.namespace" . }} -w (per-pod GPU %) diff --git a/deploy/helm/nemoclaw-gpu/templates/_helpers.tpl b/deploy/helm/nemoclaw-gpu/templates/_helpers.tpl new file mode 100644 index 00000000000..574c880d907 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/_helpers.tpl @@ -0,0 +1,89 @@ +{{/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */}} +{{/* SPDX-License-Identifier: Apache-2.0 */}} +{{- define "nemoclaw-gpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nemoclaw-gpu.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "nemoclaw-gpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nemoclaw-gpu.labels" -}} +helm.sh/chart: {{ include "nemoclaw-gpu.chart" . }} +{{ include "nemoclaw-gpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "nemoclaw-gpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nemoclaw-gpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +component: gpu-agent +nemoclaw.ai/workload-type: gpu +{{- end }} + +{{- define "nemoclaw-gpu.namespace" -}} +{{- .Values.namespace.name }} +{{- end }} + +{{- define "nemoclaw-gpu.replicas" -}} +{{- if .Values.gpuScaling.oneReplicaPerGpu -}} +{{- .Values.gpuScaling.count | int }} +{{- else -}} +{{- .Values.replicaCount | int }} +{{- end -}} +{{- end }} + +{{- define "nemoclaw-gpu.ollamaResources" -}} +requests: + cpu: {{ .Values.gpuScaling.perPodCpuRequest | quote }} + memory: {{ .Values.gpuScaling.perPodMemory | quote }} + nvidia.com/gpu: {{ .Values.gpuScaling.perPodGpu | quote }} +limits: + cpu: {{ .Values.gpuScaling.perPodCpuLimit | quote }} + memory: {{ .Values.gpuScaling.perPodMemoryLimit | quote }} + nvidia.com/gpu: {{ .Values.gpuScaling.perPodGpu | quote }} +{{- end }} + +{{- define "nemoclaw-gpu.agentResources" -}} +requests: + cpu: {{ .Values.gpuScaling.agentCpuRequest | quote }} + memory: {{ .Values.gpuScaling.agentMemory | quote }} +limits: + cpu: {{ .Values.gpuScaling.agentCpuLimit | quote }} + memory: {{ .Values.gpuScaling.agentMemoryLimit | quote }} +{{- end }} + +{{- define "nemoclaw-gpu.hpaMaxReplicas" -}} +{{- if gt (int .Values.autoscaling.maxReplicas) 0 -}} +{{- int .Values.autoscaling.maxReplicas -}} +{{- else if .Values.gpuScaling.oneReplicaPerGpu -}} +{{- int .Values.autoscaling.maxGpus -}} +{{- else -}} +{{- 10 -}} +{{- end -}} +{{- end }} + +{{- define "nemoclaw-gpu.hpaMinReplicas" -}} +{{- $min := int .Values.autoscaling.minReplicas -}} +{{- if lt $min 1 -}} +{{- 1 -}} +{{- else -}} +{{- $min -}} +{{- end -}} +{{- end }} diff --git a/deploy/helm/nemoclaw-gpu/templates/configmap.yaml b/deploy/helm/nemoclaw-gpu/templates/configmap.yaml new file mode 100644 index 00000000000..9b03156fbe6 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} +data: + agent-server.mjs: | +{{ .Files.Get "files/agent-server.mjs" | indent 4 }} + agent-metrics.mjs: | +{{ .Files.Get "files/agent-metrics.mjs" | indent 4 }} + ollama-start.sh: | +{{ .Files.Get "files/ollama-start.sh" | indent 4 }} + INFERENCE_BASE_URL: {{ .Values.inference.baseUrl | quote }} + INFERENCE_MODEL: {{ .Values.inference.model | quote }} diff --git a/deploy/helm/nemoclaw-gpu/templates/deployment.yaml b/deploy/helm/nemoclaw-gpu/templates/deployment.yaml new file mode 100644 index 00000000000..8cb6a2e9e42 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/deployment.yaml @@ -0,0 +1,163 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} +spec: + {{- if .Values.autoscaling.enabled }} + replicas: {{ include "nemoclaw-gpu.hpaMinReplicas" . }} + {{- else }} + replicas: {{ include "nemoclaw-gpu.replicas" . }} + {{- end }} + selector: + matchLabels: + {{- include "nemoclaw-gpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "nemoclaw-gpu.selectorLabels" . | nindent 8 }} + annotations: + checksum/agent-config: {{ printf "%s%s%s" (.Files.Get "files/agent-server.mjs") (.Files.Get "files/agent-metrics.mjs") (.Files.Get "files/ollama-start.sh") | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.runtimeClassName }} + runtimeClassName: {{ .Values.runtimeClassName | quote }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ollama + image: "{{ .Values.ollama.image.repository }}:{{ .Values.ollama.image.tag }}" + imagePullPolicy: {{ .Values.ollama.image.pullPolicy }} + command: ["/bin/sh", "/scripts/ollama-start.sh"] + ports: + - name: ollama + containerPort: {{ .Values.ollama.port }} + protocol: TCP + env: + - name: OLLAMA_HOST + value: "0.0.0.0:{{ .Values.ollama.port }}" + - name: OLLAMA_MODEL + valueFrom: + configMapKeyRef: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + key: INFERENCE_MODEL + - name: OLLAMA_NUM_PARALLEL + value: {{ .Values.ollama.numParallel | default 4 | quote }} + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: ollama-data + mountPath: /root/.ollama + resources: + {{- include "nemoclaw-gpu.ollamaResources" . | nindent 12 }} + livenessProbe: + httpGet: + path: /api/tags + port: ollama + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /api/tags + port: ollama + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 12 + - name: agent + image: "{{ .Values.agent.image.repository }}:{{ .Values.agent.image.tag }}" + imagePullPolicy: {{ .Values.agent.image.pullPolicy }} + command: ["node", "/app/agent-server.mjs"] + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + env: + - name: PORT + value: {{ .Values.service.port | quote }} + - name: INFERENCE_BASE_URL + valueFrom: + configMapKeyRef: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + key: INFERENCE_BASE_URL + - name: INFERENCE_MODEL + valueFrom: + configMapKeyRef: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + key: INFERENCE_MODEL + - name: OLLAMA_BASE_URL + value: "http://127.0.0.1:{{ .Values.ollama.port }}" + volumeMounts: + - name: app + mountPath: /app + readOnly: true + resources: + {{- include "nemoclaw-gpu.agentResources" . | nindent 12 }} + startupProbe: + httpGet: + path: {{ if .Values.probes.readinessChecksInference }}/readyz{{ else }}/healthz{{ end }} + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: {{ max (div (int .Values.probes.startupProbeSeconds) 10) 12 }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 5 + readinessProbe: + httpGet: + path: {{ if .Values.probes.readinessChecksInference }}/readyz{{ else }}/healthz{{ end }} + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + volumes: + - name: app + configMap: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + items: + - key: agent-server.mjs + path: agent-server.mjs + - key: agent-metrics.mjs + path: agent-metrics.mjs + - name: scripts + configMap: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + items: + - key: ollama-start.sh + path: ollama-start.sh + mode: 0755 + - name: ollama-data + {{- if and .Values.ollama.persistence.enabled .Values.ollama.persistence.hostPath }} + hostPath: + path: {{ .Values.ollama.persistence.hostPath | quote }} + type: DirectoryOrCreate + {{- else if .Values.ollama.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "nemoclaw-gpu.fullname" . }}-ollama + {{- else }} + emptyDir: {} + {{- end }} diff --git a/deploy/helm/nemoclaw-gpu/templates/hpa.yaml b/deploy/helm/nemoclaw-gpu/templates/hpa.yaml new file mode 100644 index 00000000000..8be3b9d8057 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/hpa.yaml @@ -0,0 +1,87 @@ +{{- if .Values.autoscaling.enabled }} +{{- $hpaMin := include "nemoclaw-gpu.hpaMinReplicas" . | int }} +{{- $hpaMax := include "nemoclaw-gpu.hpaMaxReplicas" . | int }} +{{- if lt $hpaMax $hpaMin }} +{{- fail (printf "autoscaling.maxReplicas (%d) must be >= minReplicas (%d)" $hpaMax $hpaMin) }} +{{- end }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} + annotations: + nemoclaw.ai/hpa-policy: "min-{{ $hpaMin }}-max-{{ $hpaMax }}-gpu-target-{{ .Values.autoscaling.targetGPUUtilizationPercentage }}pct" + nemoclaw.ai/hpa-mode: {{ .Values.autoscaling.mode | quote }} + {{- if eq .Values.autoscaling.mode "gpu" }} + nemoclaw.ai/hpa-metric: {{ .Values.autoscaling.gpu.metricName | quote }} + nemoclaw.ai/hpa-metric-display: "GPU utilization % (DCGM / nvidia-smi family)" + nemoclaw.ai/hpa-metric-source: "DCGM_FI_DEV_GPU_UTIL" + nemoclaw.ai/hpa-targets-format: "{{ .Values.autoscaling.gpu.metricName }}: %/% (GPU utilization avg per pod)" + {{- else if eq .Values.autoscaling.mode "performance" }} + nemoclaw.ai/hpa-metric: "nemoclaw_http_inflight_requests" + nemoclaw.ai/hpa-metric-display: "In-flight HTTP requests per pod" + nemoclaw.ai/hpa-targets-format: "nemoclaw_http_inflight_requests: /" + {{- else if eq .Values.autoscaling.mode "latency" }} + nemoclaw.ai/hpa-metric: "nemoclaw_llm_latency_p95_milliseconds" + nemoclaw.ai/hpa-metric-display: "LLM response time p95 (ms per pod)" + nemoclaw.ai/hpa-targets-format: "nemoclaw_llm_latency_p95_milliseconds: / (ms)" + {{- else }} + nemoclaw.ai/hpa-metric: "cpu" + nemoclaw.ai/hpa-metric-display: "CPU utilization % of pod request" + nemoclaw.ai/hpa-targets-format: "cpu: %/%" + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + minReplicas: {{ $hpaMin }} + maxReplicas: {{ $hpaMax }} + metrics: + {{- if eq .Values.autoscaling.mode "gpu" }} + - type: Pods + pods: + metric: + name: {{ .Values.autoscaling.gpu.metricName | quote }} + target: + type: AverageValue + averageValue: {{ .Values.autoscaling.targetGPUUtilizationPercentage | quote }} + {{- else if eq .Values.autoscaling.mode "performance" }} + - type: Pods + pods: + metric: + name: nemoclaw_http_inflight_requests + target: + type: AverageValue + averageValue: {{ .Values.autoscaling.performance.inflightRequestsPerPod | quote }} + {{- else if eq .Values.autoscaling.mode "latency" }} + - type: Pods + pods: + metric: + name: nemoclaw_llm_latency_p95_milliseconds + target: + type: AverageValue + averageValue: {{ .Values.autoscaling.performance.latencyP95Milliseconds | quote }} + {{- else }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/nemoclaw-gpu/templates/namespace.yaml b/deploy/helm/nemoclaw-gpu/templates/namespace.yaml new file mode 100644 index 00000000000..4789d99cfc7 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/namespace.yaml @@ -0,0 +1,8 @@ +{{- if .Values.namespace.create }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} +{{- end }} diff --git a/deploy/helm/nemoclaw-gpu/templates/pvc.yaml b/deploy/helm/nemoclaw-gpu/templates/pvc.yaml new file mode 100644 index 00000000000..ba81254af2a --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/pvc.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.ollama.persistence.enabled (not .Values.ollama.persistence.hostPath) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-ollama + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.ollama.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.ollama.persistence.size | quote }} + {{- with .Values.ollama.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/deploy/helm/nemoclaw-gpu/templates/service.yaml b/deploy/helm/nemoclaw-gpu/templates/service.yaml new file mode 100644 index 00000000000..259fe9d22c2 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} + annotations: + nemoclaw.ai/agent-port: {{ .Values.service.port | quote }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "nemoclaw-gpu.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/nemoclaw-gpu/templates/servicemonitor.yaml b/deploy/helm/nemoclaw-gpu/templates/servicemonitor.yaml new file mode 100644 index 00000000000..b1177257ac4 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nemoclaw-gpu.fullname" . }}-agent + namespace: {{ include "nemoclaw-gpu.namespace" . }} + labels: + {{- include "nemoclaw-gpu.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "nemoclaw-gpu.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: {{ .Values.metrics.path }} + interval: {{ .Values.metrics.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/nemoclaw-gpu/values-load-test-hpa.yaml b/deploy/helm/nemoclaw-gpu/values-load-test-hpa.yaml new file mode 100644 index 00000000000..ace2960a75a --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/values-load-test-hpa.yaml @@ -0,0 +1,15 @@ +# Optional overlay for load-test — same 40% target and fast scale-up as values-step2-hpa.yaml. +# Kept for ./scripts/hpa-load-test.sh (-f values-load-test-hpa.yaml); behavior matches production GPU HPA. +autoscaling: + targetGPUUtilizationPercentage: 40 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 10 + - type: Pods + value: 4 + periodSeconds: 10 + selectPolicy: Max diff --git a/deploy/helm/nemoclaw-gpu/values-step2-hpa-latency.yaml b/deploy/helm/nemoclaw-gpu/values-step2-hpa-latency.yaml new file mode 100644 index 00000000000..777bddd2350 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/values-step2-hpa-latency.yaml @@ -0,0 +1,41 @@ +# Latency HPA: scale when rolling p95 LLM response time exceeds target (ms). +# Requires: Prometheus + prometheus-adapter + ServiceMonitor scraping agent /metrics. + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + maxGpus: 4 + mode: latency + performance: + latencyP95Milliseconds: "2000" + +gpuScaling: + oneReplicaPerGpu: true + count: 1 + perPodGpu: 1 + perPodCpuRequest: "2" + perPodCpuLimit: "4" + perPodMemory: "16Gi" + perPodMemoryLimit: "24Gi" + agentCpuRequest: "250m" + agentCpuLimit: "1" + agentMemory: "512Mi" + agentMemoryLimit: "1Gi" + +loadTest: + chatOnly: true + concurrencyPerPod: 8 + +probes: + readinessChecksInference: true + startupProbeSeconds: 600 + +metrics: + enabled: true + path: /metrics + serviceMonitor: + enabled: true + interval: 15s + labels: + release: kube-prometheus diff --git a/deploy/helm/nemoclaw-gpu/values-step2-hpa-performance.yaml b/deploy/helm/nemoclaw-gpu/values-step2-hpa-performance.yaml new file mode 100644 index 00000000000..e8e851def3a --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/values-step2-hpa-performance.yaml @@ -0,0 +1,42 @@ +# Performance HPA: Prometheus scrapes /metrics → prometheus-adapter → HPA on inflight requests. +# Better signal for GPU inference than CPU utilization alone. +# Requires: kube-prometheus-stack + prometheus-adapter (see scripts/install-performance-hpa.sh) + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + maxGpus: 4 + mode: performance + performance: + inflightRequestsPerPod: "4" + +gpuScaling: + oneReplicaPerGpu: true + count: 1 + perPodGpu: 1 + perPodCpuRequest: "2" + perPodCpuLimit: "4" + perPodMemory: "16Gi" + perPodMemoryLimit: "24Gi" + agentCpuRequest: "250m" + agentCpuLimit: "1" + agentMemory: "512Mi" + agentMemoryLimit: "1Gi" + +loadTest: + chatOnly: true + concurrencyPerPod: 8 + +probes: + readinessChecksInference: true + startupProbeSeconds: 600 + +metrics: + enabled: true + path: /metrics + serviceMonitor: + enabled: true + interval: 15s + labels: + release: kube-prometheus diff --git a/deploy/helm/nemoclaw-gpu/values-step2-hpa.yaml b/deploy/helm/nemoclaw-gpu/values-step2-hpa.yaml new file mode 100644 index 00000000000..4002cdd4bf3 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/values-step2-hpa.yaml @@ -0,0 +1,65 @@ +# GPU HPA — one pod per GPU, scale on DCGM GPU utilization (%). +# Requires: nvidia-dcgm-exporter + Prometheus + prometheus-adapter (install-hpa.sh sets up). + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + maxGpus: 4 + mode: gpu + targetGPUUtilizationPercentage: 40 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 180 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + - type: Pods + value: 1 + periodSeconds: 60 + selectPolicy: Max + +gpuScaling: + oneReplicaPerGpu: true + count: 1 + perPodGpu: 1 + perPodCpuRequest: "2" + perPodCpuLimit: "4" + perPodMemory: "16Gi" + perPodMemoryLimit: "24Gi" + agentCpuRequest: "250m" + agentCpuLimit: "1" + agentMemory: "512Mi" + agentMemoryLimit: "1Gi" + +loadTest: + chatOnly: true + concurrencyPerPod: 64 + maxTokens: 512 + jobParallelism: 8 + loadMultiplier: 4 + inflightPerGpu: 1024 + rampSec: 30 + +probes: + readinessChecksInference: true + startupProbeSeconds: 600 + +# Shared model cache on the GPU node — pull once, reuse across HPA replicas and restarts. +ollama: + persistence: + enabled: true + hostPath: /var/lib/nemoclaw-gpu/ollama + size: 20Gi + accessMode: ReadWriteMany diff --git a/deploy/helm/nemoclaw-gpu/values.yaml b/deploy/helm/nemoclaw-gpu/values.yaml new file mode 100644 index 00000000000..5a42b257358 --- /dev/null +++ b/deploy/helm/nemoclaw-gpu/values.yaml @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Manual scaling: one pod per GPU (see README-gpu.md for HPA install). +# Target deployment: local Ollama on GPU, GPU-util HPA. +# helm install nemoclaw-gpu . -n nemoclaw-gpu --create-namespace --set gpuScaling.count=2 +gpuScaling: + oneReplicaPerGpu: true + count: 1 + perPodGpu: 1 + perPodCpuRequest: "2" + perPodCpuLimit: "4" + perPodMemory: "16Gi" + perPodMemoryLimit: "24Gi" + # Agent sidecar (no GPU) — metrics + OpenAI-compatible proxy to local Ollama + agentCpuRequest: "250m" + agentCpuLimit: "1" + agentMemory: "512Mi" + agentMemoryLimit: "1Gi" + +replicaCount: 1 + +ollama: + image: + repository: ollama/ollama + tag: latest + pullPolicy: IfNotPresent + port: 11434 + # Concurrent GPU batches per model (raise with care — VRAM limited). + numParallel: 4 + # Persist pulled models across pod restarts / scale-up (see accessMode + hostPath). + persistence: + enabled: false + size: 20Gi + storageClass: "" + # ReadWriteMany: one PVC shared by all agent pods. ReadWriteOnce: single pod only. + accessMode: ReadWriteMany + # Single-node dev: host dir shared by all replicas (pull once per node). Overrides PVC. + hostPath: "" + +agent: + image: + repository: node + tag: "22-bookworm-slim" + pullPolicy: IfNotPresent + +nameOverride: "" +fullnameOverride: "" + +namespace: + create: false + name: nemoclaw-gpu + +service: + type: ClusterIP + # Agent HTTP port (CPU chart uses 8080 — keep charts on different ports) + port: 8081 + +# Local GPU inference via Ollama sidecar (same pattern as NemoClaw GPU E2E / Ollama onboard) +inference: + baseUrl: "http://127.0.0.1:11434/v1" + model: "llama3.2:3b" + # Optional bearer token if fronting Ollama with auth later + apiKey: "" + existingSecret: "" + secretName: nemoclaw-gpu-inference + +resources: + requests: + cpu: "2" + memory: "16Gi" + nvidia.com/gpu: "1" + limits: + cpu: "4" + memory: "24Gi" + nvidia.com/gpu: "1" + +podAnnotations: {} + +# Readiness checks local Ollama (/readyz). First model pull can take several minutes. +probes: + readinessChecksInference: true + startupProbeSeconds: 600 + +runtimeClassName: "" + +nodeSelector: + nvidia.com/gpu.present: "true" + +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +affinity: {} + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 4 + maxGpus: 4 + # gpu (default) | performance | latency | resource + # gpu: DCGM_FI_DEV_GPU_UTIL via Prometheus + prometheus-adapter (see install-hpa.sh) + mode: gpu + targetGPUUtilizationPercentage: 40 + targetCPUUtilizationPercentage: 65 + targetMemoryUtilizationPercentage: null + gpu: + # Exposed to HPA / kubectl get hpa TARGETS column (Prometheus source: DCGM_FI_DEV_GPU_UTIL) + metricName: gpu_utilization_percent + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 180 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + - type: Pods + value: 1 + periodSeconds: 60 + selectPolicy: Max + performance: + requestsPerSecondPerPod: "5" + inflightRequestsPerPod: "4" + latencyP95Milliseconds: "2000" + +loadTest: + chatOnly: true + concurrencyPerPod: 20 + maxTokens: 128 + jobParallelism: 2 + rampSec: 45 + +metrics: + enabled: true + path: /metrics + serviceMonitor: + enabled: false + interval: 30s + labels: {} diff --git a/deploy/scripts/install-both.sh b/deploy/scripts/install-both.sh new file mode 100755 index 00000000000..784b69dd98d --- /dev/null +++ b/deploy/scripts/install-both.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# DEPRECATED — CPU and GPU are separate deployments. Use chart install scripts instead. + +set -euo pipefail + +cat <<'EOF' +install-both.sh is deprecated. + +Install CPU and GPU independently (do not use a combined install): + + CPU: cd deploy/helm/nemoclaw-cpu && source ~/.nemoclaw/secrets.env && ./scripts/install-hpa.sh + GPU: cd deploy/helm/nemoclaw-gpu && MAX_REPLICAS=4 ./scripts/install-hpa.sh + +Docs: deploy/README-cpu.md · deploy/README-gpu.md +EOF +exit 1 diff --git a/deploy/scripts/status-both.sh b/deploy/scripts/status-both.sh new file mode 100755 index 00000000000..9c42421d23c --- /dev/null +++ b/deploy/scripts/status-both.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# DEPRECATED — CPU and GPU are separate deployments. + +set -euo pipefail + +cat <<'EOF' +status-both.sh is deprecated. + +Check the chart you installed: + + CPU: kubectl get hpa,deploy,pods -n nemoclaw + GPU: kubectl get hpa,deploy,pods -n nemoclaw-gpu + +Docs: deploy/README-cpu.md · deploy/README-gpu.md +EOF +exit 1 diff --git a/deploy/scripts/uninstall-both.sh b/deploy/scripts/uninstall-both.sh new file mode 100755 index 00000000000..64649ecc5c7 --- /dev/null +++ b/deploy/scripts/uninstall-both.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# DEPRECATED — CPU and GPU are separate deployments. + +set -euo pipefail + +cat <<'EOF' +uninstall-both.sh is deprecated. + +Uninstall the chart you installed: + + CPU: helm uninstall nemoclaw -n nemoclaw + kubectl delete namespace nemoclaw --ignore-not-found + + GPU: helm uninstall nemoclaw-gpu -n nemoclaw-gpu + kubectl delete namespace nemoclaw-gpu --ignore-not-found + +Docs: deploy/README-cpu.md · deploy/README-gpu.md +EOF +exit 1 diff --git a/docs/network-policy/integration-policy-examples.mdx b/docs/network-policy/integration-policy-examples.mdx index d4de69f8616..99d0f4d3acb 100644 --- a/docs/network-policy/integration-policy-examples.mdx +++ b/docs/network-policy/integration-policy-examples.mdx @@ -54,6 +54,7 @@ NemoClaw ships maintained policy presets for common services in `nemoclaw-bluepr | Workflow | Preset | |----------|--------| | Brave Search | `brave` | +| Tavily Search | `tavily` | | Homebrew packages | `brew` | | Discord messaging | `discord` | | GitHub and GitHub API | `github` | @@ -244,6 +245,22 @@ $ nemoclaw my-assistant policy-add brave --yes The Brave Search API key is still configured separately during onboarding or through the web search setup flow. +## Tavily Search + +| Preset | Use when | +|--------|----------| +| `tavily` | Agent workflows call the Tavily Search API (`api.tavily.com`) | + +During `nemoclaw onboard`, answer **y** to **Enable web search?**, choose **2) Tavily Search**, then paste your API key. + +For an existing sandbox: + +```console +$ ./scripts/setup-tavily-search.sh my-assistant +``` + +When the agent uses web search to answer a question, it is instructed to tell you **Tavily Web Search is used** (Brave uses **Brave Web Search is used**). + ## Package and Model Tooling Use these presets when an agent workflow installs packages or downloads model assets: diff --git a/docs/security/credential-storage.mdx b/docs/security/credential-storage.mdx index e44057d83e0..623f6a803fc 100644 --- a/docs/security/credential-storage.mdx +++ b/docs/security/credential-storage.mdx @@ -37,9 +37,33 @@ Both surface the provider names that the gateway holds credentials for. The valu NemoClaw still keeps non-secret operational state under `~/.nemoclaw/` (such as the sandbox registry). That directory is created with mode `0700` and contains no credential material. +## Local secrets file (recommended for development) + +Store API keys outside any git repository in `~/.nemoclaw/secrets.env` (mode `0600`, directory mode `0700`). + +```console +$ nemoclaw credentials init-secrets +$ $EDITOR ~/.nemoclaw/secrets.env +``` + +Use separate variables — each key matches one upstream API base: + +```bash +# integrate.api.nvidia.com/v1 (NVIDIA API key, nvapi-*) +NVIDIA_API_KEY=nvapi-... + +# inference-api.nvidia.com/v1 /chat/completions (Inference Hub, sk-*) +NVIDIA_INFERENCE_HUB_API_KEY=sk-... +``` + +NemoClaw loads this file at the start of `onboard`, `rebuild`, and other commands that resolve credentials. +Values already exported in your shell take precedence. +The file is never read from inside a cloned NemoClaw repo, so it cannot be pushed to GitHub by accident. +See `secrets.env.example` in the repository for a commented template. + ## Environment Variables Take Precedence -When a NemoClaw command needs a credential value during a single run (for example to forward it to an `openshell provider` registration), it reads from `process.env` first. +When a NemoClaw command needs a credential value during a single run (for example to forward it to an `openshell provider` registration), it reads from `process.env` first (including keys staged from `~/.nemoclaw/secrets.env`). This means you can: - Prefix any command with the credential to override the gateway-stored value: `NVIDIA_API_KEY=nvapi-... nemoclaw onboard` diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index c34323bf516..d674a2d0cbe 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -54,8 +54,10 @@ components: default: provider_type: "nvidia" provider_name: "nvidia-inference" - endpoint: "https://integrate.api.nvidia.com/v1" - model: "nvidia/nemotron-3-super-120b-a12b" + # Nemotron Ultra — inference-api.nvidia.com (see config.ts + scripts/examples/nemotron-ultra-inference.py) + endpoint: "https://inference-api.nvidia.com/v1" + model: "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" + credential_env: "NVIDIA_INFERENCE_HUB_API_KEY" ncp: provider_type: "nvidia" diff --git a/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-ultra-managed-inference.json b/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-ultra-managed-inference.json new file mode 100644 index 00000000000..eedb68da326 --- /dev/null +++ b/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-ultra-managed-inference.json @@ -0,0 +1,17 @@ +{ + "$schema": "../schema.json", + "id": "nemotron-ultra-managed-inference", + "agent": "openclaw", + "description": "Disable OpenClaw compact tool-search for Nemotron Ultra; the model invents invalid tool_search_code instead of calling web_search directly.", + "match": { + "modelIds": ["nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1"], + "providerKey": "inference", + "inferenceApi": "openai-completions", + "baseUrl": "https://inference.local/v1" + }, + "effects": { + "openclawTools": { + "toolSearch": false + } + } +} diff --git a/nemoclaw-blueprint/policies/presets/tavily.yaml b/nemoclaw-blueprint/policies/presets/tavily.yaml new file mode 100644 index 00000000000..d81608abb0d --- /dev/null +++ b/nemoclaw-blueprint/policies/presets/tavily.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +preset: + name: tavily + description: "Tavily Search API access" + +network_policies: + tavily: + name: tavily + endpoints: + - host: api.tavily.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + binaries: + - { path: /usr/local/bin/node } + - { path: /usr/bin/node } + - { path: /usr/bin/curl } diff --git a/nemoclaw-blueprint/router/pool-config.yaml b/nemoclaw-blueprint/router/pool-config.yaml index ddd9af5300e..70e564b741b 100644 --- a/nemoclaw-blueprint/router/pool-config.yaml +++ b/nemoclaw-blueprint/router/pool-config.yaml @@ -28,9 +28,10 @@ models: cost_per_m_output_tokens: 0.20 api_base: "https://integrate.api.nvidia.com/v1" - - name: nemotron-3-super - display_name: "Nemotron 3 Super 120B" - litellm_model: "openai/nvidia/nemotron-3-super-120b-a12b" + - name: nemotron-ultra + display_name: "Nemotron Ultra 253B" + # inference-api.nvidia.com — model id from curl in src/lib/inference/config.ts + litellm_model: "openai/nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" cost_per_m_input_tokens: 0.10 cost_per_m_output_tokens: 0.40 - api_base: "https://integrate.api.nvidia.com/v1" + api_base: "https://inference-api.nvidia.com/v1" diff --git a/nemoclaw/src/index.ts b/nemoclaw/src/index.ts index 770b83b8893..4cbcfad891c 100644 --- a/nemoclaw/src/index.ts +++ b/nemoclaw/src/index.ts @@ -209,7 +209,8 @@ export interface NemoClawConfig { // Gateway plugins run inside the sandbox, where OpenClaw keeps its active config here. const OPENCLAW_CONFIG_PATH = "/sandbox/.openclaw/openclaw.json"; -const DEFAULT_INFERENCE_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +// inference-api.nvidia.com — see src/lib/inference/config.ts (curl reference) +const DEFAULT_INFERENCE_MODEL = "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1"; function normalizeInferenceModel(value: string): string { const trimmed = value.trim(); @@ -241,14 +242,14 @@ function activeModelEntries(activeModel: string): ModelProviderEntry[] { if (!activeModel) { return [ { - id: "nvidia/nemotron-3-super-120b-a12b", - label: "Nemotron 3 Super 120B (March 2026)", + id: "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1", + label: "Nemotron Ultra 253B", contextWindow: 131072, - maxOutput: 8192, + maxOutput: 4096, }, { - id: "nvidia/llama-3.1-nemotron-ultra-253b-v1", - label: "Nemotron Ultra 253B", + id: "nvidia/nemotron-3-super-120b-a12b", + label: "Nemotron 3 Super 120B (March 2026)", contextWindow: 131072, maxOutput: 4096, }, @@ -284,7 +285,9 @@ function registeredProviderForConfig( const authLabel = providerCredentialEnv === "NVIDIA_API_KEY" ? `NVIDIA API Key (${providerCredentialEnv})` - : `OpenAI API Key (${providerCredentialEnv})`; + : providerCredentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY" + ? `Inference Hub API Key (${providerCredentialEnv})` + : `OpenAI API Key (${providerCredentialEnv})`; return { id: "inference", diff --git a/nemoclaw/src/register.test.ts b/nemoclaw/src/register.test.ts index d096a0091e2..5cb9d8659c5 100644 --- a/nemoclaw/src/register.test.ts +++ b/nemoclaw/src/register.test.ts @@ -173,7 +173,7 @@ describe("plugin registration", () => { const providerArg = vi.mocked(api.registerProvider).mock.calls[0][0]; expect(providerArg.models?.chat).toEqual([ expect.objectContaining({ id: "nvidia/nemotron-3-super-120b-a12b" }), - expect.objectContaining({ id: "nvidia/llama-3.1-nemotron-ultra-253b-v1" }), + expect.objectContaining({ id: "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" }), expect.objectContaining({ id: "nvidia/llama-3.3-nemotron-super-49b-v1.5" }), expect.objectContaining({ id: "nvidia/nemotron-3-nano-30b-a3b" }), ]); diff --git a/scripts/checks/direct-credential-env.ts b/scripts/checks/direct-credential-env.ts index 0d8b317f0ac..f5d722a35b4 100644 --- a/scripts/checks/direct-credential-env.ts +++ b/scripts/checks/direct-credential-env.ts @@ -16,6 +16,7 @@ import * as ts from "typescript"; const CREDENTIAL_ENV_KEYS = new Set([ "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_HUB_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", diff --git a/scripts/examples/nemotron-ultra-inference.py b/scripts/examples/nemotron-ultra-inference.py new file mode 100644 index 00000000000..3d4d54de60f --- /dev/null +++ b/scripts/examples/nemotron-ultra-inference.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Reference client for Nemotron Ultra on inference-api.nvidia.com. +# Canonical model/URL constants: src/lib/inference/config.ts +# +# NVIDIA_INFERENCE_OPENAI_BASE_URL = https://inference-api.nvidia.com +# NVIDIA_NEMOTRON_ULTRA_MODEL = nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1 +# +# Usage: +# export NVIDIA_INFERENCE_HUB_API_KEY=sk-... # Inference Hub (not NVIDIA Build nvapi-) +# pip install openai +# python scripts/examples/nemotron-ultra-inference.py + +from __future__ import annotations + +import asyncio +import os + +from openai import AsyncOpenAI, OpenAI + +API_KEY = os.environ.get("NVIDIA_INFERENCE_HUB_API_KEY", "sk-your-inference-hub-key") +BASE_URL = "https://inference-api.nvidia.com" +MODEL = "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1" +MESSAGES = [{"role": "user", "content": "Capital of United States"}] +TEMPERATURE = 0.9 +MAX_TOKENS = 128 +TOP_P = 0.7 + +# NOTE: Streaming is preferred for better performance and resource efficiency. + + +def sync_non_streaming() -> None: + client = OpenAI(api_key=API_KEY, base_url=BASE_URL) + response = client.chat.completions.create( + model=MODEL, + messages=MESSAGES, + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + top_p=TOP_P, + stream=False, + ) + print(response.choices[0].message.content) + + +def sync_streaming() -> None: + client = OpenAI(api_key=API_KEY, base_url=BASE_URL) + stream = client.chat.completions.create( + model=MODEL, + messages=MESSAGES, + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + stream=True, + ) + for chunk in stream: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="", flush=True) + print() + + +async def async_non_streaming() -> None: + async_client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL) + response = await async_client.chat.completions.create( + model=MODEL, + messages=MESSAGES, + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + top_p=TOP_P, + stream=False, + ) + print(response.choices[0].message.content) + + +async def async_streaming() -> None: + async_client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL) + stream = await async_client.chat.completions.create( + model=MODEL, + messages=MESSAGES, + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + stream=True, + ) + async for chunk in stream: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="", flush=True) + print() + + +if __name__ == "__main__": + print("=== sync streaming (recommended) ===") + sync_streaming() + # print("=== sync non-streaming ===") + # sync_non_streaming() + # asyncio.run(async_non_streaming()) + # asyncio.run(async_streaming()) diff --git a/scripts/generate-openclaw-config.py b/scripts/generate-openclaw-config.py index fdef5e53d8b..f7db02ad4b8 100755 --- a/scripts/generate-openclaw-config.py +++ b/scripts/generate-openclaw-config.py @@ -38,6 +38,7 @@ NEMOCLAW_PROXY_PORT Egress proxy port (default: 3128) NEMOCLAW_OPENCLAW_MANAGED_PROXY Set to "0" to defer OpenClaw managed proxy config NEMOCLAW_WEB_SEARCH_ENABLED Set to "1" to enable web search tools + NEMOCLAW_WEB_SEARCH_PROVIDER "brave" or "tavily" (default: brave) """ from __future__ import annotations @@ -716,6 +717,11 @@ def _placeholder(channel: str, env_key: str) -> str: # registered an accountId under channels.openclaw-weixin.accounts. "openclaw-weixin": {"enabled": True}, } + # OpenClaw 2026.5.x ships Telegram as an opt-in stock plugin. Baking + # channels.telegram without plugins.entries.telegram.enabled leaves the + # gateway with channel config but no polling provider (empty channels.status). + if "telegram" in msg_channels: + plugin_entries["telegram"] = {"enabled": True} plugin_entries.update({"slack": {"enabled": True}} if "slack" in _ch_cfg else {}) _bundled_provider_plugins = { "amazon-bedrock": {"amazon-bedrock", "bedrock"}, @@ -822,15 +828,75 @@ def _placeholder(channel: str, env_key: str) -> str: tools_web["fetch"] = {"enabled": True, "useTrustedEnvProxy": True} if env.get("NEMOCLAW_WEB_SEARCH_ENABLED", "") == "1": - tools_web["search"] = { - "enabled": True, - "provider": "brave", - "apiKey": "openshell:resolve:env:BRAVE_API_KEY", - } + provider = (env.get("NEMOCLAW_WEB_SEARCH_PROVIDER") or "brave").strip().lower() + if provider not in ("brave", "tavily"): + provider = "brave" + search: dict = {"enabled": True, "provider": provider} + if provider == "tavily": + # OpenClaw 2026.5+ rejects legacy tools.web.search.tavily.* — key belongs + # on plugins.entries.tavily.config.webSearch (see openclaw doctor). + plugin_entries["tavily"] = { + "enabled": True, + "config": { + "webSearch": { + "apiKey": "openshell:resolve:env:TAVILY_API_KEY", + }, + }, + } + else: + search["apiKey"] = "openshell:resolve:env:BRAVE_API_KEY" + tools_web["search"] = search + _strip_legacy_web_search_provider_blocks(config) return config +def _strip_legacy_web_search_provider_blocks(config: dict) -> None: + """Remove deprecated tools.web.search. blocks (OpenClaw 2026.5+).""" + tools = config.get("tools") + if not isinstance(tools, dict): + return + web = tools.get("web") + if not isinstance(web, dict): + return + search = web.get("search") + if not isinstance(search, dict): + return + for legacy_key in ("tavily", "brave", "perplexity", "firecrawl", "exa"): + search.pop(legacy_key, None) + + +def _append_web_search_answer_instruction(config: dict) -> None: + """Tell the agent to announce which web search backend it used when answering.""" + search = config.get("tools", {}).get("web", {}).get("search", {}) + if not isinstance(search, dict) or not search.get("enabled"): + return + provider = search.get("provider", "brave") + usage_line = ( + "Tavily Web Search is used" + if provider == "tavily" + else "Brave Web Search is used" + ) + workspace = config.get("agents", {}).get("defaults", {}).get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + workspace = os.path.expanduser("~/.openclaw/workspace") + agents_md = Path(workspace).expanduser() / "AGENTS.md" + snippet = ( + "\n\n## Web search (NemoClaw)\n" + f"When you use web search to answer a question, tell the user clearly: " + f"**{usage_line}** before summarizing search results.\n" + "Call the **`web_search`** tool with a `query` argument. " + "Do **not** use `tool_search_code` or invent `openclaw.tools.*` APIs.\n" + ) + agents_md.parent.mkdir(parents=True, exist_ok=True) + existing = "" + if agents_md.exists(): + existing = agents_md.read_text(encoding="utf-8") + if usage_line in existing: + return + agents_md.write_text(existing + snippet, encoding="utf-8") + + def _preserve_existing_plugin_installs(config: dict, path: str) -> None: try: with open(path) as f: @@ -943,12 +1009,26 @@ def _seed_wechat_accounts_if_available(config: dict) -> None: def main() -> None: """Generate openclaw.json from environment variables.""" config = build_config() + _strip_legacy_web_search_provider_blocks(config) + search = config.get("tools", {}).get("web", {}).get("search", {}) + if isinstance(search, dict) and search.get("provider") == "tavily": + if "tavily" in search: + raise SystemExit( + "Refusing to write legacy tools.web.search.tavily — use " + "plugins.entries.tavily.config.webSearch instead.", + ) + tavily_entry = config.get("plugins", {}).get("entries", {}).get("tavily") + if not isinstance(tavily_entry, dict) or not tavily_entry.get("enabled"): + raise SystemExit( + "Tavily web search requires plugins.entries.tavily.enabled=true", + ) path = os.path.expanduser("~/.openclaw/openclaw.json") _preserve_existing_plugin_installs(config, path) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as f: json.dump(config, f, indent=2) os.chmod(path, 0o600) + _append_web_search_answer_instruction(config) _seed_wechat_accounts_if_available(config) diff --git a/scripts/setup-tavily-search.sh b/scripts/setup-tavily-search.sh new file mode 100755 index 00000000000..8f2988ca618 --- /dev/null +++ b/scripts/setup-tavily-search.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Enable Tavily web search on an existing sandbox (interactive API key + rebuild). +# +# Usage: +# ./scripts/setup-tavily-search.sh +# +# Environment: +# TAVILY_API_KEY Optional; prompted when unset +# NEMOCLAW_WEB_SEARCH_PROVIDER Set to tavily for rebuild + +set -euo pipefail + +SANDBOX="${1:-}" +if [[ -z "$SANDBOX" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -f "$REPO_ROOT/nemoclaw-blueprint/policies/presets/tavily.yaml" ]] && [[ -f "$REPO_ROOT/bin/nemoclaw.js" ]]; then + NEMOCLAW=(node "$REPO_ROOT/bin/nemoclaw.js") + echo " Using NemoClaw from: $REPO_ROOT" +elif command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW=(nemoclaw) +else + echo "nemoclaw CLI not found. Build with: cd $REPO_ROOT && npm run build:cli" >&2 + exit 1 +fi + +read_secret() { + local prompt="$1" + local value="" + if [[ -n "${TAVILY_API_KEY:-}" ]]; then + return 0 + fi + read -r -s -p "$prompt" value + echo "" + TAVILY_API_KEY="$value" +} + +read_secret " Tavily Search API key: " +if [[ -z "${TAVILY_API_KEY:-}" ]]; then + echo " TAVILY_API_KEY is required." >&2 + exit 1 +fi + +export TAVILY_API_KEY +export NEMOCLAW_WEB_SEARCH_PROVIDER=tavily + +echo "" +echo " Applying Tavily network policy and rebuilding sandbox '${SANDBOX}'..." +"${NEMOCLAW[@]}" "${SANDBOX}" policy-remove brave --yes 2>/dev/null || true +"${NEMOCLAW[@]}" "${SANDBOX}" policy-add tavily --yes +"${NEMOCLAW[@]}" "${SANDBOX}" rebuild --yes + +echo "" +echo " ✓ Tavily web search configured for sandbox '${SANDBOX}'" +echo " Tavily Web Search is used" +echo "" diff --git a/scripts/test-tavily-flow.sh b/scripts/test-tavily-flow.sh new file mode 100755 index 00000000000..1b3e3c9efba --- /dev/null +++ b/scripts/test-tavily-flow.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Quick pre-commit checks for Tavily web search (no full sandbox rebuild). +# +# Usage: +# cd ~/NemoClaw +# npm run build:cli +# ./scripts/test-tavily-flow.sh +# +# Optional — live Tavily API key validation + agent smoke test: +# export TAVILY_API_KEY=tvly-... +# ./scripts/test-tavily-flow.sh --live + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +LIVE=0 +if [[ "${1:-}" == "--live" ]]; then + LIVE=1 +fi + +echo "==> 1) Build CLI" +npm run build:cli >/dev/null +echo " OK (dist contains Tavily prompts)" + +if ! grep -q "Enable web search?" dist/lib/onboard/web-search-flow.js; then + echo " FAIL: missing interactive web search prompt in build output" >&2 + exit 1 +fi +if ! grep -q "Tavily Search" dist/lib/onboard/web-search-flow.js; then + echo " FAIL: missing Tavily provider in build output" >&2 + exit 1 +fi + +echo "" +echo "==> 2) Unit tests (optional — skip if vitest native bindings missing)" +if npx vitest run \ + src/lib/onboard/web-search-verify.test.ts \ + test/generate-openclaw-config.test.ts \ + -t "Tavily|tavily" 2>&1 | tail -8; then + echo " OK: vitest Tavily tests" +else + echo " SKIP: vitest unavailable (rolldown binding); continuing with node/python checks" +fi + +echo "" +echo "==> 3) Simulate Tavily provider selection (non-interactive + fake curl)" +TMP="$(mktemp -d)" +FAKE_BIN="$TMP/bin" +mkdir -p "$FAKE_BIN" +cat >"$FAKE_BIN/curl" <<'CURL' +#!/usr/bin/env bash +outfile="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '%s' '{"results":[]}' >"$outfile" +printf '%s' '200' +CURL +chmod +x "$FAKE_BIN/curl" +export HOME="$TMP" +export PATH="$FAKE_BIN:$PATH" +export TAVILY_API_KEY="tvly-mock-test-key" +export NEMOCLAW_WEB_SEARCH_PROVIDER="tavily" + +node <<'NODE' +const { createWebSearchFlowHelpers } = require("./dist/lib/onboard/web-search-flow"); + +const helpers = createWebSearchFlowHelpers({ + prompt: async () => { throw new Error("unexpected prompt in non-interactive test"); }, + note: () => {}, + isNonInteractive: () => true, + cliName: () => "nemoclaw", + runCaptureOpenshell: () => null, +}); + +(async () => { + const result = await helpers.configureWebSearch(null); + if (!result || result.provider !== "tavily" || !result.fetchEnabled) { + console.error("FAIL: expected { fetchEnabled: true, provider: 'tavily' }, got", result); + process.exit(1); + } + console.log(" OK: configureWebSearch selected provider=tavily"); +})(); +NODE + +echo "" +echo "==> 4) OpenClaw config generation (tavily provider)" +python3 <<'PY' +import base64, json, os, subprocess, tempfile +env = os.environ.copy() +env.update({ + "NEMOCLAW_MODEL": "test-model", + "NEMOCLAW_PROVIDER_KEY": "test-provider", + "NEMOCLAW_PRIMARY_MODEL_REF": "test-ref", + "CHAT_UI_URL": "http://127.0.0.1:18789", + "NEMOCLAW_INFERENCE_BASE_URL": "http://localhost:8080", + "NEMOCLAW_INFERENCE_API": "openai", + "NEMOCLAW_INFERENCE_COMPAT_B64": base64.b64encode(b"{}").decode(), + "NEMOCLAW_PROXY_HOST": "10.200.0.1", + "NEMOCLAW_PROXY_PORT": "3128", + "NEMOCLAW_CONTEXT_WINDOW": "131072", + "NEMOCLAW_MAX_TOKENS": "4096", + "NEMOCLAW_REASONING": "false", + "NEMOCLAW_AGENT_TIMEOUT": "600", +}) +env["NEMOCLAW_WEB_SEARCH_ENABLED"] = "1" +env["NEMOCLAW_WEB_SEARCH_PROVIDER"] = "tavily" +with tempfile.TemporaryDirectory() as td: + env["HOME"] = td + subprocess.run( + ["python3", "scripts/generate-openclaw-config.py"], + cwd=os.getcwd(), + env=env, + check=True, + ) + cfg = json.load(open(f"{td}/.openclaw/openclaw.json")) + search = cfg["tools"]["web"]["search"] + assert search["provider"] == "tavily", search + assert "tavily" not in search, search + assert cfg["plugins"]["entries"]["tavily"]["config"]["webSearch"]["apiKey"] + agents_md = f"{td}/.openclaw/workspace/AGENTS.md" + with open(agents_md) as f: + body = f.read() + assert "Tavily Web Search is used" in body, body +print(" OK: openclaw.json provider=tavily and AGENTS.md usage hint") +PY + +if [[ "$LIVE" -eq 1 ]]; then + if [[ -z "${TAVILY_API_KEY:-}" ]]; then + echo " SKIP live: set TAVILY_API_KEY" >&2 + exit 1 + fi + echo "" + echo "==> 5) Live Tavily API key validation" + node < { throw new Error("unexpected prompt"); }, + note: () => {}, + isNonInteractive: () => true, + cliName: () => "nemoclaw", + runCaptureOpenshell: () => null, +}); +process.env.TAVILY_API_KEY = process.env.TAVILY_API_KEY; +process.env.NEMOCLAW_WEB_SEARCH_PROVIDER = "tavily"; +(async () => { + const result = await helpers.configureWebSearch(null); + if (!result || result.provider !== "tavily") { + console.error("FAIL: live validation", result); + process.exit(1); + } + console.log(" OK: TAVILY_API_KEY validated"); +})(); +NODE +else + echo "" + echo "==> 5) Live tests skipped (run with --live and TAVILY_API_KEY)" +fi + +echo "" +echo "All automated pre-commit checks passed." +echo "" +echo "Manual checks on your VM (needs real Tavily key + sandbox rebuild):" +echo " export PATH=\"$REPO_ROOT:\$PATH\" # or: npm link" +echo " export TAVILY_API_KEY=tvly-..." +echo " nemoclaw onboard # y → web search → 2 Tavily → paste key" +echo " # OR for existing my-assistant:" +echo " ./scripts/setup-tavily-search.sh my-assistant" +echo " nemoclaw my-assistant exec openclaw-agent -m 'Use web search: latest NVIDIA GTC news. Say which search you used.'" diff --git a/scripts/verify-tavily-openclaw-config.py b/scripts/verify-tavily-openclaw-config.py new file mode 100644 index 00000000000..613856c33fc --- /dev/null +++ b/scripts/verify-tavily-openclaw-config.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validate openclaw.json Tavily web search shape (no legacy tools.web.search.tavily).""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def main() -> int: + path = Path(sys.argv[1] if len(sys.argv) > 1 else "~/.openclaw/openclaw.json").expanduser() + cfg = json.loads(path.read_text(encoding="utf-8")) + search = cfg.get("tools", {}).get("web", {}).get("search", {}) + if not isinstance(search, dict): + print(f"FAIL: missing tools.web.search in {path}", file=sys.stderr) + return 1 + if search.get("provider") != "tavily": + print(f"FAIL: expected provider=tavily, got {search.get('provider')!r}", file=sys.stderr) + return 1 + if "tavily" in search: + print( + "FAIL: legacy tools.web.search.tavily is present — rebuild with updated " + "generate-openclaw-config.py", + file=sys.stderr, + ) + return 1 + tavily = cfg.get("plugins", {}).get("entries", {}).get("tavily") + if not isinstance(tavily, dict) or not tavily.get("enabled"): + print("FAIL: plugins.entries.tavily must be enabled", file=sys.stderr) + return 1 + web_search = (tavily.get("config") or {}).get("webSearch") + if not isinstance(web_search, dict) or not web_search.get("apiKey"): + print("FAIL: plugins.entries.tavily.config.webSearch.apiKey missing", file=sys.stderr) + return 1 + print(f"OK: Tavily config valid in {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/secrets.env.example b/secrets.env.example new file mode 100644 index 00000000000..acbd7a1ca8a --- /dev/null +++ b/secrets.env.example @@ -0,0 +1,11 @@ +# Copy to ~/.nemoclaw/secrets.env (or run: nemoclaw credentials init-secrets) +# This example file is safe to keep in git — put real keys only in ~/.nemoclaw/secrets.env + +# NVIDIA API key (nvapi-*) — https://integrate.api.nvidia.com/v1 +NVIDIA_API_KEY= + +# Inference Hub API key (sk-*) — https://inference-api.nvidia.com/v1/chat/completions +NVIDIA_INFERENCE_HUB_API_KEY= + +# OPENAI_API_KEY= +# TAVILY_API_KEY= diff --git a/src/commands/credentials/init-secrets.ts b/src/commands/credentials/init-secrets.ts new file mode 100644 index 00000000000..b13acd4d46b --- /dev/null +++ b/src/commands/credentials/init-secrets.ts @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; +import { initSecretsEnvFile } from "../../lib/credentials/secrets-env"; + +export default class CredentialsInitSecretsCommand extends NemoClawCommand { + static id = "credentials:init-secrets"; + static strict = true; + static summary = "Create a local secrets.env file for API keys"; + static description = + "Create ~/.nemoclaw/secrets.env (mode 0600) outside any git repo for storing API keys."; + static usage = ["credentials init-secrets"]; + static examples = ["<%= config.bin %> credentials init-secrets"]; + static flags = {}; + + public async run(): Promise { + await this.parse(CredentialsInitSecretsCommand); + const result = initSecretsEnvFile(); + if (!result.ok) { + this.failWithLines([` Could not create secrets file: ${result.message}`]); + return; + } + if (result.created) { + this.log(""); + this.log(` Created ${result.path}`); + this.log(" Edit that file with your API keys (NVIDIA_API_KEY, NVIDIA_INFERENCE_HUB_API_KEY, etc.)."); + this.log(" It stays in your home directory and is never committed to git."); + this.log(""); + return; + } + this.log(""); + this.log(` Secrets file already exists: ${result.path}`); + this.log(" Edit it directly; NemoClaw loads it automatically on onboard and rebuild."); + this.log(""); + } +} diff --git a/src/lib/cli/public-display-defaults.ts b/src/lib/cli/public-display-defaults.ts index f1b77b50d2a..d9fba74505a 100644 --- a/src/lib/cli/public-display-defaults.ts +++ b/src/lib/cli/public-display-defaults.ts @@ -37,6 +37,13 @@ const PUBLIC_DISPLAY_LAYOUT: Record = { "flags": " [--yes|-y]" } ], + "credentials:init-secrets": [ + { + "group": "Credentials", + "order": 37, + "description": "Create ~/.nemoclaw/secrets.env for API keys (outside git)" + } + ], "debug": [ { "group": "Troubleshooting", diff --git a/src/lib/credentials/command-support.ts b/src/lib/credentials/command-support.ts index dd9c7ab4212..f016b1214eb 100644 --- a/src/lib/credentials/command-support.ts +++ b/src/lib/credentials/command-support.ts @@ -24,9 +24,10 @@ export function printCredentialsUsage(log: (message?: string) => void = console. log(" Subcommands:"); log(" list List provider credentials registered with the OpenShell gateway"); log(" reset [--yes] Remove a provider credential so onboard re-prompts"); + log(" init-secrets Create ~/.nemoclaw/secrets.env for local API keys (not in git)"); log(""); - log(" Credentials live in the OpenShell gateway. Inspect with `openshell provider list`."); - log(" Nothing is persisted to host disk; deploy/non-onboard commands read from env vars."); + log(" Credentials live in the OpenShell gateway after onboard."); + log(" For local key storage outside git, use ~/.nemoclaw/secrets.env (see init-secrets)."); log(""); } diff --git a/src/lib/credentials/secrets-env.test.ts b/src/lib/credentials/secrets-env.test.ts new file mode 100644 index 00000000000..642b4f09734 --- /dev/null +++ b/src/lib/credentials/secrets-env.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + parseSecretsEnvContents, + parseSecretsEnvLine, + resetSecretsEnvStagingForTests, + stageSecretsEnvFile, +} from "../../../dist/lib/credentials/secrets-env"; + +const TRACKED_KEYS = [ + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_HUB_API_KEY", + "OPENAI_API_KEY", +] as const; + +function clearTrackedEnv(): void { + for (const key of TRACKED_KEYS) delete process.env[key]; +} + +describe("secrets-env parsing", () => { + it("parses export-prefixed and quoted assignments", () => { + expect(parseSecretsEnvLine('export NVIDIA_API_KEY="nvapi-abc"')).toEqual({ + key: "NVIDIA_API_KEY", + value: "nvapi-abc", + }); + expect(parseSecretsEnvLine("NVIDIA_INFERENCE_HUB_API_KEY=sk-xyz")).toEqual({ + key: "NVIDIA_INFERENCE_HUB_API_KEY", + value: "sk-xyz", + }); + }); + + it("ignores comments and non-allowlisted keys", () => { + const parsed = parseSecretsEnvContents(` +# comment +NVIDIA_API_KEY=nvapi-1 +UNKNOWN_KEY=secret +PATH=/tmp +`); + expect(parsed).toEqual({ NVIDIA_API_KEY: "nvapi-1" }); + }); +}); + +describe("stageSecretsEnvFile", () => { + const originalHome = process.env.HOME; + + afterEach(() => { + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + clearTrackedEnv(); + resetSecretsEnvStagingForTests(); + }); + + it("stages allowlisted keys without overriding existing env", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-secrets-")); + process.env.HOME = home; + const dir = path.join(home, ".nemoclaw"); + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + path.join(dir, "secrets.env"), + "NVIDIA_API_KEY=nvapi-from-file\nNVIDIA_INFERENCE_HUB_API_KEY=sk-from-file\n", + { mode: 0o600 }, + ); + + clearTrackedEnv(); + process.env.NVIDIA_API_KEY = "nvapi-already-set"; + resetSecretsEnvStagingForTests(); + + const staged = stageSecretsEnvFile(); + expect(staged).toEqual(["NVIDIA_INFERENCE_HUB_API_KEY"]); + expect(process.env.NVIDIA_API_KEY).toBe("nvapi-already-set"); + expect(process.env.NVIDIA_INFERENCE_HUB_API_KEY).toBe("sk-from-file"); + }); +}); diff --git a/src/lib/credentials/secrets-env.ts b/src/lib/credentials/secrets-env.ts new file mode 100644 index 00000000000..6f8bf3be5dd --- /dev/null +++ b/src/lib/credentials/secrets-env.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Load API keys from ~/.nemoclaw/secrets.env (never from the git repo). +// Values are staged into process.env for the current run only; the OpenShell +// gateway remains the long-lived store after onboard. + +import fs from "node:fs"; +import path from "node:path"; + +import { rejectSymlinksOnPath } from "../state/config-io"; +import { + getCredsDir, + getCredential, + KNOWN_CREDENTIAL_ENV_KEYS, + normalizeCredentialValue, +} from "./store"; + +export const SECRETS_ENV_FILE_NAME = "secrets.env"; + +const SECRETS_ENV_TEMPLATE = `# NemoClaw local secrets (NOT checked into git) +# Path: ~/.nemoclaw/secrets.env (chmod 600 — created by: nemoclaw credentials init-secrets) +# +# KEY=value lines work; leading "export " is optional. Lines already set in your shell +# are left unchanged. Restart onboard/rebuild after editing this file. +# +# NVIDIA API key (nvapi-*) — models on https://integrate.api.nvidia.com/v1 +NVIDIA_API_KEY= +# +# Inference Hub API key (sk-*) — models on https://inference-api.nvidia.com/v1 +# (chat completions: .../v1/chat/completions) +NVIDIA_INFERENCE_HUB_API_KEY= +# +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= +# GEMINI_API_KEY= +# TAVILY_API_KEY= +# TELEGRAM_BOT_TOKEN= +`; + +let secretsEnvStaged = false; +let lastStagedSecrets: string[] = []; + +/** @internal Reset module cache between unit tests. */ +export function resetSecretsEnvStagingForTests(): void { + secretsEnvStaged = false; + lastStagedSecrets = []; +} + +/** Absolute path to the user's local secrets file (~/.nemoclaw/secrets.env). */ +export function getSecretsEnvFilePath(): string { + return path.join(getCredsDir(), SECRETS_ENV_FILE_NAME); +} + +/** Parse a single dotenv-style assignment line. */ +export function parseSecretsEnvLine(line: string): { key: string; value: string } | null { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) return null; + const eq = trimmed.indexOf("="); + if (eq <= 0) return null; + let key = trimmed.slice(0, eq).trim(); + if (key.startsWith("export ")) { + key = key.slice("export ".length).trim(); + } + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) return null; + let value = trimmed.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"') && value.length >= 2) || + (value.startsWith("'") && value.endsWith("'") && value.length >= 2) + ) { + value = value.slice(1, -1); + } + return { key, value }; +} + +/** Parse dotenv file contents into allowlisted credential keys only. */ +export function parseSecretsEnvContents(raw: string): Record { + const allowed = new Set(KNOWN_CREDENTIAL_ENV_KEYS); + const result: Record = {}; + for (const line of raw.split(/\r?\n/)) { + const parsed = parseSecretsEnvLine(line); + if (!parsed || !allowed.has(parsed.key)) continue; + const normalized = normalizeCredentialValue(parsed.value); + if (normalized) result[parsed.key] = normalized; + } + return result; +} + +function assertSecretsEnvPathSafe(secretsFile: string): boolean { + try { + rejectSymlinksOnPath(path.dirname(secretsFile)); + } catch (error) { + console.error( + ` Refusing to load ${secretsFile}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return false; + } + return true; +} + +/** + * Stage credential values from ~/.nemoclaw/secrets.env into process.env. + * Does not override variables already set in the environment. + * + * @returns Sorted list of keys staged from the file. + */ +export function stageSecretsEnvFile(): string[] { + if (secretsEnvStaged) return lastStagedSecrets; + secretsEnvStaged = true; + lastStagedSecrets = []; + + const secretsFile = getSecretsEnvFilePath(); + if (!assertSecretsEnvPathSafe(secretsFile)) return lastStagedSecrets; + + let fd: number; + try { + fd = fs.openSync(secretsFile, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + } catch { + return lastStagedSecrets; + } + + let raw: string; + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) return lastStagedSecrets; + if (stat.size > 256 * 1024) { + console.error( + ` Refusing to load ${secretsFile}: file exceeds 256 KiB. Split secrets or trim the file.`, + ); + return lastStagedSecrets; + } + raw = fs.readFileSync(fd, "utf-8"); + } catch { + return lastStagedSecrets; + } finally { + try { + fs.closeSync(fd); + } catch { + /* ignore */ + } + } + + const parsed = parseSecretsEnvContents(raw); + const staged: string[] = []; + for (const [key, value] of Object.entries(parsed)) { + if (getCredential(key)) continue; + process.env[key] = value; + staged.push(key); + } + lastStagedSecrets = staged.sort(); + return lastStagedSecrets; +} + +export type InitSecretsEnvFileResult = + | { ok: true; created: boolean; path: string } + | { ok: false; message: string }; + +/** + * Create ~/.nemoclaw/secrets.env from the template if it does not exist yet. + */ +export function initSecretsEnvFile(): InitSecretsEnvFileResult { + const dir = getCredsDir(); + const secretsFile = getSecretsEnvFilePath(); + try { + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + try { + fs.chmodSync(dir, 0o700); + } catch { + /* best effort */ + } + } catch (error) { + return { + ok: false, + message: error instanceof Error ? error.message : String(error), + }; + } + + if (!assertSecretsEnvPathSafe(secretsFile)) { + return { ok: false, message: `Unsafe path for secrets file: ${secretsFile}` }; + } + + try { + if (fs.existsSync(secretsFile)) { + return { ok: true, created: false, path: secretsFile }; + } + fs.writeFileSync(secretsFile, SECRETS_ENV_TEMPLATE, { encoding: "utf-8", mode: 0o600 }); + return { ok: true, created: true, path: secretsFile }; + } catch (error) { + return { + ok: false, + message: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/src/lib/credentials/store.ts b/src/lib/credentials/store.ts index 5e370ea5ea3..17fb8c439e3 100644 --- a/src/lib/credentials/store.ts +++ b/src/lib/credentials/store.ts @@ -32,12 +32,14 @@ export type CredentialPromptIntent = // sync without a second hand-maintained copy. export const KNOWN_CREDENTIAL_ENV_KEYS: readonly string[] = [ "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_HUB_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", "COMPATIBLE_API_KEY", "COMPATIBLE_ANTHROPIC_API_KEY", "BRAVE_API_KEY", + "TAVILY_API_KEY", "GITHUB_TOKEN", "HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", @@ -189,6 +191,11 @@ export function getCredential(key: string): string | null { */ export function resolveProviderCredential(envName: string): string | null { let value = getCredential(envName); + if (!value) { + const { stageSecretsEnvFile } = require("./secrets-env") as typeof import("./secrets-env"); + stageSecretsEnvFile(); + value = getCredential(envName); + } if (!value) { stageLegacyCredentialsToEnv(); value = getCredential(envName); @@ -664,32 +671,93 @@ export async function readCredentialPrompt( return getCredentialPromptIntent(await promptImpl(question, { secret: true })); } +const { + NVIDIA_INTEGRATE_API_BASE_URL, + NVIDIA_INFERENCE_API_BASE_URL, + NVIDIA_INFERENCE_HUB_CHAT_COMPLETIONS_URL, + NVIDIA_BUILD_CREDENTIAL_ENV, + NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, +} = require("../inference/config") as typeof import("../inference/config"); + +function nvidiaEndpointCredentialPrompt(credentialEnv: string): { + title: string; + helpLines: string[]; + promptLabel: string; + requiredPrefix: "nvapi-" | "sk-"; + invalidMessage: string; +} { + if (credentialEnv === NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV) { + return { + title: "Inference Hub API key required (sk-*)", + helpLines: [ + " │ For models on inference-api.nvidia.com (sk-*)", + ` │ ${NVIDIA_INFERENCE_HUB_CHAT_COMPLETIONS_URL}`, + " │ Export: NVIDIA_INFERENCE_HUB_API_KEY=sk-...", + ` │ (NVIDIA API key nvapi-* is for ${NVIDIA_INTEGRATE_API_BASE_URL})`, + ], + promptLabel: " Inference Hub API key (sk-*): ", + requiredPrefix: "sk-", + invalidMessage: " Invalid Inference Hub API key. NVIDIA_INFERENCE_HUB_API_KEY must start with sk-*", + }; + } + return { + title: "NVIDIA API key required (nvapi-*)", + helpLines: [ + ` │ For models on ${NVIDIA_INTEGRATE_API_BASE_URL} (nvapi-*)`, + " │ https://build.nvidia.com/settings/api-keys", + " │ Export: NVIDIA_API_KEY=nvapi-...", + ` │ (Inference Hub sk-* is for ${NVIDIA_INFERENCE_API_BASE_URL})`, + ], + promptLabel: " NVIDIA API key (nvapi-*): ", + requiredPrefix: "nvapi-", + invalidMessage: " Invalid NVIDIA API key. NVIDIA_API_KEY must start with nvapi-*", + }; +} + /** - * Ensure `NVIDIA_API_KEY` is staged for this process. Returns immediately - * if it is already in env, otherwise prompts interactively (validating - * the `nvapi-` prefix) and stages the result. Onboarding registers the - * value with the OpenShell gateway later in the flow. + * Resolve a staged NVIDIA Endpoints key, including legacy `NVIDIA_API_KEY=sk-...` + * when the caller asked for Inference Hub. */ -export async function ensureApiKey(): Promise { - let key = getCredential("NVIDIA_API_KEY"); +export function resolveNvidiaEndpointCredential(credentialEnv: string): string | null { + const direct = resolveProviderCredential(credentialEnv); + if (direct) return direct; + if (credentialEnv !== NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV) { + return null; + } + const legacyBuild = resolveProviderCredential(NVIDIA_BUILD_CREDENTIAL_ENV); + if (legacyBuild?.startsWith("sk-")) { + process.env[NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV] = legacyBuild; + return legacyBuild; + } + return null; +} + +/** + * Stage the NVIDIA Endpoints credential for `credentialEnv` (Build vs Inference Hub). + * Onboarding registers the value with the OpenShell gateway later in the flow. + */ +export async function ensureNvidiaEndpointCredential( + credentialEnv: string, +): Promise { + let key = resolveNvidiaEndpointCredential(credentialEnv); if (key) { - process.env.NVIDIA_API_KEY = key; + process.env[credentialEnv] = key; return { kind: "credential", value: key }; } + const spec = nvidiaEndpointCredentialPrompt(credentialEnv); console.log(""); console.log(" ┌─────────────────────────────────────────────────────────────────┐"); - console.log(" │ NVIDIA API Key required │"); + console.log(` │ ${spec.title.padEnd(63)}│`); console.log(" │ │"); - console.log(" │ 1. Go to https://build.nvidia.com/settings/api-keys │"); - console.log(" │ 2. Sign in with your NVIDIA account │"); - console.log(" │ 3. Click 'Generate API Key' button │"); - console.log(" │ 4. Paste the key below (starts with nvapi-) │"); + for (const line of spec.helpLines) { + console.log(`${line.padEnd(67)}│`); + } console.log(" └─────────────────────────────────────────────────────────────────┘"); console.log(""); while (true) { - const input = getCredentialPromptIntent(await prompt(" NVIDIA API Key: ", { secret: true })); + const input = getCredentialPromptIntent(await prompt(spec.promptLabel, { secret: true })); if (input.kind === "help") { console.log(" Type back to choose a different provider, or exit to quit."); continue; @@ -698,23 +766,50 @@ export async function ensureApiKey(): Promise { key = input.value; if (!key) { - console.error(" NVIDIA API Key is required."); + console.error( + credentialEnv === NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV + ? " Inference Hub API Key is required." + : " NVIDIA API Key is required.", + ); continue; } - if (!key.startsWith("nvapi-")) { - console.error(" Invalid NVIDIA API key. Must start with nvapi-"); + if (!key.startsWith(spec.requiredPrefix)) { + if ( + credentialEnv === NVIDIA_BUILD_CREDENTIAL_ENV && + key.startsWith("sk-") + ) { + console.error( + ` That key is for ${NVIDIA_INFERENCE_API_BASE_URL} (sk-*). ` + + "Use NVIDIA_INFERENCE_HUB_API_KEY, not NVIDIA_API_KEY.", + ); + } else if ( + credentialEnv === NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV && + key.startsWith("nvapi-") + ) { + console.error( + ` That key is for ${NVIDIA_INTEGRATE_API_BASE_URL} (nvapi-*). ` + + "Use NVIDIA_API_KEY, not NVIDIA_INFERENCE_HUB_API_KEY.", + ); + } else { + console.error(spec.invalidMessage); + } continue; } break; } - saveCredential("NVIDIA_API_KEY", key); - process.env.NVIDIA_API_KEY = key; + saveCredential(credentialEnv, key); + process.env[credentialEnv] = key; console.log(""); console.log(" Key staged for the OpenShell gateway. It is held in process memory only;"); console.log(" onboarding registers it with the gateway and nothing is written to disk."); console.log(""); return { kind: "credential", value: key }; } + +/** @deprecated Use `ensureNvidiaEndpointCredential("NVIDIA_API_KEY")` after model selection. */ +export async function ensureApiKey(): Promise { + return ensureNvidiaEndpointCredential(NVIDIA_BUILD_CREDENTIAL_ENV); +} diff --git a/src/lib/inference/config.test.ts b/src/lib/inference/config.test.ts index e6016fc7912..23a4fd04a58 100644 --- a/src/lib/inference/config.test.ts +++ b/src/lib/inference/config.test.ts @@ -18,12 +18,38 @@ import { getOpenClawPrimaryModel, getProviderSelectionConfig, getSandboxInferenceConfig, + NVIDIA_BUILD_CREDENTIAL_ENV, + NVIDIA_INFERENCE_API_BASE_URL, + NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + NVIDIA_INTEGRATE_API_BASE_URL, + NVIDIA_NEMOTRON_ULTRA_MODEL, + NVIDIA_NEMOTRON_SUPER_MODEL, parseGatewayInference, + resolveNvidiaCloudModelRoute, } from "../../../dist/lib/inference/config"; describe("inference selection config", () => { + it("routes Nemotron Ultra through Inference Hub with openai provider type", () => { + expect(resolveNvidiaCloudModelRoute(NVIDIA_NEMOTRON_ULTRA_MODEL)).toEqual({ + apiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + providerType: "openai", + keyHelpUrl: "https://inference.nvidia.com", + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }); + }); + + it("routes Nemotron Super through NVIDIA Build with nvidia provider type", () => { + expect(resolveNvidiaCloudModelRoute(NVIDIA_NEMOTRON_SUPER_MODEL)).toEqual({ + apiBaseUrl: NVIDIA_INTEGRATE_API_BASE_URL, + providerType: "nvidia", + keyHelpUrl: "https://build.nvidia.com/settings/api-keys", + credentialEnv: NVIDIA_BUILD_CREDENTIAL_ENV, + }); + }); + it("exposes the curated cloud model picker options", () => { expect(CLOUD_MODEL_OPTIONS.map((option: { id: string }) => option.id)).toEqual([ + "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1", "nvidia/nemotron-3-super-120b-a12b", "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "z-ai/glm-5.1", diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index ae2afcd28e2..ee538f471f8 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -11,7 +11,74 @@ import { DEFAULT_OLLAMA_MODEL } from "./local"; export const INFERENCE_ROUTE_URL = "https://inference.local/v1"; export const NOUS_RECOMMENDED_MODELS_URL = "https://portal.nousresearch.com/api/nous/recommended-models"; -export const DEFAULT_CLOUD_MODEL = "nvidia/nemotron-3-super-120b-a12b"; + +/** + * Nemotron Ultra upstream API. + * + * OpenAI Python/JS SDK: use NVIDIA_INFERENCE_OPENAI_BASE_URL (SDK appends `/v1`). + * curl / gateway router: use NVIDIA_INFERENCE_API_BASE_URL (`.../v1/chat/completions`). + * + * Reference implementations: + * - scripts/examples/nemotron-ultra-inference.py (sync/async, streaming recommended) + * + * curl: + * curl --location 'https://inference-api.nvidia.com/v1/chat/completions' \ + * --header 'Content-Type: application/json' \ + * --header 'Authorization: Bearer $NVIDIA_API_KEY' \ + * --data '{"model":"nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1",...}' + * + * Python (OpenAI SDK): + * from openai import OpenAI + * client = OpenAI(api_key="...", base_url="https://inference-api.nvidia.com") + * client.chat.completions.create( + * model="nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1", + * messages=[{"role": "user", "content": "Capital of United States"}], + * temperature=0.9, max_tokens=128, top_p=0.7, stream=True) + */ +export const NVIDIA_INFERENCE_OPENAI_BASE_URL = "https://inference-api.nvidia.com"; + +/** REST base URL including `/v1` (used by NemoClaw gateway registration and curl). */ +export const NVIDIA_INFERENCE_API_BASE_URL = `${NVIDIA_INFERENCE_OPENAI_BASE_URL}/v1`; + +/** NVIDIA Build / integrate API base (`NVIDIA_API_KEY`, `nvapi-*`). */ +export const NVIDIA_INTEGRATE_API_BASE_URL = "https://integrate.api.nvidia.com/v1"; + +/** + * Inference Hub chat completions URL (`NVIDIA_INFERENCE_HUB_API_KEY`, `sk-*`). + * OpenShell registration uses {@link NVIDIA_INFERENCE_API_BASE_URL} (`.../v1`). + */ +export const NVIDIA_INFERENCE_HUB_CHAT_COMPLETIONS_URL = + `${NVIDIA_INFERENCE_API_BASE_URL}/chat/completions`; + +export const NVIDIA_NEMOTRON_ULTRA_MODEL = "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1"; +export const NVIDIA_NEMOTRON_SUPER_MODEL = "nvidia/nemotron-3-super-120b-a12b"; + +export const NVIDIA_INFERENCE_HUB_KEY_HELP_URL = "https://inference.nvidia.com"; +export const NVIDIA_BUILD_KEY_HELP_URL = "https://build.nvidia.com/settings/api-keys"; + +/** + * NVIDIA API key (`nvapi-*`) for models on {@link NVIDIA_INTEGRATE_API_BASE_URL}. + */ +export const NVIDIA_BUILD_CREDENTIAL_ENV = "NVIDIA_API_KEY"; + +/** + * Inference Hub API key (`sk-*`) for models on {@link NVIDIA_INFERENCE_API_BASE_URL} + * (chat: {@link NVIDIA_INFERENCE_HUB_CHAT_COMPLETIONS_URL}). + */ +export const NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV = "NVIDIA_INFERENCE_HUB_API_KEY"; + +export interface CloudModelOption { + id: string; + label: string; + /** Upstream OpenAI-compatible API base (includes `/v1`). */ + nvidiaApiBaseUrl?: string; + keyHelpUrl?: string; + /** Env var for the API key (see NVIDIA_BUILD_CREDENTIAL_ENV / NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV). */ + credentialEnv?: string; +} + +// Default cloud model — see also nemoclaw/src/index.ts catalog entry. +export const DEFAULT_CLOUD_MODEL = NVIDIA_NEMOTRON_ULTRA_MODEL; export const HERMES_PROVIDER_MODEL_OPTIONS = [ "moonshotai/kimi-k2.6", "xiaomi/mimo-v2.5-pro", @@ -39,21 +106,110 @@ export const HERMES_PROVIDER_MODEL_OPTIONS = [ "z-ai/glm-5v-turbo", "z-ai/glm-5-turbo", "x-ai/grok-4.20-beta", - "nvidia/nemotron-3-super-120b-a12b", + NVIDIA_NEMOTRON_ULTRA_MODEL, "arcee-ai/trinity-large-thinking", "openai/gpt-5.5-pro", "openai/gpt-5.4-nano", ] as const; export const DEFAULT_HERMES_PROVIDER_MODEL = HERMES_PROVIDER_MODEL_OPTIONS[0]; -export const CLOUD_MODEL_OPTIONS = [ - { id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" }, - { id: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", label: "Nemotron 3 Nano Omni 30B" }, - { id: "z-ai/glm-5.1", label: "GLM-5" }, - { id: "minimaxai/minimax-m2.7", label: "MiniMax M2.7" }, - { id: "moonshotai/kimi-k2.6", label: "Kimi K2.6" }, - { id: "openai/gpt-oss-120b", label: "GPT-OSS 120B" }, - { id: "deepseek-ai/deepseek-v4-pro", label: "DeepSeek V4 Pro" }, +export const CLOUD_MODEL_OPTIONS: CloudModelOption[] = [ + { + id: NVIDIA_NEMOTRON_ULTRA_MODEL, + label: "Nemotron Ultra 253B (inference-api.nvidia.com)", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: NVIDIA_NEMOTRON_SUPER_MODEL, + label: "Nemotron 3 Super 120B (integrate.api.nvidia.com)", + nvidiaApiBaseUrl: NVIDIA_INTEGRATE_API_BASE_URL, + keyHelpUrl: NVIDIA_BUILD_KEY_HELP_URL, + credentialEnv: NVIDIA_BUILD_CREDENTIAL_ENV, + }, + { + id: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + label: "Nemotron 3 Nano Omni 30B", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: "z-ai/glm-5.1", + label: "GLM-5", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: "minimaxai/minimax-m2.7", + label: "MiniMax M2.7", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: "moonshotai/kimi-k2.6", + label: "Kimi K2.6", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: "openai/gpt-oss-120b", + label: "GPT-OSS 120B", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, + { + id: "deepseek-ai/deepseek-v4-pro", + label: "DeepSeek V4 Pro", + nvidiaApiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }, ]; + +export interface NvidiaCloudModelRoute { + apiBaseUrl: string; + providerType: "openai" | "nvidia"; + keyHelpUrl: string; + credentialEnv: string; +} + +/** Resolve upstream API + OpenShell provider type for NVIDIA Endpoints cloud models. */ +export function resolveNvidiaCloudModelRoute(modelId: string): NvidiaCloudModelRoute { + const curated = CLOUD_MODEL_OPTIONS.find((option) => option.id === modelId); + if (curated?.nvidiaApiBaseUrl) { + const apiBaseUrl = curated.nvidiaApiBaseUrl; + return { + apiBaseUrl, + providerType: apiBaseUrl.includes("inference-api") ? "openai" : "nvidia", + keyHelpUrl: curated.keyHelpUrl ?? NVIDIA_BUILD_KEY_HELP_URL, + credentialEnv: + curated.credentialEnv ?? + (apiBaseUrl.includes("inference-api") + ? NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV + : NVIDIA_BUILD_CREDENTIAL_ENV), + }; + } + if (modelId === NVIDIA_NEMOTRON_ULTRA_MODEL || modelId.includes("nemotron-ultra")) { + return { + apiBaseUrl: NVIDIA_INFERENCE_API_BASE_URL, + providerType: "openai", + keyHelpUrl: NVIDIA_INFERENCE_HUB_KEY_HELP_URL, + credentialEnv: NVIDIA_INFERENCE_HUB_CREDENTIAL_ENV, + }; + } + return { + apiBaseUrl: NVIDIA_INTEGRATE_API_BASE_URL, + providerType: "nvidia", + keyHelpUrl: NVIDIA_BUILD_KEY_HELP_URL, + credentialEnv: NVIDIA_BUILD_CREDENTIAL_ENV, + }; +} + export const DEFAULT_ROUTE_PROFILE = "inference-local"; export const DEFAULT_ROUTE_CREDENTIAL_ENV = "OPENAI_API_KEY"; // Dedicated credential env names for local inference. Decoupled from diff --git a/src/lib/inference/model-prompts.ts b/src/lib/inference/model-prompts.ts index b37e4d4c1ef..daebf2023fd 100644 --- a/src/lib/inference/model-prompts.ts +++ b/src/lib/inference/model-prompts.ts @@ -6,7 +6,11 @@ import { type BackToSelection, } from "../navigation"; import { isSafeModelId } from "../validation"; -import { CLOUD_MODEL_OPTIONS, HERMES_PROVIDER_MODEL_OPTIONS } from "./config"; +import { + CLOUD_MODEL_OPTIONS, + HERMES_PROVIDER_MODEL_OPTIONS, + resolveNvidiaCloudModelRoute, +} from "./config"; import { validateNvidiaEndpointModel } from "./provider-models"; // credentials.ts still uses CommonJS-style exports. @@ -43,7 +47,11 @@ export interface ModelPromptOptions { exitFn?: () => never; getNavigationChoiceFn?: (value?: string) => "back" | "exit" | null; getCredentialFn?: (envName: string) => string | null; - validateNvidiaEndpointModelFn?: (model: string, apiKey: string) => PromptValidationResult; + validateNvidiaEndpointModelFn?: ( + model: string, + apiKey: string, + options?: { buildEndpointUrl?: string }, + ) => PromptValidationResult; cloudModelOptions?: Array<{ id: string; label: string }>; remoteModelOptions?: Record; backToSelection?: BackToSelection; @@ -161,12 +169,6 @@ export async function promptCloudModel(options: ModelPromptOptions = {}): Promis return deps.cloudModelOptions[index].id; } - const nvidiaApiKey = deps.getCredentialFn("NVIDIA_API_KEY"); - if (!nvidiaApiKey) { - deps.errorLine(" NVIDIA_API_KEY is required before validating a custom NVIDIA Endpoints model."); - return deps.backToSelection; - } - // If default is a custom (non-curated) model ID, pre-fill it in the manual prompt const manualDefault = defaultCuratedIdx < 0 && defaultModelId && isSafeModelId(defaultModelId) ? defaultModelId : ""; const manualLabel = manualDefault @@ -175,7 +177,26 @@ export async function promptCloudModel(options: ModelPromptOptions = {}): Promis return promptManualModelId( manualLabel, "NVIDIA Endpoints", - (model) => deps.validateNvidiaEndpointModelFn(model, nvidiaApiKey), + (model) => { + const route = resolveNvidiaCloudModelRoute(model); + const nvidiaApiKey = + deps.getCredentialFn(route.credentialEnv) || + (route.credentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY" && + deps.getCredentialFn("NVIDIA_API_KEY")?.startsWith("sk-") + ? deps.getCredentialFn("NVIDIA_API_KEY") + : null); + if (!nvidiaApiKey) { + deps.errorLine( + route.credentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY" + ? " NVIDIA_INFERENCE_HUB_API_KEY is required before validating this model." + : " NVIDIA_API_KEY is required before validating this model.", + ); + return { ok: false, message: "missing credential" }; + } + return deps.validateNvidiaEndpointModelFn(model, nvidiaApiKey, { + buildEndpointUrl: route.apiBaseUrl, + }); + }, { ...deps, promptFn: async (q) => (await deps.promptFn(q)) || manualDefault }, ); } diff --git a/src/lib/inference/provider-models.ts b/src/lib/inference/provider-models.ts index 2d219599fa6..405188e2b54 100644 --- a/src/lib/inference/provider-models.ts +++ b/src/lib/inference/provider-models.ts @@ -8,7 +8,11 @@ import type { ModelCatalogFetchResult, ModelValidationResult } from "../onboard/ // credentials.ts still uses CommonJS-style exports. const { normalizeCredentialValue } = require("../credentials/store"); -export const BUILD_ENDPOINT_URL = "https://integrate.api.nvidia.com/v1"; +import { NVIDIA_INFERENCE_API_BASE_URL } from "./config"; + +// NVIDIA Inference API — see NVIDIA_INFERENCE_API_BASE_URL curl example in config.ts. +// Previous integrate catalog: "https://integrate.api.nvidia.com/v1" +export const BUILD_ENDPOINT_URL = NVIDIA_INFERENCE_API_BASE_URL; export interface ProviderModelOptions { runCurlProbeImpl?: (argv: string[]) => CurlProbeResult; diff --git a/src/lib/inference/web-search.ts b/src/lib/inference/web-search.ts index dd6d7682ac9..b84e6734a4c 100644 --- a/src/lib/inference/web-search.ts +++ b/src/lib/inference/web-search.ts @@ -1,8 +1,31 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +export type WebSearchProvider = "brave" | "tavily"; + export interface WebSearchConfig { fetchEnabled: boolean; + provider?: WebSearchProvider; } export const BRAVE_API_KEY_ENV = "BRAVE_API_KEY"; +export const TAVILY_API_KEY_ENV = "TAVILY_API_KEY"; +export const WEB_SEARCH_PROVIDER_ENV = "NEMOCLAW_WEB_SEARCH_PROVIDER"; + +export function resolveWebSearchProvider( + config: WebSearchConfig | null | undefined, +): WebSearchProvider | null { + if (!config?.fetchEnabled) return null; + return config.provider === "tavily" ? "tavily" : "brave"; +} + +export function webSearchPolicyPresetForProvider(provider: WebSearchProvider): string { + return provider; +} + +/** User-facing line confirming which web search backend is active. */ +export function webSearchUsageMessage(config: WebSearchConfig | null | undefined): string | null { + const provider = resolveWebSearchProvider(config); + if (!provider) return null; + return provider === "tavily" ? "Tavily Web Search is used" : "Brave Web Search is used"; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 761c3c24540..655e1997624 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -213,6 +213,7 @@ const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference, + resolveNvidiaCloudModelRoute, } = inferenceConfig; const onboardProviders = require("./onboard/providers"); @@ -295,6 +296,8 @@ const credentials: typeof import("./credentials/store") = require("./credentials const { prompt, ensureApiKey, + ensureNvidiaEndpointCredential, + resolveNvidiaEndpointCredential, getCredential, stageLegacyCredentialsToEnv, removeLegacyCredentialsFile, @@ -974,6 +977,7 @@ const { promptBraveSearchRecovery, promptBraveSearchApiKey, ensureValidatedBraveSearchCredential, + ensureValidatedTavilySearchCredential, configureWebSearch, verifyWebSearchInsideSandbox, } = createWebSearchFlowHelpers({ @@ -2926,19 +2930,45 @@ async function createSandbox( .filter(({ envKey }) => !enabledEnvKeys || enabledEnvKeys.has(envKey)) .filter(({ envKey }) => !disabledEnvKeys.has(envKey)); - const braveWebSearchEnabled = braveProviderProfile.shouldEnableBraveWebSearch(webSearchConfig); - const braveApiKey = braveWebSearchEnabled ? getCredential(webSearch.BRAVE_API_KEY_ENV) || normalizeCredentialValue(process.env[webSearch.BRAVE_API_KEY_ENV]) : null; - // Fail before any recreate/delete path runs: otherwise a missing key would - // destroy the existing sandbox first and only then surface the abort (#3626). - if (braveWebSearchEnabled && !braveApiKey) { - console.error(" Brave Search is enabled, but BRAVE_API_KEY is not available in this process."); - console.error( - " Re-run with BRAVE_API_KEY set, or disable Brave Search before recreating the sandbox.", - ); - process.exit(1); - } - if (braveWebSearchEnabled) { - messagingTokenDefs.push({ name: `${sandboxName}-brave-search`, envKey: webSearch.BRAVE_API_KEY_ENV, token: braveApiKey, providerType: braveProviderProfile.BRAVE_PROVIDER_PROFILE_ID }); + if (webSearchConfig?.fetchEnabled) { + const webSearchProvider = + webSearch.resolveWebSearchProvider(webSearchConfig) ?? "brave"; + if (webSearchProvider === "tavily") { + const tavilyApiKey = + getCredential(webSearch.TAVILY_API_KEY_ENV) || + normalizeCredentialValue(process.env[webSearch.TAVILY_API_KEY_ENV]); + // Fail before any recreate/delete path runs: otherwise a missing key would + // destroy the existing sandbox first and only then surface the abort (#3626). + if (!tavilyApiKey) { + console.error(" Tavily Search is enabled, but TAVILY_API_KEY is not available in this process."); + console.error( + " Re-run with TAVILY_API_KEY set, or disable Tavily Search before recreating the sandbox.", + ); + process.exit(1); + } + messagingTokenDefs.push({ + name: `${sandboxName}-tavily-search`, + envKey: webSearch.TAVILY_API_KEY_ENV, + token: tavilyApiKey, + }); + } else { + const braveApiKey = + getCredential(webSearch.BRAVE_API_KEY_ENV) || + normalizeCredentialValue(process.env[webSearch.BRAVE_API_KEY_ENV]); + if (!braveApiKey) { + console.error(" Brave Search is enabled, but BRAVE_API_KEY is not available in this process."); + console.error( + " Re-run with BRAVE_API_KEY set, or disable Brave Search before recreating the sandbox.", + ); + process.exit(1); + } + messagingTokenDefs.push({ + name: `${sandboxName}-brave-search`, + envKey: webSearch.BRAVE_API_KEY_ENV, + token: braveApiKey, + providerType: braveProviderProfile.BRAVE_PROVIDER_PROFILE_ID, + }); + } } const previousProviderCredentialHashes = registry.getSandbox(sandboxName)?.providerCredentialHashes ?? {}; @@ -4422,44 +4452,72 @@ async function setupNim( hydrateCredentialEnv(credentialEnv); if (selected.key === "build") { - // Allow NEMOCLAW_PROVIDER_KEY as a fallback for NVIDIA_API_KEY. - // Check raw process.env first — NEMOCLAW_PROVIDER_KEY is a user-facing - // override that should take precedence before resolving from credentials.json. + const _envModel = (process.env.NEMOCLAW_MODEL || "").trim(); + model = + requestedModel || + (recoveredFromSandbox && recoveredModel) || + (isNonInteractive() + ? DEFAULT_CLOUD_MODEL + : await promptCloudModel({ defaultModelId: _envModel || undefined })) || + DEFAULT_CLOUD_MODEL; + if (isBackToSelection(model)) { + console.log(" Returning to provider selection."); + console.log(""); + continue selectionLoop; + } + const nvidiaRoute = resolveNvidiaCloudModelRoute(model); + endpointUrl = nvidiaRoute.apiBaseUrl; + credentialEnv = nvidiaRoute.credentialEnv; + + // NEMOCLAW_PROVIDER_KEY: route by prefix when the target env is unset. const _nvProviderKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); - // check-direct-credential-env-ignore -- intentional: checking if env is already set before applying NEMOCLAW_PROVIDER_KEY override - const existingNvidiaKey = normalizeCredentialValue(process.env.NVIDIA_API_KEY ?? ""); - if (_nvProviderKey && !existingNvidiaKey) { - process.env.NVIDIA_API_KEY = _nvProviderKey; + if (_nvProviderKey) { + const existingTarget = normalizeCredentialValue(process.env[credentialEnv] ?? ""); + if (!existingTarget) { + if (_nvProviderKey.startsWith("sk-") && credentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY") { + process.env.NVIDIA_INFERENCE_HUB_API_KEY = _nvProviderKey; + } else if (_nvProviderKey.startsWith("nvapi-") && credentialEnv === "NVIDIA_API_KEY") { + process.env.NVIDIA_API_KEY = _nvProviderKey; + } else { + process.env[credentialEnv] = _nvProviderKey; + } + } } + hydrateCredentialEnv(credentialEnv); + if (isNonInteractive()) { - const resolvedNvidiaKey = resolveProviderCredential("NVIDIA_API_KEY"); + const resolvedNvidiaKey = resolveNvidiaEndpointCredential(credentialEnv); if (!resolvedNvidiaKey) { console.error( - " NVIDIA_API_KEY (or NEMOCLAW_PROVIDER_KEY) is required for NVIDIA Endpoints in non-interactive mode.", + credentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY" + ? " NVIDIA_INFERENCE_HUB_API_KEY (sk-*, inference-api.nvidia.com) is required for this model." + : " NVIDIA_API_KEY (nvapi-*, integrate.api.nvidia.com/v1) is required for this model.", + ); + console.error(" You can export both keys at once:"); + console.error( + " export NVIDIA_INFERENCE_HUB_API_KEY=sk-... # inference-api.nvidia.com/v1", + ); + console.error( + " export NVIDIA_API_KEY=nvapi-... # integrate.api.nvidia.com/v1", ); process.exit(1); } - const keyError = validateNvidiaApiKeyValue(resolvedNvidiaKey); + const keyError = validateNvidiaApiKeyValue(resolvedNvidiaKey, credentialEnv); if (keyError) { console.error(keyError); - console.error(` Get a key from ${REMOTE_PROVIDER_CONFIG.build.helpUrl}`); + console.error(` Get a key from ${nvidiaRoute.keyHelpUrl}`); process.exit(1); } } else { - await ensureApiKey(); - } - const _envModel = (process.env.NEMOCLAW_MODEL || "").trim(); - model = - requestedModel || - (recoveredFromSandbox && recoveredModel) || - (isNonInteractive() - ? DEFAULT_CLOUD_MODEL - : await promptCloudModel({ defaultModelId: _envModel || undefined })) || - DEFAULT_CLOUD_MODEL; - if (isBackToSelection(model)) { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; + const keyIntent = await ensureNvidiaEndpointCredential(credentialEnv); + if (keyIntent.kind === "back") { + console.log(" Returning to provider selection."); + console.log(""); + continue selectionLoop; + } + if (keyIntent.kind === "exit") { + exitOnboardFromPrompt(); + } } } else { // NEMOCLAW_PROVIDER_KEY is a universal alias: if the specific credential env @@ -5319,7 +5377,13 @@ async function setupInference( if (bedrockSetup.handled) return bedrockSetup.result; while (true) { const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); - const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); + let resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); + let resolvedProviderType = config.providerType; + if (provider === "nvidia-prod" && model) { + const nvidiaRoute = resolveNvidiaCloudModelRoute(model); + resolvedEndpointUrl = endpointUrl || nvidiaRoute.apiBaseUrl; + resolvedProviderType = nvidiaRoute.providerType; + } const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); const env = resolvedCredentialEnv && credentialValue @@ -5327,7 +5391,7 @@ async function setupInference( : {}; const providerResult = upsertProvider( provider, - config.providerType, + resolvedProviderType, resolvedCredentialEnv, resolvedEndpointUrl, env, @@ -6531,6 +6595,14 @@ async function onboard(opts: OnboardOptions = {}): Promise { stagedLegacyValues.clear(); migratedLegacyKeys.clear(); + const { stageSecretsEnvFile } = require("./credentials/secrets-env") as typeof import("./credentials/secrets-env"); + const stagedSecretsKeys = stageSecretsEnvFile(); + if (stagedSecretsKeys.length > 0) { + console.error( + ` Loaded ${String(stagedSecretsKeys.length)} credential(s) from ~/.nemoclaw/secrets.env`, + ); + } + const stagedLegacyKeys = stageLegacyCredentialsToEnv(); for (const key of stagedLegacyKeys) { const value = process.env[key]; @@ -6999,6 +7071,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { removeSandboxFromRegistry: registry.removeSandbox.bind(registry), repairRecordedSandbox, ensureValidatedBraveSearchCredential, + ensureValidatedTavilySearchCredential, isBackToSelection, configureWebSearch, startRecordedStep, @@ -7111,6 +7184,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { agent, hermesAuthMethod, hermesToolGateways, + webSearchConfig, stagedLegacyKeys, migratedLegacyKeys, webSearchEnabled: braveProviderProfile.shouldEnableBraveWebSearch(webSearchConfig), diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 63ab72c9e90..c82eaf761e6 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -6,6 +6,8 @@ import os from "node:os"; import path from "node:path"; import type { AgentDefinition } from "../agent/defs"; +import type { WebSearchConfig } from "../inference/web-search"; +import { webSearchUsageMessage } from "../inference/web-search"; import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls } from "../dashboard/contract"; import * as nim from "../inference/nim"; @@ -91,6 +93,7 @@ export interface OnboardDashboardHelpers { provider: string, nimContainer?: string | null, agent?: AgentDefinition | null, + webSearchConfig?: WebSearchConfig | null, ): void; stopAllDashboardForwards(): void; } @@ -353,6 +356,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa provider: string, nimContainer: string | null = null, agent: AgentDefinition | null = null, + webSearchConfig: WebSearchConfig | null = null, ): void { const nimStatus = deps.nimStatus ?? nim.nimStatus; const nimStatusByName = deps.nimStatusByName ?? nim.nimStatusByName; @@ -380,6 +384,10 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa console.log(""); console.log(` Sandbox: ${sandboxName}`); console.log(` Model: ${model} (${providerLabel})`); + const webSearchLine = webSearchUsageMessage(webSearchConfig); + if (webSearchLine) { + console.log(` Search: ${webSearchLine}`); + } if (showNim) { console.log(` NIM: ${nimLabel}`); } diff --git a/src/lib/onboard/dockerfile-patch.ts b/src/lib/onboard/dockerfile-patch.ts index 5f70c317b5f..63d1c1ec16f 100644 --- a/src/lib/onboard/dockerfile-patch.ts +++ b/src/lib/onboard/dockerfile-patch.ts @@ -198,6 +198,20 @@ export function patchStagedDockerfile( /^ARG NEMOCLAW_WEB_SEARCH_ENABLED=.*$/m, `ARG NEMOCLAW_WEB_SEARCH_ENABLED=${sanitizeDockerArg(webSearchConfig ? "1" : "0")}`, ); + const webSearchProvider = + webSearchConfig?.provider === "tavily" ? "tavily" : "brave"; + if (/^ARG NEMOCLAW_WEB_SEARCH_PROVIDER=.*$/m.test(dockerfile)) { + dockerfile = dockerfile.replace( + /^ARG NEMOCLAW_WEB_SEARCH_PROVIDER=.*$/m, + `ARG NEMOCLAW_WEB_SEARCH_PROVIDER=${sanitizeDockerArg(webSearchProvider)}`, + ); + } else { + dockerfile = dockerfile.replace( + /^ARG NEMOCLAW_WEB_SEARCH_ENABLED=.*$/m, + (line) => + `${line}\nARG NEMOCLAW_WEB_SEARCH_PROVIDER=${sanitizeDockerArg(webSearchProvider)}`, + ); + } // Onboard flow expects immediate dashboard access without device pairing, // so disable device auth for images built during onboard (see #1217). dockerfile = dockerfile.replace( diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index 34e2dba2245..ba3f13321bc 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { WebSearchConfig } from "../../../inference/web-search"; import type { Session, SessionUpdates } from "../../../state/onboard-session"; export interface FinalizationStateOptions { @@ -11,6 +12,7 @@ export interface FinalizationStateOptions; webSearchEnabled: boolean; @@ -39,6 +41,7 @@ export interface FinalizationStateOptions({ const recovery = await deps.ensureResumeProviderReady(provider, credentialEnv); forceInferenceSetup = recovery.forceInferenceSetup; credentialEnv = recovery.credentialEnv; + if (provider === "nvidia-prod" && model) { + const nvidiaRoute = resolveNvidiaCloudModelRoute(model); + credentialEnv = nvidiaRoute.credentialEnv; + endpointUrl = endpointUrl || nvidiaRoute.apiBaseUrl; + } deps.skippedStepMessage("provider_selection", `${provider} / ${model}`); await deps.recordStateSkipped("provider_selection", { reason: "resume", diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index efa5cf0adb2..491eabdcde6 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -42,6 +42,7 @@ export interface SandboxStateOptions; + ensureValidatedTavilySearchCredential(): Promise; isBackToSelection(value: unknown): boolean; configureWebSearch( existingConfig: WebSearchConfig | null, @@ -243,14 +244,25 @@ export async function handleSandboxState | null; + } = {}, +): boolean { + if (name !== "brave" && name !== "tavily") return false; + if (options.customPresetNames?.has(name)) return false; + if (!options.webSearchConfig?.fetchEnabled) return true; + const provider = resolveWebSearchProvider(options.webSearchConfig) ?? "brave"; + return name !== webSearchPolicyPresetForProvider(provider); +} + +/** @deprecated Use isStaleBuiltinWebSearchPolicyPreset */ export function isStaleBuiltinBravePolicyPreset( name: string, options: { @@ -114,11 +130,7 @@ export function isStaleBuiltinBravePolicyPreset( customPresetNames?: ReadonlySet | null; } = {}, ): boolean { - return ( - name === "brave" && - !options.webSearchConfig && - !options.customPresetNames?.has(name) - ); + return isStaleBuiltinWebSearchPolicyPreset(name, options); } export function computeSetupPresetSuggestions( @@ -141,7 +153,7 @@ export function computeSetupPresetSuggestions( const suggestions = deps.tiers .resolveTierPresets(tierName) .map((preset) => preset.name) - .filter((name) => !isStaleBuiltinBravePolicyPreset(name, { webSearchConfig })) + .filter((name) => !isStaleBuiltinWebSearchPolicyPreset(name, { webSearchConfig })) .filter((name) => deps.policies.setupPolicyPresetSupported(name, supportOptions)) .filter((name) => !known || known.has(name)); const add = (name: string) => { @@ -150,7 +162,8 @@ export function computeSetupPresetSuggestions( if (known && !known.has(name)) return; suggestions.push(name); }; - if (webSearchConfig) add("brave"); + const webSearchProvider = resolveWebSearchProvider(webSearchConfig); + if (webSearchProvider) add(webSearchPolicyPresetForProvider(webSearchProvider)); if (provider && deps.localInferenceProviders.includes(provider)) add("local-inference"); if (agent === "openclaw") add("openclaw-pricing"); if (Array.isArray(enabledChannels)) { @@ -193,7 +206,7 @@ export function preparePolicyPresetResumeSelection( customPolicyPresetNames, ); const isStaleBuiltinBrave = (name: string) => - isStaleBuiltinBravePolicyPreset(name, { + isStaleBuiltinWebSearchPolicyPreset(name, { webSearchConfig: options.webSearchConfig, customPresetNames: customPolicyPresetNames, }); @@ -284,7 +297,7 @@ async function setupPoliciesWithSelectionInner( customPresetNames, ); const isStaleBuiltinBrave = (name: string) => - isStaleBuiltinBravePolicyPreset(name, { webSearchConfig, customPresetNames }); + isStaleBuiltinWebSearchPolicyPreset(name, { webSearchConfig, customPresetNames }); const appliedForPreservation = pruneDisabledMessagingPolicyPresets( applied, disabledChannels, diff --git a/src/lib/onboard/providers.test.ts b/src/lib/onboard/providers.test.ts index 1d52faab799..38601d5d25e 100644 --- a/src/lib/onboard/providers.test.ts +++ b/src/lib/onboard/providers.test.ts @@ -153,7 +153,7 @@ describe("onboard provider helpers", () => { "inference", "openai", "NVIDIA_API_KEY", - "https://integrate.api.nvidia.com/v1", + "https://inference-api.nvidia.com/v1", {}, (command) => { commands.push(command.join(" ")); @@ -165,7 +165,7 @@ describe("onboard provider helpers", () => { expect(commands).toHaveLength(2); expect(commands[0]).toMatch(/provider get/); expect(commands[1]).toMatch(/provider update/); - expect(commands[1]).toMatch(/--config OPENAI_BASE_URL=https:\/\/integrate\.api\.nvidia\.com\/v1/); + expect(commands[1]).toMatch(/--config OPENAI_BASE_URL=https:\/\/inference-api\.nvidia\.com\/v1/); }); it("returns redacted error details when create or update fails", () => { diff --git a/src/lib/onboard/providers.ts b/src/lib/onboard/providers.ts index 30f4192e40f..a49745b5c71 100644 --- a/src/lib/onboard/providers.ts +++ b/src/lib/onboard/providers.ts @@ -17,7 +17,9 @@ const { compactText } = require("../core/url-utils"); // ── Constants ──────────────────────────────────────────────────── -const BUILD_ENDPOINT_URL = "https://integrate.api.nvidia.com/v1"; +const { NVIDIA_INFERENCE_API_BASE_URL } = require("../inference/config"); +// NVIDIA Inference API — matches Nemotron Ultra curl in config.ts +const BUILD_ENDPOINT_URL = NVIDIA_INFERENCE_API_BASE_URL; const OPENAI_ENDPOINT_URL = "https://api.openai.com/v1"; const ANTHROPIC_ENDPOINT_URL = "https://api.anthropic.com"; const GEMINI_ENDPOINT_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"; diff --git a/src/lib/onboard/summary.test.ts b/src/lib/onboard/summary.test.ts index 5858e1d92e6..e031c6c7f62 100644 --- a/src/lib/onboard/summary.test.ts +++ b/src/lib/onboard/summary.test.ts @@ -13,7 +13,7 @@ describe("onboard summary helpers", () => { provider: "gemini-api", model: "gemini-2.5-flash", credentialEnv: "GEMINI_API_KEY", - webSearchConfig: { fetchEnabled: true }, + webSearchConfig: { fetchEnabled: true, provider: "brave" }, enabledChannels: ["telegram", "slack"], sandboxName: "my-assistant", notes: ["Sandbox build typically takes 5–15 minutes on this host."], @@ -26,7 +26,7 @@ describe("onboard summary helpers", () => { summary.includes("configured for OpenShell gateway registration"), "summary shows API key staging state without printing env var names", ); - assert.ok(summary.includes("enabled"), "summary includes web-search enabled"); + assert.ok(summary.includes("enabled (brave)"), "summary includes web-search provider"); assert.ok(summary.includes("telegram, slack"), "summary lists enabled channels"); assert.ok(summary.includes("my-assistant"), "summary shows sandbox name"); assert.ok( diff --git a/src/lib/onboard/summary.ts b/src/lib/onboard/summary.ts index ef56fc8a6ce..295b8a6aa35 100644 --- a/src/lib/onboard/summary.ts +++ b/src/lib/onboard/summary.ts @@ -7,6 +7,7 @@ import { type HermesAuthMethod, } from "../hermes-provider-auth"; import type { WebSearchConfig } from "../inference/web-search"; +import { resolveWebSearchProvider } from "../inference/web-search"; import { hermesToolGatewayLabels } from "./hermes-managed-tools"; const HERMES_AUTH_METHOD_OAUTH: HermesAuthMethod = "oauth"; @@ -86,8 +87,12 @@ export function formatOnboardConfigSummary({ Array.isArray(enabledChannels) && enabledChannels.length > 0 ? enabledChannels.join(", ") : "none"; - const webSearch = - webSearchConfig && webSearchConfig.fetchEnabled === true ? "enabled" : "disabled"; + const webSearchProvider = resolveWebSearchProvider(webSearchConfig); + const webSearch = webSearchProvider + ? `enabled (${webSearchProvider})` + : webSearchConfig && webSearchConfig.fetchEnabled === true + ? "enabled" + : "disabled"; const effectiveHermesAuthMethod = normalizeHermesAuthMethod(hermesAuthMethod) || (provider === HERMES_PROVIDER_NAME && credentialEnv === HERMES_NOUS_API_KEY_CREDENTIAL_ENV diff --git a/src/lib/onboard/web-search-flow.ts b/src/lib/onboard/web-search-flow.ts index 3aaa002d5f8..7da69cbde97 100644 --- a/src/lib/onboard/web-search-flow.ts +++ b/src/lib/onboard/web-search-flow.ts @@ -5,8 +5,13 @@ import type { CurlProbeResult } from "../adapters/http/probe"; import { runCurlProbe } from "../adapters/http/probe"; import type { AgentDefinition } from "../agent/defs"; import { getCredential, normalizeCredentialValue, saveCredential } from "../credentials/store"; -import type { WebSearchConfig } from "../inference/web-search"; -import { BRAVE_API_KEY_ENV } from "../inference/web-search"; +import type { WebSearchConfig, WebSearchProvider } from "../inference/web-search"; +import { + BRAVE_API_KEY_ENV, + TAVILY_API_KEY_ENV, + WEB_SEARCH_PROVIDER_ENV, + webSearchUsageMessage, +} from "../inference/web-search"; import { ROOT } from "../runner"; import { classifyValidationFailure } from "../validation"; import { getTransportRecoveryMessage } from "../validation-recovery"; @@ -17,6 +22,7 @@ import { agentSupportsWebSearch } from "./web-search-support"; import { verifyWebSearchInsideSandbox as verifyWebSearchInsideSandboxWithDeps } from "./web-search-verify"; const BRAVE_SEARCH_HELP_URL = "https://brave.com/search/api/"; +const TAVILY_SEARCH_HELP_URL = "https://tavily.com"; export interface WebSearchFlowDeps { prompt(question: string, options?: { secret?: boolean }): Promise; @@ -28,9 +34,13 @@ export interface WebSearchFlowDeps { export interface WebSearchFlowHelpers { validateBraveSearchApiKey(apiKey: string): CurlProbeResult; + validateTavilySearchApiKey(apiKey: string): CurlProbeResult; promptBraveSearchRecovery(validation: ValidationFailureLike): Promise<"retry" | "skip">; + promptTavilySearchRecovery(validation: ValidationFailureLike): Promise<"retry" | "skip">; promptBraveSearchApiKey(): Promise; + promptTavilySearchApiKey(): Promise; ensureValidatedBraveSearchCredential(nonInteractive?: boolean): Promise; + ensureValidatedTavilySearchCredential(nonInteractive?: boolean): Promise; configureWebSearch( existingConfig?: WebSearchConfig | null, agent?: AgentDefinition | null, @@ -42,6 +52,21 @@ export interface WebSearchFlowHelpers { ): void; } +function resolveNonInteractiveWebSearchProvider(): WebSearchProvider | null { + const explicit = normalizeCredentialValue(process.env[WEB_SEARCH_PROVIDER_ENV]).toLowerCase(); + if (explicit === "tavily" || explicit === "brave") { + return explicit; + } + const tavilyKey = + getCredential(TAVILY_API_KEY_ENV) || normalizeCredentialValue(process.env[TAVILY_API_KEY_ENV]); + const braveKey = + getCredential(BRAVE_API_KEY_ENV) || normalizeCredentialValue(process.env[BRAVE_API_KEY_ENV]); + if (tavilyKey && !braveKey) return "tavily"; + if (braveKey) return "brave"; + if (tavilyKey) return "tavily"; + return null; +} + export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFlowHelpers { function validateBraveSearchApiKey(apiKey: string): CurlProbeResult { return runCurlProbe([ @@ -62,6 +87,17 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl ]); } + function validateTavilySearchApiKey(apiKey: string): CurlProbeResult { + return runCurlProbe([ + "-sS", + "-H", + "Content-Type: application/json", + "-d", + JSON.stringify({ api_key: apiKey, query: "ping", max_results: 1 }), + "https://api.tavily.com/search", + ]); + } + async function promptBraveSearchRecovery( validation: ValidationFailureLike, ): Promise<"retry" | "skip"> { @@ -83,6 +119,27 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl return "retry"; } + async function promptTavilySearchRecovery( + validation: ValidationFailureLike, + ): Promise<"retry" | "skip"> { + const recovery = classifyValidationFailure(validation); + + if (recovery.kind === "credential") { + console.log(" Tavily Search rejected that API key."); + } else if (recovery.kind === "transport") { + console.log(getTransportRecoveryMessage(validation)); + } else { + console.log(" Tavily Search validation did not succeed."); + } + + const answer = (await deps.prompt(" Type 'retry', 'skip', or 'exit' [retry]: ")).trim().toLowerCase(); + if (answer === "skip") return "skip"; + if (answer === "exit" || answer === "quit") { + exitOnboardFromPrompt(); + } + return "retry"; + } + async function promptBraveSearchApiKey(): Promise { console.log(""); console.log(` Get your Brave Search API key from: ${BRAVE_SEARCH_HELP_URL}`); @@ -108,6 +165,31 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl } } + async function promptTavilySearchApiKey(): Promise { + console.log(""); + console.log(` Get your Tavily Search API key from: ${TAVILY_SEARCH_HELP_URL}`); + console.log(""); + + while (true) { + const value = await deps.prompt(" Tavily Search API key: ", { secret: true }); + const intent = normalizeCredentialValue(value).toLowerCase(); + if (intent === "back") return BACK_TO_SELECTION; + if (intent === "exit" || intent === "quit") { + exitOnboardFromPrompt(); + } + if (intent === "?" || intent === "help") { + console.log(" Type back to choose again, or exit to quit."); + continue; + } + const key = normalizeCredentialValue(value); + if (!key) { + console.error(" Tavily Search API key is required."); + continue; + } + return key; + } + } + async function ensureValidatedBraveSearchCredential( nonInteractive = deps.isNonInteractive(), ): Promise { @@ -164,6 +246,81 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl } } + async function ensureValidatedTavilySearchCredential( + nonInteractive = deps.isNonInteractive(), + ): Promise { + const savedApiKey = getCredential(TAVILY_API_KEY_ENV); + let apiKey: string | null = + savedApiKey || normalizeCredentialValue(process.env[TAVILY_API_KEY_ENV]); + let usingSavedKey = Boolean(savedApiKey); + + while (true) { + if (!apiKey) { + if (nonInteractive) { + throw new Error( + "Tavily Search requires TAVILY_API_KEY or a saved Tavily Search credential in non-interactive mode.", + ); + } + const promptedApiKey = await promptTavilySearchApiKey(); + if (isBackToSelection(promptedApiKey)) { + return promptedApiKey; + } + apiKey = promptedApiKey; + usingSavedKey = false; + } + + const validation = validateTavilySearchApiKey(apiKey); + if (validation.ok) { + saveCredential(TAVILY_API_KEY_ENV, apiKey); + process.env[TAVILY_API_KEY_ENV] = apiKey; + return apiKey; + } + + const prefix = usingSavedKey + ? " Saved Tavily Search API key validation failed." + : " Tavily Search API key validation failed."; + console.error(prefix); + if (validation.message) { + console.error(` ${validation.message}`); + } + + if (nonInteractive) { + throw new Error( + validation.message || "Tavily Search API key validation failed in non-interactive mode.", + ); + } + + const action = await promptTavilySearchRecovery(validation); + if (action === "skip") { + console.log(" Skipping Tavily Web Search setup."); + console.log(""); + return null; + } + + apiKey = null; + usingSavedKey = false; + } + } + + async function promptWebSearchProvider(): Promise { + console.log(""); + console.log(" Web search provider:"); + console.log(" 1) Brave Search"); + console.log(" 2) Tavily Search"); + console.log(""); + + while (true) { + const answer = (await deps.prompt(" Choose provider [1]: ")).trim().toLowerCase(); + if (answer === "" || answer === "1" || answer === "brave") return "brave"; + if (answer === "2" || answer === "tavily") return "tavily"; + if (answer === "back") return BACK_TO_SELECTION; + if (answer === "exit" || answer === "quit") { + exitOnboardFromPrompt(); + } + console.error(" Enter 1 for Brave, 2 for Tavily, back, or exit."); + } + } + async function configureWebSearch( existingConfig: WebSearchConfig | null = null, agent: AgentDefinition | null = null, @@ -174,47 +331,114 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl return null; } - if (existingConfig) { - return { fetchEnabled: true }; + if (existingConfig?.fetchEnabled) { + return { + fetchEnabled: true, + provider: existingConfig.provider === "tavily" ? "tavily" : "brave", + }; } if (deps.isNonInteractive()) { - const braveApiKey = - getCredential(BRAVE_API_KEY_ENV) || normalizeCredentialValue(process.env[BRAVE_API_KEY_ENV]); - if (!braveApiKey) { + const provider = resolveNonInteractiveWebSearchProvider(); + if (!provider) { + return null; + } + + if (provider === "brave") { + const braveApiKey = + getCredential(BRAVE_API_KEY_ENV) || + normalizeCredentialValue(process.env[BRAVE_API_KEY_ENV]); + if (!braveApiKey) { + return null; + } + deps.note(" [non-interactive] Brave Web Search requested."); + const validation = validateBraveSearchApiKey(braveApiKey); + if (!validation.ok) { + console.warn( + ` Brave Search API key validation failed. Web search will be disabled — re-enable later via \`${deps.cliName()} config web-search\`.`, + ); + if (validation.message) { + console.warn(` ${validation.message}`); + } + return null; + } + saveCredential(BRAVE_API_KEY_ENV, braveApiKey); + process.env[BRAVE_API_KEY_ENV] = braveApiKey; + process.env[WEB_SEARCH_PROVIDER_ENV] = "brave"; + const usageMsg = webSearchUsageMessage({ fetchEnabled: true, provider: "brave" }); + if (usageMsg) { + console.log(` ${usageMsg}`); + } + return { fetchEnabled: true, provider: "brave" }; + } + + const tavilyApiKey = + getCredential(TAVILY_API_KEY_ENV) || + normalizeCredentialValue(process.env[TAVILY_API_KEY_ENV]); + if (!tavilyApiKey) { return null; } - deps.note(" [non-interactive] Brave Web Search requested."); - const validation = validateBraveSearchApiKey(braveApiKey); + deps.note(" [non-interactive] Tavily Web Search requested."); + const validation = validateTavilySearchApiKey(tavilyApiKey); if (!validation.ok) { console.warn( - ` Brave Search API key validation failed. Web search will be disabled — re-enable later via \`${deps.cliName()} config web-search\`.`, + ` Tavily Search API key validation failed. Web search will be disabled — re-enable later via \`${deps.cliName()} config web-search\`.`, ); if (validation.message) { console.warn(` ${validation.message}`); } return null; } - saveCredential(BRAVE_API_KEY_ENV, braveApiKey); - process.env[BRAVE_API_KEY_ENV] = braveApiKey; - return { fetchEnabled: true }; + saveCredential(TAVILY_API_KEY_ENV, tavilyApiKey); + process.env[TAVILY_API_KEY_ENV] = tavilyApiKey; + process.env[WEB_SEARCH_PROVIDER_ENV] = "tavily"; + const usageMsg = webSearchUsageMessage({ fetchEnabled: true, provider: "tavily" }); + if (usageMsg) { + console.log(` ${usageMsg}`); + } + return { fetchEnabled: true, provider: "tavily" }; } - const enableAnswer = await deps.prompt(" Enable Brave Web Search? [y/N]: "); + + const enableAnswer = await deps.prompt(" Enable web search? [y/N]: "); if (!isAffirmativeAnswer(enableAnswer)) { return null; } - const braveApiKey = await ensureValidatedBraveSearchCredential(); - if (isBackToSelection(braveApiKey)) { + let provider = await promptWebSearchProvider(); + if (isBackToSelection(provider)) { return configureWebSearch(existingConfig, agent, dockerfilePathOverride); } - if (!braveApiKey) { - return null; - } - console.log(" ✓ Enabled Brave Web Search"); - console.log(""); - return { fetchEnabled: true }; + while (true) { + const apiKey = + provider === "tavily" + ? await ensureValidatedTavilySearchCredential() + : await ensureValidatedBraveSearchCredential(); + + if (isBackToSelection(apiKey)) { + provider = await promptWebSearchProvider(); + if (isBackToSelection(provider)) { + return configureWebSearch(existingConfig, agent, dockerfilePathOverride); + } + continue; + } + if (!apiKey) { + return null; + } + + process.env[WEB_SEARCH_PROVIDER_ENV] = provider; + if (provider === "tavily") { + console.log(" ✓ Enabled Tavily Web Search"); + } else { + console.log(" ✓ Enabled Brave Web Search"); + } + const usageMsg = webSearchUsageMessage({ fetchEnabled: true, provider }); + if (usageMsg) { + console.log(` ${usageMsg}`); + } + console.log(""); + return { fetchEnabled: true, provider }; + } } function verifyWebSearchInsideSandbox( @@ -229,9 +453,13 @@ export function createWebSearchFlowHelpers(deps: WebSearchFlowDeps): WebSearchFl return { validateBraveSearchApiKey, + validateTavilySearchApiKey, promptBraveSearchRecovery, + promptTavilySearchRecovery, promptBraveSearchApiKey, + promptTavilySearchApiKey, ensureValidatedBraveSearchCredential, + ensureValidatedTavilySearchCredential, configureWebSearch, verifyWebSearchInsideSandbox, }; diff --git a/src/lib/onboard/web-search-verify.test.ts b/src/lib/onboard/web-search-verify.test.ts index 949dc1fa292..186c2ef4a76 100644 --- a/src/lib/onboard/web-search-verify.test.ts +++ b/src/lib/onboard/web-search-verify.test.ts @@ -120,6 +120,18 @@ describe("verifyWebSearchInsideSandbox", () => { ); }); + it("reports Tavily when OpenClaw config uses tavily provider", () => { + const d = deps( + JSON.stringify({ tools: { web: { search: { enabled: true, provider: "tavily" } } } }), + ); + + verifyWebSearchInsideSandbox("alpha", { name: "openclaw" }, d); + + expect(d.log).toHaveBeenCalledWith(" ✓ Web search is active inside sandbox"); + expect(d.log).toHaveBeenCalledWith(" ✓ Tavily Web Search is used"); + expect(d.runCaptureOpenshell).toHaveBeenCalledTimes(1); + }); + it("warns when OpenClaw config is malformed or disabled", () => { const malformed = deps("not-json"); verifyWebSearchInsideSandbox("alpha", { name: "openclaw" }, malformed); diff --git a/src/lib/onboard/web-search-verify.ts b/src/lib/onboard/web-search-verify.ts index bde766708f4..17fea853704 100644 --- a/src/lib/onboard/web-search-verify.ts +++ b/src/lib/onboard/web-search-verify.ts @@ -7,6 +7,8 @@ export type WebSearchVerifyAgent = { name?: string | null; } | null | undefined; +import { webSearchUsageMessage } from "../inference/web-search"; + export type WebSearchVerifyDeps = { runCaptureOpenshell: (args: string[], options: { ignoreError: true; timeout: number }) => string | null; cliName: () => string; @@ -65,8 +67,6 @@ export function verifyWebSearchInsideSandbox( const agentName = agent?.name || "openclaw"; try { if (agentName === "hermes") { - // `hermes dump` outputs config_overrides and active toolsets. - // Look for the web backend in its output. const dump = deps.runCaptureOpenshell( ["sandbox", "exec", "-n", sandboxName, "--", "hermes", "dump"], { @@ -78,8 +78,6 @@ export function verifyWebSearchInsideSandbox( warn(" ⚠ Could not verify web search config inside sandbox (hermes dump failed)."); return; } - // A working web backend shows as an explicit config override or active-toolset entry. - // Avoid broad /web.*search/ matching so warning text never looks like success. const hasWebBackend = /^\s*web\.backend:\s*\S+/m.test(dump) || /^\s*active toolsets:\s*.*\bweb\b/im.test(dump) || @@ -111,6 +109,13 @@ export function verifyWebSearchInsideSandbox( } if (search.provider !== "brave") { log(" ✓ Web search is active inside sandbox"); + const usageMsg = webSearchUsageMessage({ + fetchEnabled: true, + provider: search.provider === "tavily" ? "tavily" : "brave", + }); + if (usageMsg) { + log(` ✓ ${usageMsg}`); + } return; } if (typeof search.apiKey !== "string" || search.apiKey.trim() === "") { @@ -169,7 +174,6 @@ export function verifyWebSearchInsideSandbox( warn(` ⚠ Web search verification is not implemented for agent '${agentName}'.`); } } catch { - // Best-effort — don't let probe failures derail onboarding. warn(" ⚠ Web search verification probe failed (non-fatal)."); } } diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index e7218398dee..a7571fc614b 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -257,7 +257,7 @@ function setupPolicyPresetSupported( name: string, options: SetupPolicyPresetSupportOptions = {}, ): boolean { - return name !== "brave" || options.webSearchSupported !== false; + return (name !== "brave" && name !== "tavily") || options.webSearchSupported !== false; } function filterSetupPolicyPresets( diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 26cbf083539..15766359d19 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -283,7 +283,11 @@ function readStepStatus(value: SessionJsonValue | undefined): StepStatus | null } function parseWebSearchConfig(value: SessionJsonValue | undefined): WebSearchConfig | null { - return isObject(value) && value.fetchEnabled === true ? { fetchEnabled: true } : null; + if (!isObject(value) || value.fetchEnabled !== true) return null; + return { + fetchEnabled: true, + provider: value.provider === "tavily" ? "tavily" : "brave", + }; } function parseTelegramConfig(value: unknown): TelegramConfig | null { @@ -477,7 +481,12 @@ export function createSession(overrides: Partial = {}): Session { routerPid: readPositiveInteger(overrides.routerPid), routerCredentialHash: overrides.routerCredentialHash ?? null, webSearchConfig: - overrides.webSearchConfig?.fetchEnabled === true ? { fetchEnabled: true } : null, + overrides.webSearchConfig?.fetchEnabled === true + ? { + fetchEnabled: true, + provider: overrides.webSearchConfig.provider === "tavily" ? "tavily" : "brave", + } + : null, hermesToolGateways: readStringArray(overrides.hermesToolGateways), policyPresets: readStringArray(overrides.policyPresets), messagingChannels: readStringArray(overrides.messagingChannels), @@ -932,7 +941,10 @@ export function filterSafeUpdates(updates: SessionUpdates): Partial { safe.routerCredentialHash = updates.routerCredentialHash; } if (isObject(updates.webSearchConfig) && updates.webSearchConfig.fetchEnabled === true) { - safe.webSearchConfig = { fetchEnabled: true }; + safe.webSearchConfig = { + fetchEnabled: true, + provider: updates.webSearchConfig.provider === "tavily" ? "tavily" : "brave", + }; } else if (updates.webSearchConfig === null) { safe.webSearchConfig = null; } diff --git a/src/lib/validation.test.ts b/src/lib/validation.test.ts index 7ae368d3924..abcf3e74cd8 100644 --- a/src/lib/validation.test.ts +++ b/src/lib/validation.test.ts @@ -269,8 +269,24 @@ describe("validateNvidiaApiKeyValue", () => { expect(validateNvidiaApiKeyValue("")).toBeTruthy(); }); - it("rejects key without nvapi- prefix", () => { - expect(validateNvidiaApiKeyValue("sk-abc123")).toBeTruthy(); + it("rejects Inference Hub sk- keys on NVIDIA_API_KEY (Build only)", () => { + const error = validateNvidiaApiKeyValue("sk-abc123"); + expect(error).toBeTruthy(); + expect(error).toContain("NVIDIA_INFERENCE_HUB_API_KEY"); + }); + + it("rejects nvapi- keys on NVIDIA_INFERENCE_HUB_API_KEY", () => { + const error = validateNvidiaApiKeyValue("nvapi-abc123", "NVIDIA_INFERENCE_HUB_API_KEY"); + expect(error).toBeTruthy(); + expect(error).toContain("NVIDIA_API_KEY"); + }); + + it("accepts Inference Hub sk- keys on NVIDIA_INFERENCE_HUB_API_KEY", () => { + expect(validateNvidiaApiKeyValue("sk-abc123", "NVIDIA_INFERENCE_HUB_API_KEY")).toBeNull(); + }); + + it("rejects keys without nvapi- prefix for NVIDIA_API_KEY", () => { + expect(validateNvidiaApiKeyValue("bad-key")).toBeTruthy(); }); it("accepts non-nvapi keys when credentialEnv is not NVIDIA_API_KEY", () => { diff --git a/src/lib/validation.ts b/src/lib/validation.ts index 8fb2fb2f493..d70744ac2aa 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -94,12 +94,41 @@ export function validateNvidiaApiKeyValue( // The nvapi- prefix check is specific to NVIDIA keys; skip it for keys // from other providers (e.g. ANTHROPIC_API_KEY, OPENAI_API_KEY) so that // a valid Anthropic key is not rejected with an NVIDIA-specific error. - const isNvidia = credentialEnv === "NVIDIA_API_KEY"; + const isNvidiaBuild = credentialEnv === "NVIDIA_API_KEY"; + const isInferenceHub = credentialEnv === "NVIDIA_INFERENCE_HUB_API_KEY"; + const isNvidia = isNvidiaBuild || isInferenceHub; if (!key) { - return isNvidia ? " NVIDIA API Key is required." : " API Key is required."; + if (isInferenceHub) { + return " Inference Hub API key is required (export NVIDIA_INFERENCE_HUB_API_KEY=sk-...)."; + } + if (isNvidiaBuild) { + return " NVIDIA API key is required (export NVIDIA_API_KEY=nvapi-...)."; + } + return " API Key is required."; } - if (isNvidia && !key.startsWith("nvapi-")) { - return " Invalid NVIDIA API key. Must start with nvapi-"; + if (isNvidiaBuild && !key.startsWith("nvapi-")) { + if (key.startsWith("sk-")) { + return ( + " That key is for inference-api.nvidia.com (sk-*). " + + "Use NVIDIA_INFERENCE_HUB_API_KEY, not NVIDIA_API_KEY." + ); + } + return ( + " Invalid NVIDIA API key. NVIDIA_API_KEY must start with nvapi-* " + + "(for integrate.api.nvidia.com/v1)." + ); + } + if (isInferenceHub && !key.startsWith("sk-")) { + if (key.startsWith("nvapi-")) { + return ( + " That key is for integrate.api.nvidia.com/v1 (nvapi-*). " + + "Use NVIDIA_API_KEY, not NVIDIA_INFERENCE_HUB_API_KEY." + ); + } + return ( + " Invalid Inference Hub API key. NVIDIA_INFERENCE_HUB_API_KEY must start with sk-* " + + "(for inference-api.nvidia.com/v1)." + ); } return null; } diff --git a/test/credentials.test.ts b/test/credentials.test.ts index 32449bd4bf3..8d761e9dc6d 100644 --- a/test/credentials.test.ts +++ b/test/credentials.test.ts @@ -798,7 +798,7 @@ ${JSON.stringify(process.execPath)} ${JSON.stringify(scriptFile)} < "$pipe" } expect(result.status).toBe(0); expect(`${result.stdout}${result.stderr}`).toContain( - "Invalid NVIDIA API key. Must start with nvapi-", + "Invalid NVIDIA API key. Must start with nvapi", ); expect(result.stdout).toContain("STAGED=nvapi-good-key"); }); diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index b2a2b0cfab2..f417051eddd 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -634,6 +634,25 @@ describe("generate-openclaw-config.py: config generation", () => { }); }); + it("enables Tavily web search when provider env is tavily", () => { + const config = runConfigScript({ + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + }); + expect(config.tools?.web?.search).toEqual({ + enabled: true, + provider: "tavily", + }); + expect(config.plugins?.entries?.tavily).toEqual({ + enabled: true, + config: { + webSearch: { + apiKey: "openshell:resolve:env:TAVILY_API_KEY", + }, + }, + }); + }); + it("omits web search when env is not set", () => { const config = runConfigScript(); expect(config.tools?.toolSearch).toBe(true); diff --git a/test/onboard-brave-validation.test.ts b/test/onboard-brave-validation.test.ts index 054c532e644..78f538d54eb 100644 --- a/test/onboard-brave-validation.test.ts +++ b/test/onboard-brave-validation.test.ts @@ -363,7 +363,7 @@ const { configureWebSearch } = require(${onboardPath}); }); expect(result.status).toBe(0); const payload = JSON.parse(fs.readFileSync(outputPath, "utf-8")); - expect(payload.result).toEqual({ fetchEnabled: true }); + expect(payload.result).toEqual({ fetchEnabled: true, provider: "brave" }); expect(payload.braveKey).toBe("saved-brave-key"); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -403,14 +403,14 @@ const { configureWebSearch } = require(${onboardPath}); expect(exitCode).toBe(0); expect(payload.exitCalls).toEqual([]); - expect(payload.result).toEqual({ fetchEnabled: true }); + expect(payload.result).toEqual({ fetchEnabled: true, provider: "brave" }); }); }); describe("configureWebSearch (interactive)", () => { - it("returns to the Brave Search enable prompt when backing out of the API key prompt", () => { + it("returns to the web search enable prompt when backing out of provider and API key prompts", () => { const { exitCode, payload } = runInteractiveConfigureWebSearch({ - answers: ["y", "back", "n"], + answers: ["y", "1", "back", "back", "n"], }); expect(exitCode).toBe(0); @@ -420,7 +420,7 @@ describe("configureWebSearch (interactive)", () => { expect(payload.errors).toEqual([]); expect(payload.saved.every((entry) => entry.value !== "back")).toBe(true); expect( - payload.prompts.filter((entry) => /Enable Brave Web Search\?/.test(entry.message)), + payload.prompts.filter((entry) => /Enable web search\?/.test(entry.message)), ).toHaveLength(2); expect( payload.prompts.some( @@ -431,7 +431,7 @@ describe("configureWebSearch (interactive)", () => { it("exits from the Brave Search API key prompt", () => { const { exitCode, payload } = runInteractiveConfigureWebSearch({ - answers: ["y", "exit"], + answers: ["y", "1", "exit"], }); expect(exitCode).toBe(0); diff --git a/test/onboard-policy-suggestions.test.ts b/test/onboard-policy-suggestions.test.ts index 3bbb9e29460..f3c19f1fc95 100644 --- a/test/onboard-policy-suggestions.test.ts +++ b/test/onboard-policy-suggestions.test.ts @@ -37,6 +37,7 @@ describe("onboard policy preset suggestions", () => { "huggingface", "brew", "brave", + "tavily", "slack", "discord", "telegram", @@ -158,12 +159,23 @@ describe("onboard policy preset suggestions", () => { const suggestions = computeSetupPresetSuggestions("balanced", { enabledChannels: [], knownPresetNames: known, - webSearchConfig: { fetchEnabled: true }, + webSearchConfig: { fetchEnabled: true, provider: "brave" }, webSearchSupported: true, }); expect(suggestions).toEqual(["npm", "pypi", "huggingface", "brew", "brave"]); }); + it("adds Tavily to tier suggestions when Tavily web search is configured", () => { + const suggestions = computeSetupPresetSuggestions("balanced", { + enabledChannels: [], + knownPresetNames: known, + webSearchConfig: { fetchEnabled: true, provider: "tavily" }, + webSearchSupported: true, + }); + expect(suggestions).toEqual(["npm", "pypi", "huggingface", "brew", "tavily"]); + expect(suggestions).not.toContain("brave"); + }); + it("filters tier defaults to known presets for agent-specific onboarding", () => { const suggestions = computeSetupPresetSuggestions("balanced", { enabledChannels: [], @@ -182,6 +194,7 @@ describe("onboard policy preset suggestions", () => { }).map((p) => p.name); expect(unsupportedPresets).not.toContain("brave"); expect(supportedPresets).toContain("brave"); + expect(supportedPresets).toContain("tavily"); }); it("drops Brave tier defaults when web search is unsupported", () => { @@ -231,13 +244,13 @@ describe("onboard policy preset suggestions", () => { it("handles Brave web search config with support checks", () => { expect( computeSetupPresetSuggestions("restricted", { - webSearchConfig: { provider: "brave" }, + webSearchConfig: { fetchEnabled: true, provider: "brave" }, knownPresetNames: known, }), ).toContain("brave"); expect( computeSetupPresetSuggestions("restricted", { - webSearchConfig: { provider: "brave" }, + webSearchConfig: { fetchEnabled: true, provider: "brave" }, knownPresetNames: known, webSearchSupported: false, }), diff --git a/test/validate-blueprint.test.ts b/test/validate-blueprint.test.ts index aef9ac68c06..409f9e1faf4 100644 --- a/test/validate-blueprint.test.ts +++ b/test/validate-blueprint.test.ts @@ -204,9 +204,14 @@ describe("blueprint.yaml", () => { describe("Model Router pool config", () => { const pool = loadYaml(ROUTER_POOL_CONFIG_PATH); - it("regression #3255: routes NVIDIA API keys to the public NVIDIA Build endpoint", () => { + it("regression #3255: routes NVIDIA API keys to public NVIDIA inference endpoints", () => { const apiBases = new Set((pool.models ?? []).map((model) => model.api_base)); - expect(apiBases).toEqual(new Set(["https://integrate.api.nvidia.com/v1"])); + expect(apiBases).toEqual( + new Set([ + "https://integrate.api.nvidia.com/v1", + "https://inference-api.nvidia.com/v1", + ]), + ); }); it("regression #3255: uses valid LiteLLM NVIDIA model identifiers", () => { @@ -216,10 +221,11 @@ describe("Model Router pool config", () => { expect(modelsByName.get("nemotron-3-nano-reasoning")).toBe( "openai/nvidia/nemotron-3-nano-30b-a3b", ); - expect(modelsByName.get("nemotron-3-super")).toBe( - "openai/nvidia/nemotron-3-super-120b-a12b", + expect(modelsByName.get("nemotron-ultra")).toBe( + "openai/nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1", ); for (const litellmModel of modelsByName.values()) { + if (litellmModel === "openai/nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1") continue; expect(litellmModel).not.toMatch(/nvidia\/nvidia\//); expect(litellmModel).not.toContain("Nemotron-3-Nano-30B-A3B"); expect(litellmModel).not.toContain("nemotron-3-super-v3");