diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab4ec7b9f1..95c4f387a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -167,6 +167,14 @@ repos: files: ^(nemoclaw-blueprint/.*\.yaml$|nemoclaw/openclaw\.plugin\.json$|schemas/.*\.json$) priority: 10 + - id: validate-managed-inference-catalog + name: Validate managed inference catalog + entry: npm run catalog:check + language: system + pass_filenames: false + files: ^(managed-inference/(?:presets|recipes)/.*\.yaml$|managed-inference/schemas/.*\.json$|src/lib/inference/serving/(?:adapter-registry|catalog|generate-catalog|types)\.ts$) + priority: 10 + - id: repository-checks name: Repository checks entry: npm run checks:repository diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 599569dbdd..271b17923f 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -47,7 +47,7 @@ "status": "tested", "prd_priority": "P1", "ci_tested": true, - "notes": "Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw)." + "notes": "Use the standard installer and `$$nemoclaw onboard`. The automatic two-DGX Spark managed-vLLM profile is Experimental, and physical two-node end-to-end validation is pending. For the profile requirements and controls, see [Set Up vLLM on Two DGX Sparks](../inference/local-inference/set-up-vllm-on-two-dgx-sparks). For the validated single-DGX Spark walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw)." }, { "name": "Windows WSL2", diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index d3d55a9218..59add3b066 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -28,7 +28,7 @@ "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, - "src/lib/state/state-root.ts": 23, + "src/lib/state/state-root.ts": 22, "src/lib/subprocess-env.ts": 24, "src/lib/validation.ts": 25 } @@ -46,7 +46,7 @@ "src/lib/actions/sandbox/snapshot.ts": 39, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, - "src/lib/inference/vllm.ts": 23, + "src/lib/inference/vllm.ts": 21, "src/lib/onboard.ts": 225, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 22, diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index e9db60e9ca..9e7c1cd46c 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -101,7 +101,7 @@ The table comes from [`ci/platform-matrix.json`](https://github.com/NVIDIA/NemoC {/* platform-matrix:begin */} | OS | Container runtime | Status | Notes | |----|-------------------|--------|-------| -| DGX OS (Spark) | Docker | Tested | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | +| DGX OS (Spark) | Docker | Tested | Use the standard installer and `$$nemoclaw onboard`. The automatic two-DGX Spark managed-vLLM profile is Experimental, and physical two-node end-to-end validation is pending. For the profile requirements and controls, see [Set Up vLLM on Two DGX Sparks](../inference/local-inference/set-up-vllm-on-two-dgx-sparks). For the validated single-DGX Spark walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | DGX OS (Station) | Docker | Tested with limitations | Tested with limitations across qualified profiles on one physical DGX Station GB300; see [Additional Setup for DGX Station](additional-setup/dgx-station-preparation) for accepted profiles, the pending no-OTA DGX OS `7.6.x` end-to-end qualification, runtime gates, and current dual-Station and dedicated CI limitations. | | Linux | Docker | Tested | Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | Start the container runtime (Colima or Docker Desktop) before running the installer. When Homebrew is available, OpenShell uses its official formula and the gateway appears in `brew services list` as `openshell`; without Homebrew, NemoClaw uses the standalone OpenShell install and detached gateway fallback. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | diff --git a/docs/index.yml b/docs/index.yml index 8a6ee29162..88f3bfd6e8 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -101,6 +101,9 @@ navigation: - page: "Set Up vLLM" path: _build/agent-variants/inference/set-up-vllm.openclaw.generated.mdx slug: set-up-vllm + - page: "Set Up vLLM on Two DGX Sparks" + path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-sparks.openclaw.generated.mdx + slug: set-up-vllm-on-two-dgx-sparks - page: "Set Up vLLM on Two DGX Stations" path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-stations.openclaw.generated.mdx slug: set-up-vllm-on-two-dgx-stations @@ -507,6 +510,9 @@ navigation: - page: "Set Up vLLM" path: _build/agent-variants/inference/set-up-vllm.deepagents.generated.mdx slug: set-up-vllm + - page: "Set Up vLLM on Two DGX Sparks" + path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-sparks.deepagents.generated.mdx + slug: set-up-vllm-on-two-dgx-sparks - page: "Set Up vLLM on Two DGX Stations" path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-stations.deepagents.generated.mdx slug: set-up-vllm-on-two-dgx-stations @@ -835,6 +841,9 @@ navigation: - page: "Set Up vLLM" path: _build/agent-variants/inference/set-up-vllm.hermes.generated.mdx slug: set-up-vllm + - page: "Set Up vLLM on Two DGX Sparks" + path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-sparks.hermes.generated.mdx + slug: set-up-vllm-on-two-dgx-sparks - page: "Set Up vLLM on Two DGX Stations" path: _build/agent-variants/inference/set-up-vllm-on-two-dgx-stations.hermes.generated.mdx slug: set-up-vllm-on-two-dgx-stations diff --git a/docs/inference/set-up-vllm-on-two-dgx-sparks.mdx b/docs/inference/set-up-vllm-on-two-dgx-sparks.mdx new file mode 100644 index 0000000000..53848323a0 --- /dev/null +++ b/docs/inference/set-up-vllm-on-two-dgx-sparks.mdx @@ -0,0 +1,246 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Set Up vLLM on Two DGX Sparks" +sidebar-title: "Set Up vLLM on Two DGX Sparks" +description: "Qualify two trusted DGX Spark systems and start NemoClaw's Experimental distributed DeepSeek V4 Flash vLLM profile." +description-agent: "Sets up the Experimental managed vLLM profile for two DGX Spark systems, including selection, qualification, runtime ownership, and cleanup. Use when configuring NEMOCLAW_MANAGED_CLUSTER_PEERS or NEMOCLAW_SERVING_PRESET." +keywords: ["nemoclaw dual dgx spark", "two dgx spark vllm", "deepseek v4 flash"] +content: + type: "how_to" +--- + +Use this workflow to let DGX Spark Express qualify two trusted DGX Spark systems for distributed vLLM serving. +The profile serves DeepSeek V4 Flash 0731 through the managed `inference.local` route. + + +This profile is Experimental because physical two-node end-to-end (E2E) validation is pending. +Use it only for evaluation until the required physical validation is complete. + + +## Understand Profile Selection + +DGX Spark Express attempts automatic profile selection only after it selects the managed vLLM provider. +A provider other than `install-vllm`, an existing local inference route, `NEMOCLAW_VLLM_MODEL`, or `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` remains authoritative. +Without explicit inference intent, the detector derives one exact counterpart from each qualified private `/30` rail. +It does not scan a subnet, add SSH host keys, add authorized keys, or change SSH configuration. + +Both derived addresses must already have strict public-key SSH trust. +They must identify the same physical peer and the same SSH host key. +The detector performs read-only host probes only through that pinned identity. +After you confirm setup, NemoClaw repeats the read-only topology qualification. +It then writes an owner-only temporary SSH binding for each peer at `~/.nemoclaw/managed-cluster-managed-serving.json..ssh-binding/`. +These claims are host-global and shared by every gateway on the host. +Declining setup creates no binding. +After confirmation, NemoClaw attempts to remove every binding when setup fails or the durable runtime receipt is committed. +If cleanup cannot be confirmed, a new managed-vLLM install stops before mutation and preserves the claims for explicit recovery or removal. + +When no matching cluster qualifies during automatic selection, NemoClaw retains the existing single-DGX Spark managed vLLM path. +This ordinary no-match result occurs before the distributed image or model download starts. + +Existing related runtime state is not an ordinary no-match result. +NemoClaw stops when either host has a related vLLM or DGX Station container, an ambiguous runtime inspection, or a listener on port `8000` or `25000`. +An existing durable managed-cluster or dual-Station runtime receipt also stops setup. +A copied runtime binding, an orphaned discovery claim, or an SSH binding conflict also stops setup. +NemoClaw leaves the existing files, containers, and listeners untouched instead of replacing, repairing, adopting, or removing them. + +Set `NEMOCLAW_MANAGED_CLUSTER_PEERS` to a comma-separated list of exact pretrusted peers. +The current profile declares `nodeCount: 2`, so its strict selection requires one peer. +Set `NEMOCLAW_SERVING_PRESET=vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731` to request the exact serving profile. +Either setting makes qualification strict, so a missed requirement stops setup instead of selecting the single-DGX Spark profile. + +## Prepare Both Systems + +Prepare both DGX Spark systems before you run the installer. +The automatic path requires all of these conditions on both systems: + +- The host is an `arm64` DGX Spark with exactly one GB10 GPU. +- The current controller is a non-root user whose user ID (UID) matches the local host probe. +- Docker, NVIDIA Container Toolkit, Container Device Interface (CDI), and NVIDIA GPU readiness checks pass. +- One direct ConnectX-7 physical port exposes exactly two active `200,000 Mbps` Ethernet rails. +- Each rail has a maximum transmission unit (MTU) of at least `9,000` bytes and exactly one usable private IPv4 `/30` address. +- The two rails use distinct interfaces, MAC addresses, subnets, and reciprocal peer addresses. +- Each selected rail address has a dynamically resolved remote direct memory access over Converged Ethernet version 2 (RoCEv2) global identifier (GID). +- Both derived peer addresses have existing strict host-key and public-key SSH trust. +- `~/.cache/huggingface` already exists, belongs to the probed numeric Unix user ID and Unix group ID, and is writable by that user. +- Docker storage and the Hugging Face cache each report at least `1,024` available inodes. +- If `earlyoom` is installed, its active state is `inactive` on both systems. + +NemoClaw treats an active or uninspectable `earlyoom` service as an unmet two-node requirement. +It does not stop, disable, or reconfigure that service. +An explicit pair or preset stops when this requirement is not met. + +Refer to the [NVIDIA DGX Spark clustering guide](https://docs.nvidia.com/dgx/dgx-spark/spark-clustering.html) for the physical connection workflow. +The NemoClaw checks above remain the exact profile requirements. + +### Reserve Storage on Each System + +The profile verifies storage before it pulls the image or downloads the model on either system. +For a cold cache, each system needs these approximate capacities: + +| Filesystem | Required capacity | Included data | +|---|---:|---| +| Hugging Face cache | `170.936 GB` | `166.899 GB` model snapshot, `3.221 GB` staging reserve, and `0.816 GB` writable allowance. | +| Docker storage | `32.584 GB` | Three times the `9.788 GB` compressed image size plus a `3.221 GB` staging reserve. | +| Shared filesystem | `203.520 GB` | The combined cold-cache requirement when both paths share one filesystem. | + +NemoClaw reserves the selected recipe's full declared image and model download sizes, staging capacity, and writable allowance even when the exact artifacts are cached. +An unavailable capacity, ownership, directory, writability, or inode result leaves the two-node profile unqualified. + +## Restrict Network Access + +Apply network controls before you start the distributed containers. + + +The pinned image runs as root in both containers. +Each container receives all host GPUs, `/dev/infiniband`, Docker host networking, host inter-process communication, `64 GB` of shared memory, and unlimited locked memory. +The launch does not use a read-only root filesystem, drop Linux capabilities, or set `no-new-privileges`. +The serving command does not enable vLLM's `--trust-remote-code` switch. +Before it copies the pinned snapshot's required encoder module into the runtime package, it verifies that file against the recipe's exact SHA-256 digest. +The head binds HTTP port `8000` only to the qualified primary rail address. +vLLM applies bearer API key authentication to its authenticated API routes, including the inventory and chat routes that NemoClaw verifies. +The API key does not authenticate every endpoint on port `8000`. +vLLM exposes unauthenticated inference and operational endpoints on that port. +Trusted Docker administrators can inspect that API key in the head container environment. +The distributed rendezvous and RoCE traffic do not use that bearer credential. + +Treat both DGX Spark systems and every host that can reach the direct rails as one trusted runtime boundary. +Allow port `8000` only from the OpenShell Docker subnet to the selected head rail address. +Deny port `8000` on management, LAN, and internet-facing interfaces. +Restrict port `25000`, NVIDIA Collective Communications Library (NCCL), and RoCE traffic to the exact reciprocal private rail addresses. +The address-specific bind reduces listener exposure but does not replace these firewall and trusted-boundary controls. + + +The detector qualifies both ConnectX-7 rails. +This profile revision binds vLLM, NCCL, Gloo, and tensor-parallel traffic to the first qualified rail in deterministic interface order. +Do not interpret two qualified `200,000 Mbps` rails as measured aggregate serving throughput. + +## Run Automatic Express Installation + +Run the interactive installer on the system that should become the head. + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | \ + + NEMOCLAW_AGENT=openclaw \ + + + NEMOCLAW_AGENT=hermes \ + + + NEMOCLAW_AGENT=langchain-deepagents-code \ + + bash +``` + +Accept the usage notice and the **Run express install with these settings?** prompt. +DGX Spark Express prints the selected topology, immutable image, pinned model revision, cache paths, and worker-first launch order before image, model, or container mutation. + +For optional authenticated model downloads, follow [Authenticate Hugging Face Downloads](set-up-vllm#authenticate-hugging-face-downloads) before you start the installer. +The public model can download anonymously, but Hugging Face can rate-limit anonymous transfers. + +## Select an Exact Pair + +Use strict selection when you already know the pretrusted peer and want setup to stop on any missed requirement. +The following first-install example also selects the exact serving preset. + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | \ + + NEMOCLAW_AGENT=openclaw \ + + + NEMOCLAW_AGENT=hermes \ + + + NEMOCLAW_AGENT=langchain-deepagents-code \ + + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_PROVIDER=install-vllm \ + NEMOCLAW_MANAGED_CLUSTER_PEERS="" \ + NEMOCLAW_SERVING_PRESET=vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731 \ + NEMOCLAW_SANDBOX_NAME=my-assistant \ + bash +``` + +The peer must already pass strict public-key authentication and host-key checking without an interactive prompt. +The explicit target still must match the exact reciprocal rail topology. + +## Understand the Distributed Runtime + +The compiled managed inference catalog reads cluster size, profile selection, artifacts, runtime resources, serving arguments, preparation, and readiness checks from preset and recipe YAML. +Maintainers can add another compatible profile that uses the registered managed-cluster adapters by adding preset and recipe YAML. +The YAML `nodeCount` declares the required cardinality. +The TypeScript adapters validate the qualified topology against it and materialize every ranked topology node. +The build compiles those YAML definitions into the packaged runtime catalog automatically. +A profile that needs a different execution contract requires a registered adapter. + +The compiled profile pins these artifacts: + +- The image is `ghcr.io/anemll/dspark-vllm-gx10@sha256:a83948492cf13df455170fb42885f5ef4db54fefe0feff0f841ecbff464ac9d8`. +- The model is `deepseek-ai/DeepSeek-V4-Flash-0731` at revision `9e165c30e2704aec5d9d593cce3eebd58bbef1cb`. +- The served model ID is `deepseek-v4-flash-0731`. +- The context window is `1,048,576` tokens, with tensor parallelism `2` and pipeline parallelism `1`. + +NemoClaw stages the exact image and model snapshot on both systems. +It mounts the pinned Hugging Face model data read-only into each long-running container. +It resolves the selected RoCEv2 GID during qualification instead of assuming a fixed GID index. + +NemoClaw starts the headless rank-1 worker container before the rank-0 head container. +The worker must remain alive while waiting at the distributed rendezvous before the head starts. +NemoClaw passes the bearer API key only through the head container environment. +It does not add the credential or an environment-variable expansion to the vLLM process arguments. +The worker does not receive the API key. +Setup commits only after both container identities remain exact and authenticated model inventory and chat requests succeed. + +## Verify the Installed Route + +After the installer completes, inspect the live route and sandbox inference health. + +```bash +$$nemoclaw my-assistant inference get +$$nemoclaw my-assistant status +``` + +Confirm that `inference get` reports the `vllm-local` provider and `deepseek-v4-flash-0731` model. +Continue only when the `Inference` row reports `reachable`. +This result confirms inference route reachability. +It does not establish physical two-node E2E validation or product support. + +## Preserve Ownership and Uninstall State + +After runtime validation, NemoClaw writes the owner-only host-global receipt `~/.nemoclaw/managed-cluster-vllm-runtime.json`. +It copies each pinned peer SSH binding to `~/.nemoclaw/managed-cluster-vllm-runtime.json.rank-.ssh-binding/`. +The receipt contains the exact plan, ranked node identities, container IDs, peer identities, and API key fingerprint. +It does not contain the API key. + +NemoClaw stores the host-global bearer API key in the owner-only `~/.nemoclaw/dual-station-vllm-api-key` file. +The filename remains stable for managed vLLM compatibility across distributed profiles. +Users with Docker control can inspect the head container environment, so restrict Docker access to trusted administrators. + +If receipt persistence fails, NemoClaw attempts to remove only the exact containers created by that attempt and reports any rollback removal it cannot confirm. +It does not remove an exact cluster that existed before the attempt. + +The containers have no Docker restart policy. +NemoClaw does not provide automatic runtime persistence across a Docker daemon restart or host reboot. +A stopped, partial, mismatched, or unhealthy cluster remains a conflict and is not repaired automatically. + +Run a full uninstall to remove the exact receipt-owned cluster. + +```bash +$$nemoclaw uninstall +``` + +Full uninstall revalidates every exact container ID before it removes the containers. +It then retires the receipt, copied SSH bindings, and exact temporary discovery claims recorded by that receipt. +If revalidation, removal, or safe claim retirement fails, uninstall stops before other cleanup and preserves the receipt for a retry. +A gateway-scoped uninstall preserves the host-global cluster, receipt, SSH bindings, and API key while another gateway remains. +Cluster cleanup does not delete the Hugging Face model cache. + +## Related Topics + +- [Set Up vLLM](set-up-vllm) for existing servers, single-host managed setup, model selection, and download authentication. +- [Set Up vLLM on Two DGX Stations](set-up-vllm-on-two-dgx-stations) for the separate DGX Station distributed profile. +- [Uninstall NemoClaw](../../manage-sandboxes/operate-sandboxes/uninstall-nemoclaw) for full and gateway-scoped uninstall behavior. +- [Host Files and State](../../reference/host-files-and-state) for managed inference receipt and API key locations. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 1d89ed802e..a922f58bbf 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -24,8 +24,8 @@ Use a host firewall with default-deny inbound rules. Allow TCP port `8000` only from the OpenShell Docker subnet to its gateway address, keep loopback access, and deny the port on every other interface. Do not expose the port to your LAN or the internet. -The qualified dual-DGX Station path instead binds its bearer-protected `/v1` API to one selected private rail address and uses host networking on both runtime containers. -Follow [Set Up vLLM on Two DGX Stations](set-up-vllm-on-two-dgx-stations) for its isolation requirements. +The qualified two-DGX Spark and two-DGX Station paths use bearer-protected `/v1` routes and Docker host networking on both runtime containers; network isolation remains required for other endpoints and distributed traffic. +Follow [Set Up vLLM on Two DGX Sparks](set-up-vllm-on-two-dgx-sparks) or [Set Up vLLM on Two DGX Stations](set-up-vllm-on-two-dgx-stations) for their isolation requirements. ## Use an Existing Server @@ -83,7 +83,8 @@ Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PR In an interactive run, the managed path lists the supported model registry for your host profile. Press **Enter** to use the profile default or select another validated model with its registered `vllm serve` arguments. -NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, streams Hugging Face download progress, and polls `/v1/models` until the model is ready. +The single-host path pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, and starts the `nemoclaw-vllm` container on `localhost:8000`. +The distributed DGX pages document their separate container names, endpoints, and lifecycle checks. The container carries a NemoClaw ownership label so later onboarding runs can distinguish it from a bring-your-own server without trusting the container name alone. If an existing `nemoclaw-vllm` container does not carry that label, NemoClaw preserves it and asks you to remove or rename it before installing managed vLLM. @@ -102,7 +103,7 @@ The two-Station page records the distributed recipe's image and runtime versions The first run duration depends on image and model size, storage performance, and network throughput. Later runs reuse the cached image and model weights. -The managed container uses Docker's `--restart unless-stopped` policy, so Docker restarts it unless an operator explicitly stops it. +The single-host managed container uses Docker's `--restart unless-stopped` policy, so Docker restarts it unless an operator explicitly stops it. ### Authenticate Hugging Face Downloads @@ -198,6 +199,9 @@ On DGX Station, accepting the installer express prompt selects `NEMOCLAW_VLLM_MO Express uses the single-Station Ultra recipe unless a trusted reciprocal pair qualifies for distributed serving. Refer to [Set Up vLLM on Two DGX Stations](set-up-vllm-on-two-dgx-stations) for pair qualification, lifecycle receipts, runtime topology, and network controls. +On DGX Spark, Express selects the pinned two-node profile when exactly one pretrusted reciprocal pair qualifies and no inference intent or related runtime conflicts. +Refer to [Set Up vLLM on Two DGX Sparks](set-up-vllm-on-two-dgx-sparks) for automatic and explicit selection, profile requirements, and cleanup behavior. + To select the existing `deepseek-v4-flash` recipe while retaining the same one-confirmation express flow, run: ```bash @@ -331,6 +335,7 @@ If the selected image does not support an argument, the managed container exits ## Related Topics - [Choose a Local Inference Server](choose-local-inference-server) to compare vLLM with Ollama and NVIDIA NIM. +- [Set Up vLLM on Two DGX Sparks](set-up-vllm-on-two-dgx-sparks) for the Experimental distributed DeepSeek V4 Flash profile. - [Set Up vLLM on Two DGX Stations](set-up-vllm-on-two-dgx-stations) for the Deferred distributed-serving workflow. - [Configure Inference Timeouts](../manage-inference/configure-inference-timeouts) when model startup or validation needs more time. - [Verify the Inference Route](../validate-inference/verify-inference-route) after setup. diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index e7804fd7b7..77116f204b 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -75,19 +75,31 @@ It also preserves the gateway process, supervisor resources, marked Linux unit, A custom-port uninstall does not stop or remove the default gateway service or its environment file. +In this section, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway. For the default gateway, the uninstall command preserves `~/.nemoclaw/rebuild-backups/`, `~/.nemoclaw/backups/`, and `~/.nemoclaw/sandboxes.json` by default. A non-default gateway uses the corresponding entries under `~/.nemoclaw/gateways//`. When uninstall confirms that no sibling gateways remain, it also removes the shared CLI, services, images, providers, configuration, models, and swap. -During full uninstall, a managed dual-Station cleanup receipt makes NemoClaw first revalidate the recorded peer, cluster, and GPU identities and remove both exact managed vLLM containers. -Only after that pair cleanup succeeds does NemoClaw start the remaining full-uninstall steps. -If validation or pair cleanup fails, uninstall exits nonzero and keeps the receipt so you can resolve the reported SSH, Docker, or peer-host error and retry. -Pair cleanup can partially complete before an error, so inspect both Stations before retrying. +During full uninstall, a managed distributed-vLLM receipt makes NemoClaw first revalidate the recorded plan and every cluster node, GPU, container, and SSH peer identity. +It then removes only the exact receipt-owned containers. +The managed-cluster receipt is host-global under `~/.nemoclaw/`. +Current dual-DGX Station receipts and copied SSH bindings are also host-global under `~/.nemoclaw/`. +Earlier releases can leave Station receipts and bindings under `~/.nemoclaw/gateways//`; NemoClaw discovers and validates those legacy locations in place. +NemoClaw starts the remaining full-uninstall steps only after distributed runtime cleanup succeeds. +For a host-global managed-cluster receipt, successful cleanup also retires every recorded temporary discovery claim. +An interrupted transaction can leave `~/.nemoclaw/managed-cluster-managed-serving.json..ssh-binding/` without a durable runtime receipt. +In that state, full uninstall fails closed before mutation and preserves every claim for explicit recovery or removal. +These temporary claims are host-global regardless of the selected gateway. +If validation, runtime cleanup, or temporary-claim retirement fails, uninstall exits nonzero and keeps the receipt. +Resolve the reported SSH, Docker, peer-host, or claim error before you retry. +Runtime cleanup can partially complete before an error, so inspect every cluster node before retrying. When sibling gateways remain, it removes only the selected gateway's resources and port-scoped state and preserves those shared host resources. -This gateway-scoped path leaves the managed pair running and preserves its cleanup receipt and copied SSH binding, including when you pass `--destroy-user-data`. +This gateway-scoped path leaves a managed distributed runtime running. +It preserves the cleanup receipt, copied SSH bindings, temporary discovery claims, and host-global API key. +This preservation also applies when you pass `--destroy-user-data`. If the OpenShell command is unavailable or its gateway list cannot be read, uninstall cannot confirm that the selected gateway is the last one, so it uses the same scoped path and preserves the shared resources. When the command itself is unavailable, uninstall exits nonzero before OpenShell cleanup so you can restore the command and retry. The preserved `rebuild-backups/`, `backups/`, and `sandboxes.json` entries stay unless you pass `--destroy-user-data`. -That flag does not override gateway-scoped preservation of the managed pair receipt or its copied SSH binding. +That flag does not override gateway-scoped preservation of the managed runtime receipt, copied SSH bindings, or temporary discovery claims. Interactive runs prompt before they remove the preserved entries, and the default answer keeps them. For non-interactive runs using `--yes`, `NEMOCLAW_NON_INTERACTIVE=1`, or a non-TTY shell, pass `--destroy-user-data` or set `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1` to acknowledge data loss and remove the preserved entries. diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index 9e4538bfe6..f4a4535bbd 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -32,11 +32,14 @@ Share redacted diagnostics only. | `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. | | `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. | | `~/.nemoclaw/ollama-proxy-token` | Local auth token used by the host-side Ollama auth proxy. | Yes, but re-run onboarding afterward so NemoClaw recreates and registers the proxy token. | -| `~/.nemoclaw/dual-station-vllm-api-key` | Host-global, owner-only bearer key shared by gateways that route to the managed dual-Station vLLM endpoint. The head requires this key for inference requests, and the worker does not receive it. | No while the managed pair exists. Full uninstall removes it after pair cleanup succeeds; gateway-scoped uninstall preserves it while sibling gateways remain. | -| `~/.nemoclaw/dual-station-vllm-runtime.json` | Host-global, owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. A full `$$nemoclaw uninstall` removes the receipt after both exact containers are removed; gateway-scoped uninstall preserves it. | -| `~/.nemoclaw/dual-station-vllm-runtime.json.ssh-binding/` | Host-global, owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. Full uninstall removes it with the cleanup receipt after pair cleanup succeeds; gateway-scoped uninstall preserves it. | - -Earlier releases can store the runtime receipt and its SSH-binding directory under `~/.nemoclaw/gateways//` when the managed pair was created from a non-default gateway port. +| `~/.nemoclaw/dual-station-vllm-api-key` | Owner-only host-global bearer API key shared by NemoClaw-managed distributed vLLM profiles. | No while a managed distributed runtime exists. Full uninstall removes it only after managed runtime cleanup succeeds and no sibling gateway remains. | +| `~/.nemoclaw/managed-cluster-vllm-runtime.json` | Owner-only host-global managed-cluster cleanup receipt. It contains no serving API key and binds the exact plan, ranked nodes, containers, and temporary discovery-claim identities. | No while the managed cluster exists. Full uninstall removes it after every exact container is removed. | +| `~/.nemoclaw/managed-cluster-vllm-runtime.json.rank-.ssh-binding/` | Owner-only copied SSH host-key and Docker-command binding needed to reach one recorded worker rank during full uninstall. | No while the managed cluster exists. Full uninstall removes every rank binding with the cleanup receipt after cluster cleanup succeeds. | +| `~/.nemoclaw/managed-cluster-managed-serving.json..ssh-binding/` | Owner-only host-global temporary SSH host-key and Docker-command claim written for one peer after setup confirmation and repeated topology qualification. Each claim is distinct from its durable runtime binding and applies to every gateway on the host. | Only after confirming that no setup transaction is active. Without a durable runtime receipt, an orphaned claim makes a new managed-vLLM install and full uninstall fail closed without mutation. With the matching host-global managed-cluster receipt, full uninstall retires every recorded claim after it removes every exact container. If claim retirement fails, it preserves the receipt for a retry. Gateway-scoped uninstall preserves each claim. | +| `~/.nemoclaw/dual-station-vllm-runtime.json` | Owner-only host-global managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. A full `$$nemoclaw uninstall` removes the receipt after both exact containers are removed; gateway-scoped uninstall preserves it. | +| `~/.nemoclaw/dual-station-vllm-runtime.json.ssh-binding/` | Owner-only host-global copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. Full uninstall removes it with the cleanup receipt after pair cleanup succeeds; gateway-scoped uninstall preserves it. | + +Earlier releases can store the dual-Station runtime receipt and its SSH-binding directory under `~/.nemoclaw/gateways//` when the managed pair was created from a non-default gateway port. The API key remains host-global at `~/.nemoclaw/dual-station-vllm-api-key`. Current releases discover and validate the legacy receipt location without moving or duplicating its ownership state. Do not move these files or directories manually. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 160d1f7909..502553460f 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -78,7 +78,7 @@ For install requirements and the shorter setup-oriented platform view, refer to {/* platform-matrix-full:begin */} | OS | Container runtime | Status | PRD priority | CI | Notes | |----|-------------------|--------|--------------|----|-------| -| DGX OS (Spark) | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | +| DGX OS (Spark) | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. The automatic two-DGX Spark managed-vLLM profile is Experimental, and physical two-node end-to-end validation is pending. For the profile requirements and controls, see [Set Up vLLM on Two DGX Sparks](../inference/local-inference/set-up-vllm-on-two-dgx-sparks). For the validated single-DGX Spark walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | DGX OS (Station) | Docker | Tested with limitations | P1 | No | The PRD marks this platform as P1. Physical validation on one DGX Station GB300 covers generic Ubuntu 24.04 ARM64, stock DGX OS `7.5.0`, the April 2026 NVIDIA Colossus BaseOS profile, and the June 2026 NVIDIA AI Developer Tools profile. A physical no-OTA DGX OS `7.6.0` host provided the release and hardware profile used for its stable workstation-family classifier and passed read-only eligibility and runtime-command preflight. Full Station Express end-to-end qualification for the accepted no-OTA DGX OS `7.6.x` profile is pending. The profile remains subject to the same physical GB300, driver, ECC, Docker, CDI, and container GPU validation. Clean-host end-to-end validation passed on generic Ubuntu and Colossus BaseOS; stock DGX OS and AI Developer Tools completed Station Express validation. The DGX OS `7.5.0` run used released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. A dual-Station configuration has not been validated, and dedicated CI coverage is not available. Direct-GPU policies expose only the exact read-only BDF directory for each discovered display-class PCI device with NVIDIA vendor ID (`0x10de`) and GB300 device ID (`0x31c2` or `0x31c3`) plus required existing topology and module paths; they do not expose `/sys`, the PCI parent subtree, or sysfs write access. During physical validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/additional-setup/dgx-station-preparation). | | Linux | Docker | Tested | P0 | Yes | Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. When Homebrew is available, OpenShell uses its official formula and the gateway appears in `brew services list` as `openshell`; without Homebrew, NemoClaw uses the standalone OpenShell install and detached gateway fallback. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | diff --git a/managed-inference/presets/vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731.yaml b/managed-inference/presets/vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731.yaml new file mode 100644 index 0000000000..5186b79d96 --- /dev/null +++ b/managed-inference/presets/vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731.yaml @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset + +metadata: + id: vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731 + displayName: DeepSeek V4 Flash 0731 on two DGX Sparks + +spec: + selection: automatic + priority: 400 + + requirements: + all: + - readiness: + scope: everyNode + kind: qualification + id: host.platform.dgx_spark + status: qualified + - readiness: + scope: everyNode + kind: capability + id: host.platform.supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.platform.dgx_spark + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.daemon_reachable + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.runtime_supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.storage_compatible + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.nvidia_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.container_toolkit_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.cdi_healthy + state: present + - fact: cluster.nodeCount + state: present + operator: equals + value: 2 + - topologyQualification: + id: host-cluster.direct-cx7 + schemaVersion: 1 + status: qualified + + plan: + backend: vllm + recipeRef: vllm.deepseek-v4-flash-0731.spark-dual.v1 + bindings: + sparkTopology: + valueFromTopologyQualification: + id: host-cluster.direct-cx7 + schemaVersion: 1 + output: topology diff --git a/managed-inference/recipes/vllm.deepseek-v4-flash-0731.spark-dual.v1.yaml b/managed-inference/recipes/vllm.deepseek-v4-flash-0731.spark-dual.v1.yaml new file mode 100644 index 0000000000..a73d1e6bc2 --- /dev/null +++ b/managed-inference/recipes/vllm.deepseek-v4-flash-0731.spark-dual.v1.yaml @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingRecipe + +metadata: + id: vllm.deepseek-v4-flash-0731.spark-dual.v1 + displayName: DeepSeek V4 Flash 0731 on two DGX Sparks + +spec: + backend: vllm + + bindings: + sparkTopology: + type: topologyQualificationOutput + qualificationId: host-cluster.direct-cx7 + schemaVersion: 1 + outputSchema: nemoclaw.nvidia.com/managed-cluster-topology/v1 + + model: + id: deepseek-ai/DeepSeek-V4-Flash-0731 + revision: 9e165c30e2704aec5d9d593cce3eebd58bbef1cb + servedName: deepseek-v4-flash-0731 + downloadSizeBytes: 166898661074 + gated: false + installFastSafetensors: false + preparation: + ref: snapshot-copy-and-exact-text-replacement/v1 + snapshotCopy: + sourcePath: encoding/encoding_dsv4.py + digest: sha256:abc0d26120250dda0ae077dc64aa28836026e61e970854aaeb792445e6a0dde6 + targetPath: /usr/local/lib/python3.12/dist-packages/vllm/tokenizers/deepseek_v4_encoding.py + exactTextReplacement: + targetPath: /usr/local/lib/python3.12/dist-packages/vllm/tokenizers/deepseek_v4.py + expectedText: |- + elif reasoning_effort in ("max", "xhigh"): + reasoning_effort = "max" + else: + reasoning_effort = "high" + replacementText: |- + elif reasoning_effort in ("max", "xhigh"): + reasoning_effort = "max" + elif reasoning_effort == "high": + reasoning_effort = "high" + else: + reasoning_effort = "low" + + runtime: + image: ghcr.io/anemll/dspark-vllm-gx10@sha256:a83948492cf13df455170fb42885f5ef4db54fefe0feff0f841ecbff464ac9d8 + imageDownloadSizeBytes: 9787537825 + architecture: arm64 + networkMode: host + ipcMode: host + gpuRequest: all + sharedMemoryBytes: 68719476736 + pullTimeoutSeconds: 43200 + devices: + - /dev/infiniband + ulimits: + memlock: -1 + stackBytes: 67108864 + modelCache: + source: huggingface-cache + target: /cache/huggingface + temporaryFilesystems: + - target: /tmp + sizeBytes: 17179869184 + mode: "1777" + options: [rw, nosuid, nodev, exec] + environment: + CUTE_DSL_ARCH: sm_121a + DG_JIT_NVCC_COMPILER: /usr/local/cuda/bin/nvcc + DG_JIT_USE_NVRTC: "0" + FLASHINFER_CUDA_ARCH_LIST: 12.1a + FLASHINFER_DISABLE_VERSION_CHECK: "1" + FLASHINFER_WORKSPACE_BASE: /cache/huggingface/flashinfer + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + NCCL_CROSS_NIC: "1" + NCCL_CUMEM_ENABLE: "0" + NCCL_DEBUG: WARN + NCCL_IB_DISABLE: "0" + NCCL_IGNORE_CPU_AFFINITY: "1" + NCCL_NET: IB + NCCL_NVLS_ENABLE: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_CUDA_ARCH_LIST: 12.1a + TRANSFORMERS_OFFLINE: "1" + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + VLLM_B12X_W4A16_FORCE_BLOCKS_MAX_M: "16" + VLLM_B12X_W4A16_FORCE_BLOCKS_PER_SM: "0" + VLLM_CACHE_ROOT: /cache/huggingface/vllm-cache + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "256" + VLLM_USE_B12X_MOE: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_USE_FLASHINFER_SAMPLER: "1" + + execution: + materializerRef: vllm.managed-cluster/v1 + lifecycleRef: vllm.managed-cluster.lifecycle/v1 + topologyBinding: sparkTopology + nodeCount: 2 + tensorParallelSize: 2 + pipelineParallelSize: 1 + distributedExecutorBackend: mp + rendezvousPort: 25000 + + serve: + authentication: bearer + executable: /usr/local/bin/vllm + arguments: + - name: --port + value: 8000 + - name: --kv-cache-dtype + value: nvfp4_ds_mla + - name: --block-size + value: 256 + - name: --max-model-len + value: 1048576 + - name: --max-num-seqs + value: 6 + - name: --max-num-batched-tokens + value: 8192 + - name: --max-cudagraph-capture-size + value: 36 + - name: --gpu-memory-utilization + value: 0.8 + - name: --enable-prefix-caching + - name: --enable-prompt-tokens-details + - name: --async-scheduling + - name: --enable-chunked-prefill + - name: --speculative-config + value: '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic"}' + - name: --tokenizer-mode + value: deepseek_v4 + - name: --moe-backend + value: flashinfer_b12x + - name: --tool-call-parser + value: deepseek_v4 + - name: --enable-auto-tool-choice + - name: --reasoning-parser + value: deepseek_v4 + - name: --reasoning-config + value: '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}' + - name: --default-chat-template-kwargs + value: '{"thinking":true,"reasoning_effort":"low"}' + - name: --generation-config + value: vllm + - name: --enable-flashinfer-autotune + + readiness: + timeoutSeconds: 3600 + expectedModel: deepseek-v4-flash-0731 diff --git a/managed-inference/schemas/preset.schema.json b/managed-inference/schemas/preset.schema.json index 7bfc98587a..9f939b6106 100644 --- a/managed-inference/schemas/preset.schema.json +++ b/managed-inference/schemas/preset.schema.json @@ -2,45 +2,100 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/NVIDIA/NemoClaw/managed-inference/schemas/preset.schema.json", "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", - "title": "NemoClaw managed inference serving preset", "type": "object", - "required": ["apiVersion", "kind", "metadata", "spec"], + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], "properties": { - "apiVersion": { "const": "nemoclaw.nvidia.com/managed-inference/v1" }, - "kind": { "const": "ServingPreset" }, + "apiVersion": { + "const": "nemoclaw.nvidia.com/managed-inference/v1" + }, + "kind": { + "const": "ServingPreset" + }, "metadata": { - "type": "object", - "required": ["id"], - "properties": { - "id": { "$ref": "#/$defs/stableId" }, - "displayName": { "type": "string", "minLength": 1, "maxLength": 256 } - }, - "additionalProperties": false + "$ref": "#/$defs/metadata" }, "spec": { "type": "object", - "required": ["selection", "priority", "plan"], + "required": [ + "selection", + "priority", + "plan" + ], "properties": { - "selection": { "enum": ["automatic", "explicit-only", "disabled"] }, - "priority": { "type": "integer", "minimum": 0, "maximum": 10000 }, + "selection": { + "enum": [ + "automatic", + "explicit-only", + "disabled" + ] + }, + "priority": { + "type": "integer", + "minimum": -1000000, + "maximum": 1000000 + }, "requirements": { "type": "object", - "required": ["all"], + "required": [ + "all" + ], "properties": { "all": { "type": "array", - "maxItems": 64, - "items": { "$ref": "#/$defs/readinessRequirement" } + "minItems": 1, + "maxItems": 128, + "items": { + "oneOf": [ + { + "$ref": "#/$defs/readinessQualificationRequirement" + }, + { + "$ref": "#/$defs/readinessStateRequirement" + }, + { + "$ref": "#/$defs/readinessComparisonRequirement" + }, + { + "$ref": "#/$defs/factRequirement" + }, + { + "$ref": "#/$defs/topologyRequirement" + } + ] + } } }, "additionalProperties": false }, "plan": { "type": "object", - "required": ["backend", "recipeRef"], + "required": [ + "backend", + "recipeRef" + ], "properties": { - "backend": { "$ref": "#/$defs/backendId" }, - "recipeRef": { "$ref": "#/$defs/stableId" } + "backend": { + "$ref": "#/$defs/token" + }, + "recipeRef": { + "$ref": "#/$defs/stableId" + }, + "bindings": { + "type": "object", + "minProperties": 1, + "maxProperties": 16, + "propertyNames": { + "$ref": "#/$defs/bindingName" + }, + "additionalProperties": { + "$ref": "#/$defs/presetTopologyBinding" + } + } }, "additionalProperties": false } @@ -50,113 +105,253 @@ }, "additionalProperties": false, "$defs": { + "metadata": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "$ref": "#/$defs/stableId" + }, + "displayName": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "additionalProperties": false + }, "stableId": { "type": "string", - "pattern": "^[a-z][a-z0-9]*(?:[._/-][a-z0-9]+)+$", - "maxLength": 160 + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,159}$" + }, + "bindingName": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9]{0,63}$" + }, + "token": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + }, + "scope": { + "enum": [ + "controller", + "everyNode", + "anyNode" + ] }, - "backendId": { + "state": { "type": "string", - "pattern": "^[a-z][a-z0-9-]{0,63}$" + "pattern": "^[a-z][a-z0-9._-]{0,63}$" }, - "readinessRequirement": { + "readinessQualificationRequirement": { "type": "object", - "required": ["readiness"], + "required": [ + "readiness" + ], "properties": { "readiness": { - "oneOf": [ - { - "type": "object", - "required": ["scope", "kind", "id", "state"], - "properties": { - "scope": { "enum": ["controller", "everyNode"] }, - "kind": { "enum": ["observation", "capability"] }, - "id": { "$ref": "#/$defs/stableId" }, - "state": { "enum": ["present", "absent"] } - }, - "additionalProperties": false + "type": "object", + "required": [ + "scope", + "kind", + "id", + "status" + ], + "properties": { + "scope": { + "$ref": "#/$defs/scope" + }, + "kind": { + "const": "qualification" + }, + "id": { + "$ref": "#/$defs/stableId" + }, + "status": { + "$ref": "#/$defs/state" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "readinessStateRequirement": { + "type": "object", + "required": [ + "readiness" + ], + "properties": { + "readiness": { + "type": "object", + "required": [ + "scope", + "kind", + "id", + "state" + ], + "properties": { + "scope": { + "$ref": "#/$defs/scope" }, + "kind": { + "enum": [ + "observation", + "capability" + ] + }, + "id": { + "$ref": "#/$defs/stableId" + }, + "state": { + "$ref": "#/$defs/state" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "factRequirement": { + "type": "object", + "required": [ + "fact", + "state", + "operator", + "value" + ], + "properties": { + "fact": { + "$ref": "#/$defs/stableId" + }, + "state": { + "enum": [ + "present", + "absent" + ] + }, + "operator": { + "enum": [ + "equals", + "oneOf", + "atLeast", + "atMost", + "between" + ] + }, + "value": { + "oneOf": [ { - "type": "object", - "required": ["scope", "kind", "id", "comparison"], - "properties": { - "scope": { "enum": ["controller", "everyNode"] }, - "kind": { "const": "observation" }, - "id": { "$ref": "#/$defs/stableId" }, - "comparison": { - "oneOf": [ - { - "type": "object", - "required": ["operator", "value"], - "properties": { - "operator": { "const": "equals" }, - "value": { "$ref": "#/$defs/comparisonScalar" } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": ["operator", "values"], - "properties": { - "operator": { "const": "one-of" }, - "values": { - "type": "array", - "minItems": 1, - "maxItems": 16, - "uniqueItems": true, - "items": { "$ref": "#/$defs/comparisonScalar" } - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": ["operator", "value"], - "properties": { - "operator": { "const": "at-least" }, - "value": { - "type": "number", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": ["operator", "value"], - "properties": { - "operator": { "const": "version-at-least" }, - "value": { "$ref": "#/$defs/version" } - }, - "additionalProperties": false - } - ] + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" } - }, - "additionalProperties": false + ] }, { - "type": "object", - "required": ["scope", "kind", "id", "status"], - "properties": { - "scope": { "enum": ["controller", "everyNode"] }, - "kind": { "const": "qualification" }, - "id": { "$ref": "#/$defs/stableId" }, - "status": { "enum": ["qualified", "unqualified"] } - }, - "additionalProperties": false + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + } } ] } }, "additionalProperties": false }, + "topologyRequirement": { + "type": "object", + "required": [ + "topologyQualification" + ], + "properties": { + "topologyQualification": { + "type": "object", + "required": [ + "id", + "schemaVersion", + "status" + ], + "properties": { + "id": { + "$ref": "#/$defs/stableId" + }, + "schemaVersion": { + "type": "integer", + "minimum": 1 + }, + "status": { + "$ref": "#/$defs/state" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "presetTopologyBinding": { + "type": "object", + "required": [ + "valueFromTopologyQualification" + ], + "properties": { + "valueFromTopologyQualification": { + "type": "object", + "required": [ + "id", + "schemaVersion", + "output" + ], + "properties": { + "id": { + "$ref": "#/$defs/stableId" + }, + "schemaVersion": { + "type": "integer", + "minimum": 1 + }, + "output": { + "$ref": "#/$defs/token" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "comparisonScalar": { "oneOf": [ - { "$ref": "#/$defs/comparisonString" }, - { "type": "number", "minimum": 0, "maximum": 9007199254740991 }, - { "type": "boolean" } + { + "$ref": "#/$defs/comparisonString" + }, + { + "type": "number", + "minimum": 0, + "maximum": 9007199254740991 + }, + { + "type": "boolean" + } ] }, "comparisonString": { @@ -168,6 +363,116 @@ "type": "string", "pattern": "^[0-9]+(?:\\.[0-9]+){1,3}(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$", "maxLength": 64 + }, + "readinessComparisonRequirement": { + "type": "object", + "required": [ + "readiness" + ], + "properties": { + "readiness": { + "type": "object", + "required": [ + "scope", + "kind", + "id", + "comparison" + ], + "properties": { + "scope": { + "enum": [ + "controller", + "everyNode" + ] + }, + "kind": { + "const": "observation" + }, + "id": { + "$ref": "#/$defs/stableId" + }, + "comparison": { + "oneOf": [ + { + "type": "object", + "required": [ + "operator", + "value" + ], + "properties": { + "operator": { + "const": "equals" + }, + "value": { + "$ref": "#/$defs/comparisonScalar" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "operator", + "values" + ], + "properties": { + "operator": { + "const": "one-of" + }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/comparisonScalar" + } + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "operator", + "value" + ], + "properties": { + "operator": { + "const": "at-least" + }, + "value": { + "type": "number", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "operator", + "value" + ], + "properties": { + "operator": { + "const": "version-at-least" + }, + "value": { + "$ref": "#/$defs/version" + } + }, + "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } - } + }, + "title": "NemoClaw managed inference serving preset" } diff --git a/managed-inference/schemas/recipe.schema.json b/managed-inference/schemas/recipe.schema.json index 77866674d8..1cf5f6dd5b 100644 --- a/managed-inference/schemas/recipe.schema.json +++ b/managed-inference/schemas/recipe.schema.json @@ -2,53 +2,86 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/NVIDIA/NemoClaw/managed-inference/schemas/recipe.schema.json", "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", - "title": "NemoClaw managed inference serving recipe", "type": "object", - "required": ["apiVersion", "kind", "metadata", "spec"], + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], "properties": { - "apiVersion": { "const": "nemoclaw.nvidia.com/managed-inference/v1" }, - "kind": { "const": "ServingRecipe" }, - "metadata": { "$ref": "#/$defs/metadata" }, + "apiVersion": { + "const": "nemoclaw.nvidia.com/managed-inference/v1" + }, + "kind": { + "const": "ServingRecipe" + }, + "metadata": { + "$ref": "#/$defs/metadata" + }, "spec": { "type": "object", - "required": ["backend", "model", "execution"], + "required": [ + "backend", + "model", + "execution" + ], "properties": { - "backend": { "$ref": "#/$defs/backendId" }, - "providerId": { "$ref": "#/$defs/backendId" }, - "server": { + "backend": { + "$ref": "#/$defs/token" + }, + "bindings": { "type": "object", - "required": ["technology", "source"], - "properties": { - "technology": { "const": "llama.cpp" }, - "source": { - "type": "object", - "required": ["repository", "revision"], - "properties": { - "repository": { "$ref": "#/$defs/repositoryId" }, - "revision": { "$ref": "#/$defs/sourceRevision" } - }, - "additionalProperties": false - } + "minProperties": 1, + "maxProperties": 16, + "propertyNames": { + "$ref": "#/$defs/bindingName" }, - "additionalProperties": false + "additionalProperties": { + "$ref": "#/$defs/topologyBinding" + } }, "model": { "type": "object", - "required": ["id", "revision"], + "required": [ + "id", + "revision" + ], "properties": { - "id": { "type": "string", "minLength": 1, "maxLength": 256 }, + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$", + "maxLength": 256 + }, "revision": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64}|sha256:[0-9a-f]{64})$" }, - "servedName": { "$ref": "#/$defs/stableId" }, + "servedName": { + "$ref": "#/$defs/stableId" + }, + "downloadSizeBytes": { + "$ref": "#/$defs/positiveSafeInteger" + }, + "gated": { + "type": "boolean" + }, + "installFastSafetensors": { + "type": "boolean" + }, + "preparation": { + "$ref": "#/$defs/modelPreparation" + }, "files": { "type": "array", "minItems": 1, "maxItems": 32, "items": { "type": "object", - "required": ["path", "digest"], + "required": [ + "path", + "digest" + ], "properties": { "path": { "type": "string", @@ -56,15 +89,23 @@ "maxLength": 512, "pattern": "^(?!\\.{1,2}(?:/|$))(?!.*\\/\\.{1,2}(?:/|$))[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*$" }, - "digest": { "$ref": "#/$defs/sha256" }, + "digest": { + "$ref": "#/$defs/sha256" + }, "sizeBytes": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, - "format": { "const": "gguf" }, - "quantization": { "$ref": "#/$defs/artifactLabel" }, - "license": { "$ref": "#/$defs/artifactLabel" } + "format": { + "const": "gguf" + }, + "quantization": { + "$ref": "#/$defs/artifactLabel" + }, + "license": { + "$ref": "#/$defs/artifactLabel" + } }, "additionalProperties": false } @@ -79,43 +120,192 @@ "type": "string", "pattern": "^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$" }, - "architecture": { "enum": ["amd64", "arm64"] }, + "imageDownloadSizeBytes": { + "$ref": "#/$defs/positiveSafeInteger" + }, + "pullTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + }, + "architecture": { + "enum": [ + "amd64", + "arm64" + ] + }, + "networkMode": { + "$ref": "#/$defs/token" + }, + "ipcMode": { + "$ref": "#/$defs/token" + }, + "sharedMemoryBytes": { + "$ref": "#/$defs/positiveSafeInteger" + }, + "gpuRequest": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._,:=-]{0,255}$" + }, + "devices": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/absolutePath" + } + }, + "ulimits": { + "type": "object", + "required": [ + "memlock", + "stackBytes" + ], + "properties": { + "memlock": { + "oneOf": [ + { + "type": "integer", + "minimum": -1, + "maximum": 9007199254740991 + }, + { + "type": "string", + "enum": [ + "unlimited" + ] + } + ] + }, + "stackBytes": { + "$ref": "#/$defs/positiveSafeInteger" + } + }, + "additionalProperties": false + }, + "modelCache": { + "type": "object", + "required": [ + "source", + "target" + ], + "properties": { + "source": { + "$ref": "#/$defs/stableId" + }, + "target": { + "$ref": "#/$defs/absolutePath" + } + }, + "additionalProperties": false + }, + "temporaryFilesystems": { + "type": "array", + "maxItems": 16, + "items": { + "$ref": "#/$defs/temporaryFilesystem" + } + }, + "environment": { + "type": "object", + "maxProperties": 128, + "propertyNames": { + "pattern": "^[A-Z][A-Z0-9_]{0,127}$" + }, + "additionalProperties": { + "type": "string", + "maxLength": 4096 + } + }, + "components": { + "type": "object", + "maxProperties": 32, + "propertyNames": { + "$ref": "#/$defs/stableId" + }, + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, "platforms": { "type": "array", "minItems": 2, "maxItems": 2, "uniqueItems": true, - "items": { "enum": ["linux/amd64", "linux/arm64"] }, + "items": { + "enum": [ + "linux/amd64", + "linux/arm64" + ] + }, "allOf": [ - { "contains": { "const": "linux/amd64" } }, - { "contains": { "const": "linux/arm64" } } + { + "contains": { + "const": "linux/amd64" + } + }, + { + "contains": { + "const": "linux/arm64" + } + } ] }, - "containerRuntime": { "const": "docker" }, - "hosts": { "const": 1 }, + "containerRuntime": { + "const": "docker" + }, + "hosts": { + "const": 1 + }, "cuda": { "type": "object", - "required": ["baseImage", "minimumDriverVersion"], + "required": [ + "baseImage", + "minimumDriverVersion" + ], "properties": { - "baseImage": { "$ref": "#/$defs/imageDigest" }, - "minimumDriverVersion": { "$ref": "#/$defs/version" } + "baseImage": { + "$ref": "#/$defs/imageDigest" + }, + "minimumDriverVersion": { + "$ref": "#/$defs/version" + } }, "additionalProperties": false }, "gpu": { "type": "object", - "required": ["vendor", "count", "offload", "cpuFallback"], + "required": [ + "vendor", + "count", + "offload", + "cpuFallback" + ], "properties": { - "vendor": { "const": "nvidia" }, - "count": { "const": 1 }, - "offload": { "const": "full" }, - "cpuFallback": { "const": "reject" } + "vendor": { + "const": "nvidia" + }, + "count": { + "const": 1 + }, + "offload": { + "const": "full" + }, + "cpuFallback": { + "const": "reject" + } }, "additionalProperties": false }, "resources": { "type": "object", - "required": ["memoryBytes", "writableStorageBytes", "pidsLimit"], + "required": [ + "memoryBytes", + "writableStorageBytes", + "pidsLimit" + ], "properties": { "memoryBytes": { "type": "integer", @@ -127,92 +317,224 @@ "minimum": 1, "maximum": 9007199254740991 }, - "pidsLimit": { "type": "integer", "minimum": 1, "maximum": 1048576 } + "pidsLimit": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + } }, "additionalProperties": false - }, - "components": { - "type": "object", - "maxProperties": 32, - "propertyNames": { "$ref": "#/$defs/stableId" }, - "additionalProperties": { - "type": "string", - "minLength": 1, - "maxLength": 128 - } } }, "additionalProperties": false }, "execution": { "type": "object", - "required": ["materializerRef", "lifecycleRef"], + "required": [ + "materializerRef", + "lifecycleRef" + ], "properties": { - "receiptRef": { "$ref": "#/$defs/adapterId" }, - "materializerRef": { "$ref": "#/$defs/adapterId" }, - "lifecycleRef": { "$ref": "#/$defs/adapterId" } + "materializerRef": { + "$ref": "#/$defs/adapterId" + }, + "lifecycleRef": { + "$ref": "#/$defs/adapterId" + }, + "topologyBinding": { + "$ref": "#/$defs/bindingName" + }, + "nodeCount": { + "type": "integer", + "minimum": 1, + "maximum": 1024 + }, + "tensorParallelSize": { + "type": "integer", + "minimum": 1, + "maximum": 1024 + }, + "pipelineParallelSize": { + "type": "integer", + "minimum": 1, + "maximum": 1024 + }, + "distributedExecutorBackend": { + "$ref": "#/$defs/token" + }, + "rendezvousPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "receiptRef": { + "$ref": "#/$defs/adapterId" + } }, "additionalProperties": false }, "serve": { "type": "object", "properties": { - "protocol": { "const": "openai-completions" }, - "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "chatTemplate": { "$ref": "#/$defs/artifactLabel" }, - "contextSize": { "type": "integer", "minimum": 1, "maximum": 1048576 }, - "slots": { "type": "integer", "minimum": 1, "maximum": 1024 }, - "idleSleepSeconds": { "type": "integer", "minimum": -1, "maximum": 86400 }, - "limits": { - "type": "object", - "required": [ - "maxRequestBodyBytes", - "maxPromptTokens", - "maxCompletionTokens", - "requestTimeoutSeconds" - ], - "properties": { - "maxRequestBodyBytes": { "type": "integer", "minimum": 1, "maximum": 1073741824 }, - "maxPromptTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 }, - "maxCompletionTokens": { "type": "integer", "minimum": 1, "maximum": 1048576 }, - "requestTimeoutSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 } - }, - "additionalProperties": false + "authentication": { + "$ref": "#/$defs/token" + }, + "executable": { + "$ref": "#/$defs/absolutePath" }, "arguments": { "type": "array", "maxItems": 128, "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string", "pattern": "^--[a-z0-9][a-z0-9-]*$", "maxLength": 128 }, - "value": { "$ref": "#/$defs/scalar" } + "value": { + "$ref": "#/$defs/scalar" + } }, "additionalProperties": false } + }, + "protocol": { + "const": "openai-completions" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "chatTemplate": { + "$ref": "#/$defs/artifactLabel" + }, + "contextSize": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + }, + "slots": { + "type": "integer", + "minimum": 1, + "maximum": 1024 + }, + "idleSleepSeconds": { + "type": "integer", + "minimum": -1, + "maximum": 86400 + }, + "limits": { + "type": "object", + "required": [ + "maxRequestBodyBytes", + "maxPromptTokens", + "maxCompletionTokens", + "requestTimeoutSeconds" + ], + "properties": { + "maxRequestBodyBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1073741824 + }, + "maxPromptTokens": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + }, + "maxCompletionTokens": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + }, + "requestTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + } + }, + "additionalProperties": false } }, "additionalProperties": false }, "readiness": { "type": "object", + "required": [ + "timeoutSeconds", + "expectedModel" + ], "properties": { - "contractRef": { "$ref": "#/$defs/adapterId" }, - "timeoutSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, - "expectedModel": { "$ref": "#/$defs/stableId" }, + "timeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + }, + "expectedModel": { + "$ref": "#/$defs/stableId" + }, + "contractRef": { + "$ref": "#/$defs/adapterId" + }, "probes": { "type": "object", - "required": ["models", "health", "properties", "metrics"], + "required": [ + "models", + "health", + "properties", + "metrics" + ], + "properties": { + "models": { + "const": true + }, + "health": { + "const": true + }, + "properties": { + "const": true + }, + "metrics": { + "const": true + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "providerId": { + "$ref": "#/$defs/backendId" + }, + "server": { + "type": "object", + "required": [ + "technology", + "source" + ], + "properties": { + "technology": { + "const": "llama.cpp" + }, + "source": { + "type": "object", + "required": [ + "repository", + "revision" + ], "properties": { - "models": { "const": true }, - "health": { "const": true }, - "properties": { "const": true }, - "metrics": { "const": true } + "repository": { + "$ref": "#/$defs/repositoryId" + }, + "revision": { + "$ref": "#/$defs/sourceRevision" + } }, "additionalProperties": false } @@ -221,11 +543,21 @@ }, "policy": { "type": "object", - "required": ["egress", "modelSource", "modelDownloads"], + "required": [ + "egress", + "modelSource", + "modelDownloads" + ], "properties": { - "egress": { "const": "disabled" }, - "modelSource": { "const": "verified-local" }, - "modelDownloads": { "const": "disabled" } + "egress": { + "const": "disabled" + }, + "modelSource": { + "const": "verified-local" + }, + "modelDownloads": { + "const": "disabled" + } }, "additionalProperties": false }, @@ -241,13 +573,27 @@ "multimodalProjection" ], "properties": { - "ui": { "const": "disabled" }, - "slotInspection": { "const": "disabled" }, - "router": { "const": "disabled" }, - "mcpProxy": { "const": "disabled" }, - "serverTools": { "const": "disabled" }, - "agentMode": { "const": "disabled" }, - "multimodalProjection": { "const": "disabled" } + "ui": { + "const": "disabled" + }, + "slotInspection": { + "const": "disabled" + }, + "router": { + "const": "disabled" + }, + "mcpProxy": { + "const": "disabled" + }, + "serverTools": { + "const": "disabled" + }, + "agentMode": { + "const": "disabled" + }, + "multimodalProjection": { + "const": "disabled" + } }, "additionalProperties": false }, @@ -272,10 +618,17 @@ "uniqueItems": true, "items": { "type": "object", - "required": ["id", "qualificationRef"], + "required": [ + "id", + "qualificationRef" + ], "properties": { - "id": { "$ref": "#/$defs/nonPathStableId" }, - "qualificationRef": { "$ref": "#/$defs/stableId" } + "id": { + "$ref": "#/$defs/nonPathStableId" + }, + "qualificationRef": { + "$ref": "#/$defs/stableId" + } }, "additionalProperties": false } @@ -284,39 +637,229 @@ "type": "array", "minItems": 1, "maxItems": 1, - "items": { "const": "openai-completions" } - }, - "streaming": { "type": "boolean" }, - "toolCalls": { "type": "boolean" }, - "structuredOutputs": { "type": "boolean" }, - "parallelToolCalls": { "const": false }, - "responsesApi": { "const": false }, - "embeddings": { "const": false }, - "reranking": { "const": false }, - "multimodal": { "const": false } + "items": { + "const": "openai-completions" + } + }, + "streaming": { + "type": "boolean" + }, + "toolCalls": { + "type": "boolean" + }, + "structuredOutputs": { + "type": "boolean" + }, + "parallelToolCalls": { + "const": false + }, + "responsesApi": { + "const": false + }, + "embeddings": { + "const": false + }, + "reranking": { + "const": false + }, + "multimodal": { + "const": false + } }, "additionalProperties": false } }, "additionalProperties": false, - "allOf": [{ "$ref": "#/$defs/llamaCppRecipeSpec" }] + "allOf": [ + { + "$ref": "#/$defs/llamaCppRecipeSpec" + } + ] } }, "additionalProperties": false, "$defs": { "metadata": { "type": "object", - "required": ["id"], + "required": [ + "id" + ], "properties": { - "id": { "$ref": "#/$defs/stableId" }, - "displayName": { "type": "string", "minLength": 1, "maxLength": 256 } + "id": { + "$ref": "#/$defs/stableId" + }, + "displayName": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } }, "additionalProperties": false }, "stableId": { "type": "string", - "pattern": "^[a-z][a-z0-9]*(?:[._/-][a-z0-9]+)+$", - "maxLength": 160 + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,159}$" + }, + "bindingName": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9]{0,63}$" + }, + "token": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + }, + "positiveSafeInteger": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "absolutePath": { + "type": "string", + "pattern": "^/(?!.*(?:^|/)\\.\\.(?:/|$))[A-Za-z0-9._+/-]+$", + "maxLength": 4096 + }, + "relativePath": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))[A-Za-z0-9._+/-]+$", + "maxLength": 4096 + }, + "topologyBinding": { + "type": "object", + "required": [ + "type", + "qualificationId", + "schemaVersion", + "outputSchema" + ], + "properties": { + "type": { + "const": "topologyQualificationOutput" + }, + "qualificationId": { + "$ref": "#/$defs/stableId" + }, + "schemaVersion": { + "type": "integer", + "minimum": 1 + }, + "outputSchema": { + "$ref": "#/$defs/stableId" + } + }, + "additionalProperties": false + }, + "modelPreparation": { + "oneOf": [ + { + "type": "object", + "required": [ + "ref" + ], + "properties": { + "ref": { + "const": "none/v1" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "ref", + "snapshotCopy", + "exactTextReplacement" + ], + "properties": { + "ref": { + "const": "snapshot-copy-and-exact-text-replacement/v1" + }, + "snapshotCopy": { + "type": "object", + "required": [ + "sourcePath", + "digest", + "targetPath" + ], + "properties": { + "sourcePath": { + "$ref": "#/$defs/relativePath" + }, + "digest": { + "$ref": "#/$defs/sha256" + }, + "targetPath": { + "$ref": "#/$defs/absolutePath" + } + }, + "additionalProperties": false + }, + "exactTextReplacement": { + "type": "object", + "required": [ + "targetPath", + "expectedText", + "replacementText" + ], + "properties": { + "targetPath": { + "$ref": "#/$defs/absolutePath" + }, + "expectedText": { + "type": "string", + "minLength": 1, + "maxLength": 16384 + }, + "replacementText": { + "type": "string", + "minLength": 1, + "maxLength": 16384 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + ] + }, + "temporaryFilesystem": { + "type": "object", + "required": [ + "target", + "sizeBytes", + "mode", + "options" + ], + "properties": { + "target": { + "$ref": "#/$defs/absolutePath" + }, + "sizeBytes": { + "$ref": "#/$defs/positiveSafeInteger" + }, + "mode": { + "type": "string", + "pattern": "^[0-7]{4}$" + }, + "options": { + "type": "array", + "maxItems": 8, + "uniqueItems": true, + "items": { + "enum": [ + "rw", + "ro", + "nosuid", + "nodev", + "noexec", + "exec", + "noatime", + "relatime" + ] + } + } + }, + "additionalProperties": false }, "nonPathStableId": { "type": "string", @@ -361,9 +904,16 @@ }, "scalar": { "oneOf": [ - { "type": "string", "maxLength": 1024 }, - { "type": "number" }, - { "type": "boolean" } + { + "type": "string", + "maxLength": 1024 + }, + { + "type": "number" + }, + { + "type": "boolean" + } ] }, "llamaCppRecipeSpec": { @@ -371,22 +921,42 @@ "type": "object", "anyOf": [ { - "properties": { "backend": { "const": "install-llama-cpp" } }, - "required": ["backend"] + "properties": { + "backend": { + "const": "install-llama-cpp" + } + }, + "required": [ + "backend" + ] }, { - "properties": { "providerId": { "const": "llama-cpp-local" } }, - "required": ["providerId"] + "properties": { + "providerId": { + "const": "llama-cpp-local" + } + }, + "required": [ + "providerId" + ] }, { "properties": { "server": { "type": "object", - "properties": { "technology": { "const": "llama.cpp" } }, - "required": ["technology"] + "properties": { + "technology": { + "const": "llama.cpp" + } + }, + "required": [ + "technology" + ] } }, - "required": ["server"] + "required": [ + "server" + ] } ] }, @@ -405,27 +975,55 @@ "capabilities" ], "properties": { - "backend": { "const": "install-llama-cpp" }, - "providerId": { "const": "llama-cpp-local" }, + "backend": { + "const": "install-llama-cpp" + }, + "providerId": { + "const": "llama-cpp-local" + }, "server": { "type": "object", - "properties": { "technology": { "const": "llama.cpp" } }, - "required": ["technology"] + "properties": { + "technology": { + "const": "llama.cpp" + } + }, + "required": [ + "technology" + ] }, "model": { "type": "object", - "required": ["id", "revision", "servedName", "files"], + "required": [ + "id", + "revision", + "servedName", + "files" + ], "properties": { - "id": { "$ref": "#/$defs/repositoryId" }, - "revision": { "$ref": "#/$defs/sourceRevision" }, - "servedName": { "$ref": "#/$defs/nonPathStableId" }, + "id": { + "$ref": "#/$defs/repositoryId" + }, + "revision": { + "$ref": "#/$defs/sourceRevision" + }, + "servedName": { + "$ref": "#/$defs/nonPathStableId" + }, "files": { "type": "array", "minItems": 1, "maxItems": 1, "items": { "type": "object", - "required": ["path", "digest", "sizeBytes", "format", "quantization", "license"], + "required": [ + "path", + "digest", + "sizeBytes", + "format", + "quantization", + "license" + ], "properties": { "path": { "type": "string", @@ -466,21 +1064,37 @@ "anyOf": [ { "type": "object", - "properties": { "architecture": {} }, - "required": ["architecture"] + "properties": { + "architecture": {} + }, + "required": [ + "architecture" + ] }, { "type": "object", - "properties": { "components": {} }, - "required": ["components"] + "properties": { + "components": {} + }, + "required": [ + "components" + ] } ] } }, "execution": { "type": "object", - "required": ["receiptRef", "materializerRef", "lifecycleRef"], - "properties": { "receiptRef": {}, "materializerRef": {}, "lifecycleRef": {} } + "required": [ + "receiptRef", + "materializerRef", + "lifecycleRef" + ], + "properties": { + "receiptRef": {}, + "materializerRef": {}, + "lifecycleRef": {} + } }, "serve": { "type": "object", @@ -495,22 +1109,37 @@ ], "properties": { "protocol": {}, - "port": { "const": 8081 }, + "port": { + "const": 8081 + }, "chatTemplate": {}, "contextSize": {}, - "slots": { "const": 1 }, - "idleSleepSeconds": { "const": -1 }, + "slots": { + "const": 1 + }, + "idleSleepSeconds": { + "const": -1 + }, "limits": {} }, "not": { "type": "object", - "properties": { "arguments": {} }, - "required": ["arguments"] + "properties": { + "arguments": {} + }, + "required": [ + "arguments" + ] } }, "readiness": { "type": "object", - "required": ["contractRef", "timeoutSeconds", "expectedModel", "probes"], + "required": [ + "contractRef", + "timeoutSeconds", + "expectedModel", + "probes" + ], "properties": { "contractRef": {}, "timeoutSeconds": {}, @@ -518,9 +1147,15 @@ "probes": {} } }, - "policy": { "type": "object" }, - "surfaces": { "type": "object" }, - "capabilities": { "type": "object" } + "policy": { + "type": "object" + }, + "surfaces": { + "type": "object" + }, + "capabilities": { + "type": "object" + } } }, "else": { @@ -529,19 +1164,44 @@ "anyOf": [ { "type": "object", - "properties": { "providerId": {} }, - "required": ["providerId"] + "properties": { + "providerId": {} + }, + "required": [ + "providerId" + ] + }, + { + "type": "object", + "properties": { + "policy": {} + }, + "required": [ + "policy" + ] }, - { "type": "object", "properties": { "policy": {} }, "required": ["policy"] }, - { "type": "object", "properties": { "surfaces": {} }, "required": ["surfaces"] }, { "type": "object", - "properties": { "capabilities": {} }, - "required": ["capabilities"] + "properties": { + "surfaces": {} + }, + "required": [ + "surfaces" + ] + }, + { + "type": "object", + "properties": { + "capabilities": {} + }, + "required": [ + "capabilities" + ] } ] } } } - } + }, + "title": "NemoClaw managed inference serving recipe" } diff --git a/package.json b/package.json index 63d817a398..5647788d06 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,11 @@ "agent": "pi", "test": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project cli --project integration --project installer-integration --project package-contract --project plugin --project e2e-support", "test:spec": "npm test -- --reporter=tree", - "test:fast": "npm run clean:cli && vitest run --project cli --project plugin --project e2e-support", - "test:changed": "vitest run --changed --project cli --project plugin --project e2e-support", - "test:watch": "vitest watch --project cli --project plugin --project e2e-support", - "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false", - "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process", + "test:fast": "npm run clean:cli && npm run catalog:compile && vitest run --project cli --project plugin --project e2e-support", + "test:changed": "npm run catalog:compile && vitest run --changed --project cli --project plugin --project e2e-support", + "test:watch": "npm run catalog:compile && vitest watch --project cli --project plugin --project e2e-support", + "test:shuffle": "npm run catalog:compile && vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false", + "test:diagnose:leaks": "npm run catalog:compile && vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process", "test:e2e-phases:check": "npm run build:policy-boundary && node --experimental-strip-types --no-warnings tools/e2e/check-semantic-phases.mts", "test:runtime-audit": "tsx scripts/audit-test-runtime.mts", "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration", @@ -55,6 +55,8 @@ "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", "build:policy-boundary": "tsc -p nemoclaw/tsconfig.shared.json", + "catalog:compile": "tsx src/lib/inference/serving/generate-catalog.ts", + "catalog:check": "tsx src/lib/inference/serving/generate-catalog.ts --check", "build:cli": "npm run build:policy-boundary && tsc -p tsconfig.src.json && node dist/lib/core/generate-build-identity.js && node dist/lib/inference/serving/generate-catalog.js && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "clean:cli": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", "typecheck:cli": "tsc -p tsconfig.cli.json", diff --git a/scripts/install.sh b/scripts/install.sh index 2098c92568..1305d4cfaa 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4603,10 +4603,11 @@ describe_express_install() { "DGX Spark") if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then inference_summary="managed local vLLM with model ${NEMOCLAW_VLLM_MODEL}" + inference_disclosure="The explicit model remains authoritative, so this run keeps the existing single-host DGX Spark profile. Managed vLLM pulls the configured image/model and runs only its dedicated container." else - inference_summary="managed local vLLM using the DGX Spark profile default model" + inference_summary="managed vLLM with automatic DGX Spark serving-profile selection" + inference_disclosure="With no explicit inference intent or related runtime, one exactly qualified pretrusted managed cluster topology selects a matching pinned distributed profile. An ordinary no-match keeps the existing single-host DGX Spark profile; any related or ambiguous setup remains untouched and stops installation. Managed vLLM pulls the selected image/model and runs only its dedicated containers. The selected distributed profile is experimental pending physical end-to-end validation." fi - inference_disclosure="Managed vLLM pulls the configured vLLM image/model and runs a local vLLM inference container." sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts index c8ed8cbf3c..352ec59780 100644 --- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -40,20 +40,25 @@ function okWithKnownGatewayList(command: string, args: readonly string[]): RunRe : ok(); } -describe("dual-Station runtime uninstall", () => { - it("removes a managed pair before the remaining full-uninstall steps", () => { +function managedRuntimeBindingPath(receiptPath: string): string { + return receiptPath.endsWith("managed-cluster-vllm-runtime.json") + ? `${receiptPath}.rank-1.ssh-binding` + : `${receiptPath}.ssh-binding`; +} + +describe("managed distributed vLLM runtime uninstall", () => { + it.each([ + "dual-station-vllm-runtime.json", + "managed-cluster-vllm-runtime.json", + ])("removes the runtime owned by %s before the remaining full-uninstall steps", (receiptFile) => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-pair-")); const stateDir = path.join(home, ".nemoclaw"); fs.mkdirSync(stateDir, { mode: 0o700 }); - fs.writeFileSync(path.join(stateDir, "dual-station-vllm-runtime.json"), "{}\n", { - mode: 0o600, - }); - fs.writeFileSync(path.join(stateDir, "dual-station-vllm-api-key"), "ab".repeat(32), { + const receiptPath = path.join(stateDir, receiptFile); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600, }); - fs.mkdirSync(path.join(stateDir, "dual-station-vllm-runtime.json.ssh-binding"), { - mode: 0o700, - }); + fs.mkdirSync(managedRuntimeBindingPath(receiptPath), { mode: 0o700 }); const runDualStationRuntimeCleanup = vi.fn(() => ok()); const rmSync = vi.fn(); const runDocker = vi.fn(() => ok()); @@ -90,16 +95,10 @@ describe("dual-Station runtime uninstall", () => { const port = 9123; const stateDir = path.join(home, ".nemoclaw"); const legacyStateDir = path.join(stateDir, "gateways", String(port)); + const receiptPath = path.join(legacyStateDir, "dual-station-vllm-runtime.json"); fs.mkdirSync(legacyStateDir, { recursive: true, mode: 0o700 }); - fs.writeFileSync(path.join(legacyStateDir, "dual-station-vllm-runtime.json"), "{}\n", { - mode: 0o600, - }); - fs.writeFileSync(path.join(stateDir, "dual-station-vllm-api-key"), "ab".repeat(32), { - mode: 0o600, - }); - fs.mkdirSync(path.join(legacyStateDir, "dual-station-vllm-runtime.json.ssh-binding"), { - mode: 0o700, - }); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 }); + fs.mkdirSync(`${receiptPath}.ssh-binding`, { mode: 0o700 }); const runDualStationRuntimeCleanup = vi.fn(() => ok()); try { @@ -110,11 +109,7 @@ describe("dual-Station runtime uninstall", () => { { assumeYes: true, deleteModels: false, keepOpenShell: true }, { commandExists: () => true, - env: { - HOME: home, - NEMOCLAW_GATEWAY_PORT: String(port), - TMPDIR: home, - } as NodeJS.ProcessEnv, + env: { HOME: home, NEMOCLAW_GATEWAY_PORT: String(port), TMPDIR: home }, existsSync: () => false, isTty: false, log: vi.fn(), @@ -139,7 +134,10 @@ describe("dual-Station runtime uninstall", () => { ); expect(result.exitCode).toBe(0); - expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce(); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledWith( + receiptPath, + expect.objectContaining({ stdio: "inherit" }), + ); } finally { vi.unstubAllEnvs(); vi.resetModules(); @@ -147,6 +145,144 @@ describe("dual-Station runtime uninstall", () => { } }); + it("associates a canonical cluster discovery binding with its durable receipt", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-spark-claim-")); + const stateDir = path.join(home, ".nemoclaw"); + const receiptPath = path.join(stateDir, "managed-cluster-vllm-runtime.json"); + const discoveryBindingPath = path.join( + stateDir, + "managed-cluster-managed-serving.json.spark-worker.ssh-binding", + ); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 }); + fs.mkdirSync(managedRuntimeBindingPath(receiptPath), { mode: 0o700 }); + fs.mkdirSync(discoveryBindingPath, { mode: 0o700 }); + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: okWithKnownGatewayList, + runDocker: () => ok(), + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledWith( + receiptPath, + expect.objectContaining({ stdio: "inherit" }), + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it.each([ + { + title: "a noncanonical gateway cluster claim despite a host-global cluster receipt", + receiptFile: "managed-cluster-vllm-runtime.json", + bindingSegments: [ + "gateways", + "18080", + "managed-cluster-managed-serving.json.spark-worker.ssh-binding", + ], + }, + { + title: "a canonical cluster claim when only a Station receipt exists", + receiptFile: "dual-station-vllm-runtime.json", + bindingSegments: ["managed-cluster-managed-serving.json.spark-worker.ssh-binding"], + }, + ])("refuses $title", ({ receiptFile, bindingSegments }) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-spark-claim-other-")); + const stateDir = path.join(home, ".nemoclaw"); + const receiptPath = path.join(stateDir, receiptFile); + const discoveryBindingPath = path.join(stateDir, ...bindingSegments); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 }); + fs.mkdirSync(managedRuntimeBindingPath(receiptPath), { mode: 0o700 }); + fs.mkdirSync(discoveryBindingPath, { recursive: true, mode: 0o700 }); + const errors: string[] = []; + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const rmSync = vi.fn(); + const runDocker = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync, + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(runDualStationRuntimeCleanup).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(rmSync).not.toHaveBeenCalled(); + expect(errors.join("\n")).toContain( + "Managed distributed vLLM SSH binding exists without its ownership receipt", + ); + expect(fs.existsSync(discoveryBindingPath)).toBe(true); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("targets the exact Station receipt found under a stale non-default gateway root", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-stale-station-")); + const receiptPath = path.join( + home, + ".nemoclaw", + "gateways", + "18080", + "dual-station-vllm-runtime.json", + ); + fs.mkdirSync(path.dirname(receiptPath), { recursive: true, mode: 0o700 }); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 }); + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: okWithKnownGatewayList, + runDocker: () => ok(), + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledWith( + receiptPath, + expect.objectContaining({ stdio: "inherit" }), + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("preserves host-global pair ownership while sibling gateways remain", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-scoped-")); const stateDir = path.join(home, ".nemoclaw"); @@ -226,7 +362,206 @@ describe("dual-Station runtime uninstall", () => { expect(rmSync).not.toHaveBeenCalled(); expect(runDocker).not.toHaveBeenCalled(); expect(errors).toContain( - "Managed dual-Station cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", + "Managed distributed vLLM cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("refuses ambiguous Spark and Station receipts before cleanup or other mutation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-conflict-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + for (const name of ["managed-cluster-vllm-runtime.json", "dual-station-vllm-runtime.json"]) { + fs.writeFileSync(path.join(stateDir, name), "{}\n", { mode: 0o600 }); + } + const errors: string[] = []; + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const runDocker = vi.fn(() => ok()); + const rmSync = vi.fn(); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync, + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(runDualStationRuntimeCleanup).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(rmSync).not.toHaveBeenCalled(); + expect(errors).toContain( + "Both managed cluster and dual-Station managed runtime receipts exist. NemoClaw refused ambiguous cleanup before making changes.", + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it.each([ + "managed-cluster-vllm-runtime.json.rank-1.ssh-binding", + "managed-cluster-managed-serving.json.spark-worker.ssh-binding", + "dual-station-vllm-runtime.json.ssh-binding", + ])("refuses an orphaned %s before cleanup or other mutation", (bindingEntry) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-binding-orphan-")); + const stateDir = path.join(home, ".nemoclaw"); + const bindingPath = path.join(stateDir, bindingEntry); + fs.mkdirSync(bindingPath, { recursive: true, mode: 0o700 }); + const errors: string[] = []; + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const runDocker = vi.fn(() => ok()); + const rmSync = vi.fn(); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync, + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(runDualStationRuntimeCleanup).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(rmSync).not.toHaveBeenCalled(); + expect(errors.join("\n")).toContain( + "Managed distributed vLLM SSH binding exists without its ownership receipt", + ); + expect(fs.existsSync(bindingPath)).toBe(true); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it.each([ + ["Spark", "managed-cluster-vllm-runtime.json"], + ["Station", "dual-station-vllm-runtime.json"], + ])("finds the host-global %s receipt from a non-default gateway selection", async (_topology, receiptFile) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-managed-global-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(path.join(stateDir, receiptFile), "{}\n", { + mode: 0o600, + }); + fs.mkdirSync(managedRuntimeBindingPath(path.join(stateDir, receiptFile)), { + mode: 0o700, + }); + fs.writeFileSync(path.join(stateDir, "dual-station-vllm-api-key"), `${"a".repeat(64)}\n`, { + mode: 0o600, + }); + fs.mkdirSync(path.join(stateDir, "state", "mcp-lifecycle-locks"), { + recursive: true, + mode: 0o700, + }); + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + + try { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + vi.resetModules(); + const { runUninstallPlan: runFreshUninstallPlan } = await import("./run-plan"); + const result = runFreshUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home, NEMOCLAW_GATEWAY_PORT: "18080" }, + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: (command, args) => + command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify([{ name: "nemoclaw-18080" }])) + : ok(), + runDocker: () => ok(), + runDualStationRuntimeCleanup, + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce(); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it.each([ + { + shape: "symbolic link", + arrange: (stateDir: string, home: string) => { + const target = path.join(home, "redirected-state"); + fs.mkdirSync(target, { mode: 0o700 }); + fs.symlinkSync(target, stateDir, "dir"); + }, + }, + { + shape: "regular file", + arrange: (stateDir: string) => { + fs.writeFileSync(stateDir, "not a directory\n", { mode: 0o600 }); + }, + }, + ])("fails closed when the host-global managed state root is a $shape", ({ arrange }) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-unsafe-root-")); + const stateDir = path.join(home, ".nemoclaw"); + arrange(stateDir, home); + const errors: string[] = []; + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const runDocker = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(runDualStationRuntimeCleanup).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(errors.join("\n")).toContain( + "Managed distributed vLLM state root is not a real directory", ); } finally { fs.rmSync(home, { recursive: true, force: true }); diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 328e886391..48cddd8720 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -1072,11 +1072,18 @@ describe("uninstall gateway-port segregation (#3053)", () => { }, }), ); - const runtimeReceipt = path.join(stateDir, "dual-station-vllm-runtime.json"); - const runtimeBinding = `${runtimeReceipt}.ssh-binding`; + const runtimeReceipt = path.join(stateDir, "managed-cluster-vllm-runtime.json"); + const runtimeBinding = `${runtimeReceipt}.rank-1.ssh-binding`; + const discoveryBinding = path.join( + stateDir, + "managed-cluster-managed-serving.json.spark-worker.ssh-binding", + ); + const managedApiKey = path.join(stateDir, "dual-station-vllm-api-key"); fs.writeFileSync(runtimeReceipt, "{}\n", { mode: 0o600 }); fs.mkdirSync(runtimeBinding, { mode: 0o700 }); + fs.mkdirSync(discoveryBinding, { mode: 0o700 }); fs.writeFileSync(path.join(runtimeBinding, "known_hosts"), "host-key\n", { mode: 0o600 }); + fs.writeFileSync(managedApiKey, `${"a".repeat(64)}\n`, { mode: 0o600 }); const logs: string[] = []; const openshellCalls: string[][] = []; const result = runUninstallPlan( @@ -1102,6 +1109,8 @@ describe("uninstall gateway-port segregation (#3053)", () => { expect(logs.join("\n")).toContain("Sibling gateways remain"); expect(fs.existsSync(runtimeReceipt)).toBe(true); expect(fs.existsSync(runtimeBinding)).toBe(true); + expect(fs.existsSync(discoveryBinding)).toBe(true); + expect(fs.existsSync(managedApiKey)).toBe(true); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index 1eb4a92fc2..fd17027e97 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -1367,7 +1367,7 @@ describe("uninstall run plan", () => { } }); - it("removes ~/.nemoclaw wholesale when it is a symlink rather than a real directory", () => { + it("refuses to follow or remove ~/.nemoclaw when it is a symlink", () => { const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-preserve-")); const realTarget = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-uninstall-preserve-target-"), @@ -1379,12 +1379,14 @@ describe("uninstall run plan", () => { fs.writeFileSync(path.join(realTarget, "rebuild-backups"), "should not be followed"); try { const logs: string[] = []; + const errors: string[] = []; const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, { commandExists: (command) => command === "openshell", env: { HOME: tmpHome } as NodeJS.ProcessEnv, existsSync: (target: string) => target.startsWith(tmpHome) && fs.existsSync(target), + error: (line) => errors.push(line), isTty: false, log: (line) => logs.push(line), run: vi.fn(okWithKnownGatewayList), @@ -1392,10 +1394,13 @@ describe("uninstall run plan", () => { }, ); - expect(result.exitCode).toBe(0); - expect(fs.existsSync(stateDir)).toBe(false); + expect(result.exitCode).toBe(1); + expect(fs.lstatSync(stateDir).isSymbolicLink()).toBe(true); expect(fs.existsSync(realTarget)).toBe(true); - expect(logs).toContain(`Removed ${stateDir}`); + expect(errors.join("\n")).toContain( + "Managed distributed vLLM state root is not a real directory", + ); + expect(logs).not.toContain(`Removed ${stateDir}`); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); fs.rmSync(realTarget, { recursive: true, force: true }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 77aacda82f..39c7513537 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -28,10 +28,14 @@ import { import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { - DUAL_STATION_VLLM_API_KEY_FILE, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, - discoverDualStationVllmRuntimeReceiptStateDirs, -} from "../../inference/vllm-station-runtime-receipt-path"; + findManagedDistributedVllmRuntimeReceipts, + isManagedClusterDiscoveryBindingStateEntry, + isManagedClusterRuntimeBindingStateEntry, + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, + MANAGED_VLLM_API_KEY_FILE, + MCP_LIFECYCLE_LOCK_DIRNAME, +} from "../../inference/serving/managed-runtime-receipts"; import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; import { getNemoclawOpenShellGatewayUserServicePath, @@ -98,7 +102,7 @@ export interface UninstallRunDeps { rmSync?: typeof fs.rmSync; run?: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker?: (args: string[], options?: SpawnSyncOptions) => RunResult; - runDualStationRuntimeCleanup?: (options?: SpawnSyncOptions) => RunResult; + runDualStationRuntimeCleanup?: (receiptPath: string, options?: SpawnSyncOptions) => RunResult; } export interface UninstallRunOutcome { @@ -269,12 +273,52 @@ const SHARED_HOST_STATE_ENTRIES = new Set([ "source", GATEWAYS_SUBDIR, "managed_swap", - DUAL_STATION_VLLM_API_KEY_FILE, + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, `${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`, + MANAGED_VLLM_API_KEY_FILE, ...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES, ]); +function isSharedHostStateEntry(entry: string): boolean { + return ( + SHARED_HOST_STATE_ENTRIES.has(entry) || + isManagedClusterRuntimeBindingStateEntry(entry) || + isManagedClusterDiscoveryBindingStateEntry(entry) + ); +} + +function managedClusterBindingStateEntries(stateDir: string): readonly string[] { + try { + return fs + .readdirSync(stateDir) + .filter( + (entry) => + isManagedClusterRuntimeBindingStateEntry(entry) || + isManagedClusterDiscoveryBindingStateEntry(entry), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } +} + +function dormantHostGlobalLifecycleState(sharedRoot: string): boolean { + const stateDir = path.join(sharedRoot, "state"); + try { + const state = fs.lstatSync(stateDir); + if (state.isSymbolicLink() || !state.isDirectory()) return false; + const entries = fs.readdirSync(stateDir); + if (entries.length === 0) return true; + if (entries.length !== 1 || entries[0] !== MCP_LIFECYCLE_LOCK_DIRNAME) return false; + const locksDir = path.join(stateDir, MCP_LIFECYCLE_LOCK_DIRNAME); + const locks = fs.lstatSync(locksDir); + return !locks.isSymbolicLink() && locks.isDirectory() && fs.readdirSync(locksDir).length === 0; + } catch { + return false; + } +} + function removePathExcept( target: string, preserve: readonly string[], @@ -362,7 +406,7 @@ interface UninstallRuntime { rmSync: typeof fs.rmSync; run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker: (args: string[], options?: SpawnSyncOptions) => RunResult; - runDualStationRuntimeCleanup: (options?: SpawnSyncOptions) => RunResult; + runDualStationRuntimeCleanup: (receiptPath: string, options?: SpawnSyncOptions) => RunResult; warn: (message: string) => void; } @@ -400,7 +444,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { runDocker: deps.runDocker ?? defaultRunDocker, runDualStationRuntimeCleanup: deps.runDualStationRuntimeCleanup ?? - ((options = {}) => + ((receiptPath, options = {}) => defaultRun( process.execPath, [ @@ -411,6 +455,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { "inference", "vllm-station-runtime-cleanup-entry.js", ), + receiptPath, ], options, )), @@ -1230,41 +1275,91 @@ function dockerIsAvailable(runtime: UninstallRuntime): boolean { return true; } -function removeManagedDualStationRuntime( +function managedDistributedVllmStateRootStatus( + paths: UninstallPaths, + runtime: Pick, +): "absent" | "directory" | "unsafe" { + const sharedRoot = path.dirname(paths.managedSwapMarkerPath); + try { + const root = fs.lstatSync(sharedRoot); + if (root.isSymbolicLink() || !root.isDirectory()) { + runtime.error(`Managed distributed vLLM state root is not a real directory: ${sharedRoot}`); + return "unsafe"; + } + return "directory"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return "absent"; + runtime.error(`Could not inspect managed distributed vLLM state root: ${formatError(error)}`); + return "unsafe"; + } +} + +function removeManagedDistributedVllmRuntime( paths: UninstallPaths, runtime: UninstallRuntime, ): boolean { - const sharedStateDir = path.dirname(paths.managedSwapMarkerPath); - const apiKeyPath = path.join(sharedStateDir, DUAL_STATION_VLLM_API_KEY_FILE); - let receiptPaths: string[]; + const rootStatus = managedDistributedVllmStateRootStatus(paths, runtime); + if (rootStatus !== "directory") return rootStatus === "absent"; + const apiKeyPath = path.join( + path.dirname(paths.managedSwapMarkerPath), + MANAGED_VLLM_API_KEY_FILE, + ); + let state: ReturnType; try { - receiptPaths = discoverDualStationVllmRuntimeReceiptStateDirs( - sharedStateDir, - GATEWAYS_SUBDIR, - ).map((stateDir) => path.join(stateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE)); + state = findManagedDistributedVllmRuntimeReceipts({ + homeDir: runtime.env.HOME || os.homedir(), + }); } catch (error) { - runtime.error(`Could not inspect managed dual-Station rollback state: ${formatError(error)}`); + runtime.error( + `Could not inspect managed distributed vLLM rollback state: ${formatError(error)}`, + ); return false; } - let receiptFound = false; - for (const receiptPath of receiptPaths) { - try { - fs.lstatSync(receiptPath); - receiptFound = true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - runtime.error( - `Could not inspect managed dual-Station rollback state: ${formatError(error)}`, - ); - return false; - } + const receipts = [ + ...(state.managedClusterPath ? [state.managedClusterPath] : []), + ...state.stationPaths, + ]; + const bindingPaths = [ + ...state.managedClusterBindingPaths, + ...state.managedClusterDiscoveryBindingPaths, + ...state.stationBindingPaths, + ]; + const expectedBindingPaths = new Set( + state.stationPaths.map((receiptPath) => `${receiptPath}.ssh-binding`), + ); + if (state.managedClusterPath) { + const stateDir = path.dirname(state.managedClusterPath); + for (const bindingPath of [ + ...state.managedClusterBindingPaths, + ...state.managedClusterDiscoveryBindingPaths, + ]) { + if (path.dirname(bindingPath) === stateDir) expectedBindingPaths.add(bindingPath); } } - if (!receiptFound) { + const orphanBinding = bindingPaths.find((bindingPath) => !expectedBindingPaths.has(bindingPath)); + if (orphanBinding) { + runtime.error( + `Managed distributed vLLM SSH binding exists without its ownership receipt at ${orphanBinding}. NemoClaw refused uninstall before making changes. Recover or remove that state explicitly, then retry.`, + ); + return false; + } + if (receipts.length === 0) { removePath(apiKeyPath, runtime); return true; } - const result = runtime.runDualStationRuntimeCleanup({ + if (state.managedClusterPath && state.stationPaths.length > 0) { + runtime.error( + "Both managed cluster and dual-Station managed runtime receipts exist. NemoClaw refused ambiguous cleanup before making changes.", + ); + return false; + } + if (receipts.length !== 1) { + runtime.error( + "Multiple managed distributed vLLM runtime receipts exist. NemoClaw refused ambiguous cleanup before making changes.", + ); + return false; + } + const result = runtime.runDualStationRuntimeCleanup(receipts[0]!, { env: runtime.env, stdio: "inherit", }); @@ -1273,7 +1368,7 @@ function removeManagedDualStationRuntime( return true; } runtime.error( - "Managed dual-Station cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", + "Managed distributed vLLM cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", ); return false; } @@ -1468,7 +1563,15 @@ function discoverOtherGatewayEnvironments( if (!selectedIsDefault && pathEntryExists(sharedRoot, runtime)) { try { - if (fs.readdirSync(sharedRoot).some((entry) => !SHARED_HOST_STATE_ENTRIES.has(entry))) { + if ( + fs + .readdirSync(sharedRoot) + .some( + (entry) => + !isSharedHostStateEntry(entry) && + !(entry === "state" && dormantHostGlobalLifecycleState(sharedRoot)), + ) + ) { return otherGatewaysRemain([DEFAULT_GATEWAY_PORT]); } } catch { @@ -1676,7 +1779,7 @@ function executePlan( for (const [index, step] of plan.steps.entries()) { runtime.log(`[${index + 1}/${plan.steps.length}] ${planStepDisplayName(step.name, branding)}`); if (step.name === "Stopping services") { - if (!scopedToSelectedGateway && !removeManagedDualStationRuntime(paths, runtime)) { + if (!scopedToSelectedGateway && !removeManagedDistributedVllmRuntime(paths, runtime)) { return { ok: false }; } if ( @@ -1851,9 +1954,11 @@ function executePlan( ...(scopedToSelectedGateway ? [ ...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES, - DUAL_STATION_VLLM_API_KEY_FILE, + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, + ...managedClusterBindingStateEntries(paths.nemoclawStateDir), DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, `${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`, + MANAGED_VLLM_API_KEY_FILE, ] : []), ], @@ -1963,6 +2068,9 @@ export function runUninstallPlan( } const resolvedOptions = { ...options, gatewayName: expectedGatewayName }; const { paths, plan } = buildRunPlan(resolvedOptions, { ...deps, env: runtime.env }); + if (managedDistributedVllmStateRootStatus(paths, runtime) === "unsafe") { + return { exitCode: 1, plan }; + } let externallySupervised: boolean; try { externallySupervised = isExternallySupervised( diff --git a/src/lib/inference/context-window.ts b/src/lib/inference/context-window.ts index 433a091814..b18cbf4952 100644 --- a/src/lib/inference/context-window.ts +++ b/src/lib/inference/context-window.ts @@ -13,7 +13,7 @@ import { DEFAULT_CONTEXT_WINDOW } from "./config"; import { getLocalProviderHealthEndpoint, - getManagedDualStationVllmProviderBinding, + getManagedVllmProviderBinding, getOllamaWarmupCommand, probeVllmModels, type RunCaptureFn, @@ -46,9 +46,9 @@ const defaultContextWindowDeps: ContextWindowDeps = { probeVllmContextWindow: (model: string): number | null => { // Same source onboard uses: GET /v1/models on the host vLLM server and read // max_model_len (handles both NemoClaw-launched and bring-your-own vLLM). - let managedBinding: ReturnType; + let managedBinding: ReturnType; try { - managedBinding = getManagedDualStationVllmProviderBinding(); + managedBinding = getManagedVllmProviderBinding(); } catch { return null; } diff --git a/src/lib/inference/local-vllm-auth.test.ts b/src/lib/inference/local-vllm-auth.test.ts index 4c8390143a..ec77707a1b 100644 --- a/src/lib/inference/local-vllm-auth.test.ts +++ b/src/lib/inference/local-vllm-auth.test.ts @@ -14,10 +14,23 @@ interface ManagedBaseUrlOverrides { const lifecycle = vi.hoisted(() => ({ baseUrl: vi.fn<(overrides?: ManagedBaseUrlOverrides) => string | null>(), })); +const managedClusterRecovery = vi.hoisted(() => ({ + endpoint: vi.fn(), +})); +const managedKey = vi.hoisted(() => ({ + load: vi.fn(), +})); vi.mock("./vllm-station-cluster-lifecycle", () => ({ getDualStationManagedVllmBaseUrl: lifecycle.baseUrl, })); +vi.mock("./serving/managed-cluster-runtime-receipt", () => ({ + recoverInstalledManagedClusterVllmEndpoint: managedClusterRecovery.endpoint, +})); +vi.mock("./vllm-api-key", async (importOriginal) => ({ + ...(await importOriginal()), + loadManagedVllmApiKey: managedKey.load, +})); import { CONTAINER_REACHABILITY_IMAGE, @@ -25,8 +38,8 @@ import { getLocalProviderContainerReachabilityCheck, getLocalProviderHealthCheck, getLocalProviderHealthEndpoint, - getManagedDualStationVllmProviderBinding, - getManagedDualStationVllmProviderState, + getManagedVllmProviderBinding, + getManagedVllmProviderState, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, probeLocalProviderHealth, probeVllmModels, @@ -80,6 +93,10 @@ function productionManagedBaseUrlResolver( beforeEach(() => { vi.stubEnv(LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, undefined); + managedClusterRecovery.endpoint.mockReset(); + managedClusterRecovery.endpoint.mockReturnValue(null); + managedKey.load.mockReset(); + managedKey.load.mockReturnValue(API_KEY); lifecycle.baseUrl.mockReset(); lifecycle.baseUrl.mockImplementation( (overrides) => MANAGED_BASE_URL_BY_API_KEY.get(overrides?.loadApiKey?.()) ?? null, @@ -88,19 +105,87 @@ beforeEach(() => { afterEach(() => vi.unstubAllEnvs()); -describe("managed dual-Station vLLM authentication", () => { +describe("managed vLLM authentication", () => { it("keeps an explicit sandbox host override ahead of managed endpoint recovery", () => { const loadApiKeyImpl = vi.fn(() => API_KEY); expect(getLocalProviderBaseUrl("vllm-local", { hostUrl: "http://explicit-host" })).toBe( "http://explicit-host:8000/v1", ); expect( - getManagedDualStationVllmProviderBinding({ + getManagedVllmProviderBinding({ hostUrl: "http://explicit-host", loadApiKeyImpl, }), ).toBeNull(); expect(loadApiKeyImpl).not.toHaveBeenCalled(); + expect(managedClusterRecovery.endpoint).not.toHaveBeenCalled(); + }); + + it("recovers a receipt-owned managed cluster endpoint before checking Station state", () => { + const events: string[] = []; + const recoverManagedClusterVllmEndpointImpl = vi.fn(() => { + events.push("managed-cluster"); + return { baseUrl: BASE_URL, apiKey: API_KEY }; + }); + const getManagedBaseUrlImpl = vi.fn(() => { + events.push("station"); + return null; + }); + + expect( + getManagedVllmProviderBinding({ + getManagedBaseUrlImpl, + recoverManagedClusterVllmEndpointImpl, + }), + ).toEqual({ baseUrl: `${BASE_URL}/v1`, apiKey: API_KEY }); + expect(events).toEqual(["managed-cluster", "station"]); + }); + + it("does not fall through to Station when managed cluster recovery is unsafe", () => { + const getManagedBaseUrlImpl = vi.fn(() => BASE_URL); + + expect(() => + getManagedVllmProviderBinding({ + getManagedBaseUrlImpl, + recoverManagedClusterVllmEndpointImpl: () => { + throw new Error("managed cluster receipt identity changed"); + }, + }), + ).toThrow("managed cluster receipt identity changed"); + expect(getManagedBaseUrlImpl).not.toHaveBeenCalled(); + }); + + it("fails URL and validation boundaries closed when managed recovery is unsafe", () => { + managedClusterRecovery.endpoint.mockImplementation(() => { + throw new Error("managed cluster receipt identity changed"); + }); + const capture = vi.fn(() => "200"); + + expect(getLocalProviderBaseUrl("vllm-local")).toBeNull(); + expect(getLocalProviderHealthEndpoint("vllm-local")).toBeNull(); + expect(getLocalProviderHealthCheck("vllm-local")).toBeNull(); + expect(getLocalProviderContainerReachabilityCheck("vllm-local")).toBeNull(); + expect(validateLocalProvider("vllm-local", capture)).toEqual({ + ok: false, + message: + "Managed vLLM state could not be inspected safely. Re-run `nemoclaw onboard` to repair the provider.", + }); + expect(capture).not.toHaveBeenCalled(); + expect(lifecycle.baseUrl).not.toHaveBeenCalled(); + }); + + it("stops when managed cluster and Station state are both present", () => { + expect(() => + getManagedVllmProviderBinding({ + loadApiKeyImpl: () => API_KEY, + recoverManagedClusterVllmEndpointImpl: () => ({ baseUrl: BASE_URL, apiKey: API_KEY }), + getManagedBaseUrlImpl: (overrides) => { + overrides?.onManagedHeadObserved?.(); + overrides?.loadApiKey?.(); + return BASE_URL; + }, + }), + ).toThrow("Both managed cluster and Station vLLM state are present"); }); it("does not load a stale or unsafe key without a recovered managed endpoint", () => { @@ -109,7 +194,7 @@ describe("managed dual-Station vLLM authentication", () => { }); lifecycle.baseUrl.mockReturnValue(null); - expect(getManagedDualStationVllmProviderBinding({ loadApiKeyImpl })).toBeNull(); + expect(getManagedVllmProviderBinding({ loadApiKeyImpl })).toBeNull(); expect(loadApiKeyImpl).not.toHaveBeenCalled(); }); @@ -146,11 +231,11 @@ describe("managed dual-Station vLLM authentication", () => { }); it("returns one atomic provider endpoint and credential binding", () => { - expect(getManagedDualStationVllmProviderBinding({ loadApiKeyImpl: () => API_KEY })).toEqual({ + expect(getManagedVllmProviderBinding({ loadApiKeyImpl: () => API_KEY })).toEqual({ baseUrl: `${BASE_URL}/v1`, apiKey: API_KEY, }); - expect(getManagedDualStationVllmProviderState({ loadApiKeyImpl: () => API_KEY })).toEqual({ + expect(getManagedVllmProviderState({ loadApiKeyImpl: () => API_KEY })).toEqual({ kind: "ready", baseUrl: `${BASE_URL}/v1`, apiKey: API_KEY, @@ -294,7 +379,7 @@ describe("managed dual-Station vLLM authentication", () => { it("returns invalid-auth when the private key does not match the managed lifecycle", () => { expect( - getManagedDualStationVllmProviderState({ + getManagedVllmProviderState({ getManagedBaseUrlImpl: productionManagedBaseUrlResolver(OTHER_API_KEY), loadApiKeyImpl: () => API_KEY, }), @@ -312,7 +397,7 @@ describe("managed dual-Station vLLM authentication", () => { expect(runCurlProbeImpl).not.toHaveBeenCalled(); expect(result).toMatchObject({ ok: false, - endpoint: "managed dual-Station vLLM", + endpoint: "managed vLLM", failureLabel: "unhealthy", }); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index b42e036b65..5412a8830e 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -46,7 +46,11 @@ import { resetOllamaRuntimeContextWindowAutoState, resolveOllamaRuntimeContextWindow as resolveOllamaRuntimeContextWindowWithHost, } from "./ollama-runtime-context"; -import { loadDualStationVllmApiKey } from "./vllm-api-key"; +import { + type RecoveredManagedClusterVllmEndpoint, + recoverInstalledManagedClusterVllmEndpoint, +} from "./serving/managed-cluster-runtime-receipt"; +import { loadManagedVllmApiKey } from "./vllm-api-key"; import { applyVllmRuntimeContextWindow as applyVllmRuntimeContextWindowFromModels } from "./vllm-runtime-context"; import { getDualStationManagedVllmBaseUrl } from "./vllm-station-cluster-lifecycle"; @@ -254,10 +258,12 @@ export interface LocalProviderHealthProbeOptions { * state root (written by inference/ollama/proxy.ts during onboard). */ loadOllamaProxyTokenImpl?: () => string | null; - /** Reads the managed dual-Station vLLM key. Injectable so tests stay deterministic. */ + /** Reads the host-global managed vLLM key. Injectable so tests stay deterministic. */ loadVllmApiKeyImpl?: () => string | null; - /** Recovers an owned managed endpoint while validating the injected key in one lifecycle read. */ - getManagedVllmBaseUrlImpl?: ManagedDualStationVllmBaseUrlResolver; + /** Recovers a managed Station endpoint while validating the injected key. */ + getManagedVllmBaseUrlImpl?: ManagedStationVllmBaseUrlResolver; + /** Recovers a receipt-owned managed cluster endpoint. */ + recoverManagedClusterVllmEndpointImpl?: ManagedClusterVllmEndpointResolver; } function defaultLoadOllamaProxyToken(): string | null { @@ -410,39 +416,53 @@ function configuredLocalInferenceHostUrl(hostUrl?: string | null): string | null ); } -function recoveredManagedDualStationVllmBaseUrl(): string | null { - return configuredLocalInferenceHostUrl() ? null : getDualStationManagedVllmBaseUrl(); +type RecoveredManagedVllmBaseUrl = + | { readonly kind: "available"; readonly baseUrl: string | null } + | { readonly kind: "unavailable" }; + +function recoveredManagedVllmBaseUrl(): RecoveredManagedVllmBaseUrl { + if (configuredLocalInferenceHostUrl()) return { kind: "available", baseUrl: null }; + try { + return { + kind: "available", + baseUrl: getManagedVllmProviderBinding()?.baseUrl.replace(/\/v1\/?$/, "") ?? null, + }; + } catch { + return { kind: "unavailable" }; + } } -export interface ManagedDualStationVllmProviderBinding { +export interface ManagedVllmProviderBinding { baseUrl: string; apiKey: string; } -type ManagedDualStationVllmBaseUrlResolver = (overrides?: { +type ManagedStationVllmBaseUrlResolver = (overrides?: { loadApiKey?: () => string | null; onManagedHeadObserved?: () => void; }) => string | null; -export type ManagedDualStationVllmProviderState = +type ManagedClusterVllmEndpointResolver = (options?: { + loadApiKey?: () => string | null; +}) => Pick | null; + +export type ManagedVllmProviderState = | { kind: "absent" } | { kind: "invalid-auth"; reason: "missing" | "unsafe" | "mismatched" } - | ({ kind: "ready" } & ManagedDualStationVllmProviderBinding); + | ({ kind: "ready" } & ManagedVllmProviderBinding); -export interface ManagedDualStationVllmProviderBindingOptions { +export interface ManagedVllmProviderBindingOptions { hostUrl?: string | null; - getManagedBaseUrlImpl?: ManagedDualStationVllmBaseUrlResolver; + /** Compatibility seam for the Station lifecycle resolver. */ + getManagedBaseUrlImpl?: ManagedStationVllmBaseUrlResolver; loadApiKeyImpl?: () => string | null; + recoverManagedClusterVllmEndpointImpl?: ManagedClusterVllmEndpointResolver; } -/** Recover endpoint and credential as one lifecycle-validated state. */ -export function getManagedDualStationVllmProviderState( - options: ManagedDualStationVllmProviderBindingOptions = {}, -): ManagedDualStationVllmProviderState { - const configuredHostUrl = configuredLocalInferenceHostUrl(options.hostUrl); - if (configuredHostUrl) return { kind: "absent" }; - - const loadApiKey = options.loadApiKeyImpl ?? loadDualStationVllmApiKey; +function getManagedStationVllmProviderState( + options: ManagedVllmProviderBindingOptions, + loadApiKey: () => string | null, +): ManagedVllmProviderState { let keyRead = false; let managedHeadObserved = false; let apiKey: string | null = null; @@ -481,20 +501,67 @@ export function getManagedDualStationVllmProviderState( if (!managedBaseUrl || !apiKey) { return { kind: "invalid-auth", reason: authFailure ?? "mismatched" }; } - return { kind: "ready", baseUrl: `${managedBaseUrl}/v1`, apiKey }; + return { kind: "ready", baseUrl: `${managedBaseUrl.replace(/\/+$/, "")}/v1`, apiKey }; +} + +/** Recover the one owned managed endpoint and credential as a validated state. */ +export function getManagedVllmProviderState( + options: ManagedVllmProviderBindingOptions = {}, +): ManagedVllmProviderState { + if (configuredLocalInferenceHostUrl(options.hostUrl)) return { kind: "absent" }; + + const loadApiKey = options.loadApiKeyImpl ?? loadManagedVllmApiKey; + let managedClusterAuthFailure: "missing" | "unsafe" | null = null; + let managedClusterEndpoint: Pick< + RecoveredManagedClusterVllmEndpoint, + "baseUrl" | "apiKey" + > | null; + try { + managedClusterEndpoint = ( + options.recoverManagedClusterVllmEndpointImpl ?? recoverInstalledManagedClusterVllmEndpoint + )({ + loadApiKey: () => { + try { + const apiKey = loadApiKey(); + if (!apiKey) managedClusterAuthFailure = "missing"; + return apiKey; + } catch { + managedClusterAuthFailure = "unsafe"; + return null; + } + }, + }); + } catch (error) { + if (managedClusterAuthFailure) { + return { kind: "invalid-auth", reason: managedClusterAuthFailure }; + } + throw error; + } + + const stationState = getManagedStationVllmProviderState(options, loadApiKey); + if (!managedClusterEndpoint) return stationState; + if (stationState.kind !== "absent") { + throw new Error( + "Both managed cluster and Station vLLM state are present; refusing to select either endpoint.", + ); + } + return { + kind: "ready", + baseUrl: `${managedClusterEndpoint.baseUrl.replace(/\/+$/, "")}/v1`, + apiKey: managedClusterEndpoint.apiKey, + }; } -/** Compatibility binding for onboarding and context-window callers. */ -export function getManagedDualStationVllmProviderBinding( - options: ManagedDualStationVllmProviderBindingOptions = {}, -): ManagedDualStationVllmProviderBinding | null { - const state = getManagedDualStationVllmProviderState(options); +export function getManagedVllmProviderBinding( + options: ManagedVllmProviderBindingOptions = {}, +): ManagedVllmProviderBinding | null { + const state = getManagedVllmProviderState(options); if (state.kind === "absent") return null; if (state.kind === "invalid-auth") { if (state.reason !== "missing") { - throw new Error("Managed dual-Station vLLM authentication is unsafe or mismatched."); + throw new Error("Managed vLLM authentication is unsafe or mismatched."); } - throw new Error("Managed dual-Station vLLM authentication is missing."); + throw new Error("Managed vLLM authentication is missing."); } return { baseUrl: state.baseUrl, apiKey: state.apiKey }; } @@ -508,8 +575,9 @@ export function getLocalProviderBaseUrl( switch (provider) { case "vllm-local": { if (!configuredHostUrl) { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); - if (dualStationBaseUrl) return `${dualStationBaseUrl}/v1`; + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + if (managed.baseUrl) return `${managed.baseUrl}/v1`; } return `${hostUrl}:${VLLM_PORT}/v1`; } @@ -524,8 +592,9 @@ export function getLocalProviderBaseUrl( export function getLocalProviderValidationBaseUrl(provider: string): string | null { switch (provider) { case "vllm-local": { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); - return dualStationBaseUrl ? `${dualStationBaseUrl}/v1` : `http://127.0.0.1:${VLLM_PORT}/v1`; + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + return managed.baseUrl ? `${managed.baseUrl}/v1` : `http://127.0.0.1:${VLLM_PORT}/v1`; } case "ollama-local": return `http://${getResolvedOllamaHost()}:${OLLAMA_PORT}/v1`; @@ -537,9 +606,10 @@ export function getLocalProviderValidationBaseUrl(provider: string): string | nu export function getLocalProviderHealthEndpoint(provider: string): string | null { switch (provider) { case "vllm-local": { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); - return dualStationBaseUrl - ? `${dualStationBaseUrl}/v1/models` + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + return managed.baseUrl + ? `${managed.baseUrl}/v1/models` : `http://127.0.0.1:${VLLM_PORT}/v1/models`; } case "ollama-local": @@ -552,8 +622,10 @@ export function getLocalProviderHealthEndpoint(provider: string): string | null /** Lightweight endpoint used only to prove that the local service is reachable. */ export function getLocalProviderAvailabilityEndpoint(provider: string): string | null { if (provider === "vllm-local") { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); - if (dualStationBaseUrl) return `${dualStationBaseUrl}/health`; + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + if (managed.baseUrl) return `${managed.baseUrl}/health`; + return `http://127.0.0.1:${VLLM_PORT}/v1/models`; } return getLocalProviderHealthEndpoint(provider); } @@ -744,22 +816,23 @@ export function probeLocalProviderHealth( const providerLabel = getLocalProviderLabel(provider); if (!providerLabel) return null; - let managedState: ManagedDualStationVllmProviderState = { kind: "absent" }; + let managedState: ManagedVllmProviderState = { kind: "absent" }; if (provider === "vllm-local") { try { - managedState = getManagedDualStationVllmProviderState({ + managedState = getManagedVllmProviderState({ getManagedBaseUrlImpl: options.getManagedVllmBaseUrlImpl, loadApiKeyImpl: options.loadVllmApiKeyImpl, + recoverManagedClusterVllmEndpointImpl: options.recoverManagedClusterVllmEndpointImpl, }); } catch { return { ok: false, providerLabel, - endpoint: "managed dual-Station vLLM", + endpoint: "managed vLLM", failureLabel: "unhealthy", probeLabel: "vllm backend", detail: - "Local vLLM authentication state could not be inspected safely. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", + "Managed vLLM state could not be inspected safely. Re-run `nemoclaw onboard` to repair the provider.", }; } } @@ -768,12 +841,12 @@ export function probeLocalProviderHealth( return { ok: false, providerLabel, - endpoint: "managed dual-Station vLLM", + endpoint: "managed vLLM", failureLabel: missingAuth ? "unauthorized" : "unhealthy", probeLabel: "vllm backend", detail: missingAuth - ? "Local vLLM requires its managed bearer credential, but no private key is available. Re-run `nemoclaw onboard` to repair the dual-Station provider." - : "Local vLLM authentication state is unsafe or does not match the managed service. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", + ? "Managed vLLM requires its bearer credential, but no private key is available. Re-run `nemoclaw onboard` to repair the provider." + : "Managed vLLM authentication state is unsafe or does not match the service. Re-run `nemoclaw onboard` to repair the provider.", }; } const managedBinding = managedState.kind === "ready" ? managedState : null; @@ -891,9 +964,11 @@ export function probeLocalProviderHealth( export function getLocalProviderContainerReachabilityCheck(provider: string): string[] | null { switch (provider) { case "vllm-local": { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + const managedBaseUrl = managed.baseUrl; return [ - ...(dualStationBaseUrl ? ["docker", "--context", "default"] : ["docker"]), + ...(managedBaseUrl ? ["docker", "--context", "default"] : ["docker"]), "run", "--rm", "--add-host", @@ -903,11 +978,11 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st "5", "--max-time", "10", - ...(dualStationBaseUrl ? ["--noproxy", "*"] : []), + ...(managedBaseUrl ? ["--noproxy", "*"] : []), "-sf", - ...(dualStationBaseUrl ? ["-w", "%{http_code}"] : []), - dualStationBaseUrl - ? `${dualStationBaseUrl}/health` + ...(managedBaseUrl ? ["-w", "%{http_code}"] : []), + managedBaseUrl + ? `${managedBaseUrl}/health` : `http://host.openshell.internal:${VLLM_PORT}/v1/models`, ]; } @@ -961,6 +1036,13 @@ export function validateLocalProvider( const sleep = sleepFn ?? sleepSeconds; const command = getLocalProviderHealthCheck(provider); if (!command) { + if (provider === "vllm-local") { + return { + ok: false, + message: + "Managed vLLM state could not be inspected safely. Re-run `nemoclaw onboard` to repair the provider.", + }; + } return { ok: true }; } @@ -984,6 +1066,13 @@ export function validateLocalProvider( const containerCommand = getLocalProviderContainerReachabilityCheck(provider); if (!containerCommand) { + if (provider === "vllm-local") { + return { + ok: false, + message: + "Managed vLLM state could not be inspected safely. Re-run `nemoclaw onboard` to repair the provider.", + }; + } return { ok: true }; } @@ -1023,12 +1112,13 @@ export function validateLocalProvider( } } -function getContainerCheckUrl(provider: string): string { +function getContainerCheckUrl(provider: string): string | null { switch (provider) { case "vllm-local": { - const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); - return dualStationBaseUrl - ? `${dualStationBaseUrl}/health` + const managed = recoveredManagedVllmBaseUrl(); + if (managed.kind === "unavailable") return null; + return managed.baseUrl + ? `${managed.baseUrl}/health` : `http://host.openshell.internal:${VLLM_PORT}/v1/models`; } case "ollama-local": @@ -1040,8 +1130,9 @@ function getContainerCheckUrl(provider: string): string { function collectContainerDiagnostic(provider: string, capture: RunCaptureFn): string { const url = getContainerCheckUrl(provider); + if (!url) return "Managed vLLM state could not be inspected safely."; const dockerCommand = - provider === "vllm-local" && recoveredManagedDualStationVllmBaseUrl() + provider === "vllm-local" && url.endsWith("/health") ? ["docker", "--context", "default"] : ["docker"]; try { diff --git a/src/lib/inference/serving/adapter-registry.test.ts b/src/lib/inference/serving/adapter-registry.test.ts new file mode 100644 index 0000000000..3af8b1b9dc --- /dev/null +++ b/src/lib/inference/serving/adapter-registry.test.ts @@ -0,0 +1,290 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + getManagedInferenceLifecycleDescriptor, + getManagedInferenceMaterializerDescriptor, + getManagedInferencePreparationDescriptor, + getManagedInferenceRecipeRegistrationError, + getManagedInferenceTopologyQualificationDescriptor, + listManagedInferenceLifecycleDescriptors, + listManagedInferenceMaterializerDescriptors, + listManagedInferencePreparationDescriptors, + listManagedInferenceTopologyQualificationDescriptors, + MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, + MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, + NO_PREPARATION_REF, + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, +} from "./adapter-registry.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import type { ManagedInferenceServingRecipe } from "./types.js"; +import { fixtureManagedClusterSelection } from "./managed-cluster-fixture.test-support.js"; +import { + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, +} from "./managed-cluster-topology.js"; + +function shippedRecipe(): ManagedInferenceServingRecipe { + const recipe = loadManagedInferenceCatalog().recipes.find( + ({ spec }) => spec.execution.materializerRef === MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, + ); + expect(recipe).toBeDefined(); + return structuredClone(recipe as ManagedInferenceServingRecipe); +} + +describe("managed inference adapter registries", () => { + it("registers one versioned descriptor for each shipped managed cluster mechanic", () => { + expect(listManagedInferenceTopologyQualificationDescriptors()).toMatchObject([ + { + id: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + bindingOutput: "topology", + }, + ]); + expect(listManagedInferenceMaterializerDescriptors()).toMatchObject([ + { ref: MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, backend: "vllm" }, + ]); + expect(listManagedInferenceLifecycleDescriptors()).toMatchObject([ + { ref: MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, backend: "vllm" }, + ]); + expect(listManagedInferencePreparationDescriptors()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ref: NO_PREPARATION_REF, backend: "vllm" }), + expect.objectContaining({ + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + backend: "vllm", + }), + ]), + ); + }); + + it("looks up mechanics only by their stable registry references", () => { + expect( + getManagedInferenceTopologyQualificationDescriptor( + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + ), + ).toBeDefined(); + expect( + getManagedInferenceTopologyQualificationDescriptor("unknown.topology", 1), + ).toBeUndefined(); + expect( + getManagedInferenceMaterializerDescriptor(MANAGED_CLUSTER_VLLM_MATERIALIZER_REF), + ).toBeDefined(); + expect(getManagedInferenceMaterializerDescriptor("unknown.materializer/v1")).toBeUndefined(); + expect( + getManagedInferenceLifecycleDescriptor(MANAGED_CLUSTER_VLLM_LIFECYCLE_REF), + ).toBeDefined(); + expect(getManagedInferenceLifecycleDescriptor("unknown.lifecycle/v1")).toBeUndefined(); + expect( + getManagedInferencePreparationDescriptor( + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + ), + ).toBeDefined(); + expect(getManagedInferencePreparationDescriptor(NO_PREPARATION_REF)).toBeDefined(); + expect(getManagedInferencePreparationDescriptor("unknown.preparation/v1")).toBeUndefined(); + }); + + it("validates the current recipe by adapter shape without pinning its profile or model identity", () => { + const recipe = shippedRecipe(); + const changedIdentity = { + ...recipe, + metadata: { ...recipe.metadata, id: "vllm.synthetic.second-recipe" }, + spec: { + ...recipe.spec, + model: { + ...recipe.spec.model, + id: "example/AnotherModel", + revision: "b".repeat(40), + servedName: "another-model", + }, + runtime: { + ...recipe.spec.runtime, + image: `registry.example/vllm@sha256:${"c".repeat(64)}`, + }, + readiness: { ...recipe.spec.readiness, expectedModel: "another-model" }, + }, + } as ManagedInferenceServingRecipe; + + expect(getManagedInferenceRecipeRegistrationError(recipe)).toBeUndefined(); + expect(getManagedInferenceRecipeRegistrationError(changedIdentity)).toBeUndefined(); + }); + + it("rejects an unregistered or incompatible recipe mechanic", () => { + const recipe = shippedRecipe(); + const unregistered = { + ...recipe, + spec: { + ...recipe.spec, + execution: { ...recipe.spec.execution, materializerRef: "vllm.unknown/v1" }, + }, + } as ManagedInferenceServingRecipe; + const wrongShape = { + ...recipe, + spec: { + ...recipe.spec, + execution: { ...recipe.spec.execution, nodeCount: 3 }, + }, + } as ManagedInferenceServingRecipe; + + expect(getManagedInferenceRecipeRegistrationError(unregistered)).toMatch( + /unknown materializer/u, + ); + expect(getManagedInferenceRecipeRegistrationError(wrongShape)).toMatch(/TP times PP/u); + }); + + it("rejects schema-valid values that the registered adapter cannot execute", () => { + const recipe = shippedRecipe(); + const unsupportedCache = { + ...recipe, + spec: { + ...recipe.spec, + runtime: { + ...recipe.spec.runtime, + modelCache: { ...recipe.spec.runtime.modelCache, source: "synthetic-cache" }, + }, + }, + } as ManagedInferenceServingRecipe; + const rendezvousCollision = { + ...recipe, + spec: { + ...recipe.spec, + serve: { + ...recipe.spec.serve, + arguments: recipe.spec.serve.arguments.map((argument) => + argument.name === "--port" ? { ...argument, value: 25_000 } : argument, + ), + }, + }, + } as ManagedInferenceServingRecipe; + const mutableLaunchImage = { + ...recipe, + spec: { + ...recipe.spec, + model: { ...recipe.spec.model, installFastSafetensors: true }, + }, + } as ManagedInferenceServingRecipe; + const redundantPath = { + ...recipe, + spec: { + ...recipe.spec, + runtime: { + ...recipe.spec.runtime, + modelCache: { ...recipe.spec.runtime.modelCache, target: "/models//cache" }, + }, + }, + } as ManagedInferenceServingRecipe; + const shadowedCache = { + ...recipe, + spec: { + ...recipe.spec, + runtime: { + ...recipe.spec.runtime, + temporaryFilesystems: [ + ...recipe.spec.runtime.temporaryFilesystems, + { + target: "/cache", + sizeBytes: 1_073_741_824, + mode: "0700", + options: ["rw", "nosuid", "nodev"], + }, + ], + }, + }, + } as ManagedInferenceServingRecipe; + + expect(getManagedInferenceRecipeRegistrationError(unsupportedCache)).toMatch( + /Hugging Face cache source/u, + ); + expect(getManagedInferenceRecipeRegistrationError(rendezvousCollision)).toMatch( + /rendezvous port/u, + ); + expect(getManagedInferenceRecipeRegistrationError(mutableLaunchImage)).toMatch( + /cannot install fastsafetensors/u, + ); + expect(getManagedInferenceRecipeRegistrationError(redundantPath)).toMatch( + /normalized absolute/u, + ); + expect(getManagedInferenceRecipeRegistrationError(shadowedCache)).toMatch( + /cannot shadow the model cache/u, + ); + }); + + it("dispatches topology artifact validation through the registered descriptor", () => { + const artifact = fixtureManagedClusterSelection().topologyQualification; + const descriptor = getManagedInferenceTopologyQualificationDescriptor( + artifact.id, + artifact.schemaVersion, + ); + expect(descriptor).toBeDefined(); + expect(descriptor!.validateArtifact(artifact, artifact.subjectNodeIds)).toBeUndefined(); + + const changed = structuredClone(artifact); + (changed as { outputDigest: string }).outputDigest = `sha256:${"f".repeat(64)}`; + expect(descriptor!.validateArtifact(changed, artifact.subjectNodeIds)).toMatch(/digest/u); + }); + + it("keeps preparation inputs bounded without coupling them to a model ID", () => { + const recipe = shippedRecipe(); + const preparation = (recipe.spec.model as unknown as { preparation: { ref: string } }) + .preparation; + const descriptor = getManagedInferencePreparationDescriptor(preparation.ref); + const differentModel = { + ...recipe, + spec: { + ...recipe.spec, + model: { ...recipe.spec.model, id: "example/AnotherModel" }, + }, + } as ManagedInferenceServingRecipe; + const unsafePath = { + ...recipe, + spec: { + ...recipe.spec, + model: { + ...recipe.spec.model, + preparation: { + ...(recipe.spec.model as unknown as { preparation: object }).preparation, + snapshotCopy: { + ...( + recipe.spec.model as unknown as { + preparation: { snapshotCopy: object }; + } + ).preparation.snapshotCopy, + sourcePath: "../encoding.py", + }, + }, + }, + }, + } as ManagedInferenceServingRecipe; + + expect(descriptor?.validateRecipe(differentModel)).toBeUndefined(); + expect(descriptor?.validateRecipe(unsafePath)).toMatch(/copy paths/u); + }); + + it("accepts only the exact no-op preparation shape", () => { + const recipe = shippedRecipe(); + const descriptor = getManagedInferencePreparationDescriptor(NO_PREPARATION_REF); + const noPreparation = { + ...recipe, + spec: { + ...recipe.spec, + model: { ...recipe.spec.model, preparation: { ref: NO_PREPARATION_REF } }, + }, + } as ManagedInferenceServingRecipe; + const extraInput = { + ...noPreparation, + spec: { + ...noPreparation.spec, + model: { + ...noPreparation.spec.model, + preparation: { ref: NO_PREPARATION_REF, command: "true" }, + }, + }, + } as ManagedInferenceServingRecipe; + + expect(descriptor?.validateRecipe(noPreparation)).toBeUndefined(); + expect(descriptor?.validateRecipe(extraInput)).toMatch(/empty preparation/u); + }); +}); diff --git a/src/lib/inference/serving/adapter-registry.ts b/src/lib/inference/serving/adapter-registry.ts new file mode 100644 index 0000000000..910011bf69 --- /dev/null +++ b/src/lib/inference/serving/adapter-registry.ts @@ -0,0 +1,538 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + getManagedClusterTopologyArtifactError, + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, +} from "./managed-cluster-topology.js"; +import type { + ManagedInferenceServingRecipe, + ManagedInferenceTopologyQualification, + ReadinessEntityKind, + ServingCatalogRegistries, + ServingRecipe, +} from "./types.js"; + +export const MANAGED_CLUSTER_VLLM_MATERIALIZER_REF = "vllm.managed-cluster/v1" as const; +export const MANAGED_CLUSTER_VLLM_LIFECYCLE_REF = "vllm.managed-cluster.lifecycle/v1" as const; +export const SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF = + "snapshot-copy-and-exact-text-replacement/v1" as const; +export const NO_PREPARATION_REF = "none/v1" as const; +export const MANAGED_CLUSTER_HUGGING_FACE_CACHE_SOURCE = "huggingface-cache" as const; + +export function containerPathContains(parent: string, child: string): boolean { + return child === parent || child.startsWith(`${parent}/`); +} + +export interface ManagedInferenceTopologyQualificationDescriptor { + readonly id: string; + readonly schemaVersion: number; + readonly outputSchema: string; + readonly bindingOutput: string; + validateArtifact( + artifact: ManagedInferenceTopologyQualification, + expectedSubjectNodeIds?: readonly string[], + ): string | undefined; +} + +export interface ManagedInferenceMaterializerDescriptor { + readonly ref: string; + readonly backend: string; + readonly outputPlanSchema: string; + readonly topology: { + readonly qualificationId: string; + readonly schemaVersion: number; + readonly outputSchema: string; + }; + validateRecipe(recipe: ManagedInferenceServingRecipe): string | undefined; +} + +export interface ManagedInferenceLifecycleDescriptor { + readonly ref: string; + readonly backend: string; + readonly acceptedMaterializerRefs: readonly string[]; + readonly acceptedPlanSchemas: readonly string[]; + readonly secretHandlePermissions: readonly string[]; + validateRecipe(recipe: ManagedInferenceServingRecipe): string | undefined; +} + +export interface ManagedInferencePreparationDescriptor { + readonly ref: string; + readonly backend: string; + readonly phase: "container-before-exec"; + validateRecipe(recipe: ManagedInferenceServingRecipe): string | undefined; +} + +const MANAGED_CLUSTER_TOPOLOGY_OUTPUT_SCHEMA = + "nemoclaw.nvidia.com/managed-cluster-topology/v1" as const; +const MANAGED_CLUSTER_PLAN_SCHEMA = "nemoclaw.nvidia.com/managed-cluster-vllm-plan/v1" as const; +const LOWERCASE_STABLE_ID = /^[a-z0-9][a-z0-9._/-]{0,159}$/u; +const SHA256_DIGEST = /^sha256:[0-9a-f]{64}$/u; +const MANAGED_CLUSTER_MATERIALIZER_OWNED_ENVIRONMENT = new Set([ + "GLOO_SOCKET_IFNAME", + "HEADLESS", + "HF_HOME", + "MASTER_ADDR", + "MASTER_PORT", + "NCCL_IB_GID_INDEX", + "NCCL_IB_HCA", + "NCCL_SOCKET_IFNAME", + "NODE_RANK", + "TP_SOCKET_IFNAME", + "VLLM_API_KEY", + "VLLM_HOST_IP", +]); + +export function isManagedClusterMaterializerOwnedEnvironment(name: string): boolean { + return MANAGED_CLUSTER_MATERIALIZER_OWNED_ENVIRONMENT.has(name); +} + +function managedClusterTopologyBinding( + recipe: ManagedInferenceServingRecipe, +): ManagedInferenceServingRecipe["spec"]["bindings"][string] | undefined { + return recipe.spec.bindings[recipe.spec.execution.topologyBinding]; +} + +function positiveIntegerArgument( + recipe: ManagedInferenceServingRecipe, + name: string, + maximum = Number.MAX_SAFE_INTEGER, +): number | undefined { + const matches = recipe.spec.serve.arguments.filter((argument) => argument.name === name); + if (matches.length !== 1) return undefined; + const value = matches[0]!.value; + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && /^\d+$/u.test(value) + ? Number(value) + : Number.NaN; + return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : undefined; +} + +function validateManagedClusterMaterializerRecipe( + recipe: ManagedInferenceServingRecipe, +): string | undefined { + if (recipe.spec.backend !== "vllm") return "managed cluster materializer requires backend vllm"; + if (recipe.spec.execution.materializerRef !== MANAGED_CLUSTER_VLLM_MATERIALIZER_REF) { + return "recipe does not select the managed cluster materializer"; + } + const { execution } = recipe.spec; + if ( + !Number.isSafeInteger(execution.nodeCount) || + execution.nodeCount < 2 || + execution.nodeCount > 1_024 || + !Number.isSafeInteger(execution.tensorParallelSize) || + execution.tensorParallelSize < 1 || + !Number.isSafeInteger(execution.pipelineParallelSize) || + execution.pipelineParallelSize < 1 || + execution.tensorParallelSize * execution.pipelineParallelSize !== execution.nodeCount || + execution.distributedExecutorBackend !== "mp" + ) { + return "managed cluster materializer requires a bounded node count equal to TP times PP with the mp backend"; + } + if (Object.keys(recipe.spec.bindings).length !== 1) { + return "managed cluster materializer requires exactly one topology binding"; + } + if ( + recipe.spec.runtime.architecture !== "arm64" || + recipe.spec.runtime.networkMode !== "host" || + recipe.spec.runtime.ipcMode !== "host" || + recipe.spec.serve.authentication !== "bearer" + ) { + return "managed cluster materializer requires arm64 host networking, host IPC, and bearer authentication"; + } + const apiPort = positiveIntegerArgument(recipe, "--port", 65_535); + if (apiPort === undefined || positiveIntegerArgument(recipe, "--max-model-len") === undefined) { + return "managed cluster materializer requires one valid --port and one positive --max-model-len"; + } + if ( + !Number.isSafeInteger(execution.rendezvousPort) || + execution.rendezvousPort < 1 || + execution.rendezvousPort > 65_535 + ) { + return "managed cluster materializer requires a valid rendezvous port"; + } + if (apiPort === execution.rendezvousPort) { + return "managed cluster API port conflicts with the materializer rendezvous port"; + } + if (recipe.spec.readiness.expectedModel !== recipe.spec.model.servedName) { + return "managed cluster readiness must expect the recipe served model"; + } + if (!LOWERCASE_STABLE_ID.test(recipe.spec.model.servedName)) { + return "managed cluster served model name must be a lowercase stable ID"; + } + if (recipe.spec.model.installFastSafetensors) { + return "managed cluster immutable-image materializer cannot install fastsafetensors at launch"; + } + if (recipe.spec.runtime.modelCache.source !== MANAGED_CLUSTER_HUGGING_FACE_CACHE_SOURCE) { + return "managed cluster materializer requires the Hugging Face cache source"; + } + if ( + !safeAbsoluteContainerPath(recipe.spec.serve.executable) || + !safeAbsoluteContainerPath(recipe.spec.runtime.modelCache.target) || + recipe.spec.runtime.devices.some((device) => !safeAbsoluteContainerPath(device)) || + recipe.spec.runtime.temporaryFilesystems.some( + ({ target }) => !safeAbsoluteContainerPath(target), + ) + ) { + return "managed cluster runtime paths must be normalized absolute container paths"; + } + if ( + recipe.spec.runtime.temporaryFilesystems.some(({ target }) => + containerPathContains(target, recipe.spec.runtime.modelCache.target), + ) + ) { + return "managed cluster temporary filesystem cannot shadow the model cache"; + } + const resourceValues = [ + recipe.spec.model.downloadSizeBytes, + recipe.spec.runtime.imageDownloadSizeBytes, + recipe.spec.runtime.sharedMemoryBytes, + recipe.spec.runtime.ulimits.stackBytes, + ...recipe.spec.runtime.temporaryFilesystems.map(({ sizeBytes }) => sizeBytes), + ]; + if (resourceValues.some((value) => !Number.isSafeInteger(value) || value <= 0)) { + return "managed cluster recipe resource values must be positive safe integers"; + } + const memlock = recipe.spec.runtime.ulimits.memlock; + if (typeof memlock === "number" && (!Number.isSafeInteger(memlock) || memlock < -1)) { + return "managed cluster memlock value must be -1 or a non-negative safe integer"; + } + if ( + recipe.spec.serve.arguments.some( + ({ value }) => + typeof value === "string" && + (Buffer.byteLength(value, "utf8") > 16_384 || value.includes("\0")), + ) + ) { + return "managed cluster serving argument values must be bounded text without NUL bytes"; + } + if ( + Object.values(recipe.spec.runtime.environment).some( + (value) => Buffer.byteLength(value, "utf8") > 4_096 || value.includes("\0"), + ) + ) { + return "managed cluster environment values must be bounded text without NUL bytes"; + } + if ( + Object.keys(recipe.spec.runtime.environment).some((name) => + isManagedClusterMaterializerOwnedEnvironment(name), + ) + ) { + return "managed cluster recipe environment overrides a materializer-owned value"; + } + const binding = managedClusterTopologyBinding(recipe); + if ( + !binding || + binding.type !== "topologyQualificationOutput" || + binding.qualificationId !== MANAGED_CLUSTER_TOPOLOGY_ID || + binding.schemaVersion !== MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION || + binding.outputSchema !== MANAGED_CLUSTER_TOPOLOGY_OUTPUT_SCHEMA + ) { + return "managed cluster materializer topology binding is incompatible"; + } + return undefined; +} + +function validateManagedClusterLifecycleRecipe( + recipe: ManagedInferenceServingRecipe, +): string | undefined { + const materializerError = validateManagedClusterMaterializerRecipe(recipe); + if (materializerError) return materializerError; + return recipe.spec.execution.lifecycleRef === MANAGED_CLUSTER_VLLM_LIFECYCLE_REF + ? undefined + : "recipe does not select the managed cluster lifecycle"; +} + +interface SnapshotPreparationInput { + readonly ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF; + readonly snapshotCopy: { + readonly sourcePath: string; + readonly digest: string; + readonly targetPath: string; + }; + readonly exactTextReplacement: { + readonly targetPath: string; + readonly expectedText: string; + readonly replacementText: string; + }; +} + +interface NoPreparationInput { + readonly ref: typeof NO_PREPARATION_REF; +} + +type ManagedInferencePreparationInput = SnapshotPreparationInput | NoPreparationInput; + +function recipePreparation( + recipe: ManagedInferenceServingRecipe, +): ManagedInferencePreparationInput | undefined { + const preparation = (recipe.spec.model as unknown as { readonly preparation?: unknown }) + .preparation; + return typeof preparation === "object" && preparation !== null + ? (preparation as ManagedInferencePreparationInput) + : undefined; +} + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Object.keys(value).sort(); + const sortedExpected = [...expected].sort(); + return ( + keys.length === sortedExpected.length && + keys.every((key, index) => key === sortedExpected[index]) + ); +} + +function safeRelativeSnapshotPath(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 4096 && + !value.startsWith("/") && + value.split("/").every((component) => component && component !== "." && component !== "..") && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function safeAbsoluteContainerPath(value: unknown): value is string { + return ( + typeof value === "string" && + value.startsWith("/") && + value.length <= 4096 && + value + .split("/") + .slice(1) + .every((component) => component && component !== "." && component !== "..") && + !/[\u0000-\u001f\u007f]/u.test(value) + ); +} + +function validateSnapshotPreparationRecipe( + recipe: ManagedInferenceServingRecipe, +): string | undefined { + if (recipe.spec.backend !== "vllm") return "snapshot preparation requires backend vllm"; + const preparation = recipePreparation(recipe); + if ( + preparation?.ref !== SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF || + !hasExactKeys(preparation, ["exactTextReplacement", "ref", "snapshotCopy"]) + ) { + return "recipe does not select the snapshot preparation operation"; + } + if ( + !preparation.snapshotCopy || + !hasExactKeys(preparation.snapshotCopy, ["digest", "sourcePath", "targetPath"]) || + !safeRelativeSnapshotPath(preparation.snapshotCopy.sourcePath) || + !SHA256_DIGEST.test(preparation.snapshotCopy.digest) || + !safeAbsoluteContainerPath(preparation.snapshotCopy.targetPath) + ) { + return "snapshot preparation copy paths are invalid"; + } + const replacement = preparation.exactTextReplacement; + if ( + !replacement || + !hasExactKeys(replacement, ["expectedText", "replacementText", "targetPath"]) || + !safeAbsoluteContainerPath(replacement.targetPath) || + typeof replacement.expectedText !== "string" || + typeof replacement.replacementText !== "string" || + replacement.expectedText.length === 0 || + replacement.expectedText.length > 65_536 || + replacement.replacementText.length === 0 || + replacement.replacementText.length > 65_536 || + replacement.expectedText === replacement.replacementText || + replacement.expectedText.includes("\0") || + replacement.replacementText.includes("\0") + ) { + return "snapshot preparation exact-text replacement is invalid"; + } + return undefined; +} + +function validateNoPreparationRecipe(recipe: ManagedInferenceServingRecipe): string | undefined { + if (recipe.spec.backend !== "vllm") return "empty preparation requires backend vllm"; + const preparation = recipePreparation(recipe); + return preparation?.ref === NO_PREPARATION_REF && hasExactKeys(preparation, ["ref"]) + ? undefined + : "recipe does not select the empty preparation operation"; +} + +const TOPOLOGY_DESCRIPTORS = [ + { + id: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + outputSchema: MANAGED_CLUSTER_TOPOLOGY_OUTPUT_SCHEMA, + bindingOutput: "topology", + validateArtifact: getManagedClusterTopologyArtifactError, + }, +] as const satisfies readonly ManagedInferenceTopologyQualificationDescriptor[]; + +const MATERIALIZER_DESCRIPTORS = [ + { + ref: MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, + backend: "vllm", + outputPlanSchema: MANAGED_CLUSTER_PLAN_SCHEMA, + topology: { + qualificationId: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + outputSchema: MANAGED_CLUSTER_TOPOLOGY_OUTPUT_SCHEMA, + }, + validateRecipe: validateManagedClusterMaterializerRecipe, + }, +] as const satisfies readonly ManagedInferenceMaterializerDescriptor[]; + +const LIFECYCLE_DESCRIPTORS = [ + { + ref: MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, + backend: "vllm", + acceptedMaterializerRefs: [MANAGED_CLUSTER_VLLM_MATERIALIZER_REF], + acceptedPlanSchemas: [MANAGED_CLUSTER_PLAN_SCHEMA], + secretHandlePermissions: ["sshBinding"], + validateRecipe: validateManagedClusterLifecycleRecipe, + }, +] as const satisfies readonly ManagedInferenceLifecycleDescriptor[]; + +const PREPARATION_DESCRIPTORS = [ + { + ref: NO_PREPARATION_REF, + backend: "vllm", + phase: "container-before-exec", + validateRecipe: validateNoPreparationRecipe, + }, + { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + backend: "vllm", + phase: "container-before-exec", + validateRecipe: validateSnapshotPreparationRecipe, + }, +] as const satisfies readonly ManagedInferencePreparationDescriptor[]; + +function registry( + entries: readonly T[], + key: (entry: T) => string, + label: string, +): ReadonlyMap { + const result = new Map(); + for (const entry of entries) { + const id = key(entry); + if (result.has(id)) + throw new Error(`duplicate managed inference ${label} registry entry ${id}`); + result.set(id, entry); + } + return result; +} + +const TOPOLOGY_REGISTRY = registry( + TOPOLOGY_DESCRIPTORS, + ({ id, schemaVersion }) => `${id}@${String(schemaVersion)}`, + "topology qualification", +); +const MATERIALIZER_REGISTRY = registry(MATERIALIZER_DESCRIPTORS, ({ ref }) => ref, "materializer"); +const LIFECYCLE_REGISTRY = registry(LIFECYCLE_DESCRIPTORS, ({ ref }) => ref, "lifecycle"); +const PREPARATION_REGISTRY = registry(PREPARATION_DESCRIPTORS, ({ ref }) => ref, "preparation"); + +export function listManagedInferenceTopologyQualificationDescriptors(): readonly ManagedInferenceTopologyQualificationDescriptor[] { + return [...TOPOLOGY_DESCRIPTORS]; +} + +export function getManagedInferenceTopologyQualificationDescriptor( + id: string, + schemaVersion: number, +): ManagedInferenceTopologyQualificationDescriptor | undefined { + return TOPOLOGY_REGISTRY.get(`${id}@${String(schemaVersion)}`); +} + +export function listManagedInferenceMaterializerDescriptors(): readonly ManagedInferenceMaterializerDescriptor[] { + return [...MATERIALIZER_DESCRIPTORS]; +} + +export function getManagedInferenceMaterializerDescriptor( + ref: string, +): ManagedInferenceMaterializerDescriptor | undefined { + return MATERIALIZER_REGISTRY.get(ref); +} + +export function listManagedInferenceLifecycleDescriptors(): readonly ManagedInferenceLifecycleDescriptor[] { + return [...LIFECYCLE_DESCRIPTORS]; +} + +export function getManagedInferenceLifecycleDescriptor( + ref: string, +): ManagedInferenceLifecycleDescriptor | undefined { + return LIFECYCLE_REGISTRY.get(ref); +} + +export function listManagedInferencePreparationDescriptors(): readonly ManagedInferencePreparationDescriptor[] { + return [...PREPARATION_DESCRIPTORS]; +} + +export function getManagedInferencePreparationDescriptor( + ref: string, +): ManagedInferencePreparationDescriptor | undefined { + return PREPARATION_REGISTRY.get(ref); +} + +export function getManagedInferenceRecipeRegistrationError( + recipe: ManagedInferenceServingRecipe, +): string | undefined { + const materializer = getManagedInferenceMaterializerDescriptor( + recipe.spec.execution.materializerRef, + ); + if (!materializer) { + return `unknown materializer ${recipe.spec.execution.materializerRef}`; + } + const lifecycle = getManagedInferenceLifecycleDescriptor(recipe.spec.execution.lifecycleRef); + if (!lifecycle) return `unknown lifecycle ${recipe.spec.execution.lifecycleRef}`; + const preparationRef = recipePreparation(recipe)?.ref ?? ""; + const preparation = getManagedInferencePreparationDescriptor(preparationRef); + if (!preparation) return `unknown preparation ${preparationRef || "(missing)"}`; + return ( + materializer.validateRecipe(recipe) ?? + lifecycle.validateRecipe(recipe) ?? + preparation.validateRecipe(recipe) + ); +} + +const SERVING_READINESS_REGISTRY: ServingCatalogRegistries["readiness"] = new Map< + string, + ReadinessEntityKind | ReadonlySet +>([ + ["host.platform.dgx_spark", new Set(["qualification", "capability"] as const)], + ["host.platform.supported", "capability"], + ["host.docker.available", "capability"], + ["host.docker.daemon_reachable", "capability"], + ["host.docker.runtime_supported", "capability"], + ["host.docker.storage_compatible", "capability"], + ["host.gpu.nvidia_available", "capability"], + ["host.gpu.container_toolkit_available", "capability"], + ["host.gpu.cdi_healthy", "capability"], +] as const); + +export function getManagedInferenceServingCatalogRegistries(): ServingCatalogRegistries { + return { + receipts: new Set(), + materializers: new Set(MATERIALIZER_DESCRIPTORS.map(({ ref }) => ref)), + lifecycles: new Set(LIFECYCLE_DESCRIPTORS.map(({ ref }) => ref)), + readinessContracts: new Set(), + readiness: SERVING_READINESS_REGISTRY, + facts: new Set(["cluster.nodeCount"]), + topologyQualifications: new Map( + TOPOLOGY_DESCRIPTORS.map((descriptor) => [ + `${descriptor.id}@${String(descriptor.schemaVersion)}`, + { + bindingOutput: descriptor.bindingOutput, + outputSchema: descriptor.outputSchema, + }, + ]), + ), + validateRecipe: (recipe: ServingRecipe) => { + if ( + recipe.spec.execution.materializerRef !== MANAGED_CLUSTER_VLLM_MATERIALIZER_REF && + recipe.spec.execution.lifecycleRef !== MANAGED_CLUSTER_VLLM_LIFECYCLE_REF + ) { + return undefined; + } + return getManagedInferenceRecipeRegistrationError(recipe as ManagedInferenceServingRecipe); + }, + }; +} diff --git a/src/lib/inference/serving/catalog-integrity.ts b/src/lib/inference/serving/catalog-integrity.ts new file mode 100644 index 0000000000..a449c8540c --- /dev/null +++ b/src/lib/inference/serving/catalog-integrity.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +type JsonValue = null | boolean | number | string | readonly JsonValue[] | JsonObject; +type JsonObject = { readonly [key: string]: JsonValue }; + +function canonicalize(value: unknown, path: string): JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`); + return Object.is(value, -0) ? 0 : value; + } + if (Array.isArray(value)) { + return value.map((item, index) => canonicalize(item, `${path}[${String(index)}]`)); + } + if (typeof value !== "object") throw new Error(`${path} is not JSON-serializable`); + + const output: Record = {}; + for (const key of Object.keys(value).sort()) { + const child = (value as Record)[key]; + if (child === undefined) throw new Error(`${path}.${key} is undefined`); + output[key] = canonicalize(child, `${path}.${key}`); + } + return output; +} + +export function canonicalManagedInferenceJson(value: unknown): string { + return JSON.stringify(canonicalize(value, "catalog")); +} + +export function managedInferenceDigest(value: unknown): string { + return `sha256:${managedInferenceHexDigest(value)}`; +} + +export function managedInferenceHexDigest(value: unknown): string { + return createHash("sha256").update(canonicalManagedInferenceJson(value)).digest("hex"); +} + +export function managedInferenceTextDigest(value: string): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function deepFreeze(value: T): T { + if (typeof value !== "object" || value === null || Object.isFrozen(value)) return value; + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + return value; +} + +export function immutableManagedInferenceCopy(value: T): T { + return deepFreeze(JSON.parse(canonicalManagedInferenceJson(value)) as T); +} diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts new file mode 100644 index 0000000000..5255c4197a --- /dev/null +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, + MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, +} from "./adapter-registry"; +import { assertManagedInferenceCatalog } from "./catalog-loader"; +import type { CompiledServingCatalog, ServingPreset, ServingRecipe } from "./types"; + +const EMPTY_CATALOG: CompiledServingCatalog = { + schemaVersion: "1.0.0", + compilerVersion: "1.1.0", + sourceRevision: "a".repeat(40), + readinessSchemaRef: "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json", + recipes: [], + presets: [], + sources: [], + catalogDigest: `sha256:${"b".repeat(64)}`, +}; + +const INCOMPLETE_MANAGED_RECIPE: ServingRecipe = { + apiVersion: "nemoclaw.nvidia.com/managed-inference/v1", + kind: "ServingRecipe", + metadata: { id: "test.incomplete-managed-recipe" }, + spec: { + backend: "vllm", + model: { id: "test/model", revision: "c".repeat(40) }, + execution: { + materializerRef: MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, + lifecycleRef: MANAGED_CLUSTER_VLLM_LIFECYCLE_REF, + }, + }, +}; + +const INCOMPLETE_MANAGED_PRESET: ServingPreset = { + apiVersion: "nemoclaw.nvidia.com/managed-inference/v1", + kind: "ServingPreset", + metadata: { id: "test.incomplete-managed-preset" }, + spec: { + selection: "explicit-only", + priority: 1, + plan: { backend: "vllm", recipeRef: INCOMPLETE_MANAGED_RECIPE.metadata.id }, + }, +}; + +function managedCatalogValidationError(catalog: CompiledServingCatalog): string | undefined { + try { + assertManagedInferenceCatalog(catalog); + return undefined; + } catch (error) { + return (error as Error).message; + } +} + +describe("managed inference catalog loader", () => { + it("accepts an empty managed catalog", () => { + assertManagedInferenceCatalog(EMPTY_CATALOG); + + expect(EMPTY_CATALOG.recipes).toHaveLength(0); + }); + + it.each([ + ["recipe", { ...EMPTY_CATALOG, recipes: [INCOMPLETE_MANAGED_RECIPE] }], + ["preset", { ...EMPTY_CATALOG, presets: [INCOMPLETE_MANAGED_PRESET] }], + ] as const)("rejects an incomplete managed %s", (_label, catalog) => { + expect(managedCatalogValidationError(catalog)).toMatch(/Managed inference (preset|recipe)/u); + }); +}); diff --git a/src/lib/inference/serving/catalog-loader.ts b/src/lib/inference/serving/catalog-loader.ts new file mode 100644 index 0000000000..c8e7ac78ab --- /dev/null +++ b/src/lib/inference/serving/catalog-loader.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { getManagedInferenceRecipeRegistrationError } from "./adapter-registry.js"; +import { parseCompiledServingCatalogJson } from "./catalog.js"; +import { immutableManagedInferenceCopy } from "./catalog-integrity.js"; +import type { + CompiledManagedInferenceCatalog, + CompiledServingCatalog, + ManagedInferenceServingPreset, + ManagedInferenceServingRecipe, + ServingCatalogSchemas, + ServingPreset, + ServingRecipe, +} from "./types.js"; + +function readJson(path: string): object { + return JSON.parse(readFileSync(path, "utf8")) as object; +} + +function repositoryRoot(): string { + return join(__dirname, "..", "..", "..", ".."); +} + +function loadSchemas(rootDir: string): ServingCatalogSchemas { + const schemaRoot = join(rootDir, "managed-inference", "schemas"); + return { + catalog: readJson(join(schemaRoot, "catalog.schema.json")), + preset: readJson(join(schemaRoot, "preset.schema.json")), + recipe: readJson(join(schemaRoot, "recipe.schema.json")), + }; +} + +let loadedCatalog: CompiledManagedInferenceCatalog | undefined; + +function assertManagedRecipe( + recipe: ServingRecipe, +): asserts recipe is ManagedInferenceServingRecipe { + let registrationError: string | undefined; + try { + registrationError = getManagedInferenceRecipeRegistrationError( + recipe as ManagedInferenceServingRecipe, + ); + } catch { + registrationError = "does not satisfy its registered adapter contract"; + } + if (registrationError) { + throw new Error(`Managed inference recipe ${recipe.metadata.id}: ${registrationError}`); + } +} + +function assertManagedPreset( + preset: ServingPreset, +): asserts preset is ManagedInferenceServingPreset { + if (!preset.spec.requirements || !preset.spec.plan.bindings) { + throw new Error( + `Managed inference preset ${preset.metadata.id} must declare requirements and topology bindings`, + ); + } +} + +/** Narrow a generic serving catalog to the currently supported managed runtime surface. */ +export function assertManagedInferenceCatalog( + catalog: CompiledServingCatalog, +): asserts catalog is CompiledManagedInferenceCatalog { + for (const recipe of catalog.recipes) assertManagedRecipe(recipe); + for (const preset of catalog.presets) { + assertManagedPreset(preset); + const recipes = catalog.recipes.filter( + ({ metadata }) => metadata.id === preset.spec.plan.recipeRef, + ); + if (recipes.length !== 1 || recipes[0]!.spec.backend !== preset.spec.plan.backend) { + throw new Error( + `Managed inference preset ${preset.metadata.id} does not resolve one matching recipe`, + ); + } + } +} + +export function parseCompiledManagedInferenceCatalogJson( + source: string, + schemas: ServingCatalogSchemas, +): CompiledManagedInferenceCatalog { + const catalog = parseCompiledServingCatalogJson(source, schemas); + assertManagedInferenceCatalog(catalog); + return catalog; +} + +export function loadManagedInferenceCatalog(): CompiledManagedInferenceCatalog { + if (loadedCatalog) return loadedCatalog; + const rootDir = repositoryRoot(); + const source = readFileSync(join(rootDir, "dist", "managed-inference", "catalog.json"), "utf8"); + loadedCatalog = immutableManagedInferenceCopy( + parseCompiledManagedInferenceCatalogJson(source, loadSchemas(rootDir)), + ); + return loadedCatalog; +} + +export function getManagedInferenceCompiledPreset( + id: string, +): ManagedInferenceServingPreset | undefined { + return loadManagedInferenceCatalog().presets.find(({ metadata }) => metadata.id === id); +} + +export function getManagedInferenceCompiledRecipe( + id: string, +): ManagedInferenceServingRecipe | undefined { + return loadManagedInferenceCatalog().recipes.find(({ metadata }) => metadata.id === id); +} diff --git a/src/lib/inference/serving/catalog.ts b/src/lib/inference/serving/catalog.ts index 8127930276..1ebcac0f81 100644 --- a/src/lib/inference/serving/catalog.ts +++ b/src/lib/inference/serving/catalog.ts @@ -13,11 +13,14 @@ import type { ServingCatalogRegistries, ServingCatalogSchemas, ServingCatalogSource, + ServingCatalogSourceProvenance, ServingDefinitionKind, ServingPreset, ServingReadinessComparison, ServingReadinessObservationRole, + ServingReadinessRegistryEntry, ServingReadinessRequirement, + ServingReadinessRegistryValue, ServingRecipe, } from "./types"; @@ -57,9 +60,7 @@ function compareCanonicalText(left: string, right: string): number { } function normalizeForCanonicalJson(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(normalizeForCanonicalJson); - } + if (Array.isArray(value)) return value.map(normalizeForCanonicalJson); if (value !== null && typeof value === "object") { return Object.fromEntries( Object.entries(value) @@ -74,7 +75,7 @@ export function canonicalServingCatalogJson(value: unknown): string { return JSON.stringify(normalizeForCanonicalJson(value)); } -function digest(value: unknown): string { +export function servingCatalogDigest(value: unknown): string { return `sha256:${createHash("sha256").update(canonicalServingCatalogJson(value)).digest("hex")}`; } @@ -87,11 +88,7 @@ function createValidators(schemas: ServingCatalogSchemas): CatalogValidators { if (!recipe || !preset) { throw new ServingCatalogValidationError("Serving catalog schemas have invalid identifiers."); } - return { - recipe, - preset, - catalog: ajv.compile(schemas.catalog as AnySchema), - }; + return { recipe, preset, catalog: ajv.compile(schemas.catalog as AnySchema) }; } function validationDetails(validate: ValidateFunction): string { @@ -153,6 +150,35 @@ function isLlamaCppServingRecipe(recipe: ServingRecipe): recipe is LlamaCppServi return recipe.spec.providerId === "llama-cpp-local"; } +function isReadinessRegistryEntry( + registered: ServingReadinessRegistryValue, +): registered is ServingReadinessRegistryEntry { + return typeof registered === "object" && registered !== null && "kind" in registered; +} + +function readinessKindMatches( + registered: ServingReadinessRegistryValue, + kind: ServingReadinessRequirement["readiness"]["kind"], +): boolean { + if (typeof registered === "string") return registered === kind; + if (isReadinessRegistryEntry(registered)) return registered.kind === kind; + return registered.has(kind); +} + +function readinessKindLabel(registered: ServingReadinessRegistryValue): string { + if (typeof registered === "string") return registered; + if (isReadinessRegistryEntry(registered)) return registered.kind; + return [...registered].join(" or "); +} + +function readinessDescriptor( + registered: ServingReadinessRegistryValue | undefined, + kind: ServingReadinessRequirement["readiness"]["kind"], +): ServingReadinessRegistryEntry | undefined { + if (registered === undefined || !readinessKindMatches(registered, kind)) return undefined; + return isReadinessRegistryEntry(registered) ? registered : { kind }; +} + function validateRecipeSemantics( recipe: ServingRecipe, registries: ServingCatalogRegistries, @@ -173,12 +199,14 @@ function validateRecipeSemantics( `Recipe ${recipe.metadata.id} references unknown lifecycle adapter ${lifecycleRef}.`, ); } + const readinessContractRef = recipe.spec.readiness?.contractRef; if (readinessContractRef && !registries.readinessContracts.has(readinessContractRef)) { throw new ServingCatalogValidationError( `Recipe ${recipe.metadata.id} references unknown readiness contract ${readinessContractRef}.`, ); } + if (!isLlamaCppServingRecipe(recipe)) { const argumentNames = new Set(); for (const argument of recipe.spec.serve?.arguments ?? []) { @@ -228,14 +256,23 @@ function validateRecipeSemantics( ); } agents.add(agent.id); - const qualification = registries.readiness.get(agent.qualificationRef); - if (qualification?.kind !== "qualification") { + if (!readinessDescriptor(registries.readiness.get(agent.qualificationRef), "qualification")) { throw new ServingCatalogValidationError( `Recipe ${recipe.metadata.id} references unknown agent qualification ${agent.qualificationRef} for ${agent.id}.`, ); } } } + + let registrationError: string | undefined; + try { + registrationError = registries.validateRecipe?.(recipe); + } catch { + registrationError = "does not satisfy its registered adapter contract"; + } + if (registrationError) { + throw new ServingCatalogValidationError(`Recipe ${recipe.metadata.id}: ${registrationError}.`); + } } function comparisonValueType( @@ -255,60 +292,75 @@ function readinessRequirementKey(requirement: ServingReadinessRequirement): stri return `${readiness.scope}:${readiness.kind}:${readiness.id}`; } -function validatePresetReadiness( +function validatePresetRequirements( preset: ServingPreset, registries: ServingCatalogRegistries, ): void { + const requirements = new Set(); const requirementsByEntity = new Map(); for (const requirement of preset.spec.requirements?.all ?? []) { - const readiness = requirement.readiness; - const registered = registries.readiness.get(readiness.id); - if (registered === undefined) { + const requirementKey = canonicalServingCatalogJson(requirement); + if (requirements.has(requirementKey)) { + const requirementLabel = "readiness" in requirement ? "readiness requirement" : "requirement"; throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} references unknown readiness entity ${readiness.id}.`, - ); - } - if (registered.kind !== readiness.kind) { - throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} uses ${readiness.id} as ${readiness.kind}, but the readiness registry declares ${registered.kind}.`, + `Preset ${preset.metadata.id} repeats ${requirementLabel} ${requirementKey}.`, ); } + requirements.add(requirementKey); - if ("comparison" in readiness) { - if (registered.kind !== "observation") { + if ("readiness" in requirement) { + const { readiness } = requirement; + const registered = registries.readiness.get(readiness.id); + if (registered === undefined) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} cannot compare readiness entity ${readiness.id}.`, + `Preset ${preset.metadata.id} references unknown readiness entity ${readiness.id}.`, ); } - const comparisonType = comparisonValueType(readiness.comparison); - if (!registered.valueType || comparisonType !== registered.valueType) { + if (!readinessKindMatches(registered, readiness.kind)) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} compares ${readiness.id} as ${comparisonType}, but the readiness registry declares ${registered.valueType ?? "no value type"}.`, + `Preset ${preset.metadata.id} uses ${readiness.id} as ${readiness.kind}, but the readiness registry declares ${readinessKindLabel(registered)}.`, ); } - } - const key = readinessRequirementKey(requirement); - const canonicalRequirement = canonicalServingCatalogJson(requirement); - const previous = requirementsByEntity.get(key); - if (previous === canonicalRequirement) { - throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} repeats readiness requirement ${key}.`, - ); + if ("comparison" in readiness) { + const descriptor = readinessDescriptor(registered, readiness.kind); + const comparisonType = comparisonValueType(readiness.comparison); + if (!descriptor?.valueType || comparisonType !== descriptor.valueType) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} compares ${readiness.id} as ${comparisonType}, but the readiness registry declares ${descriptor?.valueType ?? "no value type"}.`, + ); + } + } + + const key = readinessRequirementKey(requirement); + const previous = requirementsByEntity.get(key); + if (previous !== undefined && previous !== requirementKey) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} has contradictory readiness requirements for ${key}.`, + ); + } + requirementsByEntity.set(key, requirementKey); + continue; } - if (previous !== undefined) { + if ("fact" in requirement) { + if (registries.facts && !registries.facts.has(requirement.fact)) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} references unknown selection fact ${requirement.fact}.`, + ); + } + continue; + } + const key = `${requirement.topologyQualification.id}@${String(requirement.topologyQualification.schemaVersion)}`; + if (registries.topologyQualifications && !registries.topologyQualifications.has(key)) { throw new ServingCatalogValidationError( - `Preset ${preset.metadata.id} has contradictory readiness requirements for ${key}.`, + `Preset ${preset.metadata.id} references unknown topology qualification ${key}.`, ); } - requirementsByEntity.set(key, canonicalRequirement); } } function canonicalReadinessComparison(comparison: ServingReadinessComparison): string { - if (comparison.operator !== "one-of") { - return canonicalServingCatalogJson(comparison); - } + if (comparison.operator !== "one-of") return canonicalServingCatalogJson(comparison); return canonicalServingCatalogJson({ ...comparison, values: [...comparison.values].sort((left, right) => @@ -342,12 +394,14 @@ function validateLlamaCppPreset( const qualified = new Set(); for (const requirement of preset.spec.requirements?.all ?? []) { - const readiness = requirement.readiness; + if (!("readiness" in requirement)) continue; + const { readiness } = requirement; if (readiness.kind === "qualification" && readiness.status === "qualified") { qualified.add(readiness.id); } const registered = registries.readiness.get(readiness.id); - if (registered?.kind !== "observation" || registered.role === undefined) continue; + if (registered === undefined || !isReadinessRegistryEntry(registered)) continue; + if (registered.kind !== "observation" || registered.role === undefined) continue; const expected = expectedComparisons.get(registered.role); if (!expected) continue; if (seenRoles.has(registered.role)) { @@ -382,6 +436,42 @@ function validateLlamaCppPreset( } } +function validateBindings( + preset: ServingPreset, + recipe: ServingRecipe, + registries: ServingCatalogRegistries, +): void { + const recipeBindingsByName = recipe.spec.bindings ?? {}; + const presetBindingsByName = preset.spec.plan.bindings ?? {}; + const recipeBindings = Object.keys(recipeBindingsByName).sort(); + const presetBindings = Object.keys(presetBindingsByName).sort(); + if ( + recipeBindings.length !== presetBindings.length || + recipeBindings.some((name, index) => name !== presetBindings[index]) + ) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} bindings do not match recipe ${recipe.metadata.id}.`, + ); + } + for (const name of recipeBindings) { + const expected = recipeBindingsByName[name]!; + const actual = presetBindingsByName[name]!.valueFromTopologyQualification; + const key = `${expected.qualificationId}@${String(expected.schemaVersion)}`; + const descriptor = registries.topologyQualifications?.get(key); + if ( + actual.id !== expected.qualificationId || + actual.schemaVersion !== expected.schemaVersion || + (descriptor && + (actual.output !== descriptor.bindingOutput || + expected.outputSchema !== descriptor.outputSchema)) + ) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} has invalid topology binding ${name}.`, + ); + } + } +} + function validateCatalogSemantics( recipes: readonly ServingRecipe[], presets: readonly ServingPreset[], @@ -413,6 +503,19 @@ function validateCatalogSemantics( } validateLlamaCppPreset(recipe, preset, registries); } + validateBindings(preset, recipe, registries); + const nodeCount = (preset.spec.requirements?.all ?? []).find( + (requirement) => + "fact" in requirement && + requirement.fact === "cluster.nodeCount" && + requirement.operator === "equals" && + typeof requirement.value === "number", + ); + if (nodeCount && "fact" in nodeCount && recipe.spec.execution.nodeCount !== nodeCount.value) { + throw new ServingCatalogValidationError( + `Preset ${preset.metadata.id} node-count requirement does not match recipe ${recipe.metadata.id}.`, + ); + } } const automaticSelectors = new Map(); @@ -447,7 +550,7 @@ export function compileTrustedServingCatalog( const validators = createValidators(options.schemas); const recipes: ServingRecipe[] = []; const presets: ServingPreset[] = []; - const sources: CompiledServingCatalog["sources"] = []; + const sources: ServingCatalogSourceProvenance[] = []; const definitionIds = new Set(); const sourcePaths = new Set(); @@ -493,10 +596,10 @@ export function compileTrustedServingCatalog( recipes.push(recipe); } else { const preset = value as unknown as ServingPreset; - validatePresetReadiness(preset, options.registries); + validatePresetRequirements(preset, options.registries); presets.push(preset); } - sources.push({ path: source.path, kind, id, digest: digest(value) }); + sources.push({ path: source.path, kind, id, digest: servingCatalogDigest(value) }); } recipes.sort((left, right) => compareCanonicalText(left.metadata.id, right.metadata.id)); @@ -512,7 +615,10 @@ export function compileTrustedServingCatalog( presets, sources, }; - const catalog: CompiledServingCatalog = { ...payload, catalogDigest: digest(payload) }; + const catalog: CompiledServingCatalog = { + ...payload, + catalogDigest: servingCatalogDigest(payload), + }; if (!validators.catalog(catalog)) { throw new ServingCatalogValidationError( `Compiled serving catalog is invalid: ${validationDetails(validators.catalog)}`, @@ -540,7 +646,7 @@ export function parseCompiledServingCatalogJson( ); } const catalog = value as CompiledServingCatalog; - const expectedDigest = digest(catalogPayload(catalog)); + const expectedDigest = servingCatalogDigest(catalogPayload(catalog)); if (catalog.catalogDigest !== expectedDigest) { throw new ServingCatalogValidationError( `Compiled serving catalog digest mismatch: expected ${expectedDigest}.`, diff --git a/src/lib/inference/serving/generate-catalog.ts b/src/lib/inference/serving/generate-catalog.ts index fe3a56b900..30d34c6f02 100644 --- a/src/lib/inference/serving/generate-catalog.ts +++ b/src/lib/inference/serving/generate-catalog.ts @@ -5,6 +5,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from import { dirname, join, relative } from "node:path"; import { resolveSourceBuildIdentity } from "../../core/version"; +import { getManagedInferenceServingCatalogRegistries } from "./adapter-registry"; import { compileTrustedServingCatalog, serializeCompiledServingCatalog } from "./catalog"; import type { ServingCatalogRegistries, @@ -12,16 +13,6 @@ import type { ServingCatalogSource, } from "./types"; -// Bootstrap only: valid while the catalog has no definitions. Production definitions must -// inject populated registries; tracked by https://github.com/NVIDIA/NemoClaw/issues/8144. -const EMPTY_REGISTRIES: ServingCatalogRegistries = { - receipts: new Set(), - materializers: new Set(), - lifecycles: new Set(), - readinessContracts: new Set(), - readiness: new Map(), -}; - function readJson(path: string): object { return JSON.parse(readFileSync(path, "utf8")) as object; } @@ -64,15 +55,23 @@ export interface GenerateServingCatalogOptions { registries?: ServingCatalogRegistries; } -export function generateServingCatalog(options: GenerateServingCatalogOptions): string { - const outputPath = - options.outputPath ?? join(options.rootDir, "dist", "managed-inference", "catalog.json"); - const catalog = compileTrustedServingCatalog({ +function compileServingCatalog(options: GenerateServingCatalogOptions) { + return compileTrustedServingCatalog({ sources: loadSources(options.rootDir), sourceRevision: resolveSourceBuildIdentity({ rootDir: options.rootDir }).sourceRevision, schemas: loadSchemas(options.rootDir), - registries: options.registries ?? EMPTY_REGISTRIES, + registries: options.registries ?? getManagedInferenceServingCatalogRegistries(), }); +} + +export function checkServingCatalog(options: GenerateServingCatalogOptions): void { + compileServingCatalog(options); +} + +export function generateServingCatalog(options: GenerateServingCatalogOptions): string { + const outputPath = + options.outputPath ?? join(options.rootDir, "dist", "managed-inference", "catalog.json"); + const catalog = compileServingCatalog(options); mkdirSync(dirname(outputPath), { recursive: true }); writeFileSync(outputPath, serializeCompiledServingCatalog(catalog)); return outputPath; @@ -80,5 +79,6 @@ export function generateServingCatalog(options: GenerateServingCatalogOptions): if (require.main === module) { const rootDir = join(__dirname, "..", "..", "..", ".."); - generateServingCatalog({ rootDir }); + if (process.argv.includes("--check")) checkServingCatalog({ rootDir }); + else generateServingCatalog({ rootDir }); } diff --git a/src/lib/inference/serving/managed-cluster-discovery-production.ts b/src/lib/inference/serving/managed-cluster-discovery-production.ts new file mode 100644 index 0000000000..a5c2ab47d3 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-discovery-production.ts @@ -0,0 +1,1182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptionsWithStringEncoding, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import type { BuildIdentity } from "../../core/version.js"; +import { getBuildIdentity } from "../../core/version.js"; +import { assessHost } from "../../onboard/preflight.js"; +import { createHostReadinessReport } from "../../readiness/host.js"; +import { collectPlatformIdentity } from "../../readiness/platform-qualification.js"; +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { managedVllmStateDir } from "../vllm-api-key.js"; +import { buildLocalManagedVllmDockerEnv, buildVllmSshTransportEnv } from "../vllm-docker-env.js"; +import type { + ManagedClusterCommandResult, + ManagedClusterConnectivityRequest, + ManagedClusterDiscoveryDeps, + ManagedClusterHostObservation, + ManagedClusterPinnedPeerTransport, + ManagedClusterReadOnlyHostTransport, +} from "./managed-cluster-discovery.js"; +import { MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE } from "./managed-cluster-runtime-receipt-path.js"; +import { + clearManagedVllmSshBinding, + encodeManagedVllmSshBindingHandoff, + managedVllmKnownHostsDigest, + managedVllmSshBindingDirectory, + type QualifiedManagedVllmSshIdentity, + strictManagedVllmSshTransportArgs, + writeManagedVllmSshBinding, +} from "./managed-cluster-ssh-binding.js"; + +const COMMAND_TIMEOUT_MS = 20_000; +const MAX_COMMAND_OUTPUT_BYTES = 4 * 1024 * 1024; +const MAX_LOCAL_FILE_BYTES = 1024 * 1024; +const MAX_DIRECTORY_ENTRIES = 4096; +const SAFE_TARGET_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const SAFE_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const SAFE_DEVICE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const MAC_PATTERN = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; +const SHA256_PATTERN = /^[a-f0-9]{64}$/; +const SAFE_SSH_KEY_TYPE_PATTERN = /^(?:ssh-|ecdsa-|sk-)[A-Za-z0-9@._+-]+$/; +const SAFE_SSH_KEY_DATA_PATTERN = /^[A-Za-z0-9+/]+={0,3}$/; + +export type ManagedClusterSpawnSync = ( + file: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, +) => { + readonly status: number | null; + readonly stdout: string | null; + readonly stderr: string | null; + readonly error?: Error; +}; + +export type ManagedClusterHostParser = (value: unknown) => ManagedClusterHostObservation; + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validatePeerTarget(raw: string): string { + if ( + raw.length === 0 || + raw.length > 286 || + raw !== raw.trim() || + /[/,:;`'"\\$(){}[\]<>|&!?*\s\u0000-\u001f\u007f]/.test(raw) + ) { + throw new Error("DGX Spark SSH peer target is invalid"); + } + const parts = raw.split("@"); + const username = parts.length === 2 ? parts[0] : ""; + const hostname = parts.at(-1) ?? ""; + if ( + parts.length > 2 || + (parts.length === 2 && !username) || + (username !== "" && !SAFE_USERNAME_PATTERN.test(username)) || + (net.isIP(hostname) !== 4 && !SAFE_TARGET_PATTERN.test(hostname)) + ) { + throw new Error("DGX Spark SSH peer target is invalid"); + } + return raw; +} + +const REMOTE_ARGV_EXECUTOR = String.raw` +import base64 +import json +import subprocess +import sys + +try: + encoded = sys.argv[1] + padding = "=" * ((4 - len(encoded) % 4) % 4) + request = json.loads(base64.urlsafe_b64decode(encoded + padding)) + argv = request["argv"] + if ( + not isinstance(argv, list) + or not argv + or len(argv) > 128 + or any(not isinstance(value, str) or not value or "\x00" in value for value in argv) + or sum(len(value) for value in argv) > 262144 + ): + raise ValueError("invalid argv") + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=15, + check=False, + shell=False, + ) + sys.stdout.buffer.write(result.stdout) + sys.stderr.buffer.write(result.stderr) + raise SystemExit(result.returncode) +except subprocess.TimeoutExpired: + raise SystemExit(124) +except Exception: + raise SystemExit(125) +`; + +const READ_FILE_SCRIPT = String.raw` +import os +import sys + +path = sys.argv[1] +fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) +try: + data = os.read(fd, 1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise SystemExit(2) + sys.stdout.buffer.write(data) +finally: + os.close(fd) +`; + +const READDIR_SCRIPT = String.raw` +import json +import os +import sys + +entries = sorted(os.listdir(sys.argv[1])) +if len(entries) > 4096: + raise SystemExit(2) +print(json.dumps(entries, separators=(",", ":"))) +`; + +const HOST_PROBE_SCRIPT = String.raw` +import csv +import ipaddress +import json +import os +from pathlib import Path +import pwd +import re +import shutil +import socket +import subprocess + +DOCKER_ENV_NAMES = ( + "DOCKER_API_VERSION", + "DOCKER_CERT_PATH", + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_HOST", + "DOCKER_TLS", + "DOCKER_TLS_VERIFY", +) + +def run(argv, timeout=10): + try: + env = None + if argv and argv[0] == "docker": + env = os.environ.copy() + for name in DOCKER_ENV_NAMES: + env.pop(name, None) + env["DOCKER_CONTEXT"] = "default" + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + shell=False, + env=env, + ) + return result.returncode, result.stdout + except Exception: + return 125, "" + +def read_text(path, default=""): + try: + return Path(path).read_text(encoding="utf-8", errors="replace")[:4096].strip() + except Exception: + return default + +def read_int(path, default=-1): + try: + return int(read_text(path)) + except Exception: + return default + +def ipv4_addresses(netdev): + code, output = run(["ip", "-j", "address", "show", "dev", netdev]) + if code != 0: + return [] + try: + payload = json.loads(output) + result = [] + for interface in payload: + for address in interface.get("addr_info", []): + if address.get("family") == "inet": + result.append({ + "address": address["local"], + "prefixLength": int(address["prefixlen"]), + }) + return result + except Exception: + return [] + +def gid_ipv4(value): + try: + mapped = ipaddress.IPv6Address(value).ipv4_mapped + return str(mapped) if mapped is not None else "" + except Exception: + return "" + +def pci_name(address): + if not address: + return "" + code, output = run(["lspci", "-D", "-s", address]) + return output.strip()[:512] if code == 0 else "" + +def physical_adapter_id(pci_address): + if re.fullmatch(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]", pci_address): + return "pci-" + pci_address.rsplit(".", 1)[0].lower() + return "unknown" + +def collect_rails(): + rails = [] + root = Path("/sys/class/infiniband") + if not root.is_dir(): + return rails + for hca in sorted(root.iterdir(), key=lambda item: item.name)[:32]: + ports = hca / "ports" + if not ports.is_dir(): + continue + for port_path in sorted(ports.iterdir(), key=lambda item: item.name)[:16]: + try: + port = int(port_path.name) + except Exception: + continue + ndevs = port_path / "gid_attrs" / "ndevs" + types = port_path / "gid_attrs" / "types" + gids = port_path / "gids" + if not ndevs.is_dir() or not types.is_dir() or not gids.is_dir(): + continue + by_netdev = {} + for ndev_file in sorted(ndevs.iterdir(), key=lambda item: item.name)[:4096]: + try: + index = int(ndev_file.name) + except Exception: + continue + netdev = read_text(ndev_file) + gid_type = read_text(types / ndev_file.name) + gid = read_text(gids / ndev_file.name).lower() + if not netdev or "roce v2" not in gid_type.lower(): + continue + address = gid_ipv4(gid) + if not address: + continue + by_netdev.setdefault(netdev, []).append({ + "index": index, + "value": gid, + "ipv4Address": address, + }) + for netdev, roce_gids in sorted(by_netdev.items()): + net_root = Path("/sys/class/net") / netdev + if not net_root.is_dir(): + continue + try: + pci_address = (net_root / "device").resolve(strict=True).name + except Exception: + pci_address = "" + rails.append({ + "physicalPortId": physical_adapter_id(pci_address), + "netdev": netdev, + "hcaDevice": hca.name, + "hcaPort": port, + "macAddress": read_text(net_root / "address").lower(), + "pciAddress": pci_address, + "pciName": pci_name(pci_address), + "state": read_text(port_path / "state"), + "operState": read_text(net_root / "operstate"), + "carrier": read_text(net_root / "carrier") == "1", + "linkLayer": read_text(port_path / "link_layer"), + "speedMbps": read_int(net_root / "speed"), + "mtu": read_int(net_root / "mtu"), + "ipv4Addresses": ipv4_addresses(netdev), + "roceV2Ipv4Gids": sorted(roce_gids, key=lambda gid: (gid["index"], gid["value"])), + }) + return rails + +def collect_gpus(): + code, output = run([ + "nvidia-smi", + "--query-gpu=index,name,uuid", + "--format=csv,noheader,nounits", + ]) + if code != 0: + return [] + result = [] + try: + for row in csv.reader(output.splitlines()): + if len(row) != 3: + return [] + result.append({"index": int(row[0].strip()), "name": row[1].strip(), "uuid": row[2].strip()}) + except Exception: + return [] + return result + +def listening_ports(): + ports = set() + complete = True + for source in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + lines = Path(source).read_text(encoding="ascii").splitlines()[1:] + for line in lines: + fields = line.split() + if len(fields) >= 4 and fields[3] == "0A": + ports.add(int(fields[1].split(":")[1], 16)) + except Exception: + complete = False + return sorted(ports), complete + +def containers(): + code, output = run(["docker", "ps", "-aq", "--no-trunc"]) + if code != 0: + return [], False + ids = [line.strip() for line in output.splitlines() if line.strip()] + if len(ids) > 256: + return [], False + if not ids: + return [], True + code, output = run(["docker", "inspect", *ids]) + if code != 0: + return [], False + try: + inspected = json.loads(output) + result = [] + for item in inspected: + state = item.get("State") or {} + health = state.get("Health") or {} + container_id = str(item.get("Id", "")) + name = str(item.get("Name", "")).lstrip("/") + image = str((item.get("Config") or {}).get("Image", "")) + labels = (item.get("Config") or {}).get("Labels") or {} + if ( + not re.fullmatch(r"[a-f0-9]{64}", container_id) + or not name or len(name) > 256 + or not image or len(image) > 1024 + or not isinstance(labels, dict) or len(labels) > 128 + or any( + not isinstance(key, str) or not key or len(key) > 256 + or not isinstance(value, str) or len(value) > 4096 + for key, value in labels.items() + ) + ): + return [], False + result.append({ + "id": container_id, + "name": name, + "image": image, + "running": bool(state.get("Running", False)), + "healthy": health.get("Status") == "healthy", + "labels": labels, + }) + return result, len(result) == len(ids) + except Exception: + return [], False + +def nearest_capacity(requested): + try: + candidate = Path(requested).expanduser() + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + if not candidate.exists(): + raise RuntimeError("no existing parent") + resolved = candidate.resolve(strict=True) + stats = os.stat(resolved) + fs_stats = os.statvfs(resolved) + return { + "requestedPath": str(Path(requested).expanduser()), + "probePath": str(resolved), + "filesystemId": str(stats.st_dev), + "availableBytes": int(fs_stats.f_bavail * fs_stats.f_frsize), + "availableInodes": int(fs_stats.f_favail), + "ownerUid": int(stats.st_uid), + "ownerGid": int(stats.st_gid), + "isDirectory": resolved.is_dir(), + "writableByUser": os.access(resolved, os.W_OK | os.X_OK), + } + except Exception: + return { + "requestedPath": str(Path(requested).expanduser()), + "probePath": None, + "filesystemId": None, + "availableBytes": None, + "availableInodes": None, + "ownerUid": None, + "ownerGid": None, + "isDirectory": False, + "writableByUser": False, + } + +uid = os.getuid() +gid = os.getgid() +account = pwd.getpwuid(uid) +home = Path(account.pw_dir).resolve() +hf_home = home / ".cache" / "huggingface" +hf_capacity = nearest_capacity(hf_home) + +docker_root = None +docker_info_code, docker_info_output = run(["docker", "info", "--format", "{{json .}}"]) +if docker_info_code == 0: + try: + docker_root = json.loads(docker_info_output).get("DockerRootDir") + except Exception: + docker_root = None +docker_capacity = nearest_capacity(docker_root or "/var/lib/docker") +observed_containers, containers_complete = containers() +ports, ports_complete = listening_ports() + +earlyoom_units = [ + Path("/etc/systemd/system/earlyoom.service"), + Path("/lib/systemd/system/earlyoom.service"), + Path("/usr/lib/systemd/system/earlyoom.service"), +] +earlyoom_service_installed = any(unit.exists() for unit in earlyoom_units) +earlyoom_installed = bool(shutil.which("earlyoom")) or earlyoom_service_installed +earlyoom_active = "unknown" if earlyoom_installed else "inactive" +earlyoom_enabled = "disabled" +if earlyoom_installed: + service_active = False if not earlyoom_service_installed else None + process_active = None + if earlyoom_service_installed and shutil.which("systemctl"): + active_code, active_output = run(["systemctl", "is-active", "earlyoom.service"]) + enabled_code, enabled_output = run(["systemctl", "is-enabled", "earlyoom.service"]) + service_active = True if active_code == 0 and active_output.strip() == "active" else False if active_output.strip() in ("inactive", "failed") else None + earlyoom_enabled = "enabled" if enabled_code == 0 and enabled_output.strip() == "enabled" else "disabled" if enabled_output.strip() in ("disabled", "masked", "static") else "unknown" + if shutil.which("pgrep"): + process_code, process_output = run(["pgrep", "-x", "earlyoom"]) + process_active = True if process_code == 0 and process_output.strip() else False if process_code == 1 else None + earlyoom_active = "active" if service_active is True or process_active is True else "inactive" if service_active is False and process_active is False else "unknown" + +result = { + "schemaVersion": 1, + "hostname": socket.gethostname(), + "nodeId": read_text("/etc/machine-id"), + "productName": read_text("/sys/class/dmi/id/product_name"), + "architecture": os.uname().machine, + "home": str(home), + "username": account.pw_name, + "uid": uid, + "gid": gid, + "gpus": collect_gpus(), + "rails": collect_rails(), + "earlyoom": { + "installed": earlyoom_installed, + "active": earlyoom_active, + "enabled": earlyoom_enabled, + }, + "runtimeInspectionComplete": containers_complete and ports_complete, + "runtimeSnapshot": {"containers": observed_containers, "listeningPorts": ports}, + "storage": { + "huggingFace": { + **hf_capacity, + "cacheRoot": str(hf_home), + }, + "docker": { + **docker_capacity, + "dockerRootDir": docker_root, + }, + }, +} +print(json.dumps(result, separators=(",", ":"), sort_keys=True)) +`; + +function defaultSpawnSync( + file: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, +): ReturnType { + const result = spawnSync(file, [...args], options); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + ...(result.error ? { error: result.error } : {}), + }; +} + +function validateArgv(argv: readonly string[]): void { + if ( + argv.length === 0 || + argv.length > 128 || + argv.some((value) => !value || value.includes("\0")) || + argv.reduce((total, value) => total + Buffer.byteLength(value, "utf8"), 0) > 262_144 + ) { + throw new Error("DGX Spark probe argv is invalid"); + } +} + +function runCommand( + spawn: ManagedClusterSpawnSync, + file: string, + args: readonly string[], + input = "", + env: Readonly> = buildVllmSshTransportEnv({ LC_ALL: "C", LANG: "C" }), +): ManagedClusterCommandResult { + const result = spawn(file, args, { + encoding: "utf8", + input, + timeout: COMMAND_TIMEOUT_MS, + maxBuffer: MAX_COMMAND_OUTPUT_BYTES, + killSignal: "SIGKILL", + windowsHide: true, + env, + }); + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + ...(result.error ? { error: result.error.message } : {}), + }; +} + +function commandSucceeded(result: ManagedClusterCommandResult, requireOutput = false): boolean { + return ( + result.status === 0 && + result.error === undefined && + (!requireOutput || result.stdout.trim().length > 0) + ); +} + +function readBoundedLocalFile(filePath: string): string { + const flags = + fs.constants.O_RDONLY | + (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0); + const descriptor = fs.openSync(filePath, flags); + try { + const metadata = fs.fstatSync(descriptor); + if (!metadata.isFile() || metadata.size > MAX_LOCAL_FILE_BYTES) { + throw new Error(`DGX Spark probe file ${filePath} is invalid`); + } + const contents = fs.readFileSync(descriptor); + if (contents.length > MAX_LOCAL_FILE_BYTES) { + throw new Error(`DGX Spark probe file ${filePath} is too large`); + } + return contents.toString("utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function readBoundedDirectory(directory: string): string[] { + const entries = fs.readdirSync(directory); + if (entries.length > MAX_DIRECTORY_ENTRIES) { + throw new Error(`DGX Spark probe directory ${directory} is too large`); + } + return entries.sort(compareStrings); +} + +function createLocalTransport(spawn: ManagedClusterSpawnSync): ManagedClusterReadOnlyHostTransport { + return { + execute(argv) { + validateArgv(argv); + return runCommand( + spawn, + argv[0]!, + argv.slice(1), + "", + buildLocalManagedVllmDockerEnv({ LC_ALL: "C", LANG: "C" }), + ); + }, + readFile: readBoundedLocalFile, + readdir: readBoundedDirectory, + }; +} + +function parseJsonCommandResult(result: ManagedClusterCommandResult, label: string): unknown { + if (!commandSucceeded(result, true)) throw new Error(`${label} failed`); + if (Buffer.byteLength(result.stdout, "utf8") > MAX_COMMAND_OUTPUT_BYTES) { + throw new Error(`${label} output is too large`); + } + try { + return JSON.parse(result.stdout); + } catch { + throw new Error(`${label} returned invalid JSON`); + } +} + +function probeHostWithTransport( + transport: ManagedClusterReadOnlyHostTransport, + parseHost: ManagedClusterHostParser, +): ManagedClusterHostObservation { + const result = transport.execute(["python3", "-c", HOST_PROBE_SCRIPT]); + return parseHost(parseJsonCommandResult(result, "DGX Spark host probe")); +} + +type SshConfiguration = ReadonlyMap; + +function parseSshConfiguration(stdout: string): SshConfiguration { + const values = new Map(); + for (const rawLine of stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + const separator = line.search(/\s/); + if (separator <= 0) throw new Error("ssh -G returned malformed effective configuration"); + const key = line.slice(0, separator).toLowerCase(); + const value = line.slice(separator).trim(); + values.set(key, [...(values.get(key) ?? []), value]); + } + return values; +} + +function oneSshConfigurationValue(values: SshConfiguration, key: string): string { + const entries = values.get(key) ?? []; + if (entries.length !== 1 || !entries[0]) { + throw new Error(`Effective SSH configuration must define exactly one ${key}`); + } + return entries[0]; +} + +function assertStrictSshConfiguration(values: SshConfiguration): void { + const exactly = (key: string, allowed: readonly string[]): boolean => { + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); + return observed.length === 1 && allowed.includes(observed[0]!); + }; + const absentOrNone = (key: string): boolean => { + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); + return observed.length === 0 || (observed.length === 1 && observed[0] === "none"); + }; + const preferred = (values.get("preferredauthentications") ?? []).flatMap((value) => + value.toLowerCase().split(",").filter(Boolean), + ); + const sendEnv = (values.get("sendenv") ?? []).map((value) => value.toLowerCase()); + if ( + !exactly("batchmode", ["yes"]) || + !exactly("stricthostkeychecking", ["yes", "true"]) || + !exactly("passwordauthentication", ["no", "false"]) || + !exactly("kbdinteractiveauthentication", ["no", "false"]) || + !exactly("numberofpasswordprompts", ["0"]) || + preferred.length !== 1 || + preferred[0] !== "publickey" || + !exactly("permitlocalcommand", ["no"]) || + !exactly("forwardagent", ["no"]) || + !exactly("forwardx11", ["no"]) || + !exactly("forwardx11trusted", ["no"]) || + !exactly("tunnel", ["false", "no"]) || + !exactly("updatehostkeys", ["false", "no"]) || + !exactly("controlmaster", ["false", "no"]) || + !absentOrNone("controlpath") || + !absentOrNone("remotecommand") || + !absentOrNone("proxycommand") || + !absentOrNone("proxyjump") || + !absentOrNone("localcommand") || + !absentOrNone("knownhostscommand") || + values.has("localforward") || + values.has("remoteforward") || + values.has("dynamicforward") || + values.has("setenv") || + !sendEnv.every((value) => value === "lang" || value === "lc_*") + ) { + throw new Error("Effective SSH configuration is unsafe for DGX Spark discovery"); + } +} + +function canonicalSshHost(value: string): boolean { + return net.isIP(value) === 4 || SAFE_TARGET_PATTERN.test(value); +} + +function trustedKnownHostLines( + spawn: ManagedClusterSpawnSync, + lookupHost: string, + files: readonly string[], +): string[] | null { + const lines = new Set(); + let positive = false; + for (const file of files) { + if (!path.isAbsolute(file) || path.normalize(file) !== file) continue; + let metadata: fs.Stats; + try { + metadata = fs.lstatSync(file); + } catch { + continue; + } + const uid = process.getuid?.(); + if ( + uid === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() || + (metadata.uid !== uid && metadata.uid !== 0) || + (metadata.mode & 0o022) !== 0 + ) { + continue; + } + const result = runCommand(spawn, "ssh-keygen", ["-F", lookupHost, "-f", file]); + if (!commandSucceeded(result, true)) continue; + for (const rawLine of result.stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || /[\u0000\r\n]/.test(line)) continue; + const fields = line.split(/\s+/); + const marker = fields[0]?.startsWith("@") ? fields.shift() : ""; + if ((marker !== "" && marker !== "@revoked") || fields.length < 3) continue; + const keyType = fields[1] ?? ""; + const keyData = fields[2] ?? ""; + if (!SAFE_SSH_KEY_TYPE_PATTERN.test(keyType) || !SAFE_SSH_KEY_DATA_PATTERN.test(keyData)) { + continue; + } + lines.add(line); + if (marker === "") positive = true; + } + } + if (!positive || lines.size === 0) return null; + const result = [...lines].sort(compareStrings); + managedVllmKnownHostsDigest(`${result.join("\n")}\n`); + return result; +} + +function inspectPretrustedTarget( + spawn: ManagedClusterSpawnSync, + rawTarget: string, +): QualifiedManagedVllmSshIdentity | null { + const target = validatePeerTarget(rawTarget); + const result = runCommand(spawn, "ssh", [ + "-G", + ...strictManagedVllmSshTransportArgs(), + "--", + target, + ]); + if (!commandSucceeded(result, true)) return null; + const config = parseSshConfiguration(result.stdout); + assertStrictSshConfiguration(config); + const resolvedHost = oneSshConfigurationValue(config, "hostname"); + const sshUser = oneSshConfigurationValue(config, "user"); + const port = Number(oneSshConfigurationValue(config, "port")); + if (!canonicalSshHost(resolvedHost) || !SAFE_USERNAME_PATTERN.test(sshUser)) { + throw new Error("Effective SSH endpoint is invalid"); + } + if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error("Effective SSH port is invalid"); + } + const requestedHost = target.slice(target.lastIndexOf("@") + 1); + if (net.isIP(requestedHost) === 4 && resolvedHost !== requestedHost) { + throw new Error("Automatic rail target was remapped by SSH configuration"); + } + const alias = config.get("hostkeyalias")?.[0]; + const baseLookupHost = alias && alias.toLowerCase() !== "none" ? alias : resolvedHost; + if (!canonicalSshHost(baseLookupHost)) throw new Error("Effective SSH host-key alias is invalid"); + const lookupHost = port === 22 ? baseLookupHost : `[${baseLookupHost}]:${String(port)}`; + const files = [ + ...(config.get("userknownhostsfile") ?? []), + ...(config.get("globalknownhostsfile") ?? []), + ].flatMap((entry) => entry.split(/\s+/).filter((value) => value && value !== "none")); + const knownHostsLines = trustedKnownHostLines(spawn, lookupHost, files); + if (!knownHostsLines) return null; + return { + requestedTarget: target, + sshTarget: target, + resolvedHost, + sshUser, + port, + lookupHost, + hostKeyDigest: managedVllmKnownHostsDigest(`${knownHostsLines.join("\n")}\n`), + knownHostsLines, + }; +} + +function assertPinnedIdentity(identity: QualifiedManagedVllmSshIdentity): void { + if ( + validatePeerTarget(identity.sshTarget) !== identity.sshTarget || + identity.requestedTarget !== identity.sshTarget || + !canonicalSshHost(identity.resolvedHost) || + !SAFE_USERNAME_PATTERN.test(identity.sshUser) || + !Number.isInteger(identity.port) || + identity.port < 1 || + identity.port > 65_535 || + !SHA256_PATTERN.test(identity.hostKeyDigest) + ) { + throw new Error("Qualified DGX Spark SSH identity is invalid"); + } + const expectedLookup = + identity.port === 22 + ? identity.resolvedHost + : `[${identity.resolvedHost}]:${String(identity.port)}`; + if (identity.lookupHost !== expectedLookup) { + throw new Error("Qualified DGX Spark SSH host-key lookup is invalid"); + } + const contents = `${identity.knownHostsLines.join("\n")}\n`; + if ( + identity.knownHostsLines.length === 0 || + identity.knownHostsLines.some( + (line) => !line || line !== line.trim() || /[\u0000\r\n]/.test(line), + ) || + managedVllmKnownHostsDigest(contents) !== identity.hostKeyDigest + ) { + throw new Error("Qualified DGX Spark SSH host-key evidence is invalid"); + } +} + +function assertTemporaryPinnedFiles( + directory: string, + knownHostsFile: string, + expectedDigest: string, +): void { + const uid = process.getuid?.(); + if (uid === undefined) throw new Error("DGX Spark discovery requires a POSIX user identity"); + const directoryMetadata = fs.lstatSync(directory); + const fileMetadata = fs.lstatSync(knownHostsFile); + if ( + directoryMetadata.isSymbolicLink() || + !directoryMetadata.isDirectory() || + directoryMetadata.uid !== uid || + (directoryMetadata.mode & 0o777) !== 0o700 || + fileMetadata.isSymbolicLink() || + !fileMetadata.isFile() || + fileMetadata.uid !== uid || + (fileMetadata.mode & 0o777) !== 0o600 || + fs.readdirSync(directory).some((entry) => entry !== path.basename(knownHostsFile)) || + managedVllmKnownHostsDigest(readBoundedLocalFile(knownHostsFile)) !== expectedDigest + ) { + throw new Error("Temporary DGX Spark SSH host-key pin is unsafe"); + } +} + +function openPinnedPeerTransport( + spawn: ManagedClusterSpawnSync, + identity: QualifiedManagedVllmSshIdentity, +): ManagedClusterPinnedPeerTransport { + assertPinnedIdentity(identity); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-cluster-")); + fs.chmodSync(directory, 0o700); + const knownHostsFile = path.join(directory, "known_hosts"); + try { + fs.writeFileSync(knownHostsFile, `${identity.knownHostsLines.join("\n")}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.chmodSync(knownHostsFile, 0o600); + assertTemporaryPinnedFiles(directory, knownHostsFile, identity.hostKeyDigest); + } catch (error) { + try { + if (fs.existsSync(knownHostsFile)) fs.unlinkSync(knownHostsFile); + fs.rmdirSync(directory); + } catch { + // The exact owner-only temporary path is retained for inspection if safe cleanup fails. + } + throw error; + } + + const execute = (argv: readonly string[]): ManagedClusterCommandResult => { + validateArgv(argv); + assertTemporaryPinnedFiles(directory, knownHostsFile, identity.hostKeyDigest); + const request = Buffer.from(JSON.stringify({ argv }), "utf8").toString("base64url"); + return runCommand( + spawn, + "ssh", + [ + "-F", + "/dev/null", + ...strictManagedVllmSshTransportArgs(), + "-o", + `UserKnownHostsFile=${knownHostsFile}`, + "-o", + "GlobalKnownHostsFile=/dev/null", + "-o", + `HostKeyAlias=${identity.lookupHost}`, + "-o", + `Hostname=${identity.resolvedHost}`, + "-o", + `User=${identity.sshUser}`, + "-o", + `Port=${String(identity.port)}`, + "--", + identity.sshTarget, + "python3", + "-", + request, + ], + REMOTE_ARGV_EXECUTOR, + ); + }; + const transport: ManagedClusterReadOnlyHostTransport = { + execute, + readFile(filePath) { + if (!path.posix.isAbsolute(filePath) || path.posix.normalize(filePath) !== filePath) { + throw new Error("Remote DGX Spark probe file path is invalid"); + } + const result = execute(["python3", "-c", READ_FILE_SCRIPT, filePath]); + if (!commandSucceeded(result)) throw new Error(`Remote file ${filePath} could not be read`); + if (Buffer.byteLength(result.stdout, "utf8") > MAX_LOCAL_FILE_BYTES) { + throw new Error(`Remote file ${filePath} is too large`); + } + return result.stdout; + }, + readdir(directoryPath) { + if ( + !path.posix.isAbsolute(directoryPath) || + path.posix.normalize(directoryPath) !== directoryPath + ) { + throw new Error("Remote DGX Spark probe directory path is invalid"); + } + const result = execute(["python3", "-c", READDIR_SCRIPT, directoryPath]); + const value = parseJsonCommandResult(result, `Remote directory ${directoryPath}`); + if ( + !Array.isArray(value) || + value.length > MAX_DIRECTORY_ENTRIES || + value.some( + (entry) => + typeof entry !== "string" || + !entry || + entry === "." || + entry === ".." || + entry.includes("/") || + entry.includes("\0"), + ) + ) { + throw new Error(`Remote directory ${directoryPath} returned invalid entries`); + } + return (value as string[]).sort(compareStrings); + }, + }; + return { + transport, + close() { + assertTemporaryPinnedFiles(directory, knownHostsFile, identity.hostKeyDigest); + fs.unlinkSync(knownHostsFile); + fs.rmdirSync(directory); + }, + }; +} + +function isRecordArray(value: unknown): value is Record[] { + return Array.isArray(value) && value.every(isRecord); +} + +function connectivityCheck( + transport: ManagedClusterReadOnlyHostTransport, + request: ManagedClusterConnectivityRequest, +): boolean { + const routeResult = transport.execute([ + "ip", + "-j", + "route", + "get", + request.peerAddress, + "from", + request.sourceAddress, + "oif", + request.netdev, + ]); + let routeValue: unknown; + try { + routeValue = parseJsonCommandResult(routeResult, "DGX Spark direct route probe"); + } catch { + return false; + } + if (!isRecordArray(routeValue) || routeValue.length !== 1) return false; + const route = routeValue[0]!; + const routeSource = route.prefsrc ?? route.src; + if ( + route.dev !== request.netdev || + routeSource !== request.sourceAddress || + route.gateway !== undefined || + (route.scope !== undefined && String(route.scope).toLowerCase() !== "link") + ) { + return false; + } + const ping = transport.execute([ + "ping", + "-c", + "1", + "-W", + "2", + "-M", + "do", + "-s", + "8972", + "-I", + request.sourceAddress, + request.peerAddress, + ]); + if (!commandSucceeded(ping)) return false; + const neighborResult = transport.execute([ + "ip", + "-j", + "neigh", + "show", + "to", + request.peerAddress, + "dev", + request.netdev, + ]); + let neighborValue: unknown; + try { + neighborValue = parseJsonCommandResult(neighborResult, "DGX Spark neighbor probe"); + } catch { + return false; + } + if (!isRecordArray(neighborValue) || neighborValue.length !== 1) return false; + const neighbor = neighborValue[0]!; + const states = Array.isArray(neighbor.state) ? neighbor.state : [neighbor.state]; + return ( + String(neighbor.dst ?? "") === request.peerAddress && + String(neighbor.dev ?? "") === request.netdev && + String(neighbor.lladdr ?? "").toLowerCase() === request.expectedPeerMac && + states.length > 0 && + states.every( + (state) => + typeof state === "string" && + /^(?:REACHABLE|STALE|DELAY|PROBE|PERMANENT|NOARP)$/i.test(state), + ) + ); +} + +function probeConnectivity( + transport: ManagedClusterReadOnlyHostTransport, + requests: readonly ManagedClusterConnectivityRequest[], +): boolean { + if ( + requests.length !== 2 || + new Set(requests.map(({ netdev }) => netdev)).size !== 2 || + requests.some( + ({ netdev, sourceAddress, peerAddress, expectedPeerMac }) => + !SAFE_DEVICE_PATTERN.test(netdev) || + net.isIP(sourceAddress) !== 4 || + net.isIP(peerAddress) !== 4 || + !MAC_PATTERN.test(expectedPeerMac), + ) + ) { + return false; + } + return requests.every((request) => connectivityCheck(transport, request)); +} + +function createCanonicalReadiness( + host: ManagedClusterHostObservation, + transport: ManagedClusterReadOnlyHostTransport, + buildIdentity: BuildIdentity, + now: Date, +): SystemReadinessReport { + const runCaptureImpl = ( + argv: readonly string[], + options?: { readonly ignoreError?: boolean }, + ): string => { + const result = transport.execute(argv); + if (!commandSucceeded(result) && options?.ignoreError !== true) { + throw new Error(`Readiness command ${argv[0] ?? "unknown"} failed`); + } + return commandSucceeded(result) ? result.stdout : ""; + }; + const releaseResult = transport.execute(["uname", "-r"]); + if (!commandSucceeded(releaseResult, true)) { + throw new Error("DGX Spark kernel release could not be observed"); + } + let procVersion = ""; + try { + procVersion = transport.readFile("/proc/version"); + } catch { + // assessHost treats an unavailable proc version as absent WSL evidence. + } + const assessment = assessHost({ + platform: "linux", + env: {}, + release: releaseResult.stdout.trim(), + procVersion, + readFileImpl: (filePath) => transport.readFile(filePath), + readdirImpl: (directory) => transport.readdir(directory), + runCaptureImpl, + gpuProbeImpl: () => host.gpus.length === 1, + }); + const platformIdentity = collectPlatformIdentity({ + readFile: (filePath) => transport.readFile(filePath), + readdir: (directory) => transport.readdir(directory), + }); + return createHostReadinessReport( + { + nemoclawVersion: buildIdentity.nemoclawVersion, + sourceRevision: buildIdentity.sourceRevision, + now: () => now, + }, + { + now: () => now, + architecture: "arm64", + assess: () => assessment, + detectHostGpuPlatform: () => "spark", + collectPlatformIdentity: () => platformIdentity, + }, + ); +} + +function claimBinding(statePath: string): boolean { + const bindingDirectory = managedVllmSshBindingDirectory(statePath); + const parent = path.dirname(bindingDirectory); + const uid = process.getuid?.(); + if (uid === undefined) throw new Error("DGX Spark binding claim requires a POSIX user identity"); + const directoryFlags = fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW; + const parentDescriptor = fs.openSync(parent, directoryFlags); + try { + const parentMetadata = fs.fstatSync(parentDescriptor); + if ( + !parentMetadata.isDirectory() || + parentMetadata.uid !== uid || + (parentMetadata.mode & 0o777) !== 0o700 + ) { + throw new Error("DGX Spark binding parent is unsafe"); + } + try { + fs.mkdirSync(bindingDirectory, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return false; + throw error; + } + + const bindingDescriptor = fs.openSync(bindingDirectory, directoryFlags); + try { + const bindingMetadata = fs.fstatSync(bindingDescriptor); + if (!bindingMetadata.isDirectory() || bindingMetadata.uid !== uid) { + throw new Error("DGX Spark binding claim is unsafe"); + } + fs.fchmodSync(bindingDescriptor, 0o700); + fs.fsyncSync(bindingDescriptor); + } finally { + fs.closeSync(bindingDescriptor); + } + + fs.fsyncSync(parentDescriptor); + return true; + } finally { + fs.closeSync(parentDescriptor); + } +} + +/** + * Construct the production discovery seams. The optional spawn adapter exists + * only so strict SSH argv behavior can be tested without network or host mutation. + */ +export function createProductionManagedClusterDiscoveryDeps( + parseHost: ManagedClusterHostParser, + spawn: ManagedClusterSpawnSync = defaultSpawnSync, +): ManagedClusterDiscoveryDeps { + return { + now: () => new Date(), + currentUid: () => process.getuid?.() ?? null, + getBuildIdentity, + localTransport: () => createLocalTransport(spawn), + probeHost: (transport) => probeHostWithTransport(transport, parseHost), + inspectPretrustedTarget: (target) => inspectPretrustedTarget(spawn, target), + openPinnedPeerTransport: (identity) => openPinnedPeerTransport(spawn, identity), + createReadiness: createCanonicalReadiness, + probeConnectivity, + claimBinding, + writeBinding: writeManagedVllmSshBinding, + clearBinding: clearManagedVllmSshBinding, + encodeBinding: encodeManagedVllmSshBindingHandoff, + resolveBindingStatePath: (nodeId) => + path.join(managedVllmStateDir(), `${MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE}.${nodeId}`), + }; +} diff --git a/src/lib/inference/serving/managed-cluster-discovery.test.ts b/src/lib/inference/serving/managed-cluster-discovery.test.ts new file mode 100644 index 0000000000..8b60116403 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-discovery.test.ts @@ -0,0 +1,962 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { + confirmManagedClusterManagedServingCapability, + createManagedClusterDiscoveryDeps, + type ManagedClusterDetectedManagedServingCapability, + type ManagedClusterDiscoveryDeps, + type ManagedClusterHostObservation, + type ManagedClusterReadOnlyHostTransport, + type ManagedClusterSpawnSync, + NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV, + NEMOCLAW_SERVING_PRESET_ENV, + parseManagedClusterHostObservation, + probeManagedClusterManagedServingCapability, +} from "./managed-cluster-discovery.js"; +import { FIXTURE_MANAGED_CLUSTER_PRESET_ID } from "./managed-cluster-fixture.test-support.js"; +import { MANAGED_CLUSTER_MANAGED_LABEL } from "./managed-cluster-materialize.js"; +import { + type ManagedVllmSshBinding, + managedVllmKnownHostsDigest, + type QualifiedManagedVllmSshIdentity, +} from "./managed-cluster-ssh-binding.js"; + +const NOW = new Date("2026-08-02T20:00:00.000Z"); +const SOURCE_REVISION = "1d6948d89b46eab739728215f9a19ef40b8f6121"; +const REQUIRED_CAPABILITIES = [ + "host.platform.supported", + "host.platform.dgx_spark", + "host.docker.available", + "host.docker.daemon_reachable", + "host.docker.runtime_supported", + "host.docker.storage_compatible", + "host.gpu.nvidia_available", + "host.gpu.container_toolkit_available", + "host.gpu.cdi_healthy", +] as const; + +type StoppedForeignContainerFixture = { + readonly signal: string; + readonly name: string; + readonly image: string; + readonly labels: Readonly>; +}; + +const STOPPED_FOREIGN_CONTAINER_FIXTURES: readonly StoppedForeignContainerFixture[] = [ + { + signal: "name", + name: "foreign-vllm-server", + image: "example.invalid/inference:latest", + labels: {}, + }, + { + signal: "image", + name: "foreign-inference", + image: "vllm/vllm-openai:latest", + labels: {}, + }, + { + signal: "managed label", + name: "foreign-inference", + image: "example.invalid/inference:latest", + labels: { [MANAGED_CLUSTER_MANAGED_LABEL]: "foreign" }, + }, +]; + +function expectDetectedCluster( + detected: ReturnType, +): ManagedClusterDetectedManagedServingCapability { + expect(detected.kind).toBe("ready"); + return detected as ManagedClusterDetectedManagedServingCapability; +} + +function readiness(overrides: Partial = {}): SystemReadinessReport { + return { + schemaVersion: "1.1.0", + status: "supported", + exitCode: 0, + mutated: false, + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, + observedAt: NOW.toISOString(), + }, + observations: [], + capabilities: REQUIRED_CAPABILITIES.map((id) => ({ + id, + state: "present" as const, + })), + qualifications: [ + { + id: "host.platform.dgx_spark", + status: "qualified", + capabilityIds: ["host.platform.dgx_spark"], + }, + ], + findings: [], + evidence: [], + ...overrides, + } as SystemReadinessReport; +} + +function capacity( + requestedPath: string, + uid: number, + gid: number, + overrides: Record = {}, +) { + return { + requestedPath, + probePath: requestedPath, + filesystemId: requestedPath.includes("docker") ? "docker-fs" : "home-fs", + availableBytes: 400_000_000_000, + availableInodes: 1_000_000, + ownerUid: uid, + ownerGid: gid, + isDirectory: true, + writableByUser: true, + ...overrides, + }; +} + +function host( + role: "local" | "peer", + overrides: Partial = {}, +): ManagedClusterHostObservation { + const local = role === "local"; + const uid = 1000; + const gid = 1000; + const home = "/home/nvidia"; + const rail = (index: 0 | 1) => { + const third = 100 + index; + const address = `192.168.${String(third)}.${local ? "1" : "2"}`; + return { + physicalPortId: local ? "cx7-local" : "cx7-peer", + netdev: `enp${String(index + 1)}s0f0np0`, + hcaDevice: `rocep${String(index + 1)}s0f0`, + hcaPort: 1, + macAddress: local + ? `02:00:00:00:00:0${String(index + 1)}` + : `02:00:00:00:01:0${String(index + 1)}`, + pciAddress: `0000:0${String(index + 1)}:00.0`, + pciName: "NVIDIA Mellanox ConnectX-7", + state: "4: ACTIVE", + operState: "up", + carrier: true, + linkLayer: "Ethernet", + speedMbps: 200_000, + mtu: 9000, + ipv4Addresses: [{ address, prefixLength: 30 }], + roceV2Ipv4Gids: [ + { + index: 3 + index, + value: `::ffff:192.168.${String(third)}.${local ? "1" : "2"}`, + ipv4Address: address, + }, + ], + }; + }; + const cacheRoot = `${home}/.cache/huggingface`; + return { + schemaVersion: 1, + hostname: local ? "spark-head" : "spark-worker", + nodeId: local ? "11111111111111111111111111111111" : "22222222222222222222222222222222", + productName: "NVIDIA DGX Spark", + architecture: "aarch64", + home, + username: "nvidia", + uid, + gid, + gpus: [ + { + index: 0, + name: "NVIDIA GB10", + uuid: local + ? "GPU-11111111-1111-1111-1111-111111111111" + : "GPU-22222222-2222-2222-2222-222222222222", + }, + ], + rails: [rail(0), rail(1)], + earlyoom: { installed: false, active: "inactive", enabled: "disabled" }, + runtimeInspectionComplete: true, + runtimeSnapshot: { containers: [], listeningPorts: [] }, + storage: { + huggingFace: { + ...capacity(cacheRoot, uid, gid), + cacheRoot, + }, + docker: { + ...capacity("/var/lib/docker", 0, 0), + dockerRootDir: "/var/lib/docker", + }, + }, + ...overrides, + }; +} + +const transport = (name: string): ManagedClusterReadOnlyHostTransport => ({ + execute: () => ({ status: 0, stdout: name, stderr: "" }), + readFile: () => name, + readdir: () => [], +}); + +function identity(target: string): QualifiedManagedVllmSshIdentity { + const knownHostsLines = [`${target} ssh-ed25519 AAAA`]; + return { + requestedTarget: target, + sshTarget: target, + resolvedHost: target, + sshUser: "nvidia", + port: 22, + lookupHost: target, + hostKeyDigest: managedVllmKnownHostsDigest(`${knownHostsLines.join("\n")}\n`), + knownHostsLines, + }; +} + +function binding(peerIdentity: QualifiedManagedVllmSshIdentity): ManagedVllmSshBinding { + return { + schemaVersion: 2, + peerTarget: peerIdentity.sshTarget, + resolvedHost: peerIdentity.resolvedHost, + sshUser: peerIdentity.sshUser, + port: peerIdentity.port, + lookupHost: peerIdentity.lookupHost, + hostKeyDigest: peerIdentity.hostKeyDigest, + bindingFile: "/state/binding.json", + dockerCliFile: "/usr/bin/docker", + dockerShimFile: "/state/docker", + dockerShimSha256: "a".repeat(64), + knownHostsFile: "/state/known_hosts", + knownHostsSha256: "b".repeat(64), + sshWrapperDirectory: "/state/bin", + sshWrapperFile: "/state/bin/ssh", + sshWrapperSha256: "c".repeat(64), + }; +} + +function fixture(overrides: Partial = {}) { + const events: string[] = []; + const localTransport = transport("local"); + const peerTransports = new Map([ + ["192.168.100.2", transport("peer-100")], + ["192.168.101.2", transport("peer-101")], + ]); + let bindingWrites = 0; + const deps: ManagedClusterDiscoveryDeps = { + now: () => NOW, + currentUid: () => 1000, + getBuildIdentity: () => ({ + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, + }), + localTransport: () => localTransport, + probeHost: (candidate) => (candidate === localTransport ? host("local") : host("peer")), + inspectPretrustedTarget: (target) => identity(target), + openPinnedPeerTransport: (peerIdentity) => { + events.push(`open:${peerIdentity.requestedTarget}`); + return { + transport: peerTransports.get(peerIdentity.requestedTarget) ?? transport("explicit-peer"), + close: () => events.push(`close:${peerIdentity.requestedTarget}`), + }; + }, + createReadiness: (observedHost) => { + events.push(`readiness:${observedHost.hostname}`); + return readiness(); + }, + probeConnectivity: (_candidate, requests) => { + events.push(`connectivity:${requests[0]?.sourceAddress ?? "missing"}`); + return true; + }, + claimBinding: () => true, + writeBinding: (_statePath, peerIdentity) => { + bindingWrites += 1; + events.push("write-binding"); + return binding(peerIdentity); + }, + clearBinding: () => events.push("clear-binding"), + encodeBinding: () => "binding-token", + resolveBindingStatePath: () => "/state/managed-cluster.json", + ...overrides, + }; + return { deps, events, bindingWrites: () => bindingWrites, localTransport }; +} + +describe("managed DGX Spark cluster discovery", () => { + it("accepts the bounded production observation and rejects malformed cache metadata", () => { + const observed = host("local"); + expect(parseManagedClusterHostObservation(observed)).toBe(observed); + expect(() => + parseManagedClusterHostObservation({ + ...observed, + storage: { + ...observed.storage, + huggingFace: { ...observed.storage.huggingFace, isDirectory: "yes" }, + }, + }), + ).toThrow("DGX Spark host observation is invalid"); + }); + + it("does nothing when another serving preset is selected", () => { + const { deps, events, bindingWrites } = fixture({ + localTransport: () => { + throw new Error("must not probe"); + }, + }); + + expect( + probeManagedClusterManagedServingCapability({ + env: { [NEMOCLAW_SERVING_PRESET_ENV]: "another-preset" }, + deps, + }), + ).toEqual({ + kind: "not-selected", + code: "no-match", + reason: "Another managed inference preset is selected.", + }); + expect(events).toEqual([]); + expect(bindingWrites()).toBe(0); + }); + + it("rejects an explicit managed cluster peer combined with another serving preset", () => { + const { deps, events, bindingWrites } = fixture({ + localTransport: () => { + throw new Error("must not probe"); + }, + }); + + expect( + probeManagedClusterManagedServingCapability({ + env: { + [NEMOCLAW_SERVING_PRESET_ENV]: "another-preset", + [NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV]: "spark-worker.local", + }, + deps, + }), + ).toMatchObject({ kind: "unavailable", code: "incompatible-selection" }); + expect(events).toEqual([]); + expect(bindingWrites()).toBe(0); + }); + + it("fails closed when an explicit preset catalog cannot be loaded", () => { + const { deps, events, bindingWrites } = fixture({ + localTransport: () => { + throw new Error("must not probe"); + }, + }); + + expect( + probeManagedClusterManagedServingCapability({ + env: { [NEMOCLAW_SERVING_PRESET_ENV]: FIXTURE_MANAGED_CLUSTER_PRESET_ID }, + deps, + loadCatalog: () => { + throw new Error("catalog unavailable"); + }, + }), + ).toMatchObject({ kind: "unavailable", code: "incompatible-selection" }); + expect(events).toEqual([]); + expect(bindingWrites()).toBe(0); + }); + + it("persists bindings only after the detected cluster is confirmed and revalidated", () => { + const { deps, events, bindingWrites } = fixture(); + + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + + expect(detected).toMatchObject({ + kind: "ready", + selectionIntent: "automatic", + sshClaims: [{ statePath: "/state/managed-cluster.json" }], + local: { hostname: "spark-head", uid: 1000 }, + peers: [{ hostname: "spark-worker", uid: 1000 }], + topology: { status: "qualified" }, + }); + expect(bindingWrites()).toBe(0); + expect(events).not.toContain("write-binding"); + const confirmed = confirmManagedClusterManagedServingCapability(detected, { + env: {}, + deps, + }); + + expect(confirmed).toMatchObject({ + kind: "ready", + sshBindings: [{ handle: "binding-token" }], + topology: { + output: { peers: [{ sshBindingHandle: "binding-token" }] }, + }, + }); + expect(bindingWrites()).toBe(1); + expect(events.filter((event) => event.startsWith("open:"))).toEqual([ + "open:192.168.100.2", + "open:192.168.101.2", + "open:192.168.100.2", + "open:192.168.101.2", + ]); + expect(events.indexOf("write-binding")).toBeGreaterThan( + events.lastIndexOf("readiness:spark-worker"), + ); + expect(events.filter((event) => event.startsWith("close:"))).toHaveLength(4); + }); + + it("returns an ordinary automatic no-match when both rails are not pretrusted", () => { + const { deps, bindingWrites } = fixture({ + inspectPretrustedTarget: () => null, + }); + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "not-selected", + code: "no-match", + }); + expect(bindingWrites()).toBe(0); + }); + + it("makes the same peer failure hard when the exact preset was requested", () => { + const { deps } = fixture({ inspectPretrustedTarget: () => null }); + + expect( + probeManagedClusterManagedServingCapability({ + env: { [NEMOCLAW_SERVING_PRESET_ENV]: FIXTURE_MANAGED_CLUSTER_PRESET_ID }, + deps, + }), + ).toMatchObject({ kind: "unavailable", code: "peer-trust-unavailable" }); + }); + + it("makes an explicit peer failure hard", () => { + const { deps } = fixture({ inspectPretrustedTarget: () => null }); + + expect( + probeManagedClusterManagedServingCapability({ + env: { [NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV]: "spark-worker.local" }, + deps, + }), + ).toMatchObject({ kind: "unavailable", code: "peer-trust-unavailable" }); + }); + + it("does not merge rail addresses that have different host-key identities", () => { + const { deps, bindingWrites } = fixture({ + inspectPretrustedTarget: (target) => { + const observed = identity(target); + return target.endsWith("101.2") ? { ...observed, hostKeyDigest: "f".repeat(64) } : observed; + }, + }); + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "not-selected", + code: "no-match", + }); + expect(bindingWrites()).toBe(0); + }); + + it.each( + STOPPED_FOREIGN_CONTAINER_FIXTURES, + )("preserves a stopped foreign vLLM setup identified by $signal", (container) => { + const base = fixture(); + const deps = { + ...base.deps, + probeHost: (candidate: ManagedClusterReadOnlyHostTransport) => + candidate === base.localTransport + ? host("local", { + runtimeSnapshot: { + containers: [ + { + id: "9".repeat(64), + name: container.name, + image: container.image, + running: false, + healthy: false, + labels: container.labels, + }, + ], + listeningPorts: [], + }, + }) + : host("peer"), + }; + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "not-selected", + code: "runtime-conflict", + }); + expect(base.bindingWrites()).toBe(0); + expect(base.events).not.toContain("write-binding"); + }); + + it("does not classify an arbitrary stopped container as a managed vLLM setup", () => { + const base = fixture(); + const deps = { + ...base.deps, + probeHost: (candidate: ManagedClusterReadOnlyHostTransport) => + candidate === base.localTransport + ? host("local", { + runtimeSnapshot: { + containers: [ + { + id: "9".repeat(64), + name: "unrelated-service", + image: "example.invalid/worker:latest", + running: false, + healthy: false, + labels: { "example.foreign": "true" }, + }, + ], + listeningPorts: [], + }, + }) + : host("peer"), + }; + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "ready", + }); + expect(base.bindingWrites()).toBe(0); + }); + + it.each([ + { + name: "an active earlyoom service", + mutate: (value: ManagedClusterHostObservation) => ({ + ...value, + earlyoom: { + installed: true, + active: "active" as const, + enabled: "enabled" as const, + }, + }), + code: "no-match", + }, + { + name: "a related runtime", + mutate: (value: ManagedClusterHostObservation) => ({ + ...value, + runtimeSnapshot: { + containers: [ + { + id: "9".repeat(64), + name: "managed-vllm", + image: "example.invalid/vllm:latest", + running: true, + healthy: true, + labels: { [MANAGED_CLUSTER_MANAGED_LABEL]: "true" }, + }, + ], + listeningPorts: [], + }, + }), + code: "runtime-conflict", + }, + { + name: "an inconclusive runtime inspection", + mutate: (value: ManagedClusterHostObservation) => ({ + ...value, + runtimeInspectionComplete: false, + }), + code: "runtime-unknown", + }, + { + name: "a missing exact cache root", + mutate: (value: ManagedClusterHostObservation) => ({ + ...value, + storage: { + ...value.storage, + huggingFace: { ...value.storage.huggingFace, probePath: value.home }, + }, + }), + code: "no-match", + }, + ])("preserves host state for $name", ({ mutate, code }) => { + const base = fixture(); + const deps = { + ...base.deps, + probeHost: (candidate: ManagedClusterReadOnlyHostTransport) => + candidate === base.localTransport ? mutate(host("local")) : host("peer"), + }; + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "not-selected", + code, + }); + expect(base.bindingWrites()).toBe(0); + expect(base.events).not.toContain("write-binding"); + }); + + it("does not claim when the exact pretrusted peer identity changes after confirmation", () => { + const base = fixture(); + let inspections = 0; + const deps: ManagedClusterDiscoveryDeps = { + ...base.deps, + inspectPretrustedTarget: (target) => { + inspections += 1; + const observed = identity(target); + return inspections > 2 + ? { + ...observed, + sshTarget: "spark-worker.local", + resolvedHost: "spark-worker.local", + } + : observed; + }, + claimBinding: () => { + base.events.push("claim-binding"); + return true; + }, + }; + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + + expect( + confirmManagedClusterManagedServingCapability(detected, { env: {}, deps }), + ).toMatchObject({ + kind: "unavailable", + code: "peer-identity-ambiguous", + }); + expect(base.events).not.toContain("claim-binding"); + expect(base.bindingWrites()).toBe(0); + }); + + it("rechecks related runtime and listeners before claiming the binding", () => { + const base = fixture(); + let runtimeAppeared = false; + const deps: ManagedClusterDiscoveryDeps = { + ...base.deps, + probeHost: (candidate) => + candidate === base.localTransport && runtimeAppeared + ? host("local", { + runtimeSnapshot: { + containers: [ + { + id: "9".repeat(64), + name: "managed-vllm", + image: "example.invalid/vllm:latest", + running: true, + healthy: true, + labels: { [MANAGED_CLUSTER_MANAGED_LABEL]: "true" }, + }, + ], + listeningPorts: [], + }, + }) + : candidate === base.localTransport + ? host("local") + : host("peer"), + claimBinding: () => { + base.events.push("claim-binding"); + return true; + }, + }; + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + runtimeAppeared = true; + + expect( + confirmManagedClusterManagedServingCapability(detected, { env: {}, deps }), + ).toMatchObject({ + kind: "unavailable", + code: "runtime-conflict", + }); + expect(base.events).not.toContain("claim-binding"); + expect(base.bindingWrites()).toBe(0); + }); + + it("preserves an existing binding as a hard automatic conflict", () => { + const { deps, events, bindingWrites } = fixture({ + claimBinding: () => false, + }); + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + + expect( + confirmManagedClusterManagedServingCapability(detected, { env: {}, deps }), + ).toMatchObject({ + kind: "unavailable", + code: "binding-conflict", + }); + expect(bindingWrites()).toBe(0); + expect(events).not.toContain("clear-binding"); + }); + + it("cleans only the new transaction binding when persistence fails", () => { + const { deps, events } = fixture({ + writeBinding: () => { + events.push("write-binding"); + throw new Error("partial write"); + }, + }); + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + + expect( + confirmManagedClusterManagedServingCapability(detected, { env: {}, deps }), + ).toMatchObject({ + kind: "unavailable", + code: "binding-persistence-failed", + }); + expect(events).toContain("clear-binding"); + expect(events.indexOf("clear-binding")).toBeGreaterThan(events.indexOf("write-binding")); + }); + + it("cleans its claimed binding when handle encoding fails", () => { + const base = fixture(); + const deps = { + ...base.deps, + encodeBinding: () => { + base.events.push("encode-binding"); + throw new Error("encode failed"); + }, + }; + const detected = expectDetectedCluster( + probeManagedClusterManagedServingCapability({ env: {}, deps }), + ); + + expect( + confirmManagedClusterManagedServingCapability(detected, { env: {}, deps }), + ).toMatchObject({ + kind: "unavailable", + code: "binding-persistence-failed", + }); + expect(base.events.filter((event) => event === "clear-binding")).toEqual(["clear-binding"]); + expect(base.events.indexOf("clear-binding")).toBeGreaterThan( + base.events.indexOf("encode-binding"), + ); + }); + + it("fails closed when the controller UID differs from the local cache owner", () => { + const { deps, bindingWrites } = fixture({ currentUid: () => 2000 }); + + expect(probeManagedClusterManagedServingCapability({ env: {}, deps })).toMatchObject({ + kind: "not-selected", + code: "no-match", + }); + expect(bindingWrites()).toBe(0); + }); +}); + +describe("production pinned peer transport", () => { + it("atomically preserves an existing binding-root owner", () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-spark-binding-")); + fs.chmodSync(parent, 0o700); + const statePath = path.join(parent, "managed-cluster.json"); + const bindingRoot = `${statePath}.ssh-binding`; + const deps = createManagedClusterDiscoveryDeps(() => ({ + status: 0, + stdout: "", + stderr: "", + })); + + try { + expect(deps.claimBinding(statePath)).toBe(true); + fs.writeFileSync(path.join(bindingRoot, "owner"), "first\n", { + mode: 0o600, + }); + expect(deps.claimBinding(statePath)).toBe(false); + expect(fs.readFileSync(path.join(bindingRoot, "owner"), "utf8")).toBe("first\n"); + } finally { + deps.clearBinding(statePath); + fs.rmSync(parent, { force: true, recursive: true }); + } + }); + + it("rejects a symlinked binding parent before creating a claim", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-spark-binding-link-")); + const parent = path.join(root, "parent"); + const alias = path.join(root, "alias"); + fs.mkdirSync(parent, { mode: 0o700 }); + fs.symlinkSync(parent, alias, "dir"); + const statePath = path.join(alias, "managed-cluster.json"); + const deps = createManagedClusterDiscoveryDeps(() => ({ + status: 0, + stdout: "", + stderr: "", + })); + + try { + expect(() => deps.claimBinding(statePath)).toThrow(); + expect(fs.existsSync(`${path.join(parent, "managed-cluster.json")}.ssh-binding`)).toBe(false); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } + }); + + it("pins local and remote host probes to the physical default Docker daemon", () => { + const calls: Array<{ + file: string; + args: readonly string[]; + options: SpawnSyncOptionsWithStringEncoding; + }> = []; + const spawn: ManagedClusterSpawnSync = (file, args, options) => { + calls.push({ file, args: [...args], options }); + return { + status: 0, + stdout: JSON.stringify(calls.length === 1 ? host("local") : host("peer")), + stderr: "", + }; + }; + const dockerNames = ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG"] as const; + const previous = new Map(dockerNames.map((name) => [name, process.env[name]])); + process.env.DOCKER_HOST = "tcp://ambient.example:2376"; + process.env.DOCKER_CONTEXT = "ambient-remote"; + process.env.DOCKER_CONFIG = "/tmp/ambient-docker-config"; + + try { + const deps = createManagedClusterDiscoveryDeps(spawn); + expect(deps.probeHost(deps.localTransport()).hostname).toBe("spark-head"); + const pinned = deps.openPinnedPeerTransport(identity("192.168.100.2")); + try { + expect(deps.probeHost(pinned.transport).hostname).toBe("spark-worker"); + } finally { + pinned.close(); + } + } finally { + for (const name of dockerNames) { + const value = previous.get(name); + value === undefined + ? Reflect.deleteProperty(process.env, name) + : Reflect.set(process.env, name, value); + } + } + + expect(calls).toHaveLength(2); + const local = calls[0]!; + expect(local.file).toBe("python3"); + expect(local.args).toHaveLength(2); + expect(local.options.env).toMatchObject({ DOCKER_CONTEXT: "default" }); + expect(local.options.env).not.toHaveProperty("DOCKER_HOST"); + expect(local.options.env).not.toHaveProperty("DOCKER_CONFIG"); + + const remote = calls[1]!; + expect(remote.file).toBe("ssh"); + expect(remote.options.env).not.toHaveProperty("DOCKER_HOST"); + expect(remote.options.env).not.toHaveProperty("DOCKER_CONTEXT"); + expect(remote.options.env).not.toHaveProperty("DOCKER_CONFIG"); + const request = JSON.parse(Buffer.from(remote.args.at(-1)!, "base64url").toString("utf8")); + expect(request.argv).toHaveLength(3); + const script = request.argv[2] as string; + expect(script).toContain("env.pop(name, None)"); + expect(script).toContain('env["DOCKER_CONTEXT"] = "default"'); + expect(script).not.toContain("MODEL_ID"); + expect(script).not.toContain("IMAGE_REF"); + }); + + it("requires direct routes, exact neighbors, and jumbo reachability on both rails", () => { + const deps = createManagedClusterDiscoveryDeps(() => ({ + status: 0, + stdout: "", + stderr: "", + })); + const requests = [ + { + netdev: "enp1s0f0np0", + sourceAddress: "192.168.100.1", + peerAddress: "192.168.100.2", + expectedPeerMac: "02:00:00:00:01:01", + }, + { + netdev: "enp2s0f0np0", + sourceAddress: "192.168.101.1", + peerAddress: "192.168.101.2", + expectedPeerMac: "02:00:00:00:01:02", + }, + ]; + let routedThroughGateway = false; + const directTransport: ManagedClusterReadOnlyHostTransport = { + execute: (argv) => { + const routeResponse = () => ({ + status: 0, + stdout: JSON.stringify([ + { + dev: argv.at(-1), + prefsrc: argv[6], + scope: "link", + ...(routedThroughGateway ? { gateway: "192.168.100.254" } : {}), + }, + ]), + stderr: "", + }); + const pingResponse = () => ({ status: 0, stdout: "", stderr: "" }); + const neighborResponse = () => { + const request = requests.find(({ peerAddress }) => peerAddress === argv[5])!; + return { + status: 0, + stdout: JSON.stringify([ + { + dst: request.peerAddress, + dev: request.netdev, + lladdr: request.expectedPeerMac, + state: ["REACHABLE"], + }, + ]), + stderr: "", + }; + }; + return argv[1] === "-j" && argv[2] === "route" + ? routeResponse() + : argv[0] === "ping" + ? pingResponse() + : neighborResponse(); + }, + readFile: () => "", + readdir: () => [], + }; + + expect(deps.probeConnectivity(directTransport, requests)).toBe(true); + routedThroughGateway = true; + expect(deps.probeConnectivity(directTransport, requests)).toBe(false); + }); + + it("uses strict SSH and a fixed argv executor without interpolated shell", () => { + const calls: Array<{ + file: string; + args: readonly string[]; + options: SpawnSyncOptionsWithStringEncoding; + }> = []; + const spawn: ManagedClusterSpawnSync = (file, args, options) => { + calls.push({ file, args: [...args], options }); + return { status: 0, stdout: "aarch64\n", stderr: "" }; + }; + const deps = createManagedClusterDiscoveryDeps(spawn); + const pinned = deps.openPinnedPeerTransport(identity("192.168.100.2")); + + try { + expect(pinned.transport.execute(["uname", "-m"])).toMatchObject({ + status: 0, + }); + } finally { + pinned.close(); + } + + expect(calls).toHaveLength(1); + const call = calls[0]!; + expect(call.file).toBe("ssh"); + expect(call.args.slice(0, 2)).toEqual(["-F", "/dev/null"]); + expect(call.args).toEqual( + expect.arrayContaining([ + "BatchMode=yes", + "StrictHostKeyChecking=yes", + "PasswordAuthentication=no", + "ProxyCommand=none", + "ProxyJump=none", + "GlobalKnownHostsFile=/dev/null", + ]), + ); + expect(call.args.slice(-4, -1)).toEqual(["192.168.100.2", "python3", "-"]); + const request = JSON.parse(Buffer.from(call.args.at(-1)!, "base64url").toString("utf8")); + expect(request).toEqual({ argv: ["uname", "-m"] }); + expect(call.options.input).toContain("subprocess.run("); + expect(call.options.input).toContain("shell=False"); + expect(call.options.input).not.toContain("uname"); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-discovery.ts b/src/lib/inference/serving/managed-cluster-discovery.ts new file mode 100644 index 0000000000..0dbd3e2542 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-discovery.ts @@ -0,0 +1,1309 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; +import path from "node:path"; + +import type { BuildIdentity } from "../../core/version.js"; +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { MANAGED_CLUSTER_VLLM_MATERIALIZER_REF } from "./adapter-registry.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { immutableManagedInferenceCopy } from "./catalog-integrity.js"; +import { createProductionManagedClusterDiscoveryDeps } from "./managed-cluster-discovery-production.js"; +import { + isRelatedManagedVllmContainer, + type ManagedClusterNodeSnapshot, + type ManagedClusterObservedContainer, +} from "./managed-cluster-lifecycle.js"; +import { + type ManagedVllmSshBinding, + type QualifiedManagedVllmSshIdentity, +} from "./managed-cluster-ssh-binding.js"; +import { + getManagedClusterTopologyArtifactError, + type ManagedClusterNodeObservation, + type ManagedClusterPeerObservation, + type ManagedClusterRailObservation, + type ManagedClusterTopologyArtifact, + type ManagedClusterTopologyOutput, + managedClusterTopologyOutputDigest, + qualifyManagedClusterTopology, +} from "./managed-cluster-topology.js"; + +export const NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV = "NEMOCLAW_MANAGED_CLUSTER_PEERS" as const; +export const NEMOCLAW_SERVING_PRESET_ENV = "NEMOCLAW_SERVING_PRESET" as const; + +const HOST_PROBE_SCHEMA_VERSION = 1; +const DIRECT_RAIL_PREFIX_LENGTH = 30; +const EXPECTED_CX7_SPEED_MBPS = 200_000; +const MINIMUM_CX7_MTU = 9_000; +const MINIMUM_AVAILABLE_INODES = 1_024; +const SAFE_TARGET_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const SAFE_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const SAFE_DEVICE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]+$/; +const MACHINE_ID_PATTERN = /^[a-f0-9]{32}$/; +const MAC_PATTERN = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; +const PCI_ADDRESS_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/; +const SHA256_PATTERN = /^[a-f0-9]{64}$/; + +export interface ManagedClusterCommandResult { + readonly status: number | null; + readonly stdout: string; + readonly stderr: string; + readonly error?: string; +} + +export interface ManagedClusterReadOnlyHostTransport { + execute(argv: readonly string[]): ManagedClusterCommandResult; + readFile(filePath: string): string; + readdir(directory: string): string[]; +} + +export interface ManagedClusterGpuObservation { + readonly index: number; + readonly name: string; + readonly uuid: string; +} + +export interface ManagedClusterIpv4Observation { + readonly address: string; + readonly prefixLength: number; +} + +export interface ManagedClusterRoceGidHostObservation { + readonly index: number; + readonly value: string; + readonly ipv4Address: string; +} + +export interface ManagedClusterCx7RailHostObservation { + readonly physicalPortId: string; + readonly netdev: string; + readonly hcaDevice: string; + readonly hcaPort: number; + readonly macAddress: string; + readonly pciAddress: string; + readonly pciName: string; + readonly state: string; + readonly operState: string; + readonly carrier: boolean; + readonly linkLayer: string; + readonly speedMbps: number; + readonly mtu: number; + readonly ipv4Addresses: readonly ManagedClusterIpv4Observation[]; + readonly roceV2Ipv4Gids: readonly ManagedClusterRoceGidHostObservation[]; +} + +export interface ManagedClusterEarlyoomObservation { + readonly installed: boolean; + readonly active: "active" | "inactive" | "unknown"; + readonly enabled: "enabled" | "disabled" | "unknown"; +} + +export interface ManagedClusterStorageCapacityObservation { + readonly requestedPath: string; + readonly probePath: string | null; + readonly filesystemId: string | null; + readonly availableBytes: number | null; + readonly availableInodes: number | null; + readonly ownerUid: number | null; + readonly ownerGid: number | null; + readonly isDirectory: boolean; + readonly writableByUser: boolean; +} + +export interface ManagedClusterStorageObservation { + readonly huggingFace: ManagedClusterStorageCapacityObservation & { + readonly cacheRoot: string; + }; + readonly docker: ManagedClusterStorageCapacityObservation & { + readonly dockerRootDir: string | null; + }; +} + +export interface ManagedClusterHostObservation { + readonly schemaVersion: 1; + readonly hostname: string; + readonly nodeId: string; + readonly productName: string; + readonly architecture: string; + readonly home: string; + readonly username: string; + readonly uid: number; + readonly gid: number; + readonly gpus: readonly ManagedClusterGpuObservation[]; + readonly rails: readonly ManagedClusterCx7RailHostObservation[]; + readonly earlyoom: ManagedClusterEarlyoomObservation; + readonly runtimeInspectionComplete: boolean; + readonly runtimeSnapshot: ManagedClusterNodeSnapshot; + readonly storage: ManagedClusterStorageObservation; +} + +export interface ManagedClusterPinnedPeerTransport { + readonly transport: ManagedClusterReadOnlyHostTransport; + close(): void; +} + +export interface ManagedClusterConnectivityRequest { + readonly netdev: string; + readonly sourceAddress: string; + readonly peerAddress: string; + readonly expectedPeerMac: string; +} + +export interface ManagedClusterDiscoveryDeps { + now(): Date; + currentUid(): number | null; + getBuildIdentity(): BuildIdentity; + localTransport(): ManagedClusterReadOnlyHostTransport; + probeHost(transport: ManagedClusterReadOnlyHostTransport): ManagedClusterHostObservation; + inspectPretrustedTarget(target: string): QualifiedManagedVllmSshIdentity | null; + openPinnedPeerTransport( + identity: QualifiedManagedVllmSshIdentity, + ): ManagedClusterPinnedPeerTransport; + createReadiness( + host: ManagedClusterHostObservation, + transport: ManagedClusterReadOnlyHostTransport, + buildIdentity: BuildIdentity, + now: Date, + ): SystemReadinessReport; + probeConnectivity( + transport: ManagedClusterReadOnlyHostTransport, + requests: readonly ManagedClusterConnectivityRequest[], + ): boolean; + /** Atomically claim a new binding root. False means an existing owner won. */ + claimBinding(statePath: string): boolean; + writeBinding(statePath: string, identity: QualifiedManagedVllmSshIdentity): ManagedVllmSshBinding; + clearBinding(statePath: string): void; + encodeBinding(binding: ManagedVllmSshBinding): string; + resolveBindingStatePath(nodeId: string): string; +} + +export type ManagedClusterManagedServingFailureCode = + | "no-match" + | "incompatible-selection" + | "invalid-peer" + | "local-host-unavailable" + | "peer-trust-unavailable" + | "peer-identity-ambiguous" + | "peer-host-unavailable" + | "peer-count" + | "host-unqualified" + | "earlyoom-active" + | "earlyoom-unknown" + | "storage-unavailable" + | "storage-insufficient" + | "runtime-conflict" + | "runtime-unknown" + | "fabric-unavailable" + | "connectivity-unavailable" + | "readiness-unavailable" + | "binding-conflict" + | "binding-persistence-failed" + | "topology-unavailable"; + +export type ManagedClusterDetectedManagedServingCapability = { + readonly kind: "ready"; + readonly selectionIntent: "automatic" | "explicit"; + readonly topology: ManagedClusterTopologyArtifact; + readonly local: ManagedClusterHostObservation; + readonly peers: readonly ManagedClusterHostObservation[]; + readonly readiness: readonly { + readonly nodeId: string; + readonly report: SystemReadinessReport; + }[]; + /** Exact transaction claims that may be persisted only after confirmation. */ + readonly sshClaims: readonly ManagedClusterSshClaim[]; +}; + +export interface ManagedClusterSshClaim { + readonly nodeId: string; + readonly statePath: string; + readonly identity: QualifiedManagedVllmSshIdentity; +} + +export interface ManagedClusterSshBinding extends ManagedClusterSshClaim { + readonly binding: ManagedVllmSshBinding; + readonly handle: string; +} + +export type ManagedClusterManagedServingCapability = + | { + readonly kind: "not-selected"; + readonly code: ManagedClusterManagedServingFailureCode; + readonly reason: string; + } + | { + readonly kind: "unavailable"; + readonly code: ManagedClusterManagedServingFailureCode; + readonly reason: string; + } + | ManagedClusterDetectedManagedServingCapability; + +export type ManagedClusterConfirmedManagedServingCapability = + ManagedClusterDetectedManagedServingCapability & { + readonly sshBindings: readonly ManagedClusterSshBinding[]; + }; + +export type ManagedClusterManagedServingConfirmation = + | Exclude + | ManagedClusterConfirmedManagedServingCapability; + +export interface ProbeManagedClusterManagedServingOptions { + readonly env?: NodeJS.ProcessEnv; + readonly deps?: ManagedClusterDiscoveryDeps; + /** @internal Catalog loader seam for fail-closed tests. */ + readonly loadCatalog?: typeof loadManagedInferenceCatalog; + readonly bindingStatePaths?: Readonly>; + readonly maxReadinessAgeMs?: number; +} + +interface QualifiedRail { + readonly host: ManagedClusterCx7RailHostObservation; + readonly address: string; + readonly peerAddress: string; + readonly subnet: string; + readonly gid: ManagedClusterRoceGidHostObservation; +} + +interface QualifiedHost { + readonly host: ManagedClusterHostObservation; + readonly gpu: ManagedClusterGpuObservation; + readonly rails: readonly [QualifiedRail, QualifiedRail]; +} + +interface ClusterPlan { + readonly nodes: readonly QualifiedHost[]; + readonly connectivity: ReadonlyMap; + readonly peerNodeIdsByRail: ReadonlyMap; +} + +type DiscoveryFailure = { + readonly code: ManagedClusterManagedServingFailureCode; + readonly reason: string; +}; + +type Selection = { + readonly strict: boolean; + readonly intent: "automatic" | "explicit"; + readonly explicitPeers: readonly string[]; +}; + +function notSelected( + code: ManagedClusterManagedServingFailureCode, + reason: string, +): ManagedClusterManagedServingCapability { + return { kind: "not-selected", code, reason }; +} + +function unavailable( + code: ManagedClusterManagedServingFailureCode, + reason: string, +): ManagedClusterManagedServingCapability { + return { kind: "unavailable", code, reason }; +} + +function disposition( + selection: Selection, + result: DiscoveryFailure, +): ManagedClusterManagedServingCapability { + if (selection.strict) return unavailable(result.code, result.reason); + const code: ManagedClusterManagedServingFailureCode = [ + "runtime-conflict", + "runtime-unknown", + "binding-conflict", + "binding-persistence-failed", + ].includes(result.code) + ? result.code + : "no-match"; + return notSelected(code, result.reason); +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validatePeerTarget(raw: string): string { + if ( + raw.length === 0 || + raw.length > 286 || + raw !== raw.trim() || + /[/,:;`'"\\$(){}[\]<>|&!?*\s\u0000-\u001f\u007f]/.test(raw) + ) { + throw new Error( + `${NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV} must contain canonical SSH hosts or user@host values`, + ); + } + const parts = raw.split("@"); + const username = parts.length === 2 ? parts[0] : ""; + const hostname = parts.at(-1) ?? ""; + if ( + parts.length > 2 || + (parts.length === 2 && !username) || + (username !== "" && !SAFE_USERNAME_PATTERN.test(username)) || + (net.isIP(hostname) !== 4 && !SAFE_TARGET_PATTERN.test(hostname)) + ) { + throw new Error( + `${NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV} must contain canonical SSH hosts or user@host values`, + ); + } + return raw; +} + +function selectionFromEnvironment( + env: NodeJS.ProcessEnv, + loadCatalog: typeof loadManagedInferenceCatalog, +): Selection | ManagedClusterManagedServingCapability { + const preset = String(env[NEMOCLAW_SERVING_PRESET_ENV] ?? "").trim(); + const peersValue = String(env[NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV] ?? "").trim(); + let peers: readonly string[] = []; + if (peersValue) { + try { + peers = peersValue.split(",").map((peer) => validatePeerTarget(peer.trim())); + if (new Set(peers).size !== peers.length || peers.length > 1_023) { + throw new Error( + `${NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV} contains duplicate or too many peers`, + ); + } + } catch (error) { + return unavailable("invalid-peer", (error as Error).message); + } + } + if (preset) { + let catalog; + try { + catalog = loadCatalog(); + } catch { + return unavailable( + "incompatible-selection", + "The selected managed inference preset catalog is unavailable.", + ); + } + const compiledPreset = catalog.presets.find(({ metadata }) => metadata.id === preset); + const recipe = compiledPreset + ? catalog.recipes.find(({ metadata }) => metadata.id === compiledPreset.spec.plan.recipeRef) + : undefined; + if (recipe?.spec.execution.materializerRef !== MANAGED_CLUSTER_VLLM_MATERIALIZER_REF) { + return peers.length > 0 + ? unavailable( + "incompatible-selection", + `${NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV} cannot be combined with another serving preset.`, + ) + : notSelected("no-match", "Another managed inference preset is selected."); + } + } + if (peers.length > 0) { + return { strict: true, intent: "explicit", explicitPeers: peers }; + } + return { + strict: Boolean(preset), + intent: preset ? "explicit" : "automatic", + explicitPeers: [], + }; +} + +function ipv4ToNumber(address: string): number { + return address + .split(".") + .map(Number) + .reduce((value, octet) => value * 256 + octet, 0); +} + +function numberToIpv4(value: number): string { + return [24, 16, 8, 0].map((shift) => Math.floor(value / 2 ** shift) % 256).join("."); +} + +function privateIpv4(address: string): boolean { + const value = ipv4ToNumber(address); + return ( + (value >= ipv4ToNumber("10.0.0.0") && value <= ipv4ToNumber("10.255.255.255")) || + (value >= ipv4ToNumber("172.16.0.0") && value <= ipv4ToNumber("172.31.255.255")) || + (value >= ipv4ToNumber("192.168.0.0") && value <= ipv4ToNumber("192.168.255.255")) + ); +} + +function slash30Counterpart(address: string, prefixLength: number): string | null { + if (prefixLength !== DIRECT_RAIL_PREFIX_LENGTH || net.isIP(address) !== 4) return null; + if (!privateIpv4(address)) return null; + const value = ipv4ToNumber(address); + const network = Math.floor(value / 4) * 4; + const host = value - network; + if (host === 1) return numberToIpv4(network + 2); + if (host === 2) return numberToIpv4(network + 1); + return null; +} + +function slash30Subnet(address: string): string { + return `${numberToIpv4(Math.floor(ipv4ToNumber(address) / 4) * 4)}/30`; +} + +function isSafeText(value: unknown, maximum = 4096): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= maximum && + value === value.trim() && + !/[\u0000-\u001f\u007f]/.test(value) + ); +} + +function isSafeInteger( + value: unknown, + minimum = 0, + maximum = Number.MAX_SAFE_INTEGER, +): value is number { + return Number.isInteger(value) && (value as number) >= minimum && (value as number) <= maximum; +} + +function validStorageCapacity(value: unknown): value is ManagedClusterStorageCapacityObservation { + if ( + !isRecord(value) || + !isSafeText(value.requestedPath) || + !path.isAbsolute(value.requestedPath) + ) { + return false; + } + return ( + (value.probePath === null || + (isSafeText(value.probePath) && path.isAbsolute(value.probePath))) && + (value.filesystemId === null || isSafeText(value.filesystemId, 128)) && + (value.availableBytes === null || isSafeInteger(value.availableBytes)) && + (value.availableInodes === null || isSafeInteger(value.availableInodes)) && + (value.ownerUid === null || isSafeInteger(value.ownerUid, 0, 2 ** 31 - 1)) && + (value.ownerGid === null || isSafeInteger(value.ownerGid, 0, 2 ** 31 - 1)) && + typeof value.isDirectory === "boolean" && + typeof value.writableByUser === "boolean" + ); +} + +function validGpu(value: unknown): value is ManagedClusterGpuObservation { + return ( + isRecord(value) && + isSafeInteger(value.index, 0, 1024) && + isSafeText(value.name, 256) && + isSafeText(value.uuid, 128) && + GPU_UUID_PATTERN.test(value.uuid) + ); +} + +function validRail(value: unknown): value is ManagedClusterCx7RailHostObservation { + if ( + !isRecord(value) || + !isSafeText(value.physicalPortId, 128) || + !SAFE_DEVICE_PATTERN.test(value.physicalPortId) || + !isSafeText(value.netdev, 64) || + !SAFE_DEVICE_PATTERN.test(value.netdev) || + !isSafeText(value.hcaDevice, 64) || + !SAFE_DEVICE_PATTERN.test(value.hcaDevice) || + !isSafeInteger(value.hcaPort, 1, 255) || + !isSafeText(value.macAddress, 17) || + !MAC_PATTERN.test(value.macAddress) || + value.macAddress === "00:00:00:00:00:00" || + !isSafeText(value.pciAddress, 32) || + !PCI_ADDRESS_PATTERN.test(value.pciAddress) || + !isSafeText(value.pciName, 512) || + !isSafeText(value.state, 64) || + !isSafeText(value.operState, 32) || + typeof value.carrier !== "boolean" || + !isSafeText(value.linkLayer, 64) || + !isSafeInteger(value.speedMbps, -1, 1_000_000) || + !isSafeInteger(value.mtu, 0, 1_000_000) || + !Array.isArray(value.ipv4Addresses) || + value.ipv4Addresses.length > 16 || + !Array.isArray(value.roceV2Ipv4Gids) || + value.roceV2Ipv4Gids.length > 64 + ) { + return false; + } + return ( + value.ipv4Addresses.every( + (address) => + isRecord(address) && + isSafeText(address.address, 15) && + net.isIP(address.address) === 4 && + isSafeInteger(address.prefixLength, 1, 32), + ) && + value.roceV2Ipv4Gids.every( + (gid) => + isRecord(gid) && + isSafeInteger(gid.index, 0, 4095) && + isSafeText(gid.value, 64) && + net.isIP(gid.value) === 6 && + gid.value !== "::" && + isSafeText(gid.ipv4Address, 15) && + net.isIP(gid.ipv4Address) === 4, + ) + ); +} + +function validContainer(value: unknown): value is ManagedClusterObservedContainer { + if ( + !isRecord(value) || + !isSafeText(value.id, 64) || + !SHA256_PATTERN.test(value.id) || + !isSafeText(value.name, 256) || + !isSafeText(value.image, 1024) || + typeof value.running !== "boolean" || + typeof value.healthy !== "boolean" || + !isRecord(value.labels) || + Object.keys(value.labels).length > 128 + ) { + return false; + } + return Object.entries(value.labels).every( + ([key, label]) => + key.length > 0 && key.length <= 256 && typeof label === "string" && label.length <= 4096, + ); +} + +export function parseManagedClusterHostObservation(value: unknown): ManagedClusterHostObservation { + if ( + !isRecord(value) || + value.schemaVersion !== HOST_PROBE_SCHEMA_VERSION || + !isSafeText(value.hostname, 256) || + !isSafeText(value.nodeId, 64) || + !MACHINE_ID_PATTERN.test(value.nodeId) || + !isSafeText(value.productName, 512) || + !isSafeText(value.architecture, 64) || + !isSafeText(value.home) || + !path.isAbsolute(value.home) || + !isSafeText(value.username, 64) || + !SAFE_USERNAME_PATTERN.test(value.username) || + !isSafeInteger(value.uid, 1, 2 ** 31 - 1) || + !isSafeInteger(value.gid, 0, 2 ** 31 - 1) || + !Array.isArray(value.gpus) || + value.gpus.length > 16 || + !value.gpus.every(validGpu) || + !Array.isArray(value.rails) || + value.rails.length > 16 || + !value.rails.every(validRail) || + !isRecord(value.earlyoom) || + typeof value.earlyoom.installed !== "boolean" || + !["active", "inactive", "unknown"].includes(String(value.earlyoom.active)) || + !["enabled", "disabled", "unknown"].includes(String(value.earlyoom.enabled)) || + typeof value.runtimeInspectionComplete !== "boolean" || + !isRecord(value.runtimeSnapshot) || + !Array.isArray(value.runtimeSnapshot.containers) || + value.runtimeSnapshot.containers.length > 256 || + !value.runtimeSnapshot.containers.every(validContainer) || + !Array.isArray(value.runtimeSnapshot.listeningPorts) || + value.runtimeSnapshot.listeningPorts.length > 65_535 || + !value.runtimeSnapshot.listeningPorts.every((port) => isSafeInteger(port, 1, 65_535)) || + new Set(value.runtimeSnapshot.listeningPorts).size !== + value.runtimeSnapshot.listeningPorts.length || + new Set(value.runtimeSnapshot.containers.map((container) => container.id)).size !== + value.runtimeSnapshot.containers.length || + !isRecord(value.storage) || + !isRecord(value.storage.huggingFace) || + !validStorageCapacity(value.storage.huggingFace) || + !isSafeText(value.storage.huggingFace.cacheRoot) || + !path.isAbsolute(value.storage.huggingFace.cacheRoot) || + !isRecord(value.storage.docker) || + !validStorageCapacity(value.storage.docker) || + (value.storage.docker.dockerRootDir !== null && + (!isSafeText(value.storage.docker.dockerRootDir) || + !path.isAbsolute(value.storage.docker.dockerRootDir))) + ) { + throw new Error("DGX Spark host observation is invalid"); + } + return value as unknown as ManagedClusterHostObservation; +} +function qualifyHost( + host: ManagedClusterHostObservation, + label: string, +): QualifiedHost | DiscoveryFailure { + if ( + !/DGX[_\s-]+Spark/i.test(host.productName) || + !/^(?:aarch64|arm64)$/i.test(host.architecture) + ) { + return { code: "host-unqualified", reason: `${label} is not an arm64 DGX Spark.` }; + } + const gpus = host.gpus.filter(({ name }) => /\bGB10\b/i.test(name)); + if (gpus.length !== 1 || host.gpus.length !== 1) { + return { code: "host-unqualified", reason: `${label} must expose exactly one GB10 GPU.` }; + } + const cx7 = host.rails.filter(({ pciName }) => /ConnectX[- ]?7|\bCX-?7\b/i.test(pciName)); + if (cx7.length !== 2 || host.rails.length !== 2) { + return { + code: "fabric-unavailable", + reason: `${label} must expose exactly two ConnectX-7 logical rails.`, + }; + } + const qualified: QualifiedRail[] = []; + for (const [index, rail] of cx7.entries()) { + if ( + !/\bACTIVE\b/i.test(rail.state) || + rail.operState.toLowerCase() !== "up" || + !rail.carrier || + rail.linkLayer.toLowerCase() !== "ethernet" || + rail.speedMbps !== EXPECTED_CX7_SPEED_MBPS || + rail.mtu < MINIMUM_CX7_MTU + ) { + return { + code: "fabric-unavailable", + reason: `${label} rail ${String(index + 1)} is not active 200G Ethernet with jumbo MTU.`, + }; + } + const addresses = rail.ipv4Addresses + .map((address) => ({ + address, + peer: slash30Counterpart(address.address, address.prefixLength), + })) + .filter( + (entry): entry is { address: ManagedClusterIpv4Observation; peer: string } => + entry.peer !== null, + ); + if (addresses.length !== 1) { + return { + code: "fabric-unavailable", + reason: `${label} rail ${String(index + 1)} must have one usable private /30 address.`, + }; + } + const selectedAddress = addresses[0]!; + const gids = rail.roceV2Ipv4Gids + .filter(({ ipv4Address }) => ipv4Address === selectedAddress.address.address) + .sort((left, right) => left.index - right.index || compareStrings(left.value, right.value)); + if (gids.length === 0) { + return { + code: "fabric-unavailable", + reason: `${label} rail ${String(index + 1)} has no usable dynamically resolved RoCEv2 GID.`, + }; + } + qualified.push({ + host: rail, + address: selectedAddress.address.address, + peerAddress: selectedAddress.peer, + subnet: slash30Subnet(selectedAddress.address.address), + gid: gids[0]!, + }); + } + if ( + new Set(qualified.map(({ host: rail }) => rail.netdev)).size !== 2 || + new Set(qualified.map(({ host: rail }) => rail.macAddress)).size !== 2 || + new Set(qualified.map(({ subnet }) => subnet)).size !== 2 || + new Set(qualified.map(({ peerAddress }) => peerAddress)).size !== 2 + ) { + return { code: "fabric-unavailable", reason: `${label} ConnectX-7 identity is ambiguous.` }; + } + qualified.sort((left, right) => compareStrings(left.subnet, right.subnet)); + return { host, gpu: gpus[0]!, rails: [qualified[0]!, qualified[1]!] }; +} + +function runtimeFailure( + host: ManagedClusterHostObservation, + label: string, +): DiscoveryFailure | null { + if (!host.runtimeInspectionComplete) { + return { code: "runtime-unknown", reason: `${label} runtime inspection is inconclusive.` }; + } + const container = host.runtimeSnapshot.containers.find(isRelatedManagedVllmContainer); + if (container) { + return { + code: "runtime-conflict", + reason: `${label} already has related container ${container.name}; it was not changed.`, + }; + } + return null; +} + +function earlyoomFailure( + host: ManagedClusterHostObservation, + label: string, +): DiscoveryFailure | null { + if (!host.earlyoom.installed) return null; + if (host.earlyoom.active === "active") { + return { + code: "earlyoom-active", + reason: `${label} has active earlyoom; NemoClaw did not stop or disable it.`, + }; + } + return host.earlyoom.active === "inactive" + ? null + : { code: "earlyoom-unknown", reason: `${label} earlyoom state is inconclusive.` }; +} + +function validCapacity(capacity: ManagedClusterStorageCapacityObservation): boolean { + return ( + capacity.probePath !== null && + capacity.filesystemId !== null && + capacity.availableBytes !== null && + capacity.availableInodes !== null && + capacity.availableInodes >= MINIMUM_AVAILABLE_INODES && + capacity.ownerUid !== null && + capacity.ownerGid !== null && + capacity.isDirectory + ); +} + +function storageFailure( + host: ManagedClusterHostObservation, + label: string, +): DiscoveryFailure | null { + const huggingFace = host.storage.huggingFace; + const docker = host.storage.docker; + if (!validCapacity(huggingFace) || !validCapacity(docker)) { + return { + code: "storage-unavailable", + reason: `${label} cache or Docker filesystem capacity could not be proven.`, + }; + } + if (!huggingFace.writableByUser) { + return { + code: "storage-unavailable", + reason: `${label} Hugging Face cache is not writable by the probed non-root user.`, + }; + } + if ( + huggingFace.probePath !== huggingFace.cacheRoot || + huggingFace.requestedPath !== huggingFace.cacheRoot || + !huggingFace.isDirectory + ) { + return { + code: "storage-unavailable", + reason: `${label} exact Hugging Face cache root must already exist as a directory.`, + }; + } + if (huggingFace.ownerUid !== host.uid || huggingFace.ownerGid !== host.gid) { + return { + code: "storage-unavailable", + reason: `${label} Hugging Face cache ownership does not match the probed non-root user.`, + }; + } + + return null; +} + +function matchClusterHosts(nodes: readonly QualifiedHost[]): ClusterPlan | DiscoveryFailure { + if ( + nodes.length < 2 || + new Set(nodes.map(({ host }) => host.nodeId)).size !== nodes.length || + new Set(nodes.map(({ gpu }) => gpu.uuid)).size !== nodes.length + ) { + return { + code: "peer-identity-ambiguous", + reason: "Managed cluster nodes or GPU identities are duplicated.", + }; + } + const peerNodeIdsByRail = new Map(); + const connectivity = new Map(); + for (const node of nodes) { + const requests: ManagedClusterConnectivityRequest[] = []; + for (const rail of node.rails) { + const matches = nodes.flatMap((candidate) => + candidate.host.nodeId === node.host.nodeId + ? [] + : candidate.rails + .filter( + (candidateRail) => + candidateRail.subnet === rail.subnet && + candidateRail.address === rail.peerAddress && + candidateRail.peerAddress === rail.address, + ) + .map((candidateRail) => ({ candidate, candidateRail })), + ); + if (matches.length !== 1) { + return { + code: "fabric-unavailable", + reason: "Managed cluster rails are not unique reciprocal /30 endpoints.", + }; + } + const match = matches[0]!; + peerNodeIdsByRail.set(rail, match.candidate.host.nodeId); + requests.push({ + netdev: rail.host.netdev, + sourceAddress: rail.address, + peerAddress: match.candidateRail.address, + expectedPeerMac: match.candidateRail.host.macAddress, + }); + } + connectivity.set(node.host.nodeId, requests); + } + return { nodes, connectivity, peerNodeIdsByRail }; +} + +function topologyRail(rail: QualifiedRail, peerNodeId: string): ManagedClusterRailObservation { + return { + adapter: "connectx-7", + path: "direct", + physicalPortId: rail.host.physicalPortId, + netdev: rail.host.netdev, + hcaDevice: rail.host.hcaDevice, + hcaPort: rail.host.hcaPort, + address: rail.address, + prefixLength: DIRECT_RAIL_PREFIX_LENGTH, + peerNodeId, + peerAddress: rail.peerAddress, + linkState: "up", + connectivity: "reachable", + roceGid: { state: "resolved", index: rail.gid.index, value: rail.gid.value }, + }; +} + +function topologyObservations( + cluster: ClusterPlan, + readiness: ReadonlyMap, + identities: ReadonlyMap, + bindingHandles: ReadonlyMap, +): { local: ManagedClusterNodeObservation; peers: readonly ManagedClusterPeerObservation[] } { + const controller = cluster.nodes[0]!; + const nodeObservation = (node: QualifiedHost): ManagedClusterNodeObservation => ({ + nodeId: node.host.nodeId, + gpuIds: [node.gpu.uuid], + readiness: readiness.get(node.host.nodeId)!, + runtimeState: "clear", + rails: node.rails.map((rail) => topologyRail(rail, cluster.peerNodeIdsByRail.get(rail)!)), + }); + return { + local: nodeObservation(controller), + peers: cluster.nodes.slice(1).map((node) => { + const identity = identities.get(node.host.nodeId)!; + return { + ...nodeObservation(node), + sshBinding: { + state: "pretrusted", + fromNodeId: controller.host.nodeId, + toNodeId: node.host.nodeId, + peerTarget: identity.sshTarget, + handle: bindingHandles.get(node.host.nodeId)!, + }, + }; + }), + }; +} + +function hostPolicyFailure( + host: ManagedClusterHostObservation, + label: string, +): DiscoveryFailure | null { + return earlyoomFailure(host, label) ?? runtimeFailure(host, label) ?? storageFailure(host, label); +} + +function samePhysicalSshIdentity( + left: QualifiedManagedVllmSshIdentity, + right: QualifiedManagedVllmSshIdentity, +): boolean { + return ( + left.sshUser === right.sshUser && + left.port === right.port && + left.hostKeyDigest === right.hostKeyDigest + ); +} + +function sameExactSshIdentity( + left: QualifiedManagedVllmSshIdentity, + right: QualifiedManagedVllmSshIdentity, +): boolean { + return ( + left.requestedTarget === right.requestedTarget && + left.sshTarget === right.sshTarget && + left.resolvedHost === right.resolvedHost && + left.sshUser === right.sshUser && + left.port === right.port && + left.lookupHost === right.lookupHost && + left.hostKeyDigest === right.hostKeyDigest && + left.knownHostsLines.length === right.knownHostsLines.length && + left.knownHostsLines.every((line, index) => line === right.knownHostsLines[index]) + ); +} + +function samePhysicalHost( + left: ManagedClusterHostObservation, + right: ManagedClusterHostObservation, +): boolean { + return ( + left.nodeId === right.nodeId && + left.gpus.length === 1 && + right.gpus.length === 1 && + left.gpus[0]?.uuid === right.gpus[0]?.uuid + ); +} + +function topologyFailureReason(result: ReturnType): string { + return result.outcome === "qualified" ? "" : result.message; +} + +function sameHostIdentity( + left: ManagedClusterHostObservation, + right: ManagedClusterHostObservation, +): boolean { + return ( + left.nodeId === right.nodeId && + left.hostname === right.hostname && + left.username === right.username && + left.uid === right.uid && + left.gid === right.gid && + left.home === right.home && + left.gpus[0]?.uuid === right.gpus[0]?.uuid && + left.storage.huggingFace.cacheRoot === right.storage.huggingFace.cacheRoot + ); +} + +function sameDetectedCluster( + detected: ManagedClusterDetectedManagedServingCapability, + revalidated: ManagedClusterDetectedManagedServingCapability, +): boolean { + return ( + detected.selectionIntent === revalidated.selectionIntent && + sameHostIdentity(detected.local, revalidated.local) && + detected.peers.length === revalidated.peers.length && + detected.peers.every((peer, index) => sameHostIdentity(peer, revalidated.peers[index]!)) && + detected.sshClaims.length === revalidated.sshClaims.length && + detected.sshClaims.every((claim, index) => { + const candidate = revalidated.sshClaims[index]; + return ( + candidate?.nodeId === claim.nodeId && + candidate.statePath === claim.statePath && + sameExactSshIdentity(claim.identity, candidate.identity) + ); + }) && + detected.topology.subjectDigest === revalidated.topology.subjectDigest && + detected.topology.outputDigest === revalidated.topology.outputDigest + ); +} + +function topologyWithBinding( + detected: ManagedClusterDetectedManagedServingCapability, + bindings: readonly ManagedClusterSshBinding[], +): ManagedClusterTopologyArtifact { + const output: ManagedClusterTopologyOutput = { + ...detected.topology.output, + peers: detected.topology.output.peers.map((peer) => ({ + ...peer, + sshBindingHandle: + bindings.find(({ nodeId }) => nodeId === peer.nodeId)?.handle ?? peer.sshBindingHandle, + })), + }; + const artifact = immutableManagedInferenceCopy({ + ...detected.topology, + output, + outputDigest: managedClusterTopologyOutputDigest(output), + }); + const error = getManagedClusterTopologyArtifactError(artifact); + if (error) throw new Error(error); + return artifact; +} + +function confirmationUnavailable( + code: ManagedClusterManagedServingFailureCode, + reason: string, +): ManagedClusterManagedServingConfirmation { + return { kind: "unavailable", code, reason }; +} + +export function probeManagedClusterManagedServingCapability( + options: ProbeManagedClusterManagedServingOptions = {}, +): ManagedClusterManagedServingCapability { + const selection = selectionFromEnvironment( + options.env ?? process.env, + options.loadCatalog ?? loadManagedInferenceCatalog, + ); + if (!("strict" in selection)) return selection; + const deps = options.deps ?? defaultManagedClusterDiscoveryDeps; + const opened: ManagedClusterPinnedPeerTransport[] = []; + let local: ManagedClusterHostObservation; + let qualifiedLocal: QualifiedHost; + let localTransport: ManagedClusterReadOnlyHostTransport; + try { + localTransport = deps.localTransport(); + local = deps.probeHost(localTransport); + const effectiveUid = deps.currentUid(); + if (effectiveUid === null || effectiveUid !== local.uid) { + return disposition(selection, { + code: "host-unqualified", + reason: "The local DGX Spark probe does not match the current non-root controller UID.", + }); + } + const candidate = qualifyHost(local, "Local DGX Spark"); + if ("code" in candidate) return disposition(selection, candidate); + qualifiedLocal = candidate; + const policyFailure = hostPolicyFailure(local, "Local DGX Spark"); + if (policyFailure) return disposition(selection, policyFailure); + } catch { + return disposition(selection, { + code: "local-host-unavailable", + reason: "The local DGX Spark read-only probe failed closed.", + }); + } + + try { + const targets = + selection.explicitPeers.length > 0 + ? selection.explicitPeers + : qualifiedLocal.rails.map(({ peerAddress }) => peerAddress); + const probes = targets.map((target) => { + const identity = deps.inspectPretrustedTarget(target); + if (!identity) return null; + const pinned = deps.openPinnedPeerTransport(identity); + opened.push(pinned); + return { identity, pinned, host: deps.probeHost(pinned.transport) }; + }); + if (probes.some((probe) => probe === null)) { + return disposition(selection, { + code: "peer-trust-unavailable", + reason: "Every managed cluster peer requires usable pre-existing SSH host-key trust.", + }); + } + + const unique = new Map< + string, + { + identity: QualifiedManagedVllmSshIdentity; + transport: ManagedClusterReadOnlyHostTransport; + host: ManagedClusterHostObservation; + } + >(); + for (const probe of probes as Array>) { + if (samePhysicalHost(local, probe.host)) { + return disposition(selection, { + code: "peer-identity-ambiguous", + reason: "A managed cluster peer resolves back to the local DGX Spark.", + }); + } + const existing = unique.get(probe.host.nodeId); + if ( + existing && + (!samePhysicalHost(existing.host, probe.host) || + !samePhysicalSshIdentity(existing.identity, probe.identity)) + ) { + return disposition(selection, { + code: "peer-identity-ambiguous", + reason: "Multiple peer targets disagree about one physical DGX Spark identity.", + }); + } + if ( + !existing || + compareStrings(probe.identity.requestedTarget, existing.identity.requestedTarget) < 0 + ) { + unique.set(probe.host.nodeId, { + identity: probe.identity, + transport: probe.pinned.transport, + host: probe.host, + }); + } + } + const selectedPeers = [...unique.values()].sort((left, right) => + compareStrings(left.host.nodeId, right.host.nodeId), + ); + if (selectedPeers.length === 0) { + return disposition(selection, { + code: "peer-count", + reason: "No distinct managed cluster peer was detected.", + }); + } + + const qualifiedPeers: QualifiedHost[] = []; + for (const [index, selected] of selectedPeers.entries()) { + const label = `Managed cluster peer ${String(index + 1)}`; + const qualifiedPeer = qualifyHost(selected.host, label); + if ("code" in qualifiedPeer) return disposition(selection, qualifiedPeer); + if (selected.host.username !== selected.identity.sshUser || selected.host.uid <= 0) { + return disposition(selection, { + code: "peer-identity-ambiguous", + reason: `${label} SSH user does not own the probed non-root cache identity.`, + }); + } + const policyFailure = hostPolicyFailure(selected.host, label); + if (policyFailure) return disposition(selection, policyFailure); + qualifiedPeers.push(qualifiedPeer); + } + + const cluster = matchClusterHosts([qualifiedLocal, ...qualifiedPeers]); + if ("code" in cluster) return disposition(selection, cluster); + const transportByNodeId = new Map([ + [local.nodeId, localTransport], + ...selectedPeers.map(({ host, transport }) => [host.nodeId, transport] as const), + ]); + for (const node of cluster.nodes) { + if ( + !deps.probeConnectivity( + transportByNodeId.get(node.host.nodeId)!, + cluster.connectivity.get(node.host.nodeId)!, + ) + ) { + return disposition(selection, { + code: "connectivity-unavailable", + reason: `Direct route, neighbor, or jumbo connectivity failed on ${node.host.hostname}.`, + }); + } + } + + const now = deps.now(); + const buildIdentity = deps.getBuildIdentity(); + const readiness = new Map(); + try { + for (const node of cluster.nodes) { + readiness.set( + node.host.nodeId, + deps.createReadiness( + node.host, + transportByNodeId.get(node.host.nodeId)!, + buildIdentity, + now, + ), + ); + } + } catch { + return disposition(selection, { + code: "readiness-unavailable", + reason: "Canonical readiness could not be generated for every managed cluster node.", + }); + } + const identities = new Map( + selectedPeers.map(({ host, identity }) => [host.nodeId, identity] as const), + ); + const temporaryHandles = new Map( + selectedPeers.map( + ({ host, identity }) => [host.nodeId, `pretrusted:${identity.hostKeyDigest}`] as const, + ), + ); + const temporary = topologyObservations(cluster, readiness, identities, temporaryHandles); + const temporaryQualification = qualifyManagedClusterTopology({ + intent: selection.intent, + evaluatedAt: now.toISOString(), + maxReadinessAgeMs: options.maxReadinessAgeMs ?? 60_000, + local: temporary.local, + peers: temporary.peers, + }); + if (temporaryQualification.outcome !== "qualified") { + return disposition(selection, { + code: "topology-unavailable", + reason: topologyFailureReason(temporaryQualification), + }); + } + + const sshClaims = selectedPeers.map(({ host, identity }) => ({ + nodeId: host.nodeId, + statePath: + options.bindingStatePaths?.[host.nodeId] ?? deps.resolveBindingStatePath(host.nodeId), + identity, + })); + return { + kind: "ready", + selectionIntent: selection.intent, + topology: temporaryQualification.artifact, + local, + peers: selectedPeers.map(({ host }) => host), + readiness: cluster.nodes.map(({ host }) => ({ + nodeId: host.nodeId, + report: readiness.get(host.nodeId)!, + })), + sshClaims, + }; + } catch { + return disposition(selection, { + code: "peer-host-unavailable", + reason: "The peer DGX Spark read-only probe failed closed.", + }); + } finally { + for (const pinned of opened.reverse()) { + try { + pinned.close(); + } catch { + // The pinned temporary directory contains public host-key material only. + } + } + } +} + +/** Revalidate the detected cluster without claiming or writing SSH bindings. */ +export function revalidateManagedClusterManagedServingCapability( + detected: ManagedClusterDetectedManagedServingCapability, + options: ProbeManagedClusterManagedServingOptions = {}, +): ManagedClusterManagedServingCapability { + const deps = options.deps ?? defaultManagedClusterDiscoveryDeps; + const revalidated = probeManagedClusterManagedServingCapability({ + ...options, + deps, + bindingStatePaths: Object.fromEntries( + detected.sshClaims.map(({ nodeId, statePath }) => [nodeId, statePath]), + ), + }); + if (revalidated.kind !== "ready") { + return confirmationUnavailable( + revalidated.code, + `The confirmed managed cluster no longer qualifies: ${revalidated.reason}`, + ); + } + if (!sameDetectedCluster(detected, revalidated)) { + return confirmationUnavailable( + "peer-identity-ambiguous", + "The confirmed managed cluster or an exact pretrusted SSH identity changed after selection.", + ); + } + return revalidated; +} + +/** Claim and persist every previously revalidated peer SSH binding. */ +export function claimManagedClusterManagedServingCapability( + revalidated: ManagedClusterDetectedManagedServingCapability, + options: Pick = {}, +): ManagedClusterManagedServingConfirmation { + const deps = options.deps ?? defaultManagedClusterDiscoveryDeps; + const claimed: string[] = []; + try { + const sshBindings: ManagedClusterSshBinding[] = []; + for (const claim of revalidated.sshClaims) { + if (!deps.claimBinding(claim.statePath)) { + throw new Error("binding-conflict"); + } + claimed.push(claim.statePath); + const binding = deps.writeBinding(claim.statePath, claim.identity); + sshBindings.push({ ...claim, binding, handle: deps.encodeBinding(binding) }); + } + return { + ...revalidated, + topology: topologyWithBinding(revalidated, sshBindings), + sshBindings, + }; + } catch (error) { + let cleanupFailed = false; + for (const statePath of claimed.reverse()) { + try { + deps.clearBinding(statePath); + } catch { + cleanupFailed = true; + } + } + if (cleanupFailed) { + return confirmationUnavailable( + "binding-persistence-failed", + "A managed cluster SSH binding failed and newly claimed state could not be cleaned safely.", + ); + } + return confirmationUnavailable( + (error as Error).message === "binding-conflict" + ? "binding-conflict" + : "binding-persistence-failed", + (error as Error).message === "binding-conflict" + ? "An existing managed cluster SSH binding was preserved and not replaced." + : "The confirmed managed cluster SSH bindings could not be persisted.", + ); + } +} + +/** Revalidate the detected cluster, then persist its SSH bindings after confirmation. */ +export function confirmManagedClusterManagedServingCapability( + detected: ManagedClusterDetectedManagedServingCapability, + options: ProbeManagedClusterManagedServingOptions = {}, +): ManagedClusterManagedServingConfirmation { + const revalidated = revalidateManagedClusterManagedServingCapability(detected, options); + return revalidated.kind === "ready" + ? claimManagedClusterManagedServingCapability(revalidated, options) + : revalidated; +} + +export type { ManagedClusterSpawnSync } from "./managed-cluster-discovery-production.js"; + +export function createManagedClusterDiscoveryDeps( + spawn?: import("./managed-cluster-discovery-production.js").ManagedClusterSpawnSync, +): ManagedClusterDiscoveryDeps { + return createProductionManagedClusterDiscoveryDeps(parseManagedClusterHostObservation, spawn); +} + +const defaultManagedClusterDiscoveryDeps = createManagedClusterDiscoveryDeps(); diff --git a/src/lib/inference/serving/managed-cluster-executor.test.ts b/src/lib/inference/serving/managed-cluster-executor.test.ts new file mode 100644 index 0000000000..3c9e7580c4 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-executor.test.ts @@ -0,0 +1,841 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NO_PREPARATION_REF } from "./adapter-registry.js"; +import { + assertManagedClusterVllmExecutorConfig, + buildManagedClusterVllmRunArgs, + createManagedClusterVllmExecutor, + inspectManagedClusterVllmNodesSync, + type ManagedClusterVllmExecutorRuntimeDeps, +} from "./managed-cluster-executor.js"; +import { fixtureManagedClusterPlan } from "./managed-cluster-fixture.test-support.js"; +import { + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_MANAGED_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + type ManagedClusterVllmPlan, + type ManagedClusterVllmRole, + type ManagedClusterVllmRolePlan, +} from "./managed-cluster-materialize.js"; +import { + createManagedVllmSshBindingFixture, + type ManagedVllmSshBindingFixture, +} from "./managed-cluster-ssh-binding.test-support.js"; + +const API_KEY = "a".repeat(64); +const FINGERPRINT = "b".repeat(64); +const TRANSACTION_ID = "c".repeat(32); +const HEAD_ID = "1".repeat(64); +const FOREIGN_ID = "2".repeat(64); +const WORKER_ID = "3".repeat(64); +const LOCAL_CACHE_ROOT = "/home/nvidia/.cache/huggingface"; +const PEER_CACHE_ROOT = "/home/spark/.cache/huggingface"; + +type DockerCaptureOptions = NonNullable< + Parameters[1] +>; + +type StoppedForeignContainerFixture = { + readonly signal: string; + readonly name: string; + readonly image: string; + readonly labels: Readonly>; +}; + +const STOPPED_FOREIGN_CONTAINER_FIXTURES: readonly StoppedForeignContainerFixture[] = [ + { + signal: "name", + name: "foreign-vllm-server", + image: "example.invalid/inference:latest", + labels: {}, + }, + { + signal: "image", + name: "foreign-inference", + image: "vllm/vllm-openai:latest", + labels: {}, + }, + { + signal: "managed label", + name: "foreign-inference", + image: "example.invalid/inference:latest", + labels: { [MANAGED_CLUSTER_MANAGED_LABEL]: "foreign" }, + }, +]; + +function bindPlan(fixture: ManagedVllmSshBindingFixture): ManagedClusterVllmPlan { + const plan = fixtureManagedClusterPlan(); + return { + ...plan, + roles: [ + plan.roles[0], + { + ...plan.roles[1], + execution: { + kind: "ssh", + expectedTarget: fixture.binding.peerTarget, + bindingHandle: fixture.token, + }, + }, + ], + }; +} + +function launchLabels(rolePlan: ManagedClusterVllmRolePlan): Record { + return { + ...rolePlan.baseLabels, + [MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL]: FINGERPRINT, + [MANAGED_CLUSTER_TRANSACTION_LABEL]: TRANSACTION_ID, + }; +} + +function inspectionRow(input: { + id: string; + name: string; + image: string; + running: boolean; + labels: Readonly>; +}): string { + return JSON.stringify([input.id, `/${input.name}`, input.image, input.running, input.labels]); +} + +function successfulDockerResult(stdout = "") { + return { + pid: 123, + output: [null, stdout, ""], + stdout, + stderr: "", + status: 0, + signal: null, + error: undefined, + }; +} + +function runtimeOverrides( + overrides: Partial = {}, +): Partial { + return { + dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(() => successfulDockerResult()), + dockerRunDetached: vi.fn(() => successfulDockerResult()), + captureListeners: vi.fn(() => ""), + createTransactionId: vi.fn(() => TRANSACTION_ID), + now: vi.fn(() => 0), + sleep: vi.fn(async () => undefined), + withLifecycleLock: vi.fn(async (operation) => await operation()), + ...overrides, + }; +} + +type DockerCaptureHandler = (args: readonly string[], options?: DockerCaptureOptions) => string; + +function fixedDockerCapture(value: string): DockerCaptureHandler { + return () => value; +} + +function unexpectedDockerCapture(args: readonly string[]): never { + throw new Error(`unexpected Docker argv: ${args.join(" ")}`); +} + +function routeDockerCapture( + args: readonly string[], + options: DockerCaptureOptions | undefined, + handlers: { + readonly quiet: DockerCaptureHandler; + readonly inspect: DockerCaptureHandler; + readonly exec: DockerCaptureHandler; + readonly fallback: DockerCaptureHandler; + }, +): string { + const handler = args.includes("--quiet") + ? handlers.quiet + : args.includes("inspect") + ? handlers.inspect + : args[0] === "exec" + ? handlers.exec + : handlers.fallback; + return handler(args, options); +} + +describe("managed-cluster vLLM executor", () => { + let bindingFixture: ManagedVllmSshBindingFixture; + let plan: ManagedClusterVllmPlan; + + beforeEach(() => { + bindingFixture = createManagedVllmSshBindingFixture("spark-worker.local"); + plan = bindPlan(bindingFixture); + }); + + afterEach(() => { + bindingFixture.cleanup(); + vi.restoreAllMocks(); + }); + + it("builds the YAML-backed role launch without a restart policy or bearer value", () => { + const head = plan.roles[0]; + const args = buildManagedClusterVllmRunArgs(head, LOCAL_CACHE_ROOT, launchLabels(head)); + const command = args.at(-1)!; + + expect(args).toEqual( + expect.arrayContaining([ + "--pull=never", + "--init", + "--network", + head.runtime.networkMode, + "--ipc", + head.runtime.ipcMode, + "--shm-size", + String(head.runtime.sharedMemoryBytes), + "--gpus", + head.runtime.gpuRequest, + "--device", + head.runtime.devices[0], + "--ulimit", + `memlock=${String(head.runtime.ulimits.memlock)}`, + `stack=${String(head.runtime.ulimits.stack)}`, + "--tmpfs", + ...head.runtime.temporaryFilesystems.map( + (filesystem) => + `${filesystem.target}:${[ + ...filesystem.options, + `size=${String(filesystem.sizeBytes)}`, + `mode=${filesystem.mode}`, + ].join(",")}`, + ), + "--volume", + `${LOCAL_CACHE_ROOT}/hub:${head.runtime.modelCache.target}/hub:ro`, + "--env", + "VLLM_API_KEY", + head.image, + ]), + ); + expect(args).not.toContain("--restart"); + expect(args).not.toContain(`${LOCAL_CACHE_ROOT}:${head.runtime.modelCache.target}`); + expect(JSON.stringify(args)).not.toContain(API_KEY); + expect(command).toContain("install -m 0644 --"); + expect(head.preparation.ref).not.toBe(NO_PREPARATION_REF); + const boundedPreparation = head.preparation as Exclude< + typeof head.preparation, + { ref: typeof NO_PREPARATION_REF } + >; + expect(command).toContain(boundedPreparation.snapshotCopy.targetPath); + expect(command).toContain(boundedPreparation.snapshotCopy.digest); + expect(command).toContain("snapshot copy source digest mismatch"); + expect(command.indexOf("snapshot copy source digest mismatch")).toBeLessThan( + command.indexOf("install -m 0644 --"), + ); + expect(command).toContain("preparation source text did not match exactly once"); + expect(command).not.toContain("--api-key"); + expect(command).not.toContain("$VLLM_API_KEY"); + expect(command).toContain("'--host' '192.168.100.10'"); + expect(command).toContain(`exec '${head.command.executable}'`); + }); + + it("dispatches the no-op preparation and YAML-backed executable without patch steps", () => { + const head: ManagedClusterVllmRolePlan = { + ...plan.roles[0], + preparation: { + ref: NO_PREPARATION_REF, + phase: "container-before-exec", + modelId: plan.model.id, + modelRevision: plan.model.revision, + modelDownloadSizeBytes: plan.roles[0].preparation.modelDownloadSizeBytes, + }, + command: { + executable: "/opt/vllm/bin/vllm", + arguments: ["serve", "synthetic/model"], + }, + }; + + const command = buildManagedClusterVllmRunArgs(head, LOCAL_CACHE_ROOT, launchLabels(head)).at( + -1, + )!; + + expect(command).toContain("exec '/opt/vllm/bin/vllm' 'serve' 'synthetic/model'"); + expect(command).not.toContain("install -m"); + expect(command).not.toContain("python3 -c"); + }); + + it("keeps the worker launch headless and free of the bearer environment key", () => { + const worker = plan.roles[1]; + const args = buildManagedClusterVllmRunArgs(worker, PEER_CACHE_ROOT, launchLabels(worker)); + + expect(args).not.toContain("VLLM_API_KEY"); + expect(args.at(-1)).toContain("'--headless'"); + expect(args.at(-1)).toContain("'--host' '192.168.100.11'"); + expect(args.at(-1)).not.toContain("--api-key"); + expect(args.at(-1)).not.toContain("$VLLM_API_KEY"); + }); + + it("rejects a changed binding handoff before any Docker operation", () => { + const changedPlan = { + ...plan, + roles: [ + plan.roles[0], + { + ...plan.roles[1], + execution: { + kind: "ssh" as const, + expectedTarget: bindingFixture.binding.peerTarget, + bindingHandle: "changed", + }, + }, + ], + }; + + expect(() => + createManagedClusterVllmExecutor( + { + plan: changedPlan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides(), + ), + ).toThrow(/executor target .* is invalid/); + }); + + it("revalidates catalog-derived commands and exposes the same synchronous inspector", () => { + const config = { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }; + expect(inspectManagedClusterVllmNodesSync(config, runtimeOverrides())).toEqual({ + nodes: [ + { nodeId: plan.roles[0].nodeId, snapshot: { containers: [], listeningPorts: [] } }, + { nodeId: plan.roles[1].nodeId, snapshot: { containers: [], listeningPorts: [] } }, + ], + }); + + const changedPlan: ManagedClusterVllmPlan = { + ...plan, + roles: [ + { + ...plan.roles[0], + command: { + ...plan.roles[0].command, + arguments: [...plan.roles[0].command.arguments, "--changed"], + }, + }, + plan.roles[1], + ], + }; + expect(() => assertManagedClusterVllmExecutorConfig({ ...config, plan: changedPlan })).toThrow( + /catalog-derived adapter contract/, + ); + }); + + it("validates selected definition digests without pinning the aggregate catalog", () => { + const config = { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }; + const catalogExpandedElsewhere: ManagedClusterVllmPlan = { + ...plan, + catalogDigest: `sha256:${"e".repeat(64)}`, + }; + + expect(() => + assertManagedClusterVllmExecutorConfig({ + ...config, + plan: catalogExpandedElsewhere, + }), + ).not.toThrow(); + expect(() => + assertManagedClusterVllmExecutorConfig({ + ...config, + plan: { ...plan, recipeDigest: `sha256:${"f".repeat(64)}` }, + }), + ).toThrow(/selected definition digests/u); + }); + + it("inspects every container plus host listeners and marks only the exact live role healthy", async () => { + const headLabels = launchLabels(plan.roles[0]); + const foreignLabels = { "example.foreign": "true" }; + const dockerCapture = vi.fn((args: readonly string[], options?: DockerCaptureOptions) => + routeDockerCapture(args, options, { + quiet: fixedDockerCapture(`${HEAD_ID}\n${FOREIGN_ID}\n`), + inspect: fixedDockerCapture( + [ + inspectionRow({ + id: HEAD_ID, + name: plan.roles[0].containerName, + image: plan.roles[0].image, + running: true, + labels: headLabels, + }), + inspectionRow({ + id: FOREIGN_ID, + name: "unrelated-service", + image: "example.invalid/foreign:latest", + running: true, + labels: foreignLabels, + }), + ].join("\n"), + ), + exec: fixedDockerCapture("ready"), + fallback: unexpectedDockerCapture, + }), + ); + const captureListeners = vi.fn( + () => + `LISTEN 0 4096 0.0.0.0:${String(plan.apiPort)} 0.0.0.0:*\nLISTEN 0 128 [::]:${String(plan.masterPort)} [::]:*\n`, + ); + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ dockerCapture, captureListeners }), + ); + + const snapshot = await executor.inspectNode(plan.roles[0]); + + expect(snapshot.listeningPorts).toEqual([plan.apiPort, plan.masterPort].sort((a, b) => a - b)); + expect(snapshot.containers).toHaveLength(2); + expect(snapshot.containers[0]).toMatchObject({ + id: HEAD_ID, + healthy: true, + }); + expect(snapshot.containers[1]).toMatchObject({ + id: FOREIGN_ID, + healthy: false, + }); + expect(dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["container", "inspect", HEAD_ID, FOREIGN_ID]), + expect.any(Object), + ); + }); + + it("fails closed when listener inspection is malformed", async () => { + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ captureListeners: vi.fn(() => "not-an-ss-row") }), + ); + + await expect(executor.inspectNode(plan.roles[0])).rejects.toThrow(/listener inspection/); + }); + + it("does not declare the worker-first boundary once a head setup appears", async () => { + const dockerCapture = vi.fn((args: readonly string[], options?: DockerCaptureOptions) => { + const roleIndex = options?.env?.DOCKER_HOST ? 1 : 0; + const rolePlan = plan.roles[roleIndex]!; + const id = roleIndex === 1 ? WORKER_ID : HEAD_ID; + return routeDockerCapture(args, options, { + quiet: fixedDockerCapture(id), + inspect: fixedDockerCapture( + inspectionRow({ + id, + name: rolePlan.containerName, + image: rolePlan.image, + running: true, + labels: launchLabels(rolePlan), + }), + ), + exec: fixedDockerCapture("ready"), + fallback: unexpectedDockerCapture, + }); + }); + let tick = 0; + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ + dockerCapture, + now: vi.fn(() => (tick += 1_000)), + }), + ); + + await expect( + executor.waitForWorkerDistributedReady({ + rolePlan: plan.roles[1], + containerId: WORKER_ID, + expectedLabels: launchLabels(plan.roles[1]), + timeoutMs: 1, + }), + ).resolves.toBe(false); + }); + + it.each( + STOPPED_FOREIGN_CONTAINER_FIXTURES, + )("keeps the worker-first boundary closed for a stopped foreign $signal", async (foreign) => { + const dockerCapture = vi.fn((args: readonly string[], options?: DockerCaptureOptions) => { + const role = options?.env?.DOCKER_HOST ? "worker" : "head"; + const id = role === "worker" ? WORKER_ID : FOREIGN_ID; + return routeDockerCapture(args, options, { + quiet: fixedDockerCapture(id), + inspect: fixedDockerCapture( + role === "worker" + ? inspectionRow({ + id, + name: plan.roles[1].containerName, + image: plan.roles[1].image, + running: true, + labels: launchLabels(plan.roles[1]), + }) + : inspectionRow({ + id, + name: foreign.name, + image: foreign.image, + running: false, + labels: foreign.labels, + }), + ), + exec: fixedDockerCapture("ready"), + fallback: unexpectedDockerCapture, + }); + }); + let tick = 0; + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ + dockerCapture, + now: vi.fn(() => (tick += 1_000)), + }), + ); + + await expect( + executor.waitForWorkerDistributedReady({ + rolePlan: plan.roles[1], + containerId: WORKER_ID, + expectedLabels: launchLabels(plan.roles[1]), + timeoutMs: 1, + }), + ).resolves.toBe(false); + }); + + it("starts one exact head with the bearer only in the Docker subprocess environment", async () => { + const labels = launchLabels(plan.roles[0]); + const dockerCapture = vi.fn((args: readonly string[], options?: DockerCaptureOptions) => + routeDockerCapture(args, options, { + quiet: fixedDockerCapture(HEAD_ID), + inspect: fixedDockerCapture( + inspectionRow({ + id: HEAD_ID, + name: plan.roles[0].containerName, + image: plan.roles[0].image, + running: true, + labels, + }), + ), + exec: fixedDockerCapture("ready"), + fallback: unexpectedDockerCapture, + }), + ); + const dockerRunDetached = vi.fn( + () => successfulDockerResult(`${HEAD_ID}\n`), + ); + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ dockerCapture, dockerRunDetached }), + ); + + const result = await executor.startContainer({ + rolePlan: plan.roles[0], + labels, + preparation: plan.roles[0].preparation, + bearerApiKey: API_KEY, + }); + + expect(result).toEqual({ ok: true, containerId: HEAD_ID }); + const [argv, options] = dockerRunDetached.mock.calls[0]!; + expect(JSON.stringify(argv)).not.toContain(API_KEY); + expect(JSON.stringify(argv)).not.toContain("--api-key"); + expect(JSON.stringify(argv)).not.toContain("$VLLM_API_KEY"); + expect(argv).toContain("VLLM_API_KEY"); + expect(options?.env?.VLLM_API_KEY).toBe(API_KEY); + expect(options?.suppressOutput).toBe(true); + expect(JSON.stringify(labels)).not.toContain(API_KEY); + }); + + it("injects staging targets but leaves cleanup construction usable without staging", async () => { + const stageNode = vi.fn(async () => ({ ok: true })); + const withStage = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + stageNode, + }, + runtimeOverrides(), + ); + const withoutStage = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides(), + ); + + expect( + await withStage.stageNode({ + rolePlan: plan.roles[1], + preparation: plan.roles[1].preparation, + }), + ).toEqual({ ok: true }); + expect(stageNode).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }), + ); + await expect( + withoutStage.stageNode({ + rolePlan: plan.roles[0], + preparation: plan.roles[0].preparation, + }), + ).resolves.toMatchObject({ + ok: false, + reason: expect.stringContaining("not configured"), + }); + }); + + it("removes only the revalidated exact container ID", async () => { + const labels = launchLabels(plan.roles[0]); + const dockerCapture = vi.fn((args: readonly string[], options?: DockerCaptureOptions) => + routeDockerCapture(args, options, { + quiet: fixedDockerCapture(HEAD_ID), + inspect: fixedDockerCapture( + inspectionRow({ + id: HEAD_ID, + name: plan.roles[0].containerName, + image: plan.roles[0].image, + running: false, + labels, + }), + ), + exec: fixedDockerCapture(""), + fallback: fixedDockerCapture(""), + }), + ); + const dockerForceRm = vi.fn(() => successfulDockerResult()); + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ dockerCapture, dockerForceRm }), + ); + + await expect(executor.removeContainer(plan.roles[0], HEAD_ID)).resolves.toEqual({ ok: true }); + expect(dockerForceRm).toHaveBeenCalledWith(HEAD_ID, expect.any(Object)); + expect(dockerForceRm).not.toHaveBeenCalledWith(plan.roles[0].containerName, expect.anything()); + }); + + it("performs bounded authenticated model and chat probes without putting the key in argv", async () => { + const cleanup = vi.fn(); + const createBearerAuthConfig = vi.fn(() => ({ + args: ["--config", "/tmp/nemoclaw-probe/auth.conf"], + trustedConfigFiles: ["/tmp/nemoclaw-probe/auth.conf"], + cleanup, + })); + const runCurlProbe = vi + .fn() + .mockReturnValueOnce({ + ok: true, + httpStatus: 200, + curlStatus: 0, + body: JSON.stringify({ data: [{ id: plan.model.servedName }] }), + stderr: "", + message: "", + }) + .mockReturnValueOnce({ + ok: true, + httpStatus: 200, + curlStatus: 0, + body: JSON.stringify({ + model: plan.model.servedName, + choices: [{ message: {} }], + }), + stderr: "", + message: "", + }); + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ createBearerAuthConfig, runCurlProbe }), + ); + const request = { + baseUrl: plan.roles[0].endpoint!, + apiKey: API_KEY, + expectedModel: plan.model.servedName, + timeoutMs: 10_000, + }; + + await expect(executor.probeModels(request)).resolves.toBe(true); + await expect(executor.probeChat(request)).resolves.toBe(true); + await expect( + executor.probeModels({ + ...request, + baseUrl: `http://attacker.invalid:${String(plan.apiPort)}`, + }), + ).resolves.toBe(false); + for (const [argv, options] of runCurlProbe.mock.calls) { + expect(JSON.stringify(argv)).not.toContain(API_KEY); + expect(argv).toContain("--config"); + expect(options.pinnedAddresses).toEqual([]); + expect(options.timeoutMs).toBeLessThanOrEqual(35_000); + } + expect(createBearerAuthConfig).toHaveBeenCalledWith(API_KEY, expect.any(Object)); + expect(cleanup).toHaveBeenCalledTimes(2); + expect(runCurlProbe).toHaveBeenCalledTimes(2); + }); + + it("rejects API probes after a caller-owned plan changes", async () => { + const mutablePlan = structuredClone(plan); + const runCurlProbe = vi.fn(); + const executor = createManagedClusterVllmExecutor( + { + plan: mutablePlan, + nodes: [ + { nodeId: mutablePlan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: mutablePlan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ runCurlProbe }), + ); + const request = { + baseUrl: mutablePlan.roles[0].endpoint!, + apiKey: API_KEY, + expectedModel: mutablePlan.model.servedName, + timeoutMs: 10_000, + }; + (mutablePlan.readiness as { expectedModel: string }).expectedModel = "mutated-model"; + + await expect(executor.probeModels(request)).rejects.toThrow("changed materialized plan"); + await expect(executor.probeChat(request)).rejects.toThrow("changed materialized plan"); + expect(runCurlProbe).not.toHaveBeenCalled(); + }); + + it("generates 32-hex transactions and serializes through the shared lifecycle lock", async () => { + let lockCalls = 0; + const withLifecycleLock = async (operation: () => Promise): Promise => { + lockCalls += 1; + return await operation(); + }; + const executor = createManagedClusterVllmExecutor( + { + plan, + nodes: [ + { nodeId: plan.roles[0].nodeId, modelCacheRoot: LOCAL_CACHE_ROOT }, + { + nodeId: plan.roles[1].nodeId, + modelCacheRoot: PEER_CACHE_ROOT, + sshBinding: bindingFixture.binding, + }, + ], + }, + runtimeOverrides({ + createTransactionId: () => "d".repeat(32), + withLifecycleLock, + }), + ); + + expect(executor.createTransactionId()).toMatch(/^[a-f0-9]{32}$/); + await expect(executor.withLifecycleLock(plan, async () => "done")).resolves.toBe("done"); + expect(lockCalls).toBe(1); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-executor.ts b/src/lib/inference/serving/managed-cluster-executor.ts new file mode 100644 index 0000000000..bb84513401 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-executor.ts @@ -0,0 +1,1140 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; +import net from "node:net"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; + +import { dockerForceRm, dockerRunDetached } from "../../adapters/docker/container.js"; +import { dockerCapture } from "../../adapters/docker/run.js"; +import { createBearerAuthConfig } from "../../adapters/http/auth-config.js"; +import { runCurlProbe } from "../../adapters/http/probe.js"; +import { + buildLocalManagedVllmDockerEnv, + buildRemoteVllmDockerEnv, + captureManagedVllmTcpListeners, +} from "../vllm-docker-env.js"; +import { withHostGlobalVllmLifecycleLock } from "../vllm-station-lifecycle-lock.js"; +import { + NO_PREPARATION_REF, + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, +} from "./adapter-registry.js"; +import { managedInferenceDigest, managedInferenceHexDigest } from "./catalog-integrity.js"; +import { + getManagedInferenceCompiledPreset, + getManagedInferenceCompiledRecipe, +} from "./catalog-loader.js"; +import { + isRelatedManagedVllmContainer, + type ManagedClusterApiProbeRequest, + type ManagedClusterContainerStartRequest, + type ManagedClusterContainerWaitRequest, + type ManagedClusterNodeSnapshot, + type ManagedClusterObservedContainer, + type ManagedClusterStageRequest, + type ManagedClusterVllmLifecycleDeps, +} from "./managed-cluster-lifecycle.js"; +import { + MANAGED_CLUSTER_ADAPTER_LABEL, + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_CLUSTER_LABEL, + MANAGED_CLUSTER_GPU_LABEL, + MANAGED_CLUSTER_IMAGE_LABEL, + MANAGED_CLUSTER_MANAGED_LABEL, + MANAGED_CLUSTER_MODEL_REVISION_LABEL, + MANAGED_CLUSTER_PLAN_LABEL, + MANAGED_CLUSTER_PRESET_LABEL, + MANAGED_CLUSTER_RECIPE_LABEL, + MANAGED_CLUSTER_ROLE_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + MANAGED_CLUSTER_VLLM_ADAPTER_ID, + type ManagedClusterVllmPlan, + type ManagedClusterVllmRolePlan, + managedClusterHeadRole, +} from "./managed-cluster-materialize.js"; +import { materializeManagedClusterVllmPreparation } from "./managed-cluster-preparation.js"; +import { + encodeManagedVllmSshBindingHandoff, + type ManagedVllmSshBinding, +} from "./managed-cluster-ssh-binding.js"; +import { + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, +} from "./managed-cluster-topology.js"; +import type { ManagedInferenceServingRecipe } from "./types.js"; + +const API_KEY_PATTERN = /^[a-f0-9]{64}$/; +const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/; +const HEX64_PATTERN = /^[a-f0-9]{64}$/; +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; +const LABEL_NAME_PATTERN = /^[a-z0-9][a-z0-9._/-]{0,255}$/; +const SAFE_DEVICE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const MAX_CONTAINERS = 256; +const DOCKER_INSPECTION_TIMEOUT_MS = 20_000; +const DOCKER_MUTATION_TIMEOUT_MS = 120_000; +const PROCESS_PROBE_TIMEOUT_MS = 10_000; +const PROCESS_PROBE_INTERVAL_MS = 1_000; +const MAX_PROCESS_WAIT_MS = 120_000; +const API_PROBE_INTERVAL_MS = 2_000; +const MAX_MODELS_PROBE_MS = 86_400_000; +const MAX_CHAT_PROBE_MS = 120_000; + +const CONTAINER_INSPECTION_FORMAT = + "[{{json .Id}},{{json .Name}},{{json .Config.Image}},{{json .State.Running}},{{json .Config.Labels}}]"; + +const PROCESS_PROBE_SCRIPT = String.raw` +import base64, glob, json, sys +expected = json.loads(base64.urlsafe_b64decode(sys.argv[1] + "==")) +for item in glob.glob("/proc/[0-9]*/cmdline"): + try: + raw = open(item, "rb").read(1024 * 1024) + argv = [part.decode("utf-8") for part in raw.split(b"\0") if part] + except (OSError, UnicodeDecodeError): + continue + for index, value in enumerate(argv): + if value == expected[0] and argv[index:] == expected: + print("ready") + raise SystemExit(0) +raise SystemExit(1) +`.trim(); + +const EXACT_TEXT_REPLACEMENT_SCRIPT = String.raw` +import base64, pathlib, sys +target = pathlib.Path(sys.argv[1]) +existing = base64.urlsafe_b64decode(sys.argv[2] + "==").decode("utf-8") +replacement = base64.urlsafe_b64decode(sys.argv[3] + "==").decode("utf-8") +content = target.read_text(encoding="utf-8") +if content.count(existing) != 1: + raise SystemExit("preparation source text did not match exactly once") +target.write_text(content.replace(existing, replacement), encoding="utf-8") +`.trim(); + +const VERIFY_FILE_SHA256_SCRIPT = String.raw` +import hashlib, pathlib, sys +target = pathlib.Path(sys.argv[1]) +expected = sys.argv[2] +actual = "sha256:" + hashlib.sha256(target.read_bytes()).hexdigest() +if actual != expected: + raise SystemExit("snapshot copy source digest mismatch") +`.trim(); + +export interface ManagedClusterExecutorStageTarget { + readonly nodeId: string; + readonly dockerEnv: Readonly>; + readonly modelCacheRoot: string; + readonly sshBinding?: ManagedVllmSshBinding; +} + +export type ManagedClusterExecutorStageNode = ( + request: ManagedClusterStageRequest, + target: ManagedClusterExecutorStageTarget, +) => Promise<{ ok: boolean; reason?: string }>; + +export interface CreateManagedClusterVllmExecutorOptions { + readonly plan: ManagedClusterVllmPlan; + readonly nodes: readonly ManagedClusterExecutorNodeTarget[]; + readonly stageNode?: ManagedClusterExecutorStageNode; +} + +export interface ManagedClusterExecutorNodeTarget { + readonly nodeId: string; + readonly modelCacheRoot: string; + readonly sshBinding?: ManagedVllmSshBinding; +} + +export interface ManagedClusterVllmExecutorRuntimeDeps { + dockerCapture: typeof dockerCapture; + dockerForceRm: typeof dockerForceRm; + dockerRunDetached: typeof dockerRunDetached; + captureListeners(rolePlan: ManagedClusterVllmRolePlan, binding?: ManagedVllmSshBinding): string; + createBearerAuthConfig: typeof createBearerAuthConfig; + createTransactionId(): string; + now(): number; + runCurlProbe: typeof runCurlProbe; + sleep(ms: number): Promise; + withLifecycleLock(operation: () => Promise): Promise; +} + +type SelectedRecipe = ManagedInferenceServingRecipe["spec"]; + +const DEFAULT_DEPS: ManagedClusterVllmExecutorRuntimeDeps = { + dockerCapture, + dockerForceRm, + dockerRunDetached, + captureListeners: (rolePlan, binding) => { + if (rolePlan.execution.kind === "ssh" && !binding) { + throw new Error("Managed cluster worker listener probe has no SSH binding"); + } + return captureManagedVllmTcpListeners( + rolePlan.role, + binding ?? ({} as ManagedVllmSshBinding), + DOCKER_INSPECTION_TIMEOUT_MS, + ); + }, + createBearerAuthConfig, + createTransactionId: () => randomBytes(16).toString("hex"), + now: () => Date.now(), + runCurlProbe, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + withLifecycleLock: (operation) => withHostGlobalVllmLifecycleLock(operation), +}; + +function normalizedAbsoluteHostPath(value: string, label: string): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > 4096 || + !path.posix.isAbsolute(value) || + path.posix.normalize(value) !== value || + value.includes(path.posix.delimiter) || + /[\u0000-\u001f\u007f]/.test(value) + ) { + throw new Error(`${label} must be a normalized absolute POSIX path`); + } + return value; +} + +function exactKeys(actual: object, expected: readonly string[], label: string): void { + const keys = Object.keys(actual).sort(); + if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) { + throw new Error(`${label} does not match the managed cluster adapter contract`); + } +} + +function recipeCommandArguments( + rolePlan: ManagedClusterVllmRolePlan, + plan: ManagedClusterVllmPlan, + recipe: SelectedRecipe, +): string[] { + const staticArguments = recipe.serve.arguments.flatMap(({ name, value }) => + value === undefined ? [name] : [name, String(value)], + ); + return [ + "serve", + recipe.model.id, + "--revision", + recipe.model.revision, + "--served-model-name", + recipe.model.servedName, + "--host", + rolePlan.fabric.address, + ...staticArguments, + "--tensor-parallel-size", + String(recipe.execution.tensorParallelSize), + "--pipeline-parallel-size", + String(recipe.execution.pipelineParallelSize), + "--distributed-executor-backend", + recipe.execution.distributedExecutorBackend, + "--nnodes", + String(recipe.execution.nodeCount), + "--node-rank", + String(rolePlan.rank), + "--master-addr", + plan.masterAddress, + "--master-port", + String(plan.masterPort), + ...(rolePlan.role === "worker" ? ["--headless"] : []), + ]; +} + +function assertRolePlan( + rolePlan: ManagedClusterVllmRolePlan, + plan: ManagedClusterVllmPlan, + recipe: SelectedRecipe, +): void { + const isHead = rolePlan.role === "head"; + const environment = { + ...recipe.runtime.environment, + HF_HOME: recipe.runtime.modelCache.target, + VLLM_HOST_IP: rolePlan.fabric.address, + NCCL_IB_HCA: `${rolePlan.fabric.hcaDevice}:${String(rolePlan.fabric.hcaPort)}`, + NCCL_SOCKET_IFNAME: rolePlan.fabric.netdev, + TP_SOCKET_IFNAME: rolePlan.fabric.netdev, + GLOO_SOCKET_IFNAME: rolePlan.fabric.netdev, + NCCL_IB_GID_INDEX: String(rolePlan.fabric.roceGidIndex), + MASTER_ADDR: plan.masterAddress, + MASTER_PORT: String(plan.masterPort), + NODE_RANK: String(rolePlan.rank), + HEADLESS: isHead ? "" : "1", + }; + const baseLabels = { + [MANAGED_CLUSTER_MANAGED_LABEL]: "true", + [MANAGED_CLUSTER_ADAPTER_LABEL]: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + [MANAGED_CLUSTER_PRESET_LABEL]: plan.presetId, + [MANAGED_CLUSTER_RECIPE_LABEL]: plan.recipeId, + [MANAGED_CLUSTER_ROLE_LABEL]: rolePlan.role, + [MANAGED_CLUSTER_CLUSTER_LABEL]: plan.clusterId, + [MANAGED_CLUSTER_PLAN_LABEL]: plan.planId, + [MANAGED_CLUSTER_GPU_LABEL]: rolePlan.gpuId, + [MANAGED_CLUSTER_IMAGE_LABEL]: recipe.runtime.image, + [MANAGED_CLUSTER_MODEL_REVISION_LABEL]: recipe.model.revision, + }; + const preparation = materializeManagedClusterVllmPreparation({ + ...recipe.model, + modelCacheTarget: recipe.runtime.modelCache.target, + }); + const fabric = { + primaryRailIndex: rolePlan.fabric.primaryRailIndex, + netdev: rolePlan.fabric.netdev, + hcaDevice: rolePlan.fabric.hcaDevice, + hcaPort: rolePlan.fabric.hcaPort, + address: rolePlan.fabric.address, + roceGidIndex: rolePlan.fabric.roceGidIndex, + roceGidValue: rolePlan.fabric.roceGidValue, + }; + const expected = { + role: rolePlan.role, + rank: rolePlan.rank, + nodeId: rolePlan.nodeId, + gpuId: rolePlan.gpuId, + containerName: `nemoclaw-vllm-cluster-rank-${String(rolePlan.rank)}`, + execution: rolePlan.execution, + image: recipe.runtime.image, + runtime: { + architecture: recipe.runtime.architecture, + networkMode: recipe.runtime.networkMode, + ipcMode: recipe.runtime.ipcMode, + sharedMemoryBytes: recipe.runtime.sharedMemoryBytes, + gpuRequest: recipe.runtime.gpuRequest, + devices: recipe.runtime.devices, + imageDownloadSizeBytes: recipe.runtime.imageDownloadSizeBytes, + pullTimeoutSeconds: recipe.runtime.pullTimeoutSeconds, + ulimits: { + memlock: recipe.runtime.ulimits.memlock, + stack: recipe.runtime.ulimits.stackBytes, + }, + modelCache: recipe.runtime.modelCache, + temporaryFilesystems: recipe.runtime.temporaryFilesystems, + }, + preparation, + fabric, + environment, + command: { + executable: recipe.serve.executable, + arguments: recipeCommandArguments(rolePlan, plan, recipe), + }, + endpoint: isHead ? `http://${plan.masterAddress}:${String(plan.apiPort)}` : null, + baseLabels, + }; + if ( + !isDeepStrictEqual(rolePlan, expected) || + !rolePlan.nodeId || + rolePlan.nodeId.length > 256 || + !/^GPU-[A-Za-z0-9-]+$/.test(rolePlan.gpuId) || + !Number.isSafeInteger(rolePlan.fabric.primaryRailIndex) || + rolePlan.fabric.primaryRailIndex < 0 || + !SAFE_DEVICE_PATTERN.test(rolePlan.fabric.netdev) || + !SAFE_DEVICE_PATTERN.test(rolePlan.fabric.hcaDevice) || + net.isIP(rolePlan.fabric.address) !== 4 || + net.isIP(rolePlan.fabric.roceGidValue) !== 6 || + !Number.isInteger(rolePlan.fabric.hcaPort) || + !Number.isInteger(rolePlan.fabric.roceGidIndex) + ) { + throw new Error(`${rolePlan.role} role does not match the catalog-derived adapter contract`); + } +} + +function recipeApiPort(recipe: SelectedRecipe): number | undefined { + const ports = recipe.serve.arguments + .filter(({ name }) => name === "--port") + .map(({ value }) => + typeof value === "number" + ? value + : typeof value === "string" && /^\d{1,5}$/u.test(value) + ? Number(value) + : Number.NaN, + ); + return ports.length === 1 && Number.isInteger(ports[0]) ? ports[0] : undefined; +} + +/** Read-only validation shared by installer, receipt, recovery, and cleanup. */ +export function assertManagedClusterVllmExecutorConfig( + config: CreateManagedClusterVllmExecutorOptions, +): void { + exactKeys( + config, + ["nodes", "plan", ...(Object.hasOwn(config, "stageNode") ? ["stageNode"] : [])], + "Managed cluster executor configuration", + ); + const { plan } = config; + if ( + config.nodes.length !== plan.roles.length || + new Set(config.nodes.map(({ nodeId }) => nodeId)).size !== config.nodes.length || + config.nodes.some(({ nodeId }) => !plan.roles.some((role) => role.nodeId === nodeId)) + ) { + throw new Error("Managed cluster executor node targets do not match the plan"); + } + for (const target of config.nodes) { + normalizedAbsoluteHostPath(target.modelCacheRoot, `${target.nodeId} model cache root`); + const rolePlan = plan.roles.find(({ nodeId }) => nodeId === target.nodeId)!; + if ( + (rolePlan.execution.kind === "local" && target.sshBinding !== undefined) || + (rolePlan.execution.kind === "ssh" && + (!target.sshBinding || + rolePlan.execution.expectedTarget !== target.sshBinding.peerTarget || + rolePlan.execution.bindingHandle !== + encodeManagedVllmSshBindingHandoff(target.sshBinding))) + ) { + throw new Error(`Managed cluster executor target ${target.nodeId} is invalid`); + } + } + const compiledPreset = getManagedInferenceCompiledPreset(plan.presetId); + const compiledRecipe = getManagedInferenceCompiledRecipe(plan.recipeId); + if ( + !compiledPreset || + !compiledRecipe || + managedInferenceDigest(compiledPreset) !== plan.presetDigest || + managedInferenceDigest(compiledRecipe) !== plan.recipeDigest || + compiledPreset.spec.plan.recipeRef !== compiledRecipe.metadata.id || + compiledPreset.spec.plan.backend !== compiledRecipe.spec.backend + ) { + throw new Error( + "Managed cluster executor input does not match its selected definition digests", + ); + } + const recipe = compiledRecipe.spec; + const topologyIdentity = { + id: plan.topologyId, + schemaVersion: plan.topologySchemaVersion, + subjectDigest: plan.topologySubjectDigest, + outputDigest: plan.topologyOutputDigest, + }; + const expectedClusterId = managedInferenceHexDigest(topologyIdentity); + const expectedPlanId = managedInferenceHexDigest({ + adapterId: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + preset: { id: plan.presetId, digest: plan.presetDigest }, + recipe: { id: plan.recipeId, digest: plan.recipeDigest }, + topology: topologyIdentity, + }); + const expectedPlan = { + schemaVersion: 1, + adapterId: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + catalogDigest: plan.catalogDigest, + presetId: compiledPreset.metadata.id, + presetDigest: managedInferenceDigest(compiledPreset), + recipeId: compiledRecipe.metadata.id, + recipeDigest: managedInferenceDigest(compiledRecipe), + topologyId: MANAGED_CLUSTER_TOPOLOGY_ID, + topologySchemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + topologySubjectDigest: plan.topologySubjectDigest, + topologyOutputDigest: plan.topologyOutputDigest, + clusterId: expectedClusterId, + planId: expectedPlanId, + model: { + id: recipe.model.id, + revision: recipe.model.revision, + servedName: recipe.model.servedName, + }, + authentication: recipe.serve.authentication, + apiPort: recipeApiPort(recipe), + masterAddress: plan.masterAddress, + masterPort: recipe.execution.rendezvousPort, + readiness: { + timeoutMs: recipe.readiness.timeoutSeconds * 1000, + expectedModel: recipe.readiness.expectedModel, + }, + roles: plan.roles, + }; + if ( + !isDeepStrictEqual(plan, expectedPlan) || + !SHA256_PATTERN.test(plan.catalogDigest) || + !SHA256_PATTERN.test(plan.presetDigest) || + !SHA256_PATTERN.test(plan.recipeDigest) || + plan.masterAddress !== managedClusterHeadRole(plan).fabric.address || + !HEX64_PATTERN.test(plan.clusterId) || + !HEX64_PATTERN.test(plan.planId) || + !SHA256_PATTERN.test(plan.topologySubjectDigest) || + !SHA256_PATTERN.test(plan.topologyOutputDigest) || + plan.roles.length !== recipe.execution.nodeCount || + !isDeepStrictEqual(managedClusterHeadRole(plan).execution, { kind: "local" }) || + plan.roles.some( + (rolePlan, index) => + rolePlan.rank !== index || + (index === 0 ? rolePlan.role !== "head" : rolePlan.role !== "worker"), + ) + ) { + throw new Error("Managed cluster executor input does not match its qualified binding and plan"); + } + for (const rolePlan of plan.roles) assertRolePlan(rolePlan, plan, recipe); +} + +function shellQuote(value: string): string { + if (value.includes("\0")) throw new Error("Managed cluster command value contains a NUL byte"); + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function base64url(value: string): string { + return Buffer.from(value, "utf8").toString("base64url"); +} + +function preparationCommand(rolePlan: ManagedClusterVllmRolePlan): string { + const preparation = rolePlan.preparation; + const command = ["set -Eeuo pipefail"]; + if (preparation.ref === SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF) { + command.push( + `test -f ${shellQuote(preparation.snapshotCopy.sourcePath)}`, + [ + "python3", + "-c", + shellQuote(VERIFY_FILE_SHA256_SCRIPT), + shellQuote(preparation.snapshotCopy.sourcePath), + shellQuote(preparation.snapshotCopy.digest), + ].join(" "), + `install -m 0644 -- ${shellQuote(preparation.snapshotCopy.sourcePath)} ${shellQuote(preparation.snapshotCopy.targetPath)}`, + [ + "python3", + "-c", + shellQuote(EXACT_TEXT_REPLACEMENT_SCRIPT), + shellQuote(preparation.exactTextReplacement.targetPath), + shellQuote(base64url(preparation.exactTextReplacement.expectedText)), + shellQuote(base64url(preparation.exactTextReplacement.replacementText)), + ].join(" "), + ); + } else if (preparation.ref !== NO_PREPARATION_REF) { + throw new Error("Unsupported managed cluster preparation"); + } + const executable = shellQuote(rolePlan.command.executable); + const commandArguments = rolePlan.command.arguments.map(shellQuote).join(" "); + command.push(`exec ${executable} ${commandArguments}`); + return command.join(" && "); +} + +function validateLaunchLabels( + rolePlan: ManagedClusterVllmRolePlan, + labels: Readonly>, +): void { + const expectedKeys = [ + ...Object.keys(rolePlan.baseLabels), + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + ].sort(); + exactKeys(labels, expectedKeys, `${rolePlan.role} launch labels`); + for (const [name, value] of Object.entries(labels)) { + if ( + !LABEL_NAME_PATTERN.test(name) || + typeof value !== "string" || + value.length === 0 || + value.length > 4096 || + /[\u0000\r\n]/.test(value) + ) { + throw new Error(`${rolePlan.role} launch labels are unsafe`); + } + } + if ( + !Object.entries(rolePlan.baseLabels).every(([name, value]) => labels[name] === value) || + !HEX64_PATTERN.test(labels[MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL] ?? "") || + !TRANSACTION_ID_PATTERN.test(labels[MANAGED_CLUSTER_TRANSACTION_LABEL] ?? "") + ) { + throw new Error(`${rolePlan.role} launch labels do not match its plan`); + } +} + +/** Build the exact, non-secret Docker run argv for one materialized role. */ +export function buildManagedClusterVllmRunArgs( + rolePlan: ManagedClusterVllmRolePlan, + modelCacheRoot: string, + labels: Readonly>, +): string[] { + const cacheRoot = normalizedAbsoluteHostPath(modelCacheRoot, `${rolePlan.role} model cache root`); + validateLaunchLabels(rolePlan, labels); + const args = [ + "--pull=never", + "--init", + "--network", + rolePlan.runtime.networkMode, + "--ipc", + rolePlan.runtime.ipcMode, + "--shm-size", + String(rolePlan.runtime.sharedMemoryBytes), + "--gpus", + rolePlan.runtime.gpuRequest, + "--ulimit", + `memlock=${rolePlan.runtime.ulimits.memlock === "unlimited" ? "-1" : String(rolePlan.runtime.ulimits.memlock)}`, + "--ulimit", + `stack=${String(rolePlan.runtime.ulimits.stack)}`, + ]; + for (const device of rolePlan.runtime.devices) args.push("--device", device); + for (const temporaryFilesystem of rolePlan.runtime.temporaryFilesystems) { + args.push( + "--tmpfs", + `${temporaryFilesystem.target}:${[ + ...temporaryFilesystem.options, + `size=${String(temporaryFilesystem.sizeBytes)}`, + `mode=${temporaryFilesystem.mode}`, + ].join(",")}`, + ); + } + args.push("--volume", `${cacheRoot}/hub:${rolePlan.runtime.modelCache.target}/hub:ro`); + for (const [name, value] of Object.entries(rolePlan.environment).sort(([left], [right]) => + left.localeCompare(right), + )) { + args.push("--env", `${name}=${value}`); + } + if (rolePlan.role === "head") args.push("--env", "VLLM_API_KEY"); + for (const [name, value] of Object.entries(labels).sort(([left], [right]) => + left.localeCompare(right), + )) { + args.push("--label", `${name}=${value}`); + } + args.push( + "--name", + rolePlan.containerName, + "--entrypoint", + "/bin/bash", + rolePlan.image, + "-lc", + preparationCommand(rolePlan), + ); + return args; +} + +function parseJsonField(value: string, label: string): T { + try { + return JSON.parse(value) as T; + } catch { + throw new Error(`${label} returned malformed JSON`); + } +} + +function parseLabels(value: unknown): Readonly> { + if (value === null) return {}; + if (typeof value !== "object" || Array.isArray(value) || Object.keys(value).length > 128) { + throw new Error("Docker container labels are malformed"); + } + const labels: Record = {}; + for (const [name, item] of Object.entries(value)) { + if ( + !LABEL_NAME_PATTERN.test(name) || + typeof item !== "string" || + item.length > 4096 || + /[\u0000\r\n]/.test(item) + ) { + throw new Error("Docker container labels are malformed"); + } + labels[name] = item; + } + return labels; +} + +interface InspectedContainer { + id: string; + name: string; + image: string; + running: boolean; + labels: Readonly>; +} + +function parseContainerRows(output: string, expectedIds: readonly string[]): InspectedContainer[] { + const lines = output.split(/\r?\n/).filter(Boolean); + if (lines.length !== expectedIds.length) { + throw new Error("Docker container inspection was incomplete or ambiguous"); + } + const containers = lines.map((line, index) => { + const fields = parseJsonField(line, "Docker container inspection"); + if (!Array.isArray(fields) || fields.length !== 5) { + throw new Error("Docker container inspection was malformed"); + } + const [id, rawName, image, running, rawLabels] = fields; + const labels = parseLabels(rawLabels); + const name = + typeof rawName === "string" && rawName.startsWith("/") ? rawName.slice(1) : rawName; + if ( + typeof id !== "string" || + !CONTAINER_ID_PATTERN.test(id) || + id !== expectedIds[index] || + typeof name !== "string" || + name.length === 0 || + name.length > 256 || + typeof image !== "string" || + image.length === 0 || + image.length > 1024 || + typeof running !== "boolean" + ) { + throw new Error("Docker container inspection was malformed"); + } + return { id, name, image, running, labels }; + }); + if (new Set(containers.map(({ id }) => id)).size !== containers.length) { + throw new Error("Docker container inspection returned duplicate IDs"); + } + return containers; +} + +function parseListeningPorts(output: string): number[] { + if (!output.trim()) return []; + const ports = output + .trim() + .split(/\r?\n/) + .map((line) => { + const fields = line.trim().split(/\s+/); + const local = fields[3]; + const match = local?.match(/:([0-9]{1,5})$/); + const port = match ? Number(match[1]) : Number.NaN; + if (fields[0] !== "LISTEN" || !Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error("Host listener inspection was malformed"); + } + return port; + }); + return [...new Set(ports)].sort((left, right) => left - right); +} + +function labelsMatch( + actual: Readonly>, + expected: Readonly>, +): boolean { + return Object.entries(expected).every(([name, value]) => actual[name] === value); +} + +function mutationSucceeded(result: ReturnType): boolean { + return result.status === 0 && !result.error && !result.signal; +} + +function targetFor( + rolePlan: ManagedClusterVllmRolePlan, + config: CreateManagedClusterVllmExecutorOptions, +): ManagedClusterExecutorNodeTarget { + const target = config.nodes.find(({ nodeId }) => nodeId === rolePlan.nodeId); + if (!target) throw new Error(`Managed cluster node ${rolePlan.nodeId} has no executor target`); + return target; +} + +function roleDockerEnv( + rolePlan: ManagedClusterVllmRolePlan, + config: CreateManagedClusterVllmExecutorOptions, +): Record { + const target = targetFor(rolePlan, config); + const env = + rolePlan.execution.kind === "local" + ? buildLocalManagedVllmDockerEnv() + : buildRemoteVllmDockerEnv(target.sshBinding!); + delete env.VLLM_API_KEY; + return env; +} + +function processMatchesRole( + rolePlan: ManagedClusterVllmRolePlan, + containerId: string, + env: Record, + deps: ManagedClusterVllmExecutorRuntimeDeps, +): boolean { + const expected = base64url( + JSON.stringify([rolePlan.command.executable, ...rolePlan.command.arguments]), + ); + try { + return ( + deps + .dockerCapture(["exec", containerId, "python3", "-c", PROCESS_PROBE_SCRIPT, expected], { + env, + timeout: PROCESS_PROBE_TIMEOUT_MS, + }) + .trim() === "ready" + ); + } catch { + return false; + } +} + +function inspectRoleNode( + rolePlan: ManagedClusterVllmRolePlan, + config: CreateManagedClusterVllmExecutorOptions, + deps: ManagedClusterVllmExecutorRuntimeDeps, +): ManagedClusterNodeSnapshot { + const env = roleDockerEnv(rolePlan, config); + const rawIds = deps.dockerCapture(["container", "ls", "--all", "--no-trunc", "--quiet"], { + env, + timeout: DOCKER_INSPECTION_TIMEOUT_MS, + }); + const ids = rawIds.split(/\r?\n/).filter(Boolean); + if ( + ids.length > MAX_CONTAINERS || + ids.some((id) => !CONTAINER_ID_PATTERN.test(id)) || + new Set(ids).size !== ids.length + ) { + throw new Error(`${rolePlan.role} Docker container inventory is malformed or too large`); + } + const inspected = ids.length + ? parseContainerRows( + deps.dockerCapture( + ["container", "inspect", "--format", CONTAINER_INSPECTION_FORMAT, ...ids], + { env, timeout: DOCKER_INSPECTION_TIMEOUT_MS }, + ), + ids, + ) + : []; + const containers: ManagedClusterObservedContainer[] = inspected.map((container) => ({ + ...container, + healthy: + container.running && + container.name === rolePlan.containerName && + container.image === rolePlan.image && + labelsMatch(container.labels, rolePlan.baseLabels) && + processMatchesRole(rolePlan, container.id, env, deps), + })); + const listeningPorts = parseListeningPorts( + deps.captureListeners(rolePlan, targetFor(rolePlan, config).sshBinding), + ); + return { containers, listeningPorts }; +} + +export interface ManagedClusterVllmNodeSnapshots { + readonly nodes: readonly { + readonly nodeId: string; + readonly snapshot: ManagedClusterNodeSnapshot; + }[]; +} + +/** Synchronous read-only recovery seam backed by the production inspector. */ +export function inspectManagedClusterVllmNodesSync( + config: CreateManagedClusterVllmExecutorOptions, + overrides: Partial = {}, +): ManagedClusterVllmNodeSnapshots { + const deps = { ...DEFAULT_DEPS, ...overrides }; + assertManagedClusterVllmExecutorConfig(config); + return { + nodes: config.plan.roles.map((rolePlan) => ({ + nodeId: rolePlan.nodeId, + snapshot: inspectRoleNode(rolePlan, config, deps), + })), + }; +} + +function exactWaitObservation( + snapshot: ManagedClusterNodeSnapshot, + request: ManagedClusterContainerWaitRequest, +): ManagedClusterObservedContainer | null { + const matches = snapshot.containers.filter(({ id }) => id === request.containerId); + if (matches.length !== 1) return null; + const observed = matches[0]!; + return observed.name === request.rolePlan.containerName && + observed.image === request.rolePlan.image && + labelsMatch(observed.labels, request.expectedLabels) + ? observed + : null; +} + +function workerFirstBoundaryIsClear( + snapshot: ManagedClusterNodeSnapshot, + plan: ManagedClusterVllmPlan, +): boolean { + return ( + !snapshot.listeningPorts.some((port) => port === plan.apiPort || port === plan.masterPort) && + !snapshot.containers.some(isRelatedManagedVllmContainer) + ); +} + +async function waitForRoleProcess( + request: ManagedClusterContainerWaitRequest, + config: CreateManagedClusterVllmExecutorOptions, + deps: ManagedClusterVllmExecutorRuntimeDeps, + stableChecks: number, + requireHeadAbsent = false, +): Promise { + const deadline = deps.now() + Math.min(Math.max(1, request.timeoutMs), MAX_PROCESS_WAIT_MS); + let consecutive = 0; + do { + try { + const observed = exactWaitObservation( + inspectRoleNode(request.rolePlan, config, deps), + request, + ); + const headAbsent = + !requireHeadAbsent || + workerFirstBoundaryIsClear( + inspectRoleNode(managedClusterHeadRole(config.plan), config, deps), + config.plan, + ); + consecutive = observed?.running && observed.healthy && headAbsent ? consecutive + 1 : 0; + if (consecutive >= stableChecks) return true; + } catch { + consecutive = 0; + } + const remaining = deadline - deps.now(); + if (remaining <= 0) return false; + await deps.sleep(Math.min(PROCESS_PROBE_INTERVAL_MS, remaining)); + } while (deps.now() <= deadline); + return false; +} + +function validApiBody(body: string, expectedModel: string, kind: "models" | "chat"): boolean { + try { + const parsed = JSON.parse(body) as { data?: unknown; model?: unknown; choices?: unknown }; + if (kind === "models") { + const item = Array.isArray(parsed.data) && parsed.data.length === 1 ? parsed.data[0] : null; + return ( + !!item && + typeof item === "object" && + !Array.isArray(item) && + (item as { id?: unknown }).id === expectedModel + ); + } + return ( + parsed.model === expectedModel && + Array.isArray(parsed.choices) && + parsed.choices.length === 1 && + typeof parsed.choices[0] === "object" && + parsed.choices[0] !== null + ); + } catch { + return false; + } +} + +async function probeAuthenticatedApi( + request: ManagedClusterApiProbeRequest, + kind: "models" | "chat", + deps: ManagedClusterVllmExecutorRuntimeDeps, +): Promise { + if (!API_KEY_PATTERN.test(request.apiKey)) return false; + const maximum = kind === "models" ? MAX_MODELS_PROBE_MS : MAX_CHAT_PROBE_MS; + const deadline = deps.now() + Math.min(Math.max(1, request.timeoutMs), maximum); + do { + const remaining = Math.max(1, deadline - deps.now()); + const maxTimeSeconds = Math.max(1, Math.min(30, Math.ceil(remaining / 1000))); + let authConfig: ReturnType | undefined; + try { + authConfig = deps.createBearerAuthConfig(request.apiKey, { + prefix: "nemoclaw-managed-cluster-vllm-probe", + }); + const baseUrl = request.baseUrl.replace(/\/+$/, ""); + const body = + kind === "chat" + ? JSON.stringify({ + model: request.expectedModel, + messages: [{ role: "user", content: "Reply with OK." }], + max_tokens: 1, + temperature: 0, + }) + : null; + const result = deps.runCurlProbe( + [ + "-sS", + "--connect-timeout", + "3", + "--max-time", + String(maxTimeSeconds), + ...(body ? ["-H", "Content-Type: application/json", "-d", body] : []), + ...authConfig.args, + `${baseUrl}/v1/${kind === "models" ? "models" : "chat/completions"}`, + ], + { + trustedConfigFiles: authConfig.trustedConfigFiles, + pinnedAddresses: [], + timeoutMs: (maxTimeSeconds + 5) * 1000, + }, + ); + if (result.ok && validApiBody(result.body, request.expectedModel, kind)) { + return true; + } + } catch { + // Retry until the caller-owned, bounded readiness deadline. + } finally { + authConfig?.cleanup(); + } + const retryRemaining = deadline - deps.now(); + if (retryRemaining <= 0) return false; + await deps.sleep(Math.min(API_PROBE_INTERVAL_MS, retryRemaining)); + } while (deps.now() <= deadline); + return false; +} + +/** + * Bind the code-owned lifecycle to every exact Docker daemon and each qualified, + * owner-only SSH handoff. Staging stays caller-injected so this module does not + * depend on installer/model-download orchestration. + */ +export function createManagedClusterVllmExecutor( + config: CreateManagedClusterVllmExecutorOptions, + overrides: Partial = {}, +): ManagedClusterVllmLifecycleDeps { + const deps = { ...DEFAULT_DEPS, ...overrides }; + assertManagedClusterVllmExecutorConfig(config); + const capturedPlan = structuredClone(config.plan); + const assertPlan = (plan: ManagedClusterVllmPlan): void => { + if (!isDeepStrictEqual(plan, capturedPlan)) { + throw new Error("Managed cluster executor refused a changed materialized plan"); + } + assertManagedClusterVllmExecutorConfig(config); + }; + const assertRole = (rolePlan: ManagedClusterVllmRolePlan): void => { + assertPlan(config.plan); + if ( + !isDeepStrictEqual( + rolePlan, + config.plan.roles.find(({ nodeId }) => nodeId === rolePlan.nodeId), + ) + ) { + throw new Error("Managed cluster executor refused a role outside its materialized plan"); + } + }; + + return { + async inspectNode(rolePlan) { + assertRole(rolePlan); + return inspectRoleNode(rolePlan, config, deps); + }, + async stageNode(request) { + assertRole(request.rolePlan); + if (!isDeepStrictEqual(request.preparation, request.rolePlan.preparation)) { + return { ok: false, reason: `${request.rolePlan.role} preparation contract changed` }; + } + if (!config.stageNode) { + return { ok: false, reason: "managed cluster model and image staging is not configured" }; + } + const dockerEnv = roleDockerEnv(request.rolePlan, config); + const target = targetFor(request.rolePlan, config); + return await config.stageNode(request, { + nodeId: request.rolePlan.nodeId, + dockerEnv, + modelCacheRoot: target.modelCacheRoot, + ...(target.sshBinding ? { sshBinding: target.sshBinding } : {}), + }); + }, + async startContainer(request: ManagedClusterContainerStartRequest) { + assertRole(request.rolePlan); + if (!isDeepStrictEqual(request.preparation, request.rolePlan.preparation)) { + return { ok: false, reason: `${request.rolePlan.role} preparation contract changed` }; + } + const isHead = request.rolePlan.role === "head"; + if ( + (isHead && !API_KEY_PATTERN.test(request.bearerApiKey ?? "")) || + (!isHead && request.bearerApiKey !== undefined) || + (request.bearerApiKey && Object.values(request.labels).includes(request.bearerApiKey)) + ) { + return { ok: false, reason: `${request.rolePlan.role} bearer-key boundary is invalid` }; + } + let args: string[]; + try { + args = buildManagedClusterVllmRunArgs( + request.rolePlan, + targetFor(request.rolePlan, config).modelCacheRoot, + request.labels, + ); + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } + if (request.bearerApiKey && JSON.stringify(args).includes(request.bearerApiKey)) { + return { ok: false, reason: "managed cluster bearer key entered Docker argv" }; + } + const env = roleDockerEnv(request.rolePlan, config); + if (isHead) env.VLLM_API_KEY = request.bearerApiKey!; + let result: ReturnType | null = null; + try { + result = deps.dockerRunDetached(args, { + env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }); + } catch { + // Reconcile an exact transaction-owned create below. + } + let observed: ManagedClusterObservedContainer | null = null; + try { + const snapshot = inspectRoleNode(request.rolePlan, config, deps); + const candidates = snapshot.containers.filter( + (container) => + container.name === request.rolePlan.containerName && + container.image === request.rolePlan.image && + labelsMatch(container.labels, request.labels), + ); + if (candidates.length === 1) observed = candidates[0]!; + } catch { + // A missing exact observation leaves the mutation uncommitted here. + } + const capturedId = String(result?.stdout ?? "").trim(); + const exactId = CONTAINER_ID_PATTERN.test(capturedId) ? capturedId : observed?.id; + if ( + exactId && + observed?.id === exactId && + observed.running && + (result === null || mutationSucceeded(result)) + ) { + return { ok: true, containerId: exactId }; + } + return { + ok: false, + ...(exactId ? { containerId: exactId } : {}), + reason: `${request.rolePlan.role} Docker create did not commit one exact running container`, + }; + }, + async waitForContainerReady(request) { + assertRole(request.rolePlan); + return await waitForRoleProcess(request, config, deps, 1); + }, + async waitForWorkerDistributedReady(request) { + assertRole(request.rolePlan); + if (request.rolePlan.role !== "worker") return false; + // Two spaced exact-process observations prove the worker remained alive + // while rank 0 was still absent, the worker-first rendezvous boundary. + return await waitForRoleProcess(request, config, deps, 2, true); + }, + async removeContainer(rolePlan, exactContainerId) { + assertRole(rolePlan); + if (!CONTAINER_ID_PATTERN.test(exactContainerId)) { + return { ok: false, reason: `${rolePlan.role} cleanup container ID is invalid` }; + } + const before = inspectRoleNode(rolePlan, config, deps); + const matches = before.containers.filter(({ id }) => id === exactContainerId); + if ( + matches.length !== 1 || + matches[0]!.name !== rolePlan.containerName || + matches[0]!.image !== rolePlan.image || + !labelsMatch(matches[0]!.labels, rolePlan.baseLabels) + ) { + return { ok: false, reason: `${rolePlan.role} exact cleanup ownership changed` }; + } + const env = roleDockerEnv(rolePlan, config); + let removed = false; + try { + removed = mutationSucceeded( + deps.dockerForceRm(exactContainerId, { + env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }), + ); + } catch { + // Reconcile the exact ID below. + } + if (!removed) { + try { + const after = inspectRoleNode(rolePlan, config, deps); + removed = !after.containers.some(({ id }) => id === exactContainerId); + } catch { + removed = false; + } + } + return removed + ? { ok: true } + : { ok: false, reason: `${rolePlan.role} exact container removal failed` }; + }, + async probeModels(request) { + assertPlan(config.plan); + if ( + request.baseUrl !== managedClusterHeadRole(config.plan).endpoint || + request.expectedModel !== config.plan.readiness.expectedModel + ) { + return false; + } + return await probeAuthenticatedApi(request, "models", deps); + }, + async probeChat(request) { + assertPlan(config.plan); + if ( + request.baseUrl !== managedClusterHeadRole(config.plan).endpoint || + request.expectedModel !== config.plan.readiness.expectedModel + ) { + return false; + } + return await probeAuthenticatedApi(request, "chat", deps); + }, + createTransactionId: deps.createTransactionId, + async withLifecycleLock(plan, operation) { + assertPlan(plan); + return await deps.withLifecycleLock(operation); + }, + }; +} diff --git a/src/lib/inference/serving/managed-cluster-fixture.test-support.ts b/src/lib/inference/serving/managed-cluster-fixture.test-support.ts new file mode 100644 index 0000000000..32c947a145 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-fixture.test-support.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MANAGED_CLUSTER_VLLM_MATERIALIZER_REF } from "./adapter-registry.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { managedInferenceDigest } from "./catalog-integrity.js"; +import type { ResolvedManagedInferenceSelection } from "./types.js"; +import { + type ManagedClusterVllmPlan, + materializeManagedClusterVllmPlan, +} from "./managed-cluster-materialize.js"; +import { + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + type ManagedClusterTopologyOutput, + managedClusterTopologyOutputDigest, + managedClusterTopologySubjectDigest, +} from "./managed-cluster-topology.js"; + +function fixtureCatalogDefinitions() { + const catalog = loadManagedInferenceCatalog(); + for (const compiledPreset of catalog.presets) { + const compiledRecipe = catalog.recipes.find( + ({ metadata }) => metadata.id === compiledPreset.spec.plan.recipeRef, + ); + if (compiledRecipe?.spec.execution.materializerRef === MANAGED_CLUSTER_VLLM_MATERIALIZER_REF) { + return { catalog, compiledPreset, compiledRecipe }; + } + } + throw new Error("managed inference fixture catalog is incomplete"); +} + +export const FIXTURE_MANAGED_CLUSTER_PRESET_ID = + fixtureCatalogDefinitions().compiledPreset.metadata.id; + +export function fixtureManagedClusterSelection(): ResolvedManagedInferenceSelection { + const { catalog, compiledPreset, compiledRecipe } = fixtureCatalogDefinitions(); + const preset = structuredClone(compiledPreset); + const recipe = structuredClone(compiledRecipe); + const subjectNodeIds = ["spark-head", "spark-worker"] as const; + const output: ManagedClusterTopologyOutput = { + controllerNodeId: "spark-head", + nodes: [ + { nodeId: "spark-head", gpuId: "GPU-head", rank: 0, role: "head" }, + { nodeId: "spark-worker", gpuId: "GPU-worker", rank: 1, role: "worker" }, + ], + rails: [ + { + index: 0, + endpoints: [ + { + nodeId: "spark-head", + netdev: "enp1s0f0np0", + hcaDevice: "rocep1s0f0", + hcaPort: 1, + address: "192.168.100.10", + prefixLength: 24, + peerAddress: "192.168.100.11", + roceGid: { index: 3, value: "::ffff:c0a8:640a" }, + }, + { + nodeId: "spark-worker", + netdev: "enp1s0f1np1", + hcaDevice: "rocep1s0f1", + hcaPort: 1, + address: "192.168.100.11", + prefixLength: 24, + peerAddress: "192.168.100.10", + roceGid: { index: 6, value: "::ffff:c0a8:640b" }, + }, + ], + }, + { + index: 1, + endpoints: [ + { + nodeId: "spark-head", + netdev: "enP2p1s0f0np0", + hcaDevice: "roceP2p1s0f0", + hcaPort: 1, + address: "192.168.101.10", + prefixLength: 24, + peerAddress: "192.168.101.11", + roceGid: { index: 4, value: "::ffff:c0a8:650a" }, + }, + { + nodeId: "spark-worker", + netdev: "enP2p1s0f1np1", + hcaDevice: "roceP2p1s0f1", + hcaPort: 1, + address: "192.168.101.11", + prefixLength: 24, + peerAddress: "192.168.101.10", + roceGid: { index: 7, value: "::ffff:c0a8:650b" }, + }, + ], + }, + ], + masterAddress: "192.168.100.10", + peers: [ + { + nodeId: "spark-worker", + target: "spark-worker.local", + sshBindingHandle: "state/managed-cluster/peer", + }, + ], + }; + return { + outcome: "selected", + selection: "automatic", + catalogDigest: catalog.catalogDigest, + presetDigest: managedInferenceDigest(compiledPreset), + recipeDigest: managedInferenceDigest(compiledRecipe), + preset, + recipe, + topologyQualification: { + id: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + status: "qualified", + subjectNodeIds, + subjectDigest: managedClusterTopologySubjectDigest(subjectNodeIds), + outputDigest: managedClusterTopologyOutputDigest(output), + output, + }, + }; +} + +export function fixtureManagedClusterPlan(): ManagedClusterVllmPlan { + return materializeManagedClusterVllmPlan(fixtureManagedClusterSelection()); +} diff --git a/src/lib/inference/serving/managed-cluster-identifiers.ts b/src/lib/inference/serving/managed-cluster-identifiers.ts new file mode 100644 index 0000000000..ab1247e11a --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-identifiers.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Shared bounded identity contract for cluster nodes and hardware observations. */ +export const MANAGED_CLUSTER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; diff --git a/src/lib/inference/serving/managed-cluster-installer.test.ts b/src/lib/inference/serving/managed-cluster-installer.test.ts new file mode 100644 index 0000000000..a8a4004da8 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-installer.test.ts @@ -0,0 +1,769 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { + ManagedClusterConfirmedManagedServingCapability, + ManagedClusterDetectedManagedServingCapability, +} from "./managed-cluster-discovery.js"; +import type { + CreateManagedClusterVllmExecutorOptions, + ManagedClusterExecutorStageNode, +} from "./managed-cluster-executor.js"; +import { fixtureManagedClusterSelection } from "./managed-cluster-fixture.test-support.js"; +import { + type ManagedClusterInstallerEffects, + tryInstallManagedClusterManagedVllm, +} from "./managed-cluster-installer.js"; +import type { ManagedClusterVllmLifecycleDeps } from "./managed-cluster-lifecycle.js"; + +const API_KEY = "a".repeat(64); +const HEAD_ID = "b".repeat(64); +const WORKER_ID = "c".repeat(64); + +function readyCapability(): ManagedClusterDetectedManagedServingCapability { + const selection = fixtureManagedClusterSelection(); + const host = (nodeId: string, hostname: string, home: string, uid: number) => ({ + nodeId, + hostname, + home, + uid, + gid: uid, + runtimeSnapshot: { containers: [], listeningPorts: [] }, + storage: { + huggingFace: { + cacheRoot: `${home}/.cache/huggingface`, + filesystemId: `${hostname}-home`, + availableBytes: 400_000_000_000, + }, + docker: { + filesystemId: `${hostname}-docker`, + availableBytes: 400_000_000_000, + }, + }, + }); + return { + kind: "ready", + selectionIntent: "automatic", + topology: selection.topologyQualification, + local: host("spark-head", "spark-a", "/home/alice", 1000), + peers: [host("spark-worker", "spark-b", "/home/bob", 1001)], + readiness: [], + sshClaims: [ + { + nodeId: "spark-worker", + statePath: "/state/managed-cluster-managed-serving.json.spark-worker", + identity: { sshTarget: "spark-b" }, + }, + ], + } as unknown as ManagedClusterDetectedManagedServingCapability; +} + +function confirmedCapability( + detected: ManagedClusterDetectedManagedServingCapability, +): ManagedClusterConfirmedManagedServingCapability { + return { + ...detected, + sshBindings: [ + { + ...detected.sshClaims[0], + binding: { peerTarget: "spark-b" }, + handle: "binding", + }, + ], + } as unknown as ManagedClusterConfirmedManagedServingCapability; +} + +function effects(): ManagedClusterInstallerEffects { + return { + prerequisites: vi.fn(() => ({ ok: true })), + pullImage: vi.fn(async () => ({ ok: true })), + downloadModel: vi.fn(async () => ({ ok: true })), + printDownloadAuthentication: vi.fn(), + }; +} + +function successfulStart(reusedExisting = false) { + return { + ok: true as const, + reusedExisting, + baseUrl: "http://192.168.100.10:8000", + containers: [ + { nodeId: "spark-head", containerId: HEAD_ID }, + { nodeId: "spark-worker", containerId: WORKER_ID }, + ], + apiKeyFingerprint: "d".repeat(64), + }; +} + +describe("managed-cluster vLLM installer selection", () => { + beforeEach(() => vi.restoreAllMocks()); + + it("leaves non-Spark and conflict-free explicit legacy vLLM intent untouched", async () => { + const probeCapability = vi.fn(() => ({ + kind: "not-selected" as const, + code: "no-match" as const, + reason: "no related distributed runtime", + })); + await expect( + tryInstallManagedClusterManagedVllm( + { platform: "station", nonInteractive: true, promptFn: vi.fn() }, + effects(), + { probeCapability }, + ), + ).resolves.toEqual({ kind: "not-selected" }); + await expect( + tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: { NEMOCLAW_VLLM_MODEL: "nvidia/Qwen3.6-35B-A3B-NVFP4" }, + nonInteractive: true, + promptFn: vi.fn(), + }, + effects(), + { probeCapability }, + ), + ).resolves.toEqual({ kind: "not-selected" }); + expect(probeCapability).toHaveBeenCalledOnce(); + }); + + it("does not let explicit legacy intent bypass a related-runtime conflict", async () => { + const installEffects = effects(); + const result = await tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: { NEMOCLAW_VLLM_MODEL: "nvidia/Qwen3.6-35B-A3B-NVFP4" }, + nonInteractive: true, + promptFn: vi.fn(), + }, + installEffects, + { + probeCapability: () => ({ + kind: "not-selected", + code: "runtime-conflict", + reason: "existing related setup was preserved", + }), + error: vi.fn(), + }, + ); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + }); + + it("defers qualified explicit legacy intent without claiming binding state", async () => { + const capability = readyCapability(); + const clearBinding = vi.fn(); + const revalidateCapability = vi.fn(); + const claimCapability = vi.fn(); + const resolveSelection = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: { NEMOCLAW_VLLM_MODEL: "nvidia/Qwen3.6-35B-A3B-NVFP4" }, + nonInteractive: true, + promptFn: vi.fn(), + }, + effects(), + { + probeCapability: () => capability, + revalidateCapability, + claimCapability, + clearBinding, + resolveSelection, + }, + ); + expect(result).toEqual({ kind: "not-selected" }); + expect(resolveSelection).not.toHaveBeenCalled(); + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(clearBinding).not.toHaveBeenCalled(); + }); + + it("falls back only for an ordinary automatic no-match", async () => { + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => ({ + kind: "not-selected", + code: "no-match", + reason: "no exact cluster", + }), + }, + ); + expect(result).toEqual({ kind: "not-selected" }); + }); + + it("stops on durable distributed ownership before capability probing or effects", async () => { + const installEffects = effects(); + const probeCapability = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + installEffects, + { + assertNoRuntimeReceipts: () => { + throw new Error("managed runtime receipt already exists"); + }, + probeCapability, + error: vi.fn(), + }, + ); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(probeCapability).not.toHaveBeenCalled(); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + }); + + it("stops before effects when a related runtime is already present", async () => { + const installEffects = effects(); + const error = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + installEffects, + { + probeCapability: () => ({ + kind: "not-selected", + code: "runtime-conflict", + reason: "existing related setup was preserved", + }), + error, + }, + ); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(error).toHaveBeenCalledWith(expect.stringContaining("preserved")); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + }); + + it("admits the selected recipe port before prompting or claiming binding state", async () => { + const selection = fixtureManagedClusterSelection(); + const port = Number( + selection.recipe.spec.serve.arguments.find(({ name }) => name === "--port")?.value, + ); + const base = readyCapability(); + const capability = { + ...base, + local: { + ...base.local, + runtimeSnapshot: { ...base.local.runtimeSnapshot, listeningPorts: [port] }, + }, + } as ManagedClusterDetectedManagedServingCapability; + const promptFn = vi.fn(async () => "yes"); + const revalidateCapability = vi.fn(); + const claimCapability = vi.fn(); + const installEffects = effects(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn }, + installEffects, + { + probeCapability: () => capability, + resolveSelection: () => selection, + revalidateCapability, + claimCapability, + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(promptFn).not.toHaveBeenCalled(); + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(installEffects.prerequisites).not.toHaveBeenCalled(); + }); + + it("budgets the selected model and image at full size before prompting", async () => { + const selection = fixtureManagedClusterSelection(); + const base = readyCapability(); + const capability = { + ...base, + selectionIntent: "explicit", + local: { + ...base.local, + storage: { + ...base.local.storage, + huggingFace: { ...base.local.storage.huggingFace, availableBytes: 1 }, + }, + }, + } as ManagedClusterDetectedManagedServingCapability; + const promptFn = vi.fn(async () => "yes"); + const claimCapability = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn }, + effects(), + { + probeCapability: () => capability, + resolveSelection: () => selection, + claimCapability, + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(promptFn).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + }); + + it("rechecks the selected port after consent and before claiming binding state", async () => { + const capability = readyCapability(); + const selection = fixtureManagedClusterSelection(); + const port = Number( + selection.recipe.spec.serve.arguments.find(({ name }) => name === "--port")?.value, + ); + const revalidated = { + ...capability, + peers: capability.peers.map((peer) => ({ + ...peer, + runtimeSnapshot: { ...peer.runtimeSnapshot, listeningPorts: [port] }, + })), + } as ManagedClusterDetectedManagedServingCapability; + const claimCapability = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn: async () => "yes" }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => revalidated, + claimCapability, + resolveSelection: () => selection, + assertNoRuntimeReceipts: vi.fn(), + log: vi.fn(), + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(claimCapability).not.toHaveBeenCalled(); + }); + + it("does not fall through to legacy setup when storage changes after consent", async () => { + const capability = readyCapability(); + const revalidated = { + ...capability, + local: { + ...capability.local, + storage: { + ...capability.local.storage, + huggingFace: { ...capability.local.storage.huggingFace, availableBytes: 1 }, + }, + }, + } as ManagedClusterDetectedManagedServingCapability; + const claimCapability = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn: async () => "yes" }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => revalidated, + claimCapability, + resolveSelection: () => fixtureManagedClusterSelection(), + assertNoRuntimeReceipts: vi.fn(), + log: vi.fn(), + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(claimCapability).not.toHaveBeenCalled(); + }); + + it("does not fall through to legacy setup when selection changes after consent", async () => { + const capability = readyCapability(); + const resolveSelection = vi + .fn() + .mockReturnValueOnce(fixtureManagedClusterSelection()) + .mockReturnValueOnce({ + outcome: "no-match", + code: "requirements-not-met", + message: "selected requirements changed", + }); + const claimCapability = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn: async () => "yes" }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability, + resolveSelection, + assertNoRuntimeReceipts: vi.fn(), + log: vi.fn(), + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(resolveSelection).toHaveBeenCalledTimes(2); + expect(claimCapability).not.toHaveBeenCalled(); + }); + + it("revalidates only after consent and stops before effects when the cluster changed", async () => { + const capability = readyCapability(); + const installEffects = effects(); + const clearBinding = vi.fn(); + const revalidateCapability = vi.fn(() => ({ + kind: "unavailable" as const, + code: "runtime-conflict" as const, + reason: "a related listener appeared after confirmation", + })); + const claimCapability = vi.fn(); + const promptFn = vi.fn(async () => { + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(installEffects.prerequisites).not.toHaveBeenCalled(); + return "yes"; + }); + const assertNoRuntimeReceipts = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn }, + installEffects, + { + probeCapability: () => capability, + revalidateCapability, + claimCapability, + resolveSelection: () => fixtureManagedClusterSelection(), + assertNoRuntimeReceipts, + clearBinding, + log: vi.fn(), + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(assertNoRuntimeReceipts).toHaveBeenCalledTimes(2); + expect(revalidateCapability).toHaveBeenCalledOnce(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(installEffects.prerequisites).not.toHaveBeenCalled(); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + expect(clearBinding).not.toHaveBeenCalled(); + }); + + it("applies selected-model access preflight before prompting or effects", async () => { + const capability = readyCapability(); + const installEffects = effects(); + const promptFn = vi.fn(async () => "yes"); + const assertGatedModelAccess = vi.fn(() => { + throw new Error("selected model access is unavailable"); + }); + const revalidateCapability = vi.fn(); + const claimCapability = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn }, + installEffects, + { + probeCapability: () => capability, + resolveSelection: () => fixtureManagedClusterSelection(), + assertGatedModelAccess, + revalidateCapability, + claimCapability, + log: vi.fn(), + error: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(assertGatedModelAccess).toHaveBeenCalledOnce(); + expect(promptFn).not.toHaveBeenCalled(); + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(installEffects.prerequisites).not.toHaveBeenCalled(); + expect(installEffects.pullImage).not.toHaveBeenCalled(); + expect(installEffects.downloadModel).not.toHaveBeenCalled(); + }); + + it("stages both exact nodes, launches, persists ownership, and retires temporary binding state", async () => { + const capability = readyCapability(); + const confirmed = confirmedCapability(capability); + const selection = fixtureManagedClusterSelection(); + const installEffects = effects(); + const beforeInstall = vi.fn(); + const clearBinding = vi.fn(); + const persistReceipt = vi.fn(); + const stageCalls: string[] = []; + let capturedStage: ManagedClusterExecutorStageNode | undefined; + const executor = {} as ManagedClusterVllmLifecycleDeps; + const createExecutor = vi.fn((config: CreateManagedClusterVllmExecutorOptions) => { + capturedStage = config.stageNode; + return executor; + }); + const start = vi.fn(async (plan) => { + stageCalls.push(plan.roles[1].nodeId); + await capturedStage!( + { rolePlan: plan.roles[1], preparation: plan.roles[1].preparation }, + { + nodeId: plan.roles[1].nodeId, + dockerEnv: { DOCKER_HOST: "ssh://spark-b" }, + modelCacheRoot: capability.peers[0].storage.huggingFace.cacheRoot, + sshBinding: confirmed.sshBindings[0].binding, + }, + ); + stageCalls.push(plan.roles[0].nodeId); + await capturedStage!( + { rolePlan: plan.roles[0], preparation: plan.roles[0].preparation }, + { + nodeId: plan.roles[0].nodeId, + dockerEnv: {}, + modelCacheRoot: capability.local.storage.huggingFace.cacheRoot, + }, + ); + return successfulStart(); + }); + + const result = await tryInstallManagedClusterManagedVllm( + { + platform: "spark", + env: {}, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }, + installEffects, + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmed, + resolveSelection: () => selection, + createExecutor: createExecutor as never, + start: start as never, + ensureApiKey: () => API_KEY, + persistReceipt, + clearBinding, + log: vi.fn(), + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: true } }); + expect(capturedStage).toBeDefined(); + expect(stageCalls).toEqual(["spark-worker", "spark-head"]); + expect(beforeInstall).toHaveBeenCalledWith("deepseek-v4-flash-0731"); + expect(installEffects.pullImage).toHaveBeenCalledTimes(2); + expect(installEffects.downloadModel).toHaveBeenNthCalledWith( + 1, + expect.anything(), + expect.objectContaining({ revision: selection.recipe.spec.model.revision }), + { DOCKER_HOST: "ssh://spark-b" }, + { + hostCacheDir: capability.peers[0].storage.huggingFace.cacheRoot, + userIdentity: "1001:1001", + }, + ); + expect(start).toHaveBeenCalledWith(expect.anything(), API_KEY, executor); + expect(persistReceipt).toHaveBeenCalledWith( + expect.objectContaining({ + nodes: [ + expect.objectContaining({ + nodeId: "spark-head", + containerId: HEAD_ID, + cacheRoot: "/home/alice/.cache/huggingface", + }), + expect.objectContaining({ + nodeId: "spark-worker", + containerId: WORKER_ID, + cacheRoot: "/home/bob/.cache/huggingface", + }), + ], + }), + ); + expect(clearBinding).toHaveBeenCalledWith(capability.sshClaims[0].statePath); + }); + + it("keeps a successful receipt-owned install when temporary binding retirement fails", async () => { + const capability = readyCapability(); + const persistReceipt = vi.fn(); + const clearBinding = vi.fn(() => { + throw new Error("temporary binding busy"); + }); + const warn = vi.fn(); + + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmedCapability(capability), + resolveSelection: () => fixtureManagedClusterSelection(), + createExecutor: () => ({}) as ManagedClusterVllmLifecycleDeps, + start: async () => successfulStart(), + ensureApiKey: () => API_KEY, + persistReceipt, + clearBinding, + log: vi.fn(), + warn, + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: true } }); + expect(persistReceipt).toHaveBeenCalledOnce(); + expect(clearBinding).toHaveBeenCalledWith(capability.sshClaims[0].statePath); + expect(persistReceipt.mock.invocationCallOrder[0]).toBeLessThan( + clearBinding.mock.invocationCallOrder[0]!, + ); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("temporary managed cluster SSH state could not be retired"), + ); + }); + + it("cleans only a newly-created exact cluster when receipt persistence fails", async () => { + const capability = readyCapability(); + const selection = fixtureManagedClusterSelection(); + const cleanup = vi.fn(async () => ({ + ok: true as const, + removedContainerIds: [HEAD_ID, WORKER_ID], + })); + const clearBinding = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmedCapability(capability), + resolveSelection: () => selection, + createExecutor: () => ({}) as ManagedClusterVllmLifecycleDeps, + start: async () => successfulStart(false), + cleanup, + ensureApiKey: () => API_KEY, + persistReceipt: () => { + throw new Error("disk full"); + }, + clearBinding, + log: vi.fn(), + error: vi.fn(), + }, + ); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(cleanup).toHaveBeenCalledOnce(); + expect(clearBinding).toHaveBeenCalledWith(capability.sshClaims[0].statePath); + }); + + it("retains the claimed binding when receipt-failure rollback is incomplete", async () => { + const capability = readyCapability(); + const clearBinding = vi.fn(); + const warn = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmedCapability(capability), + resolveSelection: () => fixtureManagedClusterSelection(), + createExecutor: () => ({}) as ManagedClusterVllmLifecycleDeps, + start: async () => successfulStart(false), + cleanup: async () => ({ + ok: false, + reason: "worker cleanup failed", + removedContainerIds: [HEAD_ID], + }), + ensureApiKey: () => API_KEY, + persistReceipt: () => { + throw new Error("disk full"); + }, + clearBinding, + log: vi.fn(), + error: vi.fn(), + warn, + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(clearBinding).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("retained managed cluster SSH ownership"), + ); + }); + + it("retains the claimed binding when lifecycle rollback leaves a container", async () => { + const capability = readyCapability(); + const clearBinding = vi.fn(); + const warn = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmedCapability(capability), + resolveSelection: () => fixtureManagedClusterSelection(), + createExecutor: () => ({}) as ManagedClusterVllmLifecycleDeps, + start: async () => ({ + ok: false, + code: "start-failed", + reason: "head start failed", + rollbackErrors: ["worker cleanup failed"], + }), + ensureApiKey: () => API_KEY, + clearBinding, + log: vi.fn(), + error: vi.fn(), + warn, + }, + ); + + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(clearBinding).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("retained managed cluster SSH ownership"), + ); + }); + + it("does not remove an exact reused cluster when receipt persistence fails", async () => { + const capability = readyCapability(); + const cleanup = vi.fn(); + const clearBinding = vi.fn(); + await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: true, promptFn: vi.fn() }, + effects(), + { + probeCapability: () => capability, + revalidateCapability: () => capability, + claimCapability: () => confirmedCapability(capability), + resolveSelection: () => fixtureManagedClusterSelection(), + createExecutor: () => ({}) as ManagedClusterVllmLifecycleDeps, + start: async () => successfulStart(true), + cleanup, + ensureApiKey: () => API_KEY, + persistReceipt: () => { + throw new Error("receipt changed"); + }, + clearBinding, + log: vi.fn(), + error: vi.fn(), + }, + ); + expect(cleanup).not.toHaveBeenCalled(); + expect(clearBinding).not.toHaveBeenCalled(); + }); + + it("does not claim or clear binding state when the operator declines", async () => { + const capability = readyCapability(); + const clearBinding = vi.fn(); + const revalidateCapability = vi.fn(); + const claimCapability = vi.fn(); + const result = await tryInstallManagedClusterManagedVllm( + { platform: "spark", env: {}, nonInteractive: false, promptFn: async () => "no" }, + effects(), + { + probeCapability: () => capability, + revalidateCapability, + claimCapability, + resolveSelection: () => fixtureManagedClusterSelection(), + clearBinding, + log: vi.fn(), + }, + ); + expect(result).toEqual({ kind: "handled", result: { ok: false } }); + expect(revalidateCapability).not.toHaveBeenCalled(); + expect(claimCapability).not.toHaveBeenCalled(); + expect(clearBinding).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-installer.ts b/src/lib/inference/serving/managed-cluster-installer.ts new file mode 100644 index 0000000000..a49e3b40e6 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-installer.ts @@ -0,0 +1,641 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isAffirmativeAnswer } from "../../onboard/prompt-helpers.js"; +import type { VllmProfile } from "../vllm.js"; +import { ensureManagedVllmApiKey } from "../vllm-api-key.js"; +import { assertGatedModelAccess, VLLM_EXTRA_ARGS_ENV, type VllmModelDef } from "../vllm-models.js"; +import { imageStorageRequirementBytes, modelStorageRequirementBytes } from "../vllm-storage.js"; +import { + type ManagedInferenceResolution, + type ManagedInferenceResolverInput, + type ManagedInferenceServingRecipe, +} from "./types.js"; +import { + claimManagedClusterManagedServingCapability, + type ManagedClusterConfirmedManagedServingCapability, + type ManagedClusterDetectedManagedServingCapability, + type ManagedClusterHostObservation, + type ManagedClusterStorageCapacityObservation, + NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV, + NEMOCLAW_SERVING_PRESET_ENV, + probeManagedClusterManagedServingCapability, + revalidateManagedClusterManagedServingCapability, +} from "./managed-cluster-discovery.js"; +import { + type CreateManagedClusterVllmExecutorOptions, + createManagedClusterVllmExecutor, + type ManagedClusterExecutorStageNode, +} from "./managed-cluster-executor.js"; +import { + cleanupManagedClusterManagedVllm, + type StartManagedClusterVllmResult, + startAutomaticManagedClusterVllm, +} from "./managed-cluster-lifecycle.js"; +import { + type ManagedClusterVllmPlan, + managedClusterHeadRole, + materializeManagedClusterVllmPlan, +} from "./managed-cluster-materialize.js"; +import { + type PersistManagedClusterVllmRuntimeReceiptInput, + persistManagedClusterVllmRuntimeReceipt, +} from "./managed-cluster-runtime-receipt.js"; +import { clearManagedVllmSshBinding } from "./managed-cluster-ssh-binding.js"; +import type { ManagedClusterTopologyOutput } from "./managed-cluster-topology.js"; +import { assertNoManagedDistributedVllmRuntimeReceipts } from "./managed-runtime-receipts.js"; +import { resolveManagedInferenceServing } from "./resolver.js"; + +export interface ManagedClusterInstallerOptions { + readonly env?: NodeJS.ProcessEnv; + readonly nonInteractive: boolean; + readonly platform: VllmProfile["platform"]; + readonly promptFn: (question: string) => Promise; + readonly beforeInstall?: (modelId: string) => void; +} + +export interface ManagedClusterInstallerEffects { + readonly prerequisites: () => { ok: boolean; reason?: string }; + readonly pullImage: ( + profile: VllmProfile, + dockerEnv: Record, + ) => Promise<{ ok: boolean; reason?: string }>; + readonly downloadModel: ( + profile: VllmProfile, + model: VllmModelDef, + dockerEnv: Record, + target: { hostCacheDir: string; userIdentity: string }, + ) => Promise<{ ok: boolean; reason?: string }>; + readonly printDownloadAuthentication: (nonInteractive: boolean) => void; +} + +export type ManagedClusterInstallerResult = + | { readonly kind: "not-selected" } + | { readonly kind: "handled"; readonly result: { readonly ok: boolean } }; + +interface ManagedClusterInstallerDeps { + readonly probeCapability: typeof probeManagedClusterManagedServingCapability; + readonly revalidateCapability: typeof revalidateManagedClusterManagedServingCapability; + readonly claimCapability: typeof claimManagedClusterManagedServingCapability; + readonly resolveSelection: ( + input: ManagedInferenceResolverInput, + ) => ManagedInferenceResolution; + readonly materializePlan: typeof materializeManagedClusterVllmPlan; + readonly createExecutor: typeof createManagedClusterVllmExecutor; + readonly start: typeof startAutomaticManagedClusterVllm; + readonly cleanup: typeof cleanupManagedClusterManagedVllm; + readonly persistReceipt: typeof persistManagedClusterVllmRuntimeReceipt; + readonly ensureApiKey: typeof ensureManagedVllmApiKey; + readonly assertNoRuntimeReceipts: typeof assertNoManagedDistributedVllmRuntimeReceipts; + readonly clearBinding: typeof clearManagedVllmSshBinding; + readonly assertGatedModelAccess: typeof assertGatedModelAccess; + readonly log: (line?: string) => void; + readonly error: (line: string) => void; + readonly warn: (line: string) => void; +} + +const DEFAULT_DEPS: ManagedClusterInstallerDeps = { + probeCapability: probeManagedClusterManagedServingCapability, + revalidateCapability: revalidateManagedClusterManagedServingCapability, + claimCapability: claimManagedClusterManagedServingCapability, + resolveSelection: resolveManagedInferenceServing, + materializePlan: materializeManagedClusterVllmPlan, + createExecutor: createManagedClusterVllmExecutor, + start: startAutomaticManagedClusterVllm, + cleanup: cleanupManagedClusterManagedVllm, + persistReceipt: persistManagedClusterVllmRuntimeReceipt, + ensureApiKey: ensureManagedVllmApiKey, + assertNoRuntimeReceipts: assertNoManagedDistributedVllmRuntimeReceipts, + clearBinding: clearManagedVllmSshBinding, + assertGatedModelAccess, + log: (line = "") => console.log(line), + error: (line) => console.error(line), + warn: (line) => console.warn(line), +}; + +const VLLM_WRITABLE_ALLOWANCE_BYTES = 816_000_000n; + +type SelectedRecipeAdmissionFailure = { + readonly code: + | "runtime-conflict" + | "runtime-unknown" + | "storage-unavailable" + | "storage-insufficient"; + readonly reason: string; +}; + +function recipeApiPort(recipe: ManagedInferenceServingRecipe): number | null { + const ports = recipe.spec.serve.arguments + .filter(({ name }) => name === "--port") + .map(({ value }) => Number(value)); + return ports.length === 1 && + Number.isSafeInteger(ports[0]) && + ports[0]! > 0 && + ports[0]! <= 65_535 + ? ports[0]! + : null; +} + +function selectedHostStorageFailure( + host: ManagedClusterHostObservation, + label: string, + recipe: ManagedInferenceServingRecipe, +): SelectedRecipeAdmissionFailure | null { + const requirements = new Map(); + const available = new Map(); + const add = (capacity: ManagedClusterStorageCapacityObservation, required: bigint): boolean => { + if (capacity.filesystemId === null || capacity.availableBytes === null) return false; + requirements.set( + capacity.filesystemId, + (requirements.get(capacity.filesystemId) ?? 0n) + required, + ); + const bytes = BigInt(capacity.availableBytes); + const prior = available.get(capacity.filesystemId); + available.set(capacity.filesystemId, prior === undefined || bytes < prior ? bytes : prior); + return true; + }; + if ( + !add( + host.storage.huggingFace, + modelStorageRequirementBytes(recipe.spec.model.downloadSizeBytes) + + VLLM_WRITABLE_ALLOWANCE_BYTES, + ) || + !add( + host.storage.docker, + imageStorageRequirementBytes(recipe.spec.runtime.imageDownloadSizeBytes), + ) + ) { + return { + code: "storage-unavailable", + reason: `${label} cache or Docker filesystem capacity could not be proven.`, + }; + } + for (const [filesystemId, required] of requirements) { + if ((available.get(filesystemId) ?? -1n) < required) { + return { + code: "storage-insufficient", + reason: `${label} filesystem ${filesystemId} lacks capacity for the selected image, model, staging, and writable allowance.`, + }; + } + } + return null; +} + +function selectedRecipeAdmissionFailure( + capability: ManagedClusterDetectedManagedServingCapability, + recipe: ManagedInferenceServingRecipe, +): SelectedRecipeAdmissionFailure | null { + const apiPort = recipeApiPort(recipe); + if (apiPort === null) { + return { code: "runtime-unknown", reason: "The selected recipe serving port is invalid." }; + } + for (const [index, host] of [capability.local, ...capability.peers].entries()) { + const label = index === 0 ? "Local DGX Spark" : `Managed cluster peer ${String(index)}`; + const occupied = host.runtimeSnapshot.listeningPorts.find( + (port) => port === apiPort || port === recipe.spec.execution.rendezvousPort, + ); + if (occupied !== undefined) { + return { + code: "runtime-conflict", + reason: `${label} port ${String(occupied)} is already in use; its listener was not changed.`, + }; + } + const storage = selectedHostStorageFailure(host, label, recipe); + if (storage) return storage; + } + return null; +} + +function structuredArguments(recipe: ManagedInferenceServingRecipe): string[] { + return recipe.spec.serve.arguments.flatMap(({ name, value }) => + value === undefined ? [name] : [name, String(value)], + ); +} + +function requiredPositiveIntegerArgument( + recipe: ManagedInferenceServingRecipe, + name: string, +): number { + const matches = recipe.spec.serve.arguments.filter((argument) => argument.name === name); + const value = matches.length === 1 ? Number(matches[0]?.value) : Number.NaN; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`Selected serving recipe must define one positive ${name} argument.`); + } + return value; +} + +function managedProfile( + plan: ManagedClusterVllmPlan, + recipe: ManagedInferenceServingRecipe, +): VllmProfile { + const head = managedClusterHeadRole(plan); + return { + name: recipe.metadata.displayName ?? recipe.metadata.id, + platform: "spark", + image: head.image, + imageDownloadSizeBytes: head.runtime.imageDownloadSizeBytes, + defaultModel: managedModel(plan, recipe), + containerName: head.containerName, + dockerRunFlags: [], + pullTimeoutSec: recipe.spec.runtime.pullTimeoutSeconds, + loadTimeoutSec: Math.ceil(plan.readiness.timeoutMs / 1000), + modelDownloadSizeBytes: head.preparation.modelDownloadSizeBytes, + }; +} + +function managedModel( + plan: ManagedClusterVllmPlan, + recipe: ManagedInferenceServingRecipe, +): VllmModelDef { + return { + id: plan.model.id, + label: recipe.metadata.displayName ?? recipe.metadata.id, + envValue: plan.model.servedName, + downloadSizeBytes: managedClusterHeadRole(plan).preparation.modelDownloadSizeBytes, + maxModelLen: requiredPositiveIntegerArgument(recipe, "--max-model-len"), + revision: plan.model.revision, + servedModelId: plan.model.servedName, + modelArgs: structuredArguments(recipe), + gated: recipe.spec.model.gated, + platforms: ["spark"], + installFastSafetensors: recipe.spec.model.installFastSafetensors, + }; +} + +function selectionIntent(env: NodeJS.ProcessEnv) { + const configuredPreset = String(env[NEMOCLAW_SERVING_PRESET_ENV] ?? "").trim(); + const configuredModel = String(env.NEMOCLAW_VLLM_MODEL ?? "").trim(); + const extraArguments = String(env[VLLM_EXTRA_ARGS_ENV] ?? "").trim(); + return { + ...(configuredPreset ? { preset: configuredPreset } : {}), + ...(configuredModel ? { vllmModel: configuredModel } : {}), + ...(extraArguments ? { vllmExtraArguments: [extraArguments] } : {}), + }; +} + +function automaticIntentDefersToLegacy(env: NodeJS.ProcessEnv): boolean { + const clusterIntent = + String(env[NEMOCLAW_MANAGED_CLUSTER_PEERS_ENV] ?? "").trim() || + String(env[NEMOCLAW_SERVING_PRESET_ENV] ?? "").trim(); + if (clusterIntent) return false; + return Boolean( + String(env.NEMOCLAW_VLLM_MODEL ?? "").trim() || String(env[VLLM_EXTRA_ARGS_ENV] ?? "").trim(), + ); +} + +function printSummary( + capability: ManagedClusterDetectedManagedServingCapability, + plan: ManagedClusterVllmPlan, + deps: ManagedClusterInstallerDeps, +): void { + const rails = capability.topology.output.rails + .map(({ endpoints }) => + endpoints + .map(({ address, prefixLength }) => `${address}/${String(prefixLength)}`) + .join(" to "), + ) + .join(", "); + const head = managedClusterHeadRole(plan); + deps.log(); + deps.log(` vLLM (${String(plan.roles.length)}-node DGX Spark cluster, experimental):`); + deps.log(` Serving profile: ${plan.presetId}`); + deps.log(` Recipe: ${plan.recipeId}`); + deps.log(` Image: ${head.image}`); + deps.log(` Model: ${plan.model.id}@${plan.model.revision}`); + deps.log(` Served model: ${plan.model.servedName}`); + deps.log( + ` Topology: ${[capability.local, ...capability.peers] + .map(({ hostname }) => hostname) + .join(" + ")}`, + ); + deps.log(` Direct rails: ${rails}`); + deps.log( + ` RoCEv2 GIDs: ${capability.topology.output.rails + .map(({ endpoints }) => endpoints.map(({ roceGid }) => String(roceGid.index)).join("/")) + .join(", ")}`, + ); + deps.log( + ` Model caches: ${[capability.local, ...capability.peers] + .map(({ storage }) => storage.huggingFace.cacheRoot) + .join(", ")}`, + ); + deps.log(" Launch order: descending worker ranks, then head"); + deps.log(" Restart policy: none; a stopped cluster requires explicit cleanup"); + deps.log(" Experimental: physical end-to-end validation is pending"); +} + +function resolutionFailure( + resolution: Exclude< + ManagedInferenceResolution, + { outcome: "selected" } + >, + selectionIntent: ManagedClusterDetectedManagedServingCapability["selectionIntent"], + allowAutomaticFallback: boolean, + deps: ManagedClusterInstallerDeps, +): ManagedClusterInstallerResult { + if ( + allowAutomaticFallback && + selectionIntent === "automatic" && + resolution.outcome === "no-match" + ) { + return { kind: "not-selected" }; + } + deps.error(` Managed-cluster vLLM setup unavailable: ${resolution.message}`); + return { kind: "handled", result: { ok: false } }; +} + +function admissionFailure( + failure: SelectedRecipeAdmissionFailure, + selectionIntent: ManagedClusterDetectedManagedServingCapability["selectionIntent"], + allowAutomaticFallback: boolean, + deps: ManagedClusterInstallerDeps, +): ManagedClusterInstallerResult { + if ( + allowAutomaticFallback && + selectionIntent === "automatic" && + (failure.code === "storage-unavailable" || failure.code === "storage-insufficient") + ) { + return { kind: "not-selected" }; + } + deps.error(` Managed-cluster vLLM setup stopped: ${failure.reason}`); + return { kind: "handled", result: { ok: false } }; +} + +function receiptInput( + capability: ManagedClusterConfirmedManagedServingCapability, + plan: ManagedClusterVllmPlan, + started: Extract, +): PersistManagedClusterVllmRuntimeReceiptInput { + const hosts = new Map( + [capability.local, ...capability.peers].map((host) => [host.nodeId, host] as const), + ); + return { + plan, + apiKeyFingerprint: started.apiKeyFingerprint, + nodes: plan.roles.map((rolePlan) => { + const host = hosts.get(rolePlan.nodeId)!; + const owned = started.containers.find(({ nodeId }) => nodeId === rolePlan.nodeId)!; + const transport = capability.sshBindings.find(({ nodeId }) => nodeId === rolePlan.nodeId); + return { + nodeId: rolePlan.nodeId, + cacheRoot: host.storage.huggingFace.cacheRoot, + containerId: owned.containerId, + ...(transport + ? { sshBinding: transport.binding, discoveryStatePath: transport.statePath } + : {}), + }; + }), + }; +} + +/** Select and run the automatic managed cluster profile without changing the legacy DGX Spark path. */ +export async function tryInstallManagedClusterManagedVllm( + options: ManagedClusterInstallerOptions, + effects: ManagedClusterInstallerEffects, + overrides: Partial = {}, +): Promise { + if (options.platform !== "spark") return { kind: "not-selected" }; + const env = options.env ?? process.env; + const deferToLegacy = automaticIntentDefersToLegacy(env); + + const deps = { ...DEFAULT_DEPS, ...overrides }; + try { + deps.assertNoRuntimeReceipts(); + } catch (error) { + deps.error(` Managed vLLM setup stopped: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + + const detected = deps.probeCapability({ env }); + if (detected.kind === "not-selected" && detected.code === "no-match") { + return { kind: "not-selected" }; + } + if (detected.kind !== "ready") { + deps.error(` Managed-cluster vLLM setup stopped: ${detected.reason}`); + return { kind: "handled", result: { ok: false } }; + } + + let confirmedBinding: ManagedClusterConfirmedManagedServingCapability | null = null; + let retainBinding = false; + try { + // Explicit legacy model/argument intent keeps the single-host DGX Spark path, but + // only after read-only discovery has proved that doing so will not overlap + // a related distributed runtime or ambiguous binding. + if (deferToLegacy) return { kind: "not-selected" }; + + const previewResolution = deps.resolveSelection({ + readinessReports: detected.readiness, + topologyQualifications: [detected.topology], + intent: selectionIntent(env), + }); + if (previewResolution.outcome !== "selected") { + return resolutionFailure(previewResolution, detected.selectionIntent, true, deps); + } + const previewAdmission = selectedRecipeAdmissionFailure(detected, previewResolution.recipe); + if (previewAdmission) { + return admissionFailure(previewAdmission, detected.selectionIntent, true, deps); + } + + let previewPlan: ManagedClusterVllmPlan; + try { + previewPlan = deps.materializePlan(previewResolution); + } catch (error) { + deps.error(` Managed-cluster vLLM setup stopped: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + try { + deps.assertGatedModelAccess(managedModel(previewPlan, previewResolution.recipe), env); + } catch (error) { + deps.error(` Managed-cluster vLLM setup stopped: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + printSummary(detected, previewPlan, deps); + effects.printDownloadAuthentication(options.nonInteractive); + deps.log(); + + const proceed = + options.nonInteractive || isAffirmativeAnswer(await options.promptFn(" Continue? [y/N]: ")); + if (!proceed) return { kind: "handled", result: { ok: false } }; + + try { + deps.assertNoRuntimeReceipts(); + } catch (error) { + deps.error(` Managed vLLM setup stopped: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + + const revalidated = deps.revalidateCapability(detected, { env }); + if (revalidated.kind !== "ready") { + deps.error(` Managed-cluster vLLM setup stopped: ${revalidated.reason}`); + return { kind: "handled", result: { ok: false } }; + } + + const revalidatedResolution = deps.resolveSelection({ + readinessReports: revalidated.readiness, + topologyQualifications: [revalidated.topology], + intent: selectionIntent(env), + }); + if (revalidatedResolution.outcome !== "selected") { + return resolutionFailure(revalidatedResolution, revalidated.selectionIntent, false, deps); + } + const revalidatedAdmission = selectedRecipeAdmissionFailure( + revalidated, + revalidatedResolution.recipe, + ); + if (revalidatedAdmission) { + return admissionFailure(revalidatedAdmission, revalidated.selectionIntent, false, deps); + } + if ( + revalidatedResolution.presetDigest !== previewResolution.presetDigest || + revalidatedResolution.recipeDigest !== previewResolution.recipeDigest + ) { + deps.error(" Managed-cluster vLLM setup stopped: the selected profile changed."); + return { kind: "handled", result: { ok: false } }; + } + + const confirmation = deps.claimCapability(revalidated); + if (confirmation.kind !== "ready") { + deps.error(` Managed-cluster vLLM setup stopped: ${confirmation.reason}`); + return { kind: "handled", result: { ok: false } }; + } + confirmedBinding = confirmation; + const resolution = { + ...revalidatedResolution, + topologyQualification: confirmation.topology, + }; + + let plan: ManagedClusterVllmPlan; + try { + plan = deps.materializePlan(resolution); + } catch (error) { + deps.error(` Managed-cluster vLLM setup stopped: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + if ( + plan.presetId !== previewPlan.presetId || + plan.recipeId !== previewPlan.recipeId || + plan.model.id !== previewPlan.model.id || + plan.model.revision !== previewPlan.model.revision || + plan.model.servedName !== previewPlan.model.servedName || + plan.roles.length !== previewPlan.roles.length || + plan.roles.some((role, index) => role.image !== previewPlan.roles[index]?.image) + ) { + deps.error(" Managed-cluster vLLM setup stopped: the presented serving plan changed."); + return { kind: "handled", result: { ok: false } }; + } + + const profile = managedProfile(plan, resolution.recipe); + const model = managedModel(plan, resolution.recipe); + options.beforeInstall?.(plan.model.servedName); + + const prerequisites = effects.prerequisites(); + if (!prerequisites.ok) { + deps.error(` vLLM install failed: ${prerequisites.reason ?? "prerequisites unavailable"}`); + return { kind: "handled", result: { ok: false } }; + } + + let apiKey: string; + try { + apiKey = deps.ensureApiKey(); + } catch (error) { + deps.error(` vLLM install failed: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } + + const hosts = new Map( + [confirmation.local, ...confirmation.peers].map((host) => [host.nodeId, host] as const), + ); + const stageNode: ManagedClusterExecutorStageNode = async (_request, target) => { + const host = hosts.get(target.nodeId)!; + deps.log(` ==> Staging pinned vLLM image and model on ${host.hostname}`); + const pull = await effects.pullImage(profile, { ...target.dockerEnv }); + if (!pull.ok) return pull; + return await effects.downloadModel( + profile, + model, + { ...target.dockerEnv }, + { + hostCacheDir: target.modelCacheRoot, + userIdentity: `${String(host.uid)}:${String(host.gid)}`, + }, + ); + }; + const executorOptions: CreateManagedClusterVllmExecutorOptions = { + plan, + nodes: plan.roles.map((rolePlan) => { + const host = hosts.get(rolePlan.nodeId)!; + const binding = confirmation.sshBindings.find(({ nodeId }) => nodeId === rolePlan.nodeId); + return { + nodeId: rolePlan.nodeId, + modelCacheRoot: host.storage.huggingFace.cacheRoot, + ...(binding ? { sshBinding: binding.binding } : {}), + }; + }), + stageNode, + }; + const executor = deps.createExecutor(executorOptions); + retainBinding = true; + const started = await deps.start(plan, apiKey, executor); + if (!started.ok) { + retainBinding = started.rollbackErrors.length > 0; + deps.error(` vLLM install failed: ${started.reason}`); + for (const warning of started.rollbackErrors) + deps.warn(` vLLM rollback warning: ${warning}`); + return { kind: "handled", result: { ok: false } }; + } + + try { + deps.persistReceipt(receiptInput(confirmation, plan, started)); + retainBinding = false; + } catch (error) { + if (!started.reusedExisting) { + const cleanup = await deps.cleanup(plan, apiKey, executor); + const expected = new Set(started.containers.map(({ containerId }) => containerId)); + if ( + cleanup.ok && + cleanup.removedContainerIds.length === expected.size && + new Set(cleanup.removedContainerIds).size === expected.size && + cleanup.removedContainerIds.every((id) => expected.has(id)) + ) { + retainBinding = false; + } else { + deps.warn( + ` vLLM rollback warning: ${cleanup.ok ? "exact cluster cleanup was incomplete" : cleanup.reason}`, + ); + } + } + deps.error( + ` vLLM install failed: could not persist managed cluster cleanup ownership: ${(error as Error).message}`, + ); + return { kind: "handled", result: { ok: false } }; + } + + deps.log( + ` ✓ vLLM ready across ${String(plan.roles.length)} DGX Spark systems at ${started.baseUrl}`, + ); + return { kind: "handled", result: { ok: true } }; + } catch (error) { + deps.error(` Managed-cluster vLLM setup failed closed: ${(error as Error).message}`); + return { kind: "handled", result: { ok: false } }; + } finally { + if (confirmedBinding && retainBinding) { + deps.warn( + ` vLLM rollback warning: retained managed cluster SSH ownership state at ${confirmedBinding.sshBindings + .map(({ statePath }) => statePath) + .join( + ", ", + )} because exact container rollback is incomplete. Resolve the related runtime state before retrying setup or uninstall.`, + ); + } else if (confirmedBinding) { + for (const { statePath } of confirmedBinding.sshBindings) { + try { + deps.clearBinding(statePath); + } catch (error) { + deps.warn( + ` vLLM cleanup warning: temporary managed cluster SSH state could not be retired: ${(error as Error).message}`, + ); + } + } + } + } +} diff --git a/src/lib/inference/serving/managed-cluster-lifecycle.test.ts b/src/lib/inference/serving/managed-cluster-lifecycle.test.ts new file mode 100644 index 0000000000..94e0d69486 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-lifecycle.test.ts @@ -0,0 +1,502 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { fixtureManagedClusterPlan } from "./managed-cluster-fixture.test-support.js"; +import { + classifyManagedClusterExistingState, + cleanupManagedClusterManagedVllm, + type ManagedClusterContainerStartRequest, + type ManagedClusterNodeSnapshot, + type ManagedClusterObservedContainer, + type ManagedClusterVllmLifecycleDeps, + managedClusterVllmApiKeyFingerprint, + startAutomaticManagedClusterVllm, +} from "./managed-cluster-lifecycle.js"; +import { + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_MANAGED_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + type ManagedClusterVllmPlan, + type ManagedClusterVllmRole, + type ManagedClusterVllmRolePlan, +} from "./managed-cluster-materialize.js"; + +const API_KEY = "a".repeat(64); +const TRANSACTION_ID = "b".repeat(32); +const HEAD_ID = "1".repeat(64); +const WORKER_ID = "2".repeat(64); + +type StoppedForeignContainerFixture = { + readonly signal: string; + readonly name: string; + readonly image: string; + readonly labels: Readonly>; +}; + +const STOPPED_FOREIGN_CONTAINER_FIXTURES: readonly StoppedForeignContainerFixture[] = [ + { + signal: "name", + name: "foreign-vllm-server", + image: "example.invalid/inference:latest", + labels: {}, + }, + { + signal: "image", + name: "foreign-inference", + image: "vllm/vllm-openai:latest", + labels: {}, + }, + { + signal: "managed label", + name: "foreign-inference", + image: "example.invalid/inference:latest", + labels: { [MANAGED_CLUSTER_MANAGED_LABEL]: "foreign" }, + }, +]; + +type Harness = ReturnType; + +function managedContainer( + plan: ManagedClusterVllmPlan, + role: ManagedClusterVllmRole, + overrides: Partial = {}, +): ManagedClusterObservedContainer { + const rolePlan = plan.roles.find((candidate) => candidate.role === role)!; + return { + id: role === "head" ? HEAD_ID : WORKER_ID, + name: rolePlan.containerName, + image: rolePlan.image, + running: true, + healthy: true, + labels: { + ...rolePlan.baseLabels, + [MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL]: managedClusterVllmApiKeyFingerprint(API_KEY), + [MANAGED_CLUSTER_TRANSACTION_LABEL]: TRANSACTION_ID, + }, + ...overrides, + }; +} + +function classifiedSnapshots( + plan: ManagedClusterVllmPlan, + snapshots: Record, +) { + return plan.roles.map((rolePlan) => ({ + nodeId: rolePlan.nodeId, + snapshot: snapshots[rolePlan.nodeId], + })); +} + +function createHarness(plan: ManagedClusterVllmPlan) { + const events: string[] = []; + const snapshots: Record = Object.fromEntries( + plan.roles.map(({ nodeId }) => [nodeId, { containers: [], listeningPorts: [] }]), + ); + const inspectNode = vi.fn(async (rolePlan: ManagedClusterVllmRolePlan) => { + events.push(`inspect:${rolePlan.role}`); + return snapshots[rolePlan.nodeId]; + }); + const stageNode = vi.fn(async ({ rolePlan }: { rolePlan: ManagedClusterVllmRolePlan }) => { + events.push(`stage:${rolePlan.role}`); + return { ok: true }; + }); + const startContainer = vi.fn(async (request: ManagedClusterContainerStartRequest) => { + const { rolePlan, labels } = request; + events.push(`start:${rolePlan.role}`); + const id = rolePlan.role === "head" ? HEAD_ID : WORKER_ID; + snapshots[rolePlan.nodeId] = { + ...snapshots[rolePlan.nodeId], + containers: [ + { + id, + name: rolePlan.containerName, + image: rolePlan.image, + running: true, + healthy: true, + labels, + }, + ], + }; + return { ok: true, containerId: id }; + }); + const waitForContainerReady = vi.fn(async (request) => { + events.push(`wait:${request.rolePlan.role}`); + return true; + }); + const waitForWorkerDistributedReady = vi.fn(async (request) => { + events.push(`distributed:${request.rolePlan.role}`); + return true; + }); + const removeContainer = vi.fn(async (rolePlan: ManagedClusterVllmRolePlan, id: string) => { + events.push(`remove:${rolePlan.role}:${id}`); + snapshots[rolePlan.nodeId] = { + ...snapshots[rolePlan.nodeId], + containers: snapshots[rolePlan.nodeId]!.containers.filter((container) => container.id !== id), + }; + return { ok: true }; + }); + const probeModels = vi.fn(async () => { + events.push("probe:models"); + return true; + }); + const probeChat = vi.fn(async () => { + events.push("probe:chat"); + return true; + }); + const deps: ManagedClusterVllmLifecycleDeps = { + inspectNode, + stageNode, + startContainer, + waitForContainerReady, + waitForWorkerDistributedReady, + removeContainer, + probeModels, + probeChat, + createTransactionId: () => TRANSACTION_ID, + withLifecycleLock: async (_plan, operation) => await operation(), + }; + return { + deps, + events, + snapshots, + inspectNode, + stageNode, + startContainer, + removeContainer, + probeModels, + probeChat, + }; +} + +describe("automatic managed-cluster vLLM lifecycle", () => { + let plan: ManagedClusterVllmPlan; + let harness: Harness; + + beforeEach(() => { + plan = fixtureManagedClusterPlan(); + harness = createHarness(plan); + }); + + it("inspects both nodes before staging either node", async () => { + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result.ok).toBe(true); + const firstStage = harness.events.findIndex((event) => event.startsWith("stage:")); + expect(harness.events.slice(0, firstStage)).toEqual( + expect.arrayContaining(["inspect:head", "inspect:worker"]), + ); + }); + + it("preserves singleton, Station, and related external setups", async () => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [ + { + id: "9".repeat(64), + name: "nemoclaw-vllm", + image: "vllm/vllm-openai:latest", + running: false, + healthy: false, + labels: { "com.nvidia.nemoclaw.managed-vllm": "true" }, + }, + ], + listeningPorts: [], + }; + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ ok: false, code: "conflict" }); + expect(harness.stageNode).not.toHaveBeenCalled(); + expect(harness.startContainer).not.toHaveBeenCalled(); + expect(harness.removeContainer).not.toHaveBeenCalled(); + }); + + it.each( + STOPPED_FOREIGN_CONTAINER_FIXTURES, + )("preserves a stopped foreign vLLM setup identified by $signal", async (container) => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [ + { + id: "9".repeat(64), + name: container.name, + image: container.image, + running: false, + healthy: false, + labels: container.labels, + }, + ], + listeningPorts: [], + }; + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ ok: false, code: "conflict" }); + expect(harness.stageNode).not.toHaveBeenCalled(); + expect(harness.startContainer).not.toHaveBeenCalled(); + expect(harness.removeContainer).not.toHaveBeenCalled(); + }); + + it("does not classify an arbitrary stopped container as a managed vLLM setup", () => { + const snapshots = { + [plan.roles[0].nodeId]: { + containers: [ + { + id: "9".repeat(64), + name: "unrelated-service", + image: "example.invalid/worker:latest", + running: false, + healthy: false, + labels: { "example.foreign": "true" }, + }, + ], + listeningPorts: [], + }, + [plan.roles[1].nodeId]: { containers: [], listeningPorts: [] }, + }; + + expect( + classifyManagedClusterExistingState( + plan, + managedClusterVllmApiKeyFingerprint(API_KEY), + classifiedSnapshots(plan, snapshots), + ), + ).toEqual({ outcome: "clear" }); + }); + + it("reuses only one exact healthy cluster with the same transaction", async () => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [managedContainer(plan, "head")], + listeningPorts: [8000], + }; + harness.snapshots[plan.roles[1].nodeId] = { + containers: [managedContainer(plan, "worker")], + listeningPorts: [25000], + }; + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ + ok: true, + reusedExisting: true, + containers: [ + { nodeId: plan.roles[0].nodeId, containerId: HEAD_ID }, + { nodeId: plan.roles[1].nodeId, containerId: WORKER_ID }, + ], + }); + expect(harness.probeModels).toHaveBeenCalledOnce(); + expect(harness.probeChat).toHaveBeenCalledOnce(); + expect(harness.stageNode).not.toHaveBeenCalled(); + expect(harness.startContainer).not.toHaveBeenCalled(); + }); + + it("does not implicitly repair a stopped or partial managed deployment", async () => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [managedContainer(plan, "head", { running: false, healthy: false })], + listeningPorts: [], + }; + harness.snapshots[plan.roles[1].nodeId] = { + containers: [managedContainer(plan, "worker")], + listeningPorts: [], + }; + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ ok: false, code: "conflict" }); + expect(harness.startContainer).not.toHaveBeenCalled(); + expect(harness.removeContainer).not.toHaveBeenCalled(); + }); + + it("starts and prepares rank 1 before rank 0 without exposing its API key", async () => { + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result.ok).toBe(true); + expect(harness.events.indexOf("start:worker")).toBeLessThan( + harness.events.indexOf("distributed:worker"), + ); + expect(harness.events.indexOf("distributed:worker")).toBeLessThan( + harness.events.indexOf("start:head"), + ); + const workerRequest = harness.startContainer.mock.calls[0]![0]; + const headRequest = harness.startContainer.mock.calls[1]![0]; + expect(workerRequest).not.toHaveProperty("bearerApiKey"); + expect(headRequest.bearerApiKey).toBe(API_KEY); + expect(workerRequest.preparation.phase).toBe("container-before-exec"); + expect(headRequest.preparation.phase).toBe("container-before-exec"); + expect(JSON.stringify(workerRequest.labels)).not.toContain(API_KEY); + expect(JSON.stringify(headRequest.labels)).not.toContain(API_KEY); + }); + + it("retains SSH ownership when a failed worker create leaves runtime state", async () => { + harness.startContainer.mockImplementation(async ({ rolePlan, labels }) => { + harness.snapshots[plan.roles[1].nodeId] = { + containers: [ + { + id: WORKER_ID, + name: rolePlan.containerName, + image: rolePlan.image, + running: true, + healthy: true, + labels, + }, + ], + listeningPorts: [], + }; + throw new Error("Docker create outcome was ambiguous"); + }); + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ ok: false, code: "start-failed" }); + const failed = result as Extract; + expect(failed.rollbackErrors).toContain( + "managed cluster post-failure runtime state could not be proven clear; SSH ownership state was retained", + ); + expect(harness.removeContainer).not.toHaveBeenCalled(); + }); + + it("does not retain SSH ownership when a failed create is proven mutation-free", async () => { + harness.startContainer.mockImplementation(async () => { + throw new Error("Docker create failed before mutation"); + }); + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ + ok: false, + code: "start-failed", + rollbackErrors: [], + }); + }); + + it("rolls back only exact transaction-created IDs after API failure", async () => { + harness.probeChat.mockImplementation(async () => { + harness.events.push("probe:chat"); + return false; + }); + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ + ok: false, + code: "health-failed", + rollbackErrors: [], + }); + expect(harness.removeContainer.mock.calls.map((call) => call[1])).toEqual([HEAD_ID, WORKER_ID]); + expect(harness.events).toContain("probe:models"); + expect(harness.events).toContain("probe:chat"); + }); + + it("leaves a container untouched when transaction ownership changes before rollback", async () => { + harness.probeChat.mockImplementation(async () => { + const worker = harness.snapshots[plan.roles[1].nodeId]!.containers[0]!; + harness.snapshots[plan.roles[1].nodeId] = { + ...harness.snapshots[plan.roles[1].nodeId], + containers: [ + { + ...worker, + labels: { + ...worker.labels, + [MANAGED_CLUSTER_TRANSACTION_LABEL]: "c".repeat(32), + }, + }, + ], + }; + return false; + }); + + const result = await startAutomaticManagedClusterVllm(plan, API_KEY, harness.deps); + + expect(result).toMatchObject({ ok: false, code: "health-failed" }); + const failed = result as Extract; + expect(failed.rollbackErrors).toContain( + "worker rollback ownership changed; container was left untouched", + ); + expect(harness.removeContainer.mock.calls.map((call) => call[1])).toEqual([HEAD_ID]); + expect(harness.snapshots[plan.roles[1].nodeId]!.containers).toHaveLength(1); + }); + + it("classifies port conflicts on either node as nonselectable", () => { + expect( + classifyManagedClusterExistingState( + plan, + managedClusterVllmApiKeyFingerprint(API_KEY), + classifiedSnapshots(plan, { + [plan.roles[0].nodeId]: { containers: [], listeningPorts: [] }, + [plan.roles[1].nodeId]: { containers: [], listeningPorts: [25000] }, + }), + ), + ).toEqual({ + outcome: "conflict", + reason: `${plan.roles[1].nodeId} port 25000 is already in use`, + }); + }); + + it("cleans up only a complete exact cluster and retains all cache state", async () => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [managedContainer(plan, "head")], + listeningPorts: [], + }; + harness.snapshots[plan.roles[1].nodeId] = { + containers: [managedContainer(plan, "worker")], + listeningPorts: [], + }; + + const result = await cleanupManagedClusterManagedVllm(plan, API_KEY, harness.deps); + + expect(result).toEqual({ + ok: true, + removedContainerIds: [HEAD_ID, WORKER_ID], + }); + expect(harness.stageNode).not.toHaveBeenCalled(); + }); + + it("retries cleanup after one receipt-owned container was already removed", async () => { + harness.snapshots[plan.roles[0].nodeId] = { + containers: [managedContainer(plan, "head")], + listeningPorts: [], + }; + harness.snapshots[plan.roles[1].nodeId] = { + containers: [managedContainer(plan, "worker")], + listeningPorts: [], + }; + let workerAttempts = 0; + harness.removeContainer.mockImplementation(async (rolePlan, id) => { + const shouldFailWorker = rolePlan.role === "worker" && workerAttempts === 0; + workerAttempts += Number(rolePlan.role === "worker"); + const removeOwnedContainer = () => { + harness.snapshots[rolePlan.nodeId] = { + ...harness.snapshots[rolePlan.nodeId], + containers: harness.snapshots[rolePlan.nodeId]!.containers.filter( + (container) => container.id !== id, + ), + }; + return { ok: true } as const; + }; + return shouldFailWorker + ? ({ ok: false, reason: "worker daemon unavailable" } as const) + : removeOwnedContainer(); + }); + const ownership = { + containers: [ + { nodeId: plan.roles[0].nodeId, containerId: HEAD_ID }, + { nodeId: plan.roles[1].nodeId, containerId: WORKER_ID }, + ], + }; + + await expect( + cleanupManagedClusterManagedVllm(plan, API_KEY, harness.deps, ownership), + ).resolves.toEqual({ + ok: false, + reason: "worker daemon unavailable", + removedContainerIds: [HEAD_ID], + }); + await expect( + cleanupManagedClusterManagedVllm(plan, API_KEY, harness.deps, ownership), + ).resolves.toEqual({ + ok: true, + removedContainerIds: [WORKER_ID], + alreadyAbsentContainerIds: [HEAD_ID], + }); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-lifecycle.ts b/src/lib/inference/serving/managed-cluster-lifecycle.ts new file mode 100644 index 0000000000..15013e92ae --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-lifecycle.ts @@ -0,0 +1,868 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHmac } from "node:crypto"; + +import { + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_MANAGED_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + MANAGED_CLUSTER_VLLM_PROJECT_ID, + type ManagedClusterVllmPlan, + type ManagedClusterVllmRolePlan, + managedClusterHeadRole, +} from "./managed-cluster-materialize.js"; + +const API_KEY_PATTERN = /^[a-f0-9]{64}$/; +const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/; +const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; +const FINGERPRINT_CONTEXT = "nemoclaw-managed-cluster-vllm-api-key\0"; +const STATION_CONTAINER_NAMES = new Set(["nemoclaw-vllm", "nemoclaw-vllm-worker"]); +const STATION_LABEL_PREFIX = "com.nvidia.nemoclaw.vllm-"; +const COMPOSE_PROJECT_LABEL = "com.docker.compose.project"; +const COMPOSE_SERVICE_LABEL = "com.docker.compose.service"; +const VLLM_TOKEN_PATTERN = /(?:^|[./:_-])vllm(?:$|[./:@_-])/i; + +export interface ManagedClusterObservedContainer { + readonly id: string; + readonly name: string; + readonly image: string; + readonly running: boolean; + /** True only after the executor's bounded role-specific readiness check. */ + readonly healthy: boolean; + readonly labels: Readonly>; +} + +export interface ManagedClusterNodeSnapshot { + /** All containers visible to the node daemon, including stopped containers. */ + readonly containers: readonly ManagedClusterObservedContainer[]; + /** Host-network listeners. The executor must inspect all requested ports. */ + readonly listeningPorts: readonly number[]; +} + +export type ManagedClusterExistingState = + | { readonly outcome: "clear" } + | { + readonly outcome: "reuse"; + readonly containers: readonly ManagedClusterOwnedContainer[]; + readonly transactionId: string; + } + | { readonly outcome: "conflict"; readonly reason: string } + | { readonly outcome: "unknown"; readonly reason: string }; + +export interface ManagedClusterStageRequest { + readonly rolePlan: ManagedClusterVllmRolePlan; + /** Verify or fetch only the pinned model snapshot and immutable image. */ + readonly preparation: ManagedClusterVllmRolePlan["preparation"]; +} + +export interface ManagedClusterContainerStartRequest { + readonly rolePlan: ManagedClusterVllmRolePlan; + readonly labels: Readonly>; + /** + * The executor performs this code-owned preparation inside the newly + * created container, then directly execs the role command. Copy/replace + * operations must match exactly or creation fails before vLLM starts. + */ + readonly preparation: ManagedClusterVllmRolePlan["preparation"]; + /** Present only for the head. The executor must not persist it in labels or the plan. */ + readonly bearerApiKey?: string; +} + +export interface ManagedClusterContainerStartResult { + readonly ok: boolean; + readonly containerId?: string; + readonly reason?: string; +} + +export interface ManagedClusterContainerWaitRequest { + readonly rolePlan: ManagedClusterVllmRolePlan; + readonly containerId: string; + readonly expectedLabels: Readonly>; + readonly timeoutMs: number; +} + +export interface ManagedClusterApiProbeRequest { + readonly baseUrl: string; + readonly apiKey: string; + readonly expectedModel: string; + readonly timeoutMs: number; +} + +/** + * Production integration supplies one executor that resolves the opaque SSH + * binding in each worker role plan and performs Docker operations using argv, + * never a caller-built command string. + */ +export interface ManagedClusterVllmLifecycleDeps { + inspectNode(rolePlan: ManagedClusterVllmRolePlan): Promise; + stageNode(request: ManagedClusterStageRequest): Promise<{ ok: boolean; reason?: string }>; + startContainer( + request: ManagedClusterContainerStartRequest, + ): Promise; + waitForContainerReady(request: ManagedClusterContainerWaitRequest): Promise; + /** Prove a worker process is alive and waiting at the distributed rendezvous. */ + waitForWorkerDistributedReady(request: ManagedClusterContainerWaitRequest): Promise; + removeContainer( + rolePlan: ManagedClusterVllmRolePlan, + exactContainerId: string, + ): Promise<{ ok: boolean; reason?: string }>; + probeModels(request: ManagedClusterApiProbeRequest): Promise; + probeChat(request: ManagedClusterApiProbeRequest): Promise; + createTransactionId(): string; + withLifecycleLock(plan: ManagedClusterVllmPlan, operation: () => Promise): Promise; +} + +export interface ManagedClusterRuntimeInspection { + readonly state: ManagedClusterExistingState; + readonly snapshots?: readonly ManagedClusterRoleSnapshot[]; +} + +export type CleanupManagedClusterVllmResult = + | { + readonly ok: true; + readonly removedContainerIds: readonly string[]; + readonly alreadyAbsentContainerIds?: readonly string[]; + } + | { + readonly ok: false; + readonly reason: string; + readonly removedContainerIds: readonly string[]; + }; + +export interface ManagedClusterCleanupOwnership { + readonly containers: readonly ManagedClusterOwnedContainer[]; +} + +export interface ManagedClusterOwnedContainer { + readonly nodeId: string; + readonly containerId: string; +} + +export interface ManagedClusterRoleSnapshot { + readonly nodeId: string; + readonly snapshot: ManagedClusterNodeSnapshot; +} + +export type StartManagedClusterVllmResult = + | { + readonly ok: true; + readonly reusedExisting: boolean; + readonly baseUrl: string; + readonly containers: readonly ManagedClusterOwnedContainer[]; + readonly apiKeyFingerprint: string; + } + | { + readonly ok: false; + readonly code: "conflict" | "unknown" | "staging-failed" | "start-failed" | "health-failed"; + readonly reason: string; + readonly rollbackErrors: readonly string[]; + }; + +interface RoleObservation { + rolePlan: ManagedClusterVllmRolePlan; + container: ManagedClusterObservedContainer; + expectedLabels: Readonly>; +} + +interface CreatedContainer { + rolePlan: ManagedClusterVllmRolePlan; + containerId: string; + expectedLabels: Readonly>; +} + +function labelsMatch( + actual: Readonly>, + expected: Readonly>, +): boolean { + return Object.entries(expected).every(([key, value]) => actual[key] === value); +} + +function exactRoleObservation( + snapshot: ManagedClusterNodeSnapshot, + rolePlan: ManagedClusterVllmRolePlan, + apiKeyFingerprint: string, +): RoleObservation | null { + const matches = snapshot.containers.filter(({ name }) => name === rolePlan.containerName); + if (matches.length !== 1) return null; + const container = matches[0]!; + const transactionId = container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL] ?? ""; + const expectedLabels = { + ...rolePlan.baseLabels, + [MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL]: apiKeyFingerprint, + [MANAGED_CLUSTER_TRANSACTION_LABEL]: transactionId, + }; + if ( + !CONTAINER_ID_PATTERN.test(container.id) || + !TRANSACTION_ID_PATTERN.test(transactionId) || + container.image !== rolePlan.image || + !labelsMatch(container.labels, expectedLabels) + ) { + return null; + } + return { rolePlan, container, expectedLabels }; +} + +/** Recognize only containers that declare or visibly identify a vLLM runtime. */ +export function isRelatedManagedVllmContainer(container: ManagedClusterObservedContainer): boolean { + if (STATION_CONTAINER_NAMES.has(container.name)) return true; + if (VLLM_TOKEN_PATTERN.test(container.name) || VLLM_TOKEN_PATTERN.test(container.image)) { + return true; + } + if (Object.hasOwn(container.labels, MANAGED_CLUSTER_MANAGED_LABEL)) return true; + if (container.labels[COMPOSE_PROJECT_LABEL] === MANAGED_CLUSTER_VLLM_PROJECT_ID) return true; + if (container.labels[COMPOSE_SERVICE_LABEL] === "vllm-cluster") return true; + return Object.keys(container.labels).some((key) => key.startsWith(STATION_LABEL_PREFIX)); +} + +function invalidSnapshotReason(snapshot: ManagedClusterNodeSnapshot, node: string): string | null { + if ( + snapshot.listeningPorts.some((port) => !Number.isInteger(port) || port < 1 || port > 65_535) + ) { + return `${node} listener inspection is malformed`; + } + const ids = snapshot.containers.map(({ id }) => id); + if (ids.some((id) => !CONTAINER_ID_PATTERN.test(id)) || new Set(ids).size !== ids.length) { + return `${node} container inspection is malformed or ambiguous`; + } + return null; +} + +/** Pure, fail-closed classification shared by discovery and lifecycle preflight. */ +export function classifyManagedClusterExistingState( + plan: ManagedClusterVllmPlan, + apiKeyFingerprint: string, + snapshots: readonly ManagedClusterRoleSnapshot[], +): ManagedClusterExistingState { + if (!/^[a-f0-9]{64}$/.test(apiKeyFingerprint)) { + return { outcome: "unknown", reason: "managed cluster API key fingerprint is invalid" }; + } + if ( + snapshots.length !== plan.roles.length || + new Set(snapshots.map(({ nodeId }) => nodeId)).size !== snapshots.length || + snapshots.some(({ nodeId }) => !plan.roles.some((role) => role.nodeId === nodeId)) + ) { + return { outcome: "unknown", reason: "managed cluster node inspection is incomplete" }; + } + for (const { nodeId, snapshot } of snapshots) { + const invalid = invalidSnapshotReason(snapshot, nodeId); + if (invalid) return { outcome: "unknown", reason: invalid }; + } + + const observations = plan.roles.map((rolePlan) => { + const snapshot = snapshots.find(({ nodeId }) => nodeId === rolePlan.nodeId)!.snapshot; + return exactRoleObservation(snapshot, rolePlan, apiKeyFingerprint); + }); + const allContainers = snapshots.flatMap(({ snapshot }) => snapshot.containers); + const exactIds = new Set(observations.flatMap((entry) => (entry ? [entry.container.id] : []))); + const unexpectedRelated = allContainers.find( + (container) => isRelatedManagedVllmContainer(container) && !exactIds.has(container.id), + ); + if (unexpectedRelated) { + return { + outcome: "conflict", + reason: `existing related container ${unexpectedRelated.name} is not part of the exact managed cluster`, + }; + } + + if (observations.some(Boolean)) { + if (observations.some((observation) => !observation)) { + return { outcome: "conflict", reason: "managed cluster deployment is incomplete" }; + } + const exact = observations as RoleObservation[]; + const transactions = new Set( + exact.map(({ container }) => container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL]), + ); + if (transactions.size !== 1) { + return { outcome: "conflict", reason: "managed cluster transaction labels do not match" }; + } + if (exact.some(({ container }) => !container.running || !container.healthy)) { + return { + outcome: "conflict", + reason: "managed cluster deployment is stopped, incomplete, or unhealthy", + }; + } + return { + outcome: "reuse", + containers: exact.map(({ rolePlan, container }) => ({ + nodeId: rolePlan.nodeId, + containerId: container.id, + })), + transactionId: exact[0]!.container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL]!, + }; + } + + const dedicatedNames = new Set(plan.roles.map(({ containerName }) => containerName)); + const dedicatedNameExists = allContainers.some(({ name }) => dedicatedNames.has(name)); + if (dedicatedNameExists) { + return { outcome: "conflict", reason: "managed cluster container name ownership is foreign" }; + } + for (const { nodeId, snapshot } of snapshots) { + const occupied = snapshot.listeningPorts.find( + (port) => port === plan.apiPort || port === plan.masterPort, + ); + if (occupied !== undefined) { + return { + outcome: "conflict", + reason: `${nodeId} port ${String(occupied)} is already in use`, + }; + } + } + return { outcome: "clear" }; +} + +/** Domain-separated non-secret ownership binding for the managed endpoint key. */ +export function managedClusterVllmApiKeyFingerprint(apiKey: string): string { + if (!API_KEY_PATTERN.test(apiKey)) { + throw new Error("Managed cluster vLLM API key must be 64 lowercase hexadecimal characters."); + } + return createHmac("sha256", Buffer.from(apiKey, "hex")).update(FINGERPRINT_CONTEXT).digest("hex"); +} + +async function inspectAll( + plan: ManagedClusterVllmPlan, + deps: Pick, +): Promise { + try { + return await Promise.all( + plan.roles.map(async (rolePlan) => ({ + nodeId: rolePlan.nodeId, + snapshot: await deps.inspectNode(rolePlan), + })), + ); + } catch { + return null; + } +} + +/** Inspect every daemon before any image, cache, or container mutation. */ +export async function preflightManagedClusterVllm( + plan: ManagedClusterVllmPlan, + apiKey: string, + deps: Pick, +): Promise { + let fingerprint: string; + try { + fingerprint = managedClusterVllmApiKeyFingerprint(apiKey); + } catch (error) { + return { outcome: "unknown", reason: (error as Error).message }; + } + const snapshots = await inspectAll(plan, deps); + return snapshots + ? classifyManagedClusterExistingState(plan, fingerprint, snapshots) + : { outcome: "unknown", reason: "could not inspect every managed cluster container daemon" }; +} + +/** Plan- and key-specific read-only inspection for installer/runtime recovery. */ +export async function inspectManagedClusterManagedRuntime( + plan: ManagedClusterVllmPlan, + apiKey: string, + deps: Pick, +): Promise { + let fingerprint: string; + try { + fingerprint = managedClusterVllmApiKeyFingerprint(apiKey); + } catch (error) { + return { state: { outcome: "unknown", reason: (error as Error).message } }; + } + const snapshots = await inspectAll(plan, deps); + return snapshots + ? { state: classifyManagedClusterExistingState(plan, fingerprint, snapshots), snapshots } + : { + state: { + outcome: "unknown", + reason: "could not inspect every managed cluster container daemon", + }, + }; +} + +function labelsForStart( + rolePlan: ManagedClusterVllmRolePlan, + apiKeyFingerprint: string, + transactionId: string, +): Readonly> { + return { + ...rolePlan.baseLabels, + [MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL]: apiKeyFingerprint, + [MANAGED_CLUSTER_TRANSACTION_LABEL]: transactionId, + }; +} + +function exactCreatedContainer( + snapshot: ManagedClusterNodeSnapshot, + created: CreatedContainer, +): ManagedClusterObservedContainer | null { + const matches = snapshot.containers.filter(({ id }) => id === created.containerId); + if (matches.length !== 1) return null; + const container = matches[0]!; + return container.name === created.rolePlan.containerName && + container.image === created.rolePlan.image && + labelsMatch(container.labels, created.expectedLabels) + ? container + : null; +} + +async function rollbackCreated( + created: readonly CreatedContainer[], + deps: ManagedClusterVllmLifecycleDeps, +): Promise { + const errors: string[] = []; + for (const item of [...created].reverse()) { + let snapshot: ManagedClusterNodeSnapshot; + try { + snapshot = await deps.inspectNode(item.rolePlan); + } catch { + errors.push(`${item.rolePlan.role} rollback ownership could not be inspected`); + continue; + } + if (!exactCreatedContainer(snapshot, item)) { + errors.push(`${item.rolePlan.role} rollback ownership changed; container was left untouched`); + continue; + } + try { + const removed = await deps.removeContainer(item.rolePlan, item.containerId); + if (!removed.ok) { + errors.push(removed.reason ?? `${item.rolePlan.role} rollback removal failed`); + } + } catch { + errors.push(`${item.rolePlan.role} rollback removal failed`); + } + } + return errors; +} + +async function rollbackCreatedAndProveClear( + plan: ManagedClusterVllmPlan, + apiKeyFingerprint: string, + created: readonly CreatedContainer[], + deps: ManagedClusterVllmLifecycleDeps, +): Promise { + const rollbackErrors = await rollbackCreated(created, deps); + const snapshots = await inspectAll(plan, deps); + if ( + snapshots && + classifyManagedClusterExistingState(plan, apiKeyFingerprint, snapshots).outcome === "clear" + ) { + return []; + } + return [ + ...rollbackErrors, + "managed cluster post-failure runtime state could not be proven clear; SSH ownership state was retained", + ]; +} + +async function startRole( + rolePlan: ManagedClusterVllmRolePlan, + apiKey: string, + apiKeyFingerprint: string, + transactionId: string, + timeoutMs: number, + deps: ManagedClusterVllmLifecycleDeps, +): Promise< + | { ok: true; created: CreatedContainer } + | { ok: false; reason: string; created?: CreatedContainer } +> { + const expectedLabels = labelsForStart(rolePlan, apiKeyFingerprint, transactionId); + let started: ManagedClusterContainerStartResult; + try { + started = await deps.startContainer({ + rolePlan, + labels: expectedLabels, + preparation: rolePlan.preparation, + ...(rolePlan.role === "head" ? { bearerApiKey: apiKey } : {}), + }); + } catch { + return { ok: false, reason: `${rolePlan.role} container start failed` }; + } + const containerId = started.containerId ?? ""; + if (!CONTAINER_ID_PATTERN.test(containerId)) { + return { + ok: false, + reason: started.reason ?? `${rolePlan.role} container start returned no exact container ID`, + }; + } + const created = { rolePlan, containerId, expectedLabels }; + if (!started.ok) { + return { + ok: false, + reason: started.reason ?? `${rolePlan.role} container start failed`, + created, + }; + } + let ready = false; + try { + const waitRequest = { + rolePlan, + containerId, + expectedLabels, + timeoutMs, + }; + ready = + rolePlan.role === "worker" + ? await deps.waitForWorkerDistributedReady(waitRequest) + : await deps.waitForContainerReady(waitRequest); + } catch { + ready = false; + } + if (!ready) + return { ok: false, reason: `${rolePlan.role} container did not become ready`, created }; + + let snapshot: ManagedClusterNodeSnapshot; + try { + snapshot = await deps.inspectNode(rolePlan); + } catch { + return { + ok: false, + reason: `${rolePlan.role} container ownership could not be revalidated`, + created, + }; + } + const observed = exactCreatedContainer(snapshot, created); + if (!observed?.running || !observed.healthy) { + return { ok: false, reason: `${rolePlan.role} container ownership or health changed`, created }; + } + return { ok: true, created }; +} + +function failure( + code: Extract["code"], + reason: string, + rollbackErrors: readonly string[] = [], +): StartManagedClusterVllmResult { + return { ok: false, code, reason, rollbackErrors }; +} + +async function probeManagedApi( + plan: ManagedClusterVllmPlan, + apiKey: string, + deps: Pick, +): Promise { + const baseUrl = managedClusterHeadRole(plan).endpoint; + if (!baseUrl) return false; + const request = { + baseUrl, + apiKey, + expectedModel: plan.readiness.expectedModel, + timeoutMs: plan.readiness.timeoutMs, + }; + try { + if (!(await deps.probeModels(request))) return false; + return await deps.probeChat({ ...request, timeoutMs: Math.min(request.timeoutMs, 120_000) }); + } catch { + return false; + } +} + +async function startNewCluster( + plan: ManagedClusterVllmPlan, + apiKey: string, + apiKeyFingerprint: string, + deps: ManagedClusterVllmLifecycleDeps, +): Promise { + const staged = await Promise.all( + [...plan.roles] + .sort((left, right) => right.rank - left.rank) + .map(async (rolePlan) => { + try { + return await deps.stageNode({ rolePlan, preparation: rolePlan.preparation }); + } catch { + return { ok: false, reason: `${rolePlan.role} staging failed` }; + } + }), + ); + const failedStage = staged.find((result) => !result.ok); + if (failedStage) { + return failure("staging-failed", failedStage.reason ?? "managed cluster staging failed"); + } + + const afterStageSnapshots = await inspectAll(plan, deps); + if (!afterStageSnapshots) { + return failure("unknown", "could not re-inspect every daemon after staging"); + } + const afterStage = classifyManagedClusterExistingState( + plan, + apiKeyFingerprint, + afterStageSnapshots, + ); + if (afterStage.outcome !== "clear") { + return failure( + afterStage.outcome === "unknown" ? "unknown" : "conflict", + `managed cluster ownership changed during staging: ${ + "reason" in afterStage ? afterStage.reason : afterStage.outcome + }`, + ); + } + + const transactionId = deps.createTransactionId(); + if (!TRANSACTION_ID_PATTERN.test(transactionId)) { + return failure("unknown", "managed cluster lifecycle transaction ID is invalid"); + } + + const created: CreatedContainer[] = []; + for (const rolePlan of [...plan.roles].sort((left, right) => right.rank - left.rank)) { + const result = await startRole( + rolePlan, + apiKey, + apiKeyFingerprint, + transactionId, + plan.readiness.timeoutMs, + deps, + ); + if (!result.ok) { + if (result.created) created.push(result.created); + return failure( + "start-failed", + result.reason, + await rollbackCreatedAndProveClear(plan, apiKeyFingerprint, created, deps), + ); + } + created.push(result.created); + } + + if (!(await probeManagedApi(plan, apiKey, deps))) { + return failure( + "health-failed", + "managed cluster models or chat health check failed", + await rollbackCreatedAndProveClear(plan, apiKeyFingerprint, created, deps), + ); + } + + const finalSnapshots = await inspectAll(plan, deps); + const finalState = finalSnapshots + ? classifyManagedClusterExistingState(plan, apiKeyFingerprint, finalSnapshots) + : null; + if ( + !finalState || + finalState.outcome !== "reuse" || + finalState.transactionId !== transactionId || + finalState.containers.length !== created.length || + created.some( + ({ rolePlan, containerId }) => + !finalState.containers.some( + (owned) => owned.nodeId === rolePlan.nodeId && owned.containerId === containerId, + ), + ) + ) { + return failure( + "health-failed", + "managed cluster ownership changed before lifecycle commit", + await rollbackCreatedAndProveClear(plan, apiKeyFingerprint, created, deps), + ); + } + return { + ok: true, + reusedExisting: false, + baseUrl: managedClusterHeadRole(plan).endpoint!, + containers: created.map(({ rolePlan, containerId }) => ({ + nodeId: rolePlan.nodeId, + containerId, + })), + apiKeyFingerprint, + }; +} + +/** + * Automatic lifecycle: exact healthy reuse or clean worker-first creation. + * Stopped, partial, mismatched, Station, singleton, and foreign deployments + * are never repaired or replaced. + */ +export async function startAutomaticManagedClusterVllm( + plan: ManagedClusterVllmPlan, + apiKey: string, + deps: ManagedClusterVllmLifecycleDeps, +): Promise { + let apiKeyFingerprint: string; + try { + apiKeyFingerprint = managedClusterVllmApiKeyFingerprint(apiKey); + } catch (error) { + return failure("unknown", (error as Error).message); + } + + try { + return await deps.withLifecycleLock(plan, async () => { + const preflight = await preflightManagedClusterVllm(plan, apiKey, deps); + if (preflight.outcome === "unknown" || preflight.outcome === "conflict") { + return failure(preflight.outcome, preflight.reason); + } + if (preflight.outcome === "reuse") { + if (!(await probeManagedApi(plan, apiKey, deps))) { + return failure( + "conflict", + "existing managed cluster API is unhealthy; no repair attempted", + ); + } + return { + ok: true, + reusedExisting: true, + baseUrl: managedClusterHeadRole(plan).endpoint!, + containers: preflight.containers, + apiKeyFingerprint, + }; + } + return await startNewCluster(plan, apiKey, apiKeyFingerprint, deps); + }); + } catch (error) { + return failure("unknown", `managed cluster lifecycle failed: ${(error as Error).message}`); + } +} + +function exactClusterForCleanup( + plan: ManagedClusterVllmPlan, + apiKeyFingerprint: string, + snapshots: readonly ManagedClusterRoleSnapshot[], +): readonly RoleObservation[] | null { + const observations = plan.roles.map((rolePlan) => { + const snapshot = snapshots.find(({ nodeId }) => nodeId === rolePlan.nodeId)?.snapshot; + return snapshot ? exactRoleObservation(snapshot, rolePlan, apiKeyFingerprint) : null; + }); + if (observations.some((observation) => !observation)) return null; + const exact = observations as RoleObservation[]; + const transactions = new Set( + exact.map(({ container }) => container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL]), + ); + if (transactions.size !== 1 || !exact[0]?.container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL]) { + return null; + } + const exactIds = new Set(exact.map(({ container }) => container.id)); + const related = snapshots + .flatMap(({ snapshot }) => snapshot.containers) + .find((container) => isRelatedManagedVllmContainer(container) && !exactIds.has(container.id)); + return related ? null : exact; +} + +function receiptOwnedTargetsForCleanup( + plan: ManagedClusterVllmPlan, + apiKeyFingerprint: string, + snapshots: readonly ManagedClusterRoleSnapshot[], + ownership: ManagedClusterCleanupOwnership, +): + | { + readonly ok: true; + readonly observations: readonly RoleObservation[]; + readonly alreadyAbsentContainerIds: readonly string[]; + } + | { readonly ok: false; readonly reason: string } { + const expectedIds = ownership.containers.map(({ containerId }) => containerId); + if ( + ownership.containers.length !== plan.roles.length || + expectedIds.some((id) => !CONTAINER_ID_PATTERN.test(id)) || + new Set(expectedIds).size !== expectedIds.length || + new Set(ownership.containers.map(({ nodeId }) => nodeId)).size !== + ownership.containers.length || + ownership.containers.some(({ nodeId }) => !plan.roles.some((role) => role.nodeId === nodeId)) + ) { + return { ok: false, reason: "managed cluster cleanup receipt identities are invalid" }; + } + + const observations: RoleObservation[] = []; + const alreadyAbsentContainerIds: string[] = []; + let transactionId: string | null = null; + for (const rolePlan of plan.roles) { + const snapshot = snapshots.find(({ nodeId }) => nodeId === rolePlan.nodeId)?.snapshot; + const expectedId = ownership.containers.find( + ({ nodeId }) => nodeId === rolePlan.nodeId, + )?.containerId; + if (!snapshot || !expectedId) { + return { ok: false, reason: "managed cluster cleanup receipt is incomplete" }; + } + const invalid = invalidSnapshotReason(snapshot, rolePlan.nodeId); + if (invalid) return { ok: false, reason: invalid }; + const related = snapshot.containers.filter(isRelatedManagedVllmContainer); + const expected = snapshot.containers.find(({ id }) => id === expectedId); + if (!expected) { + if (related.length > 0) { + return { + ok: false, + reason: `${rolePlan.nodeId} receipt-owned container is absent but related runtime state exists`, + }; + } + alreadyAbsentContainerIds.push(expectedId); + continue; + } + const observation = exactRoleObservation(snapshot, rolePlan, apiKeyFingerprint); + if ( + !observation || + observation.container.id !== expectedId || + related.some(({ id }) => id !== expectedId) + ) { + return { ok: false, reason: `${rolePlan.nodeId} receipt-owned container identity changed` }; + } + const observedTransaction = observation.container.labels[MANAGED_CLUSTER_TRANSACTION_LABEL]!; + if (transactionId !== null && transactionId !== observedTransaction) { + return { ok: false, reason: "managed cluster receipt-owned transaction identity changed" }; + } + transactionId = observedTransaction; + observations.push(observation); + } + return { ok: true, observations, alreadyAbsentContainerIds }; +} + +/** Remove only a complete, plan/key/transaction-owned cluster. Model caches remain. */ +export async function cleanupManagedClusterManagedVllm( + plan: ManagedClusterVllmPlan, + apiKey: string, + deps: Pick< + ManagedClusterVllmLifecycleDeps, + "inspectNode" | "removeContainer" | "withLifecycleLock" + >, + ownership?: ManagedClusterCleanupOwnership, +): Promise { + let fingerprint: string; + try { + fingerprint = managedClusterVllmApiKeyFingerprint(apiKey); + } catch (error) { + return { ok: false, reason: (error as Error).message, removedContainerIds: [] }; + } + try { + return await deps.withLifecycleLock(plan, async () => { + const snapshots = await inspectAll(plan, deps); + if (!snapshots) { + return { + ok: false, + reason: "could not inspect every managed cluster container daemon", + removedContainerIds: [], + }; + } + const owned = ownership + ? receiptOwnedTargetsForCleanup(plan, fingerprint, snapshots, ownership) + : null; + if (owned && !owned.ok) { + return { ok: false, reason: owned.reason, removedContainerIds: [] }; + } + const cluster = ownership ? null : exactClusterForCleanup(plan, fingerprint, snapshots); + if (!ownership && !cluster) { + return { + ok: false, + reason: "managed cluster cleanup requires one complete exact owned cluster", + removedContainerIds: [], + }; + } + const observations = owned?.ok ? owned.observations : cluster!; + const removedContainerIds: string[] = []; + for (const observation of observations) { + const removed = await deps.removeContainer(observation.rolePlan, observation.container.id); + if (!removed.ok) { + return { + ok: false, + reason: removed.reason ?? `${observation.rolePlan.role} cleanup failed`, + removedContainerIds, + }; + } + removedContainerIds.push(observation.container.id); + } + return { + ok: true, + removedContainerIds, + ...(owned?.ok && owned.alreadyAbsentContainerIds.length > 0 + ? { alreadyAbsentContainerIds: owned.alreadyAbsentContainerIds } + : {}), + }; + }); + } catch (error) { + return { + ok: false, + reason: `managed cluster cleanup failed: ${(error as Error).message}`, + removedContainerIds: [], + }; + } +} diff --git a/src/lib/inference/serving/managed-cluster-materialize.test.ts b/src/lib/inference/serving/managed-cluster-materialize.test.ts new file mode 100644 index 0000000000..8e4c81b9ec --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-materialize.test.ts @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + NO_PREPARATION_REF, + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, +} from "./adapter-registry.js"; +import { managedInferenceDigest } from "./catalog-integrity.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { fixtureManagedClusterSelection } from "./managed-cluster-fixture.test-support.js"; +import { + MANAGED_CLUSTER_PRESET_LABEL, + MANAGED_CLUSTER_RECIPE_LABEL, + materializeManagedClusterVllmPlan, +} from "./managed-cluster-materialize.js"; +import { + type ManagedClusterTopologyOutput, + managedClusterTopologyOutputDigest, + managedClusterTopologySubjectDigest, +} from "./managed-cluster-topology.js"; +import type { + CompiledManagedInferenceCatalog, + ManagedInferenceServingPreset, + ManagedInferenceServingRecipe, + ResolvedManagedInferenceSelection, +} from "./types.js"; + +interface SyntheticProfile { + readonly catalog: CompiledManagedInferenceCatalog; + readonly selection: ResolvedManagedInferenceSelection; +} + +function selectionWithDigests(): ResolvedManagedInferenceSelection { + const selection = fixtureManagedClusterSelection(); + return { + ...selection, + presetDigest: managedInferenceDigest(selection.preset), + recipeDigest: managedInferenceDigest(selection.recipe), + }; +} + +function threeNodeTopology(): ResolvedManagedInferenceSelection["topologyQualification"] { + const endpoint = (nodeId: string, peerAddress: string, address: string, index: number) => ({ + nodeId, + netdev: `eth${String(index)}`, + hcaDevice: `roce${String(index)}`, + hcaPort: 1, + address, + prefixLength: 30, + peerAddress, + roceGid: { index: 3 + index, value: `fe80::${address.replaceAll(".", ":")}` }, + }); + const output: ManagedClusterTopologyOutput = { + controllerNodeId: "spark-head", + nodes: [ + { nodeId: "spark-head", gpuId: "GPU-head", rank: 0, role: "head" }, + { nodeId: "spark-worker-a", gpuId: "GPU-worker-a", rank: 1, role: "worker" }, + { nodeId: "spark-worker-b", gpuId: "GPU-worker-b", rank: 2, role: "worker" }, + ], + rails: [ + { + index: 0, + endpoints: [ + endpoint("spark-head", "192.168.100.2", "192.168.100.1", 0), + endpoint("spark-worker-a", "192.168.100.1", "192.168.100.2", 0), + ], + }, + { + index: 1, + endpoints: [ + endpoint("spark-worker-a", "192.168.101.2", "192.168.101.1", 1), + endpoint("spark-worker-b", "192.168.101.1", "192.168.101.2", 0), + ], + }, + { + index: 2, + endpoints: [ + endpoint("spark-head", "192.168.102.2", "192.168.102.1", 1), + endpoint("spark-worker-b", "192.168.102.1", "192.168.102.2", 1), + ], + }, + ], + masterAddress: "192.168.100.1", + peers: [ + { nodeId: "spark-worker-a", target: "worker-a.local", sshBindingHandle: "binding-a" }, + { nodeId: "spark-worker-b", target: "worker-b.local", sshBindingHandle: "binding-b" }, + ], + }; + const subjectNodeIds = output.nodes.map(({ nodeId }) => nodeId); + return { + ...fixtureManagedClusterSelection().topologyQualification, + subjectNodeIds, + subjectDigest: managedClusterTopologySubjectDigest(subjectNodeIds), + output, + outputDigest: managedClusterTopologyOutputDigest(output), + }; +} + +function syntheticSecondProfile( + temporaryFilesystemTarget = "/dev/shm-scratch", + nodeCount = 2, +): SyntheticProfile { + const baseCatalog = loadManagedInferenceCatalog(); + const baseSelection = selectionWithDigests(); + const topology = nodeCount === 3 ? threeNodeTopology() : baseSelection.topologyQualification; + const basePreset = baseCatalog.presets.find( + ({ metadata }) => metadata.id === baseSelection.preset.metadata.id, + ); + const baseRecipe = baseCatalog.recipes.find( + ({ metadata }) => metadata.id === baseSelection.recipe.metadata.id, + ); + expect(basePreset).toBeDefined(); + expect(baseRecipe).toBeDefined(); + const presetTemplate = basePreset as ManagedInferenceServingPreset; + const recipeTemplate = baseRecipe as ManagedInferenceServingRecipe; + + const recipe: ManagedInferenceServingRecipe = { + ...recipeTemplate, + metadata: { + id: "vllm.synthetic-model.managed-cluster.v1", + displayName: "Synthetic model on a compatible cluster", + }, + spec: { + ...recipeTemplate.spec, + model: { + ...recipeTemplate.spec.model, + id: "example-org/Synthetic-Model", + revision: "b".repeat(40), + servedName: "synthetic-model", + downloadSizeBytes: 1_234_567, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "assets/synthetic_tokenizer.py", + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/synthetic-vllm/tokenizers/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/synthetic-vllm/parsers/reasoning.py", + expectedText: "MODE = 'legacy'", + replacementText: "MODE = 'compatible'", + }, + }, + }, + runtime: { + ...recipeTemplate.spec.runtime, + image: `registry.example.test/inference/vllm@sha256:${"c".repeat(64)}`, + imageDownloadSizeBytes: 7_654_321, + pullTimeoutSeconds: 7_200, + sharedMemoryBytes: 8_589_934_592, + gpuRequest: "device=all", + devices: ["/dev/infiniband", "/dev/synthetic"], + ulimits: { memlock: "unlimited", stackBytes: 33_554_432 }, + modelCache: { source: "huggingface-cache", target: "/models/cache" }, + temporaryFilesystems: [ + { + target: temporaryFilesystemTarget, + sizeBytes: 4_294_967_296, + mode: "0700", + options: ["rw", "nosuid", "nodev"], + }, + ], + environment: { SYNTHETIC_PROFILE: "enabled" }, + }, + execution: { + ...recipeTemplate.spec.execution, + nodeCount, + tensorParallelSize: nodeCount, + }, + serve: { + ...recipeTemplate.spec.serve, + executable: "/opt/vllm/bin/vllm", + arguments: [ + { name: "--port", value: 9_001 }, + { name: "--max-model-len", value: 4_096 }, + { name: "--generation-config", value: "auto" }, + ], + }, + readiness: { timeoutSeconds: 900, expectedModel: "synthetic-model" }, + }, + }; + const preset: ManagedInferenceServingPreset = { + ...presetTemplate, + metadata: { + id: "vllm.synthetic-profile.managed-cluster", + displayName: "Synthetic profile on a compatible cluster", + }, + spec: { + ...presetTemplate.spec, + priority: 399, + plan: { ...presetTemplate.spec.plan, recipeRef: recipe.metadata.id }, + }, + }; + const recipeDigest = managedInferenceDigest(recipe); + const presetDigest = managedInferenceDigest(preset); + const recipeSourceFile = "managed-inference/recipes/vllm.synthetic-model.managed-cluster.v1.yaml"; + const presetSourceFile = "managed-inference/presets/vllm.synthetic-profile.managed-cluster.yaml"; + const sources = [ + ...baseCatalog.sources, + { + path: presetSourceFile, + kind: "ServingPreset" as const, + id: preset.metadata.id, + digest: presetDigest, + }, + { + path: recipeSourceFile, + kind: "ServingRecipe" as const, + id: recipe.metadata.id, + digest: recipeDigest, + }, + ]; + const catalogContents = { + compilerVersion: baseCatalog.compilerVersion, + presets: [...baseCatalog.presets, preset], + recipes: [...baseCatalog.recipes, recipe], + readinessSchemaRef: baseCatalog.readinessSchemaRef, + schemaVersion: baseCatalog.schemaVersion, + sources, + sourceRevision: baseCatalog.sourceRevision, + } as const; + const catalog: CompiledManagedInferenceCatalog = { + ...catalogContents, + catalogDigest: managedInferenceDigest(catalogContents), + }; + return { + catalog, + selection: { + outcome: "selected", + selection: "automatic", + catalogDigest: catalog.catalogDigest, + presetDigest, + recipeDigest, + preset, + recipe, + topologyQualification: topology, + }, + }; +} + +describe("managed-cluster vLLM materializer", () => { + it("creates deterministic plans entirely from the selected catalog definitions", () => { + const selection = selectionWithDigests(); + const plan = materializeManagedClusterVllmPlan(selection); + const recipe = selection.recipe.spec; + + expect(materializeManagedClusterVllmPlan(selection)).toEqual(plan); + expect(Object.isFrozen(plan)).toBe(true); + expect(plan).toMatchObject({ + catalogDigest: selection.catalogDigest, + presetId: selection.preset.metadata.id, + presetDigest: selection.presetDigest, + recipeId: selection.recipe.metadata.id, + recipeDigest: selection.recipeDigest, + model: { + id: recipe.model.id, + revision: recipe.model.revision, + servedName: recipe.model.servedName, + }, + authentication: recipe.serve.authentication, + readiness: { + timeoutMs: recipe.readiness.timeoutSeconds * 1_000, + expectedModel: recipe.readiness.expectedModel, + }, + }); + expect(plan.roles[0]).toMatchObject({ + image: recipe.runtime.image, + runtime: { + architecture: recipe.runtime.architecture, + networkMode: recipe.runtime.networkMode, + ipcMode: recipe.runtime.ipcMode, + sharedMemoryBytes: recipe.runtime.sharedMemoryBytes, + gpuRequest: recipe.runtime.gpuRequest, + devices: recipe.runtime.devices, + imageDownloadSizeBytes: recipe.runtime.imageDownloadSizeBytes, + pullTimeoutSeconds: recipe.runtime.pullTimeoutSeconds, + ulimits: { + memlock: recipe.runtime.ulimits.memlock, + stack: recipe.runtime.ulimits.stackBytes, + }, + modelCache: recipe.runtime.modelCache, + temporaryFilesystems: recipe.runtime.temporaryFilesystems, + }, + command: { executable: recipe.serve.executable }, + }); + const configuredPort = recipe.serve.arguments.find(({ name }) => name === "--port")?.value; + expect(plan.apiPort).toBe(configuredPort); + expect(plan.roles[0].endpoint).toBe(`http://${plan.masterAddress}:${String(configuredPort)}`); + }); + + it("uses role-local topology values and starts rank 1 headless", () => { + const plan = materializeManagedClusterVllmPlan(selectionWithDigests()); + + expect(plan.roles[0].environment).toMatchObject({ + VLLM_HOST_IP: "192.168.100.10", + NCCL_IB_HCA: "rocep1s0f0:1", + NCCL_SOCKET_IFNAME: "enp1s0f0np0", + NCCL_IB_GID_INDEX: "3", + NODE_RANK: "0", + }); + expect(plan.roles[1].environment).toMatchObject({ + VLLM_HOST_IP: "192.168.100.11", + NCCL_IB_HCA: "rocep1s0f1:1", + NCCL_SOCKET_IFNAME: "enp1s0f1np1", + NCCL_IB_GID_INDEX: "6", + NODE_RANK: "1", + HEADLESS: "1", + }); + expect(plan.roles[0].command.arguments).toEqual( + expect.arrayContaining(["--host", "192.168.100.10"]), + ); + expect(plan.roles[1].command.arguments).toEqual( + expect.arrayContaining(["--host", "192.168.100.11", "--headless"]), + ); + expect(plan.roles[0].command.arguments).not.toContain("--headless"); + expect(plan.roles[1].command.arguments).toEqual( + expect.arrayContaining([ + "--tensor-parallel-size", + "2", + "--pipeline-parallel-size", + "1", + "--distributed-executor-backend", + "mp", + "--nnodes", + "2", + "--node-rank", + "1", + "--master-port", + String(selectionWithDigests().recipe.spec.execution.rendezvousPort), + ]), + ); + }); + + it("passes every recipe serving argument without embedding an API key", () => { + const selection = selectionWithDigests(); + const plan = materializeManagedClusterVllmPlan(selection); + const headArguments = plan.roles[0].command.arguments; + + for (const argument of selection.recipe.spec.serve.arguments) { + const index = headArguments.indexOf(argument.name); + expect(index).toBeGreaterThan(-1); + } + for (const argument of selection.recipe.spec.serve.arguments.filter( + ({ value }) => value !== undefined, + )) { + const index = headArguments.indexOf(argument.name); + expect(headArguments[index + 1]).toBe(String(argument.value)); + } + expect(headArguments).not.toContain("--api-key"); + expect(plan.roles[1].command.arguments).not.toContain("--api-key"); + }); + + it("rejects a selected definition changed after catalog resolution", () => { + const selection = selectionWithDigests(); + (selection.recipe.spec.runtime as { image: string }).image = + `registry.example.test/vllm@sha256:${"d".repeat(64)}`; + + expect(() => materializeManagedClusterVllmPlan(selection)).toThrow(/definition digest/u); + }); + + it("rejects stale topology subject and output digests", () => { + const staleSubject = selectionWithDigests(); + (staleSubject.topologyQualification as { subjectDigest: string }).subjectDigest = + `sha256:${"f".repeat(64)}`; + expect(() => materializeManagedClusterVllmPlan(staleSubject)).toThrow(/subject digest/u); + + const staleOutput = selectionWithDigests(); + (staleOutput.topologyQualification.output.peers[0] as { target: string }).target = + "other-worker.local"; + expect(() => materializeManagedClusterVllmPlan(staleOutput)).toThrow(/output digest/u); + }); + + it("rejects an inconsistent master address even with a recomputed digest", () => { + const selection = selectionWithDigests(); + const artifact = selection.topologyQualification; + (artifact.output as { masterAddress: string }).masterAddress = "192.168.100.99"; + (artifact as { outputDigest: string }).outputDigest = managedClusterTopologyOutputDigest( + artifact.output, + ); + + expect(() => materializeManagedClusterVllmPlan(selection)).toThrow(/master address/u); + }); + + it("selects each role endpoint by direct reachability to the qualified master", () => { + const selection = selectionWithDigests(); + const artifact = selection.topologyQualification; + (artifact.output as { masterAddress: string }).masterAddress = "192.168.101.10"; + (artifact as { outputDigest: string }).outputDigest = managedClusterTopologyOutputDigest( + artifact.output, + ); + + const plan = materializeManagedClusterVllmPlan(selection); + + expect(plan.roles[0].fabric.address).toBe("192.168.101.10"); + expect(plan.roles[1].fabric.address).toBe("192.168.101.11"); + }); + + it("materializes bounded preparation operations from recipe data", () => { + const selection = selectionWithDigests(); + const preparation = materializeManagedClusterVllmPlan(selection).roles[0].preparation; + const configured = selection.recipe.spec.model.preparation; + expect(configured.ref).not.toBe(NO_PREPARATION_REF); + const boundedConfigured = configured as Extract< + typeof configured, + { ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF } + >; + + expect(preparation).toMatchObject({ + ref: boundedConfigured.ref, + phase: "container-before-exec", + modelId: selection.recipe.spec.model.id, + modelRevision: selection.recipe.spec.model.revision, + modelDownloadSizeBytes: selection.recipe.spec.model.downloadSizeBytes, + snapshotCopy: { + digest: boundedConfigured.snapshotCopy.digest, + targetPath: boundedConfigured.snapshotCopy.targetPath, + }, + exactTextReplacement: boundedConfigured.exactTextReplacement, + }); + expect(preparation.ref).not.toBe(NO_PREPARATION_REF); + const boundedPreparation = preparation as Extract< + typeof preparation, + { ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF } + >; + expect(boundedPreparation.snapshotCopy.sourcePath).toContain(boundedPreparation.modelRevision); + expect(boundedPreparation.snapshotCopy.sourcePath).toContain( + boundedConfigured.snapshotCopy.sourcePath, + ); + }); + + it("materializes a synthetic second profile without materializer code changes", () => { + const { catalog, selection } = syntheticSecondProfile(); + const plan = materializeManagedClusterVllmPlan(selection, { catalog }); + + expect(plan).toMatchObject({ + presetId: selection.preset.metadata.id, + recipeId: selection.recipe.metadata.id, + model: { + id: "example-org/Synthetic-Model", + servedName: "synthetic-model", + }, + apiPort: 9_001, + readiness: { timeoutMs: 900_000, expectedModel: "synthetic-model" }, + }); + expect(plan.roles[0]).toMatchObject({ + image: selection.recipe.spec.runtime.image, + runtime: { + sharedMemoryBytes: 8_589_934_592, + gpuRequest: "device=all", + pullTimeoutSeconds: 7_200, + ulimits: { memlock: "unlimited", stack: 33_554_432 }, + modelCache: { source: "huggingface-cache", target: "/models/cache" }, + }, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: expect.stringContaining( + "/models/cache/hub/models--example-org--Synthetic-Model/", + ), + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/synthetic-vllm/tokenizers/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/synthetic-vllm/parsers/reasoning.py", + expectedText: "MODE = 'legacy'", + replacementText: "MODE = 'compatible'", + }, + }, + command: { executable: "/opt/vllm/bin/vllm" }, + baseLabels: { + [MANAGED_CLUSTER_PRESET_LABEL]: selection.preset.metadata.id, + [MANAGED_CLUSTER_RECIPE_LABEL]: selection.recipe.metadata.id, + }, + }); + expect(plan.roles[0].environment).toMatchObject({ + HF_HOME: "/models/cache", + SYNTHETIC_PROFILE: "enabled", + }); + expect(plan.roles[0].command.arguments).toEqual( + expect.arrayContaining(["--port", "9001", "--max-model-len", "4096"]), + ); + + const originalSelection = selectionWithDigests(); + const originalPlan = materializeManagedClusterVllmPlan(originalSelection); + const originalFromExpandedCatalog = materializeManagedClusterVllmPlan( + { ...originalSelection, catalogDigest: catalog.catalogDigest }, + { catalog }, + ); + expect(originalFromExpandedCatalog.planId).toBe(originalPlan.planId); + }); + + it("rejects a larger catalog profile only when its fabric cannot reach the master", () => { + const { catalog, selection } = syntheticSecondProfile("/dev/shm-scratch", 3); + + expect(selection.recipe.spec.execution).toMatchObject({ + nodeCount: 3, + tensorParallelSize: 3, + }); + expect(() => materializeManagedClusterVllmPlan(selection, { catalog })).toThrow( + /spark-worker-b has no direct fabric endpoint to master address/u, + ); + }); + + it("rejects a temporary filesystem that shadows the model cache", () => { + const { catalog, selection } = syntheticSecondProfile("/models"); + + expect(() => materializeManagedClusterVllmPlan(selection, { catalog })).toThrow( + /temporary filesystem cannot shadow the model cache/u, + ); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-materialize.ts b/src/lib/inference/serving/managed-cluster-materialize.ts new file mode 100644 index 0000000000..5038fd3988 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-materialize.ts @@ -0,0 +1,695 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + containerPathContains, + getManagedInferenceLifecycleDescriptor, + getManagedInferenceMaterializerDescriptor, + getManagedInferenceRecipeRegistrationError, + getManagedInferenceTopologyQualificationDescriptor, + isManagedClusterMaterializerOwnedEnvironment, + MANAGED_CLUSTER_VLLM_MATERIALIZER_REF, +} from "./adapter-registry.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { + immutableManagedInferenceCopy, + managedInferenceDigest, + managedInferenceHexDigest, +} from "./catalog-integrity.js"; +import { + type CompiledManagedInferenceCatalog, + isManagedInferenceMaterializerOwnedArgument, + type ManagedInferenceServingPreset, + type ManagedInferenceServingRecipe, + type ResolvedManagedInferenceSelection, +} from "./types.js"; +import { + type ManagedClusterVllmPreparationPlan, + materializeManagedClusterVllmPreparation, +} from "./managed-cluster-preparation.js"; +import { + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + type ManagedClusterTopologyOutput, + type ManagedClusterTopologyRailEndpoint, +} from "./managed-cluster-topology.js"; + +/** Stable adapter identity; profile identities and values come from the selected catalog entries. */ +export const MANAGED_CLUSTER_VLLM_ADAPTER_ID = MANAGED_CLUSTER_VLLM_MATERIALIZER_REF; +export const MANAGED_CLUSTER_VLLM_PROJECT_ID = "nemoclaw-vllm-cluster"; + +export const MANAGED_CLUSTER_MANAGED_LABEL = "com.nvidia.nemoclaw.managed-vllm"; +export const MANAGED_CLUSTER_ADAPTER_LABEL = "com.nvidia.nemoclaw.serving-adapter"; +export const MANAGED_CLUSTER_PRESET_LABEL = "com.nvidia.nemoclaw.serving-preset"; +export const MANAGED_CLUSTER_RECIPE_LABEL = "com.nvidia.nemoclaw.serving-recipe"; +export const MANAGED_CLUSTER_ROLE_LABEL = "com.nvidia.nemoclaw.serving-role"; +export const MANAGED_CLUSTER_CLUSTER_LABEL = "com.nvidia.nemoclaw.serving-cluster"; +export const MANAGED_CLUSTER_PLAN_LABEL = "com.nvidia.nemoclaw.serving-plan"; +export const MANAGED_CLUSTER_GPU_LABEL = "com.nvidia.nemoclaw.serving-gpu"; +export const MANAGED_CLUSTER_IMAGE_LABEL = "com.nvidia.nemoclaw.serving-image"; +export const MANAGED_CLUSTER_MODEL_REVISION_LABEL = "com.nvidia.nemoclaw.serving-model-revision"; +export const MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL = + "com.nvidia.nemoclaw.serving-api-key-fingerprint"; +export const MANAGED_CLUSTER_TRANSACTION_LABEL = "com.nvidia.nemoclaw.serving-transaction"; + +const SAFE_DEVICE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/u; +const SAFE_ABSOLUTE_PATH_PATTERN = /^\/(?!.*(?:^|\/)\.\.(?:\/|$))[A-Za-z0-9._+/-]+$/u; +const SAFE_GPU_REQUEST_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._,:=-]{0,255}$/u; +const SAFE_ENVIRONMENT_NAME_PATTERN = /^[A-Z][A-Z0-9_]{0,127}$/u; +const SAFE_MODEL_ID_PATTERN = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/u; +const SAFE_MODEL_REVISION_PATTERN = /^[0-9a-f]{40,64}$/u; +const SAFE_STABLE_ID_PATTERN = /^[a-z0-9][a-z0-9._/-]{0,159}$/u; +const PINNED_IMAGE_PATTERN = /^[a-z0-9.-]+(?::[0-9]+)?(?:\/[a-z0-9._-]+)+@sha256:[0-9a-f]{64}$/u; +const SHA256_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const TMPFS_OPTIONS = new Set([ + "rw", + "ro", + "nosuid", + "nodev", + "noexec", + "exec", + "noatime", + "relatime", +]); +export type ManagedClusterVllmRole = "head" | "worker"; + +export interface ManagedClusterVllmRolePlan { + readonly role: ManagedClusterVllmRole; + readonly rank: number; + readonly nodeId: string; + readonly gpuId: string; + readonly containerName: string; + readonly execution: + | { readonly kind: "local" } + | { + readonly kind: "ssh"; + readonly expectedTarget: string; + readonly bindingHandle: string; + }; + readonly image: string; + readonly runtime: { + readonly architecture: string; + readonly networkMode: string; + readonly ipcMode: string; + readonly sharedMemoryBytes: number; + readonly gpuRequest: string; + readonly devices: readonly string[]; + readonly imageDownloadSizeBytes: number; + readonly pullTimeoutSeconds: number; + readonly ulimits: { + readonly memlock: number | string; + readonly stack: number; + }; + readonly modelCache: { + readonly source: string; + readonly target: string; + }; + readonly temporaryFilesystems: readonly { + readonly target: string; + readonly sizeBytes: number; + readonly mode: string; + readonly options: readonly string[]; + }[]; + }; + readonly preparation: ManagedClusterVllmPreparationPlan; + readonly fabric: { + readonly primaryRailIndex: number; + readonly netdev: string; + readonly hcaDevice: string; + readonly hcaPort: number; + readonly address: string; + readonly roceGidIndex: number; + readonly roceGidValue: string; + }; + readonly environment: Readonly>; + readonly command: { + readonly executable: string; + readonly arguments: readonly string[]; + }; + readonly endpoint: string | null; + readonly baseLabels: Readonly>; +} + +export interface ManagedClusterVllmPlan { + readonly schemaVersion: 1; + readonly adapterId: typeof MANAGED_CLUSTER_VLLM_ADAPTER_ID; + readonly catalogDigest: string; + readonly presetId: string; + readonly presetDigest: string; + readonly recipeId: string; + readonly recipeDigest: string; + readonly topologyId: typeof MANAGED_CLUSTER_TOPOLOGY_ID; + readonly topologySchemaVersion: typeof MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION; + readonly topologySubjectDigest: string; + readonly topologyOutputDigest: string; + readonly clusterId: string; + readonly planId: string; + readonly model: { + readonly id: string; + readonly revision: string; + readonly servedName: string; + }; + readonly authentication: string; + readonly apiPort: number; + readonly masterAddress: string; + readonly masterPort: number; + readonly readiness: { + readonly timeoutMs: number; + readonly expectedModel: string; + }; + readonly roles: readonly ManagedClusterVllmRolePlan[]; +} + +export interface ManagedClusterVllmMaterializeOptions { + /** Explicit catalog input keeps the materializer testable with additional YAML-compiled profiles. */ + readonly catalog?: CompiledManagedInferenceCatalog; +} + +interface CatalogSelection { + readonly preset: ManagedInferenceServingPreset; + readonly recipe: ManagedInferenceServingRecipe; +} + +interface ParsedServingArguments { + readonly apiPort: number; + readonly arguments: readonly string[]; +} + +function fail(message: string): never { + throw new Error(`Cannot materialize managed-cluster vLLM: ${message}`); +} + +function positiveSafeInteger(value: number, maximum = Number.MAX_SAFE_INTEGER): boolean { + return Number.isSafeInteger(value) && value > 0 && value <= maximum; +} + +function safeAbsolutePath(value: string): boolean { + return ( + value.length <= 4_096 && + SAFE_ABSOLUTE_PATH_PATTERN.test(value) && + path.posix.normalize(value) === value + ); +} + +function selectedDefinition( + definitions: readonly TDefinition[], + selected: TDefinition, + selectedDigest: string, + label: string, +): TDefinition { + const matches = definitions.filter(({ metadata }) => metadata.id === selected.metadata.id); + if (matches.length !== 1) { + fail(`selected ${label} ${selected.metadata.id} is not unique in the compiled catalog`); + } + const compiled = matches[0]!; + if ( + !SHA256_PATTERN.test(selectedDigest) || + managedInferenceDigest(compiled) !== selectedDigest || + managedInferenceDigest(selected) !== selectedDigest + ) { + fail(`selected ${label} ${selected.metadata.id} does not match its definition digest`); + } + return compiled; +} + +function assertCatalogSelection( + selection: ResolvedManagedInferenceSelection, + catalog: CompiledManagedInferenceCatalog, +): CatalogSelection { + const { catalogDigest, ...catalogContents } = catalog; + if ( + selection.catalogDigest !== catalogDigest || + !SHA256_PATTERN.test(catalogDigest) || + managedInferenceDigest(catalogContents) !== catalogDigest + ) { + fail("the resolved selection does not match the compiled catalog digest"); + } + if (!/^[0-9a-f]{40,64}$/u.test(catalog.sourceRevision)) { + fail("the compiled catalog provenance is invalid"); + } + + const preset = selectedDefinition( + catalog.presets, + selection.preset, + selection.presetDigest, + "preset", + ); + const recipe = selectedDefinition( + catalog.recipes, + selection.recipe, + selection.recipeDigest, + "recipe", + ); + const bindingName = recipe.spec.execution.topologyBinding; + const recipeBinding = recipe.spec.bindings[bindingName]; + const presetBinding = preset.spec.plan.bindings[bindingName]?.valueFromTopologyQualification; + const topologyDescriptor = recipeBinding + ? getManagedInferenceTopologyQualificationDescriptor( + recipeBinding.qualificationId, + recipeBinding.schemaVersion, + ) + : undefined; + + if ( + preset.spec.plan.recipeRef !== recipe.metadata.id || + preset.spec.plan.backend !== recipe.spec.backend + ) { + fail("the selected preset does not reference the selected recipe and backend"); + } + if ( + !recipeBinding || + !presetBinding || + !topologyDescriptor || + presetBinding.output !== topologyDescriptor.bindingOutput || + presetBinding.id !== recipeBinding.qualificationId || + presetBinding.schemaVersion !== recipeBinding.schemaVersion + ) { + fail(`preset and recipe topology binding ${bindingName} is incompatible`); + } + + const materializer = getManagedInferenceMaterializerDescriptor( + recipe.spec.execution.materializerRef, + ); + if (!materializer || materializer.ref !== MANAGED_CLUSTER_VLLM_MATERIALIZER_REF) { + fail(`recipe selects unsupported materializer ${recipe.spec.execution.materializerRef}`); + } + const lifecycle = getManagedInferenceLifecycleDescriptor(recipe.spec.execution.lifecycleRef); + if ( + !lifecycle || + lifecycle.backend !== recipe.spec.backend || + !lifecycle.acceptedMaterializerRefs.includes(materializer.ref) || + !lifecycle.acceptedPlanSchemas.includes(materializer.outputPlanSchema) + ) { + fail(`recipe selects incompatible lifecycle ${recipe.spec.execution.lifecycleRef}`); + } + const registrationError = getManagedInferenceRecipeRegistrationError(recipe); + if (registrationError) fail(registrationError); + + const topology = selection.topologyQualification; + if ( + topology.status !== "qualified" || + topology.id !== recipeBinding.qualificationId || + topology.schemaVersion !== recipeBinding.schemaVersion || + topology.id !== materializer.topology.qualificationId || + topology.schemaVersion !== materializer.topology.schemaVersion || + recipeBinding.outputSchema !== materializer.topology.outputSchema + ) { + fail("topology artifact is incompatible with the selected recipe binding"); + } + if (topologyDescriptor.outputSchema !== recipeBinding.outputSchema) { + fail("topology artifact has no compatible registered validator"); + } + const topologyError = topologyDescriptor.validateArtifact(topology); + if (topologyError) fail(topologyError); + if (topology.output.nodes.length !== recipe.spec.execution.nodeCount) { + fail("topology node count does not match the recipe execution shape"); + } + + return { preset, recipe }; +} + +function assertRecipeValues(recipe: ManagedInferenceServingRecipe): void { + const { model, readiness, runtime, serve } = recipe.spec; + if ( + runtime.architecture !== "arm64" || + runtime.networkMode !== "host" || + runtime.ipcMode !== "host" || + serve.authentication !== "bearer" + ) { + fail("recipe runtime and authentication do not match adapter v1"); + } + if ( + !PINNED_IMAGE_PATTERN.test(runtime.image) || + !positiveSafeInteger(runtime.imageDownloadSizeBytes) || + !positiveSafeInteger(runtime.pullTimeoutSeconds, 86_400) || + !positiveSafeInteger(runtime.sharedMemoryBytes) || + !SAFE_GPU_REQUEST_PATTERN.test(runtime.gpuRequest) || + !safeAbsolutePath(serve.executable) + ) { + fail("recipe runtime contains an invalid executable or resource value"); + } + if ( + runtime.devices.length > 32 || + new Set(runtime.devices).size !== runtime.devices.length || + runtime.devices.some((device) => !safeAbsolutePath(device)) + ) { + fail("recipe runtime device bindings are invalid"); + } + const { memlock, stackBytes } = runtime.ulimits; + if ( + !( + (typeof memlock === "number" && Number.isSafeInteger(memlock) && memlock >= -1) || + memlock === "unlimited" + ) || + !positiveSafeInteger(stackBytes) + ) { + fail("recipe runtime ulimits are invalid"); + } + if ( + !SAFE_STABLE_ID_PATTERN.test(runtime.modelCache.source) || + !safeAbsolutePath(runtime.modelCache.target) + ) { + fail("recipe model-cache binding is invalid"); + } + const temporaryTargets = new Set(); + for (const temporaryFilesystem of runtime.temporaryFilesystems) { + if ( + temporaryTargets.has(temporaryFilesystem.target) || + !safeAbsolutePath(temporaryFilesystem.target) || + containerPathContains(temporaryFilesystem.target, runtime.modelCache.target) || + !positiveSafeInteger(temporaryFilesystem.sizeBytes) || + !/^[0-7]{4}$/u.test(temporaryFilesystem.mode) || + temporaryFilesystem.options.length > 8 || + new Set(temporaryFilesystem.options).size !== temporaryFilesystem.options.length || + temporaryFilesystem.options.some((option) => !TMPFS_OPTIONS.has(option)) + ) { + fail("recipe temporary-filesystem configuration is invalid"); + } + temporaryTargets.add(temporaryFilesystem.target); + } + const environmentEntries = Object.entries(runtime.environment); + if ( + environmentEntries.length > 128 || + environmentEntries.some( + ([name, value]) => + !SAFE_ENVIRONMENT_NAME_PATTERN.test(name) || + isManagedClusterMaterializerOwnedEnvironment(name) || + Buffer.byteLength(value, "utf8") > 4_096 || + value.includes("\0"), + ) + ) { + fail("recipe runtime environment is invalid or overrides adapter-owned values"); + } + if ( + !SAFE_MODEL_ID_PATTERN.test(model.id) || + !SAFE_MODEL_REVISION_PATTERN.test(model.revision) || + !SAFE_STABLE_ID_PATTERN.test(model.servedName) || + !positiveSafeInteger(model.downloadSizeBytes) || + !positiveSafeInteger(readiness.timeoutSeconds, 86_400) || + readiness.expectedModel !== model.servedName + ) { + fail("recipe model identity or readiness contract is invalid"); + } +} + +function servingArguments(recipe: ManagedInferenceServingRecipe): ParsedServingArguments { + const seen = new Set(); + const staticArguments: string[] = []; + let apiPort: number | undefined; + for (const argument of recipe.spec.serve.arguments) { + if (!/^--[a-z0-9][a-z0-9-]*$/u.test(argument.name)) fail("a serve argument is invalid"); + if (seen.has(argument.name)) fail(`serve argument ${argument.name} is duplicated`); + if (isManagedInferenceMaterializerOwnedArgument(argument.name)) { + fail(`serve argument ${argument.name} is owned by the materializer`); + } + seen.add(argument.name); + staticArguments.push(argument.name); + if (argument.value !== undefined) { + const value = String(argument.value); + if (Buffer.byteLength(value, "utf8") > 16_384 || value.includes("\0")) { + fail(`serve argument ${argument.name} has an invalid value`); + } + staticArguments.push(value); + } + if (argument.name === "--port") { + const parsed = + typeof argument.value === "number" + ? argument.value + : typeof argument.value === "string" && /^\d{1,5}$/u.test(argument.value) + ? Number(argument.value) + : Number.NaN; + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 65_535) { + fail("serve argument --port must contain a valid TCP port"); + } + apiPort = parsed; + } + } + if (apiPort === undefined) fail("recipe must define one --port serve argument"); + return { apiPort, arguments: staticArguments }; +} + +function commandArguments( + recipe: ManagedInferenceServingRecipe, + topology: ResolvedManagedInferenceSelection["topologyQualification"], + staticArguments: readonly string[], + rank: number, + hostAddress: string, +): string[] { + return [ + "serve", + recipe.spec.model.id, + "--revision", + recipe.spec.model.revision, + "--served-model-name", + recipe.spec.model.servedName, + "--host", + hostAddress, + ...staticArguments, + "--tensor-parallel-size", + String(recipe.spec.execution.tensorParallelSize), + "--pipeline-parallel-size", + String(recipe.spec.execution.pipelineParallelSize), + "--distributed-executor-backend", + recipe.spec.execution.distributedExecutorBackend, + "--nnodes", + String(recipe.spec.execution.nodeCount), + "--node-rank", + String(rank), + "--master-addr", + topology.output.masterAddress, + "--master-port", + String(recipe.spec.execution.rendezvousPort), + ...(rank > 0 ? ["--headless"] : []), + ]; +} + +function endpointsForNode( + output: ManagedClusterTopologyOutput, + nodeId: string, +): readonly (ManagedClusterTopologyRailEndpoint & { readonly railIndex: number })[] { + const endpoints = output.rails + .flatMap((rail) => + rail.endpoints + .filter((endpoint) => endpoint.nodeId === nodeId) + .map((endpoint) => ({ ...endpoint, railIndex: rail.index })), + ) + .sort((left, right) => left.railIndex - right.railIndex); + if ( + endpoints.length === 0 || + endpoints.some( + (endpoint) => + !SAFE_DEVICE_PATTERN.test(endpoint.hcaDevice) || !SAFE_DEVICE_PATTERN.test(endpoint.netdev), + ) + ) { + fail(`node ${nodeId} has no valid fabric endpoints`); + } + return endpoints; +} + +interface RolePlanInput { + readonly selection: ResolvedManagedInferenceSelection; + readonly preset: ManagedInferenceServingPreset; + readonly recipe: ManagedInferenceServingRecipe; + readonly serving: ParsedServingArguments; + readonly preparation: ManagedClusterVllmPreparationPlan; + readonly node: ManagedClusterTopologyOutput["nodes"][number]; + readonly clusterId: string; + readonly planId: string; +} + +function rolePlan(input: RolePlanInput): ManagedClusterVllmRolePlan { + const { clusterId, node, planId, preparation, preset, recipe, selection, serving } = input; + const output = selection.topologyQualification.output; + const { nodeId, rank, role } = node; + const fabricEndpoints = endpointsForNode(output, nodeId); + const primaryEndpoint = fabricEndpoints.find((endpoint) => + role === "head" + ? endpoint.address === output.masterAddress + : endpoint.peerAddress === output.masterAddress, + ); + if (!primaryEndpoint) { + fail(`node ${nodeId} has no direct fabric endpoint to master address ${output.masterAddress}`); + } + const peer = output.peers.find((candidate) => candidate.nodeId === nodeId); + if (rank > 0 && !peer) fail(`worker rank ${String(rank)} is missing its SSH binding`); + const baseLabels = { + [MANAGED_CLUSTER_MANAGED_LABEL]: "true", + [MANAGED_CLUSTER_ADAPTER_LABEL]: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + [MANAGED_CLUSTER_PRESET_LABEL]: preset.metadata.id, + [MANAGED_CLUSTER_RECIPE_LABEL]: recipe.metadata.id, + [MANAGED_CLUSTER_ROLE_LABEL]: role, + [MANAGED_CLUSTER_CLUSTER_LABEL]: clusterId, + [MANAGED_CLUSTER_PLAN_LABEL]: planId, + [MANAGED_CLUSTER_GPU_LABEL]: node.gpuId, + [MANAGED_CLUSTER_IMAGE_LABEL]: recipe.spec.runtime.image, + [MANAGED_CLUSTER_MODEL_REVISION_LABEL]: recipe.spec.model.revision, + }; + const environment = { + ...recipe.spec.runtime.environment, + HF_HOME: recipe.spec.runtime.modelCache.target, + VLLM_HOST_IP: primaryEndpoint.address, + NCCL_IB_HCA: `${primaryEndpoint.hcaDevice}:${String(primaryEndpoint.hcaPort)}`, + NCCL_SOCKET_IFNAME: primaryEndpoint.netdev, + TP_SOCKET_IFNAME: primaryEndpoint.netdev, + GLOO_SOCKET_IFNAME: primaryEndpoint.netdev, + NCCL_IB_GID_INDEX: String(primaryEndpoint.roceGid.index), + MASTER_ADDR: output.masterAddress, + MASTER_PORT: String(recipe.spec.execution.rendezvousPort), + NODE_RANK: String(rank), + HEADLESS: role === "worker" ? "1" : "", + }; + const runtime = recipe.spec.runtime; + + return { + role, + rank, + nodeId, + gpuId: node.gpuId, + containerName: `${MANAGED_CLUSTER_VLLM_PROJECT_ID}-rank-${String(rank)}`, + execution: + role === "head" + ? { kind: "local" } + : { + kind: "ssh", + expectedTarget: peer!.target, + bindingHandle: peer!.sshBindingHandle, + }, + image: runtime.image, + runtime: { + architecture: runtime.architecture, + networkMode: runtime.networkMode, + ipcMode: runtime.ipcMode, + sharedMemoryBytes: runtime.sharedMemoryBytes, + gpuRequest: runtime.gpuRequest, + devices: runtime.devices, + imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, + pullTimeoutSeconds: runtime.pullTimeoutSeconds, + ulimits: { memlock: runtime.ulimits.memlock, stack: runtime.ulimits.stackBytes }, + modelCache: runtime.modelCache, + temporaryFilesystems: runtime.temporaryFilesystems, + }, + preparation, + fabric: { + primaryRailIndex: primaryEndpoint.railIndex, + netdev: primaryEndpoint.netdev, + hcaDevice: primaryEndpoint.hcaDevice, + hcaPort: primaryEndpoint.hcaPort, + address: primaryEndpoint.address, + roceGidIndex: primaryEndpoint.roceGid.index, + roceGidValue: primaryEndpoint.roceGid.value, + }, + environment, + command: { + executable: recipe.spec.serve.executable, + arguments: commandArguments( + recipe, + selection.topologyQualification, + serving.arguments, + rank, + primaryEndpoint.address, + ), + }, + endpoint: role === "head" ? `http://${output.masterAddress}:${String(serving.apiPort)}` : null, + baseLabels, + }; +} + +/** Compile one resolved, qualified catalog selection into immutable role-local plans. */ +export function materializeManagedClusterVllmPlan( + selection: ResolvedManagedInferenceSelection, + options: ManagedClusterVllmMaterializeOptions = {}, +): ManagedClusterVllmPlan { + let snapshot: ResolvedManagedInferenceSelection; + let catalog: CompiledManagedInferenceCatalog; + try { + snapshot = immutableManagedInferenceCopy(selection); + catalog = options.catalog + ? immutableManagedInferenceCopy(options.catalog) + : loadManagedInferenceCatalog(); + } catch { + fail("the resolved selection or catalog is not immutable JSON data"); + } + + const selected = assertCatalogSelection(snapshot, catalog); + const catalogSelection = { ...snapshot, ...selected }; + assertRecipeValues(selected.recipe); + const serving = servingArguments(selected.recipe); + let preparation: ManagedClusterVllmPreparationPlan; + try { + preparation = materializeManagedClusterVllmPreparation({ + ...selected.recipe.spec.model, + modelCacheTarget: selected.recipe.spec.runtime.modelCache.target, + }); + } catch (error) { + fail(error instanceof Error ? error.message : "model preparation is invalid"); + } + + const topologyIdentity = { + id: snapshot.topologyQualification.id, + schemaVersion: snapshot.topologyQualification.schemaVersion, + subjectDigest: snapshot.topologyQualification.subjectDigest, + outputDigest: snapshot.topologyQualification.outputDigest, + }; + const clusterId = managedInferenceHexDigest(topologyIdentity); + const planId = managedInferenceHexDigest({ + adapterId: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + preset: { id: selected.preset.metadata.id, digest: snapshot.presetDigest }, + recipe: { id: selected.recipe.metadata.id, digest: snapshot.recipeDigest }, + topology: topologyIdentity, + }); + const output = snapshot.topologyQualification.output; + return immutableManagedInferenceCopy({ + schemaVersion: 1, + adapterId: MANAGED_CLUSTER_VLLM_ADAPTER_ID, + catalogDigest: snapshot.catalogDigest, + presetId: selected.preset.metadata.id, + presetDigest: snapshot.presetDigest, + recipeId: selected.recipe.metadata.id, + recipeDigest: snapshot.recipeDigest, + topologyId: MANAGED_CLUSTER_TOPOLOGY_ID, + topologySchemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + topologySubjectDigest: snapshot.topologyQualification.subjectDigest, + topologyOutputDigest: snapshot.topologyQualification.outputDigest, + clusterId, + planId, + model: { + id: selected.recipe.spec.model.id, + revision: selected.recipe.spec.model.revision, + servedName: selected.recipe.spec.model.servedName, + }, + authentication: selected.recipe.spec.serve.authentication, + apiPort: serving.apiPort, + masterAddress: output.masterAddress, + masterPort: selected.recipe.spec.execution.rendezvousPort, + readiness: { + timeoutMs: selected.recipe.spec.readiness.timeoutSeconds * 1000, + expectedModel: selected.recipe.spec.readiness.expectedModel, + }, + roles: output.nodes.map((node) => + rolePlan({ + selection: catalogSelection, + preset: selected.preset, + recipe: selected.recipe, + serving, + preparation, + node, + clusterId, + planId, + }), + ), + }); +} + +export function managedClusterHeadRole(plan: ManagedClusterVllmPlan): ManagedClusterVllmRolePlan { + const head = plan.roles.find((role) => role.rank === 0 && role.role === "head"); + if (!head) fail("plan has no rank-zero head role"); + return head; +} + +export function managedClusterWorkerRoles( + plan: ManagedClusterVllmPlan, +): readonly ManagedClusterVllmRolePlan[] { + return plan.roles.filter((role) => role.rank > 0 && role.role === "worker"); +} diff --git a/src/lib/inference/serving/managed-cluster-preparation.test.ts b/src/lib/inference/serving/managed-cluster-preparation.test.ts new file mode 100644 index 0000000000..717cddbf4b --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-preparation.test.ts @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + NO_PREPARATION_REF, + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, +} from "./adapter-registry.js"; +import { materializeManagedClusterVllmPreparation } from "./managed-cluster-preparation.js"; + +const MODEL = { + id: "example-org/Synthetic-Model", + revision: "a".repeat(64), + downloadSizeBytes: 1_234_567, + modelCacheTarget: "/models/alternate-cache", +} as const; + +describe("managed cluster vLLM preparation materializer", () => { + it("materializes the registered no-op without executable input", () => { + const preparation = materializeManagedClusterVllmPreparation({ + ...MODEL, + preparation: { ref: NO_PREPARATION_REF }, + }); + + expect(preparation).toEqual({ + ref: NO_PREPARATION_REF, + phase: "container-before-exec", + modelId: MODEL.id, + modelRevision: MODEL.revision, + modelDownloadSizeBytes: MODEL.downloadSizeBytes, + }); + expect(preparation).not.toHaveProperty("command"); + }); + + it("derives the snapshot root and opaque target paths from recipe data", () => { + const preparation = materializeManagedClusterVllmPreparation({ + ...MODEL, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "artifacts/tokenizer.py", + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/alternate-vllm/tokenizers/model.py", + }, + exactTextReplacement: { + targetPath: "/opt/alternate-vllm/parsers/reasoning.py", + expectedText: "MODE = 'before'", + replacementText: "MODE = 'after'", + }, + }, + }); + + expect(preparation).toEqual({ + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + phase: "container-before-exec", + modelId: MODEL.id, + modelRevision: MODEL.revision, + modelDownloadSizeBytes: MODEL.downloadSizeBytes, + snapshotCopy: { + sourcePath: `/models/alternate-cache/hub/models--example-org--Synthetic-Model/snapshots/${MODEL.revision}/artifacts/tokenizer.py`, + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/alternate-vllm/tokenizers/model.py", + }, + exactTextReplacement: { + targetPath: "/opt/alternate-vllm/parsers/reasoning.py", + expectedText: "MODE = 'before'", + replacementText: "MODE = 'after'", + }, + }); + expect(preparation).not.toHaveProperty("command"); + }); + + it.each([ + { + label: "snapshot traversal", + error: /snapshot copy source path must be a normalized relative POSIX path/u, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "../escape.py", + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/vllm/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/vllm/reasoning.py", + expectedText: "before", + replacementText: "after", + }, + }, + }, + { + label: "relative container target", + error: /snapshot copy target path must be a normalized absolute container path/u, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "safe.py", + digest: `sha256:${"4".repeat(64)}`, + targetPath: "relative/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/vllm/reasoning.py", + expectedText: "before", + replacementText: "after", + }, + }, + }, + { + label: "invalid snapshot digest", + error: /snapshot copy digest must be an exact SHA-256 digest/u, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "safe.py", + digest: "sha256:not-a-digest", + targetPath: "/opt/vllm/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/vllm/reasoning.py", + expectedText: "before", + replacementText: "after", + }, + }, + }, + { + label: "unchanged replacement", + error: /exact-text replacement must change the matched text/u, + preparation: { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: "safe.py", + digest: `sha256:${"4".repeat(64)}`, + targetPath: "/opt/vllm/copied.py", + }, + exactTextReplacement: { + targetPath: "/opt/vllm/reasoning.py", + expectedText: "same", + replacementText: "same", + }, + }, + }, + ])("rejects bounded operation input with $label", ({ error, preparation }) => { + expect(() => materializeManagedClusterVllmPreparation({ ...MODEL, preparation })).toThrow( + error, + ); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-preparation.ts b/src/lib/inference/serving/managed-cluster-preparation.ts new file mode 100644 index 0000000000..a416031c81 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-preparation.ts @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + NO_PREPARATION_REF, + SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, +} from "./adapter-registry.js"; + +const MODEL_ID = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/u; +const MODEL_REVISION = /^[0-9a-f]{40,64}$/u; +const SHA256_DIGEST = /^sha256:[0-9a-f]{64}$/u; +const MAX_PATH_BYTES = 4_096; +const MAX_REPLACEMENT_BYTES = 64 * 1_024; + +export type ManagedClusterVllmPreparationRecipe = + | { readonly ref: typeof NO_PREPARATION_REF } + | { + readonly ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF; + readonly snapshotCopy: { + /** Normalized path relative to the immutable model snapshot. */ + readonly sourcePath: string; + /** Digest required before model-snapshot code can enter the runtime package. */ + readonly digest: string; + /** Normalized absolute destination inside the pinned vLLM image. */ + readonly targetPath: string; + }; + readonly exactTextReplacement: { + /** Normalized absolute file inside the pinned vLLM package. */ + readonly targetPath: string; + readonly expectedText: string; + readonly replacementText: string; + }; + }; + +interface ManagedClusterVllmPreparationPlanBase { + readonly phase: "container-before-exec"; + readonly modelId: string; + readonly modelRevision: string; + readonly modelDownloadSizeBytes: number; +} + +export type ManagedClusterVllmPreparationPlan = + | (ManagedClusterVllmPreparationPlanBase & { readonly ref: typeof NO_PREPARATION_REF }) + | (ManagedClusterVllmPreparationPlanBase & { + readonly ref: typeof SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF; + readonly snapshotCopy: { + readonly sourcePath: string; + readonly digest: string; + readonly targetPath: string; + }; + readonly exactTextReplacement: { + readonly targetPath: string; + readonly expectedText: string; + readonly replacementText: string; + }; + }); + +interface PreparationModel { + readonly id: string; + readonly revision: string; + readonly downloadSizeBytes: number; + readonly preparation: unknown; + readonly modelCacheTarget: string; +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function exactKeys( + value: Readonly>, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const sorted = [...expected].sort(); + if (actual.length !== sorted.length || actual.some((key, index) => key !== sorted[index])) { + throw new Error(`${label} fields are invalid`); + } +} + +function boundedText(value: unknown, label: string, maximum = MAX_REPLACEMENT_BYTES): string { + if ( + typeof value !== "string" || + value.length === 0 || + Buffer.byteLength(value, "utf8") > maximum || + value.includes("\0") + ) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function snapshotRelativePath(value: unknown): string { + const candidate = boundedText(value, "snapshot copy source path", MAX_PATH_BYTES); + if ( + path.posix.isAbsolute(candidate) || + path.posix.normalize(candidate) !== candidate || + candidate === "." || + candidate.startsWith("../") || + candidate.includes(":") + ) { + throw new Error("snapshot copy source path must be a normalized relative POSIX path"); + } + return candidate; +} + +function containerPath(value: unknown, label: string): string { + const candidate = boundedText(value, label, MAX_PATH_BYTES); + if ( + !path.posix.isAbsolute(candidate) || + path.posix.normalize(candidate) !== candidate || + candidate === "/" + ) { + throw new Error(`${label} must be a normalized absolute container path`); + } + return candidate; +} + +function preparationRecipe(value: unknown): ManagedClusterVllmPreparationRecipe { + if (!isRecord(value)) throw new Error("model preparation must be an object"); + if (value.ref === NO_PREPARATION_REF) { + exactKeys(value, ["ref"], "model preparation"); + return { ref: NO_PREPARATION_REF }; + } + exactKeys(value, ["exactTextReplacement", "ref", "snapshotCopy"], "model preparation"); + if (value.ref !== SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF) { + throw new Error(`unsupported model preparation ${String(value.ref)}`); + } + if (!isRecord(value.snapshotCopy)) throw new Error("snapshot copy preparation is invalid"); + exactKeys( + value.snapshotCopy, + ["digest", "sourcePath", "targetPath"], + "snapshot copy preparation", + ); + if (!isRecord(value.exactTextReplacement)) { + throw new Error("exact-text replacement preparation is invalid"); + } + exactKeys( + value.exactTextReplacement, + ["expectedText", "replacementText", "targetPath"], + "exact-text replacement preparation", + ); + const expectedText = boundedText( + value.exactTextReplacement.expectedText, + "exact-text replacement expected text", + ); + const replacementText = boundedText( + value.exactTextReplacement.replacementText, + "exact-text replacement replacement text", + ); + if (expectedText === replacementText) { + throw new Error("exact-text replacement must change the matched text"); + } + if ( + typeof value.snapshotCopy.digest !== "string" || + !SHA256_DIGEST.test(value.snapshotCopy.digest) + ) { + throw new Error("snapshot copy digest must be an exact SHA-256 digest"); + } + return { + ref: SNAPSHOT_COPY_AND_EXACT_TEXT_REPLACEMENT_PREPARATION_REF, + snapshotCopy: { + sourcePath: snapshotRelativePath(value.snapshotCopy.sourcePath), + digest: value.snapshotCopy.digest, + targetPath: containerPath(value.snapshotCopy.targetPath, "snapshot copy target path"), + }, + exactTextReplacement: { + targetPath: containerPath( + value.exactTextReplacement.targetPath, + "exact-text replacement target path", + ), + expectedText, + replacementText, + }, + }; +} + +/** Materialize one allowlisted, non-executable preparation operation from recipe data. */ +export function materializeManagedClusterVllmPreparation( + model: PreparationModel, +): ManagedClusterVllmPreparationPlan { + if (!MODEL_ID.test(model.id) || !MODEL_REVISION.test(model.revision)) { + throw new Error("model preparation identity is invalid"); + } + if (!Number.isSafeInteger(model.downloadSizeBytes) || model.downloadSizeBytes <= 0) { + throw new Error("model preparation download size is invalid"); + } + const modelCacheTarget = containerPath(model.modelCacheTarget, "model cache target"); + const preparation = preparationRecipe(model.preparation); + const base = { + phase: "container-before-exec" as const, + modelId: model.id, + modelRevision: model.revision, + modelDownloadSizeBytes: model.downloadSizeBytes, + }; + if (preparation.ref === NO_PREPARATION_REF) { + return { ...base, ref: NO_PREPARATION_REF }; + } + const snapshotRoot = `${modelCacheTarget}/hub/models--${model.id.replaceAll( + "/", + "--", + )}/snapshots/${model.revision}`; + return { + ...base, + ref: preparation.ref, + snapshotCopy: { + sourcePath: `${snapshotRoot}/${preparation.snapshotCopy.sourcePath}`, + digest: preparation.snapshotCopy.digest, + targetPath: preparation.snapshotCopy.targetPath, + }, + exactTextReplacement: preparation.exactTextReplacement, + }; +} diff --git a/src/lib/inference/serving/managed-cluster-runtime-receipt-path.ts b/src/lib/inference/serving/managed-cluster-runtime-receipt-path.ts new file mode 100644 index 0000000000..628369b5a3 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-runtime-receipt-path.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MANAGED_CLUSTER_ID_PATTERN } from "./managed-cluster-identifiers.js"; + +export const MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE = "managed-cluster-vllm-runtime.json"; +export const MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE = "managed-cluster-managed-serving.json"; + +const RUNTIME_BINDING_PREFIX = `${MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE}.rank-`; +const DISCOVERY_BINDING_PREFIX = `${MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE}.`; +const BINDING_SUFFIX = ".ssh-binding"; + +export function isManagedClusterRuntimeBindingStateEntry(entry: string): boolean { + if (!entry.startsWith(RUNTIME_BINDING_PREFIX) || !entry.endsWith(BINDING_SUFFIX)) return false; + const rank = entry.slice(RUNTIME_BINDING_PREFIX.length, -BINDING_SUFFIX.length); + // Rank 0 is local; only remote SSH execution roles own copied binding directories. + return /^(?:[1-9]\d{0,3})$/u.test(rank) && Number(rank) <= 1_023; +} + +export function isManagedClusterDiscoveryBindingStateEntry(entry: string): boolean { + if (!entry.startsWith(DISCOVERY_BINDING_PREFIX) || !entry.endsWith(BINDING_SUFFIX)) return false; + const nodeId = entry.slice(DISCOVERY_BINDING_PREFIX.length, -BINDING_SUFFIX.length); + return MANAGED_CLUSTER_ID_PATTERN.test(nodeId); +} diff --git a/src/lib/inference/serving/managed-cluster-runtime-receipt.ts b/src/lib/inference/serving/managed-cluster-runtime-receipt.ts new file mode 100644 index 0000000000..fc0537dbb4 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-runtime-receipt.ts @@ -0,0 +1,735 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { ensureLocalAdapterStateDir } from "../local-adapter-lifecycle"; +import { loadManagedVllmApiKey, managedVllmStateDir } from "../vllm-api-key"; +import { managedInferenceHexDigest } from "./catalog-integrity"; +import { + assertManagedClusterVllmExecutorConfig, + createManagedClusterVllmExecutor, + inspectManagedClusterVllmNodesSync, + type ManagedClusterVllmNodeSnapshots, +} from "./managed-cluster-executor"; +import { MANAGED_CLUSTER_ID_PATTERN } from "./managed-cluster-identifiers"; +import { + classifyManagedClusterExistingState, + cleanupManagedClusterManagedVllm, + type ManagedClusterVllmLifecycleDeps, + managedClusterVllmApiKeyFingerprint, +} from "./managed-cluster-lifecycle"; +import { type ManagedClusterVllmPlan, managedClusterHeadRole } from "./managed-cluster-materialize"; +import { MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE } from "./managed-cluster-runtime-receipt-path"; +import { + clearManagedVllmSshBinding, + copyManagedVllmSshBinding, + encodeManagedVllmSshBindingHandoff, + loadManagedVllmSshBindingForStatePath, + loadManagedVllmSshBindingHandoff, + type ManagedVllmSshBinding, +} from "./managed-cluster-ssh-binding"; + +const MAX_RECEIPT_BYTES = 128 * 1024; +const SHA256 = /^[a-f0-9]{64}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const BINDING_HANDLE = /^[A-Za-z0-9_-]{1,8192}$/; +const RECEIPT_KEYS = ["apiKeyFingerprint", "nodes", "plan", "planDigest", "schemaVersion"] as const; +const RECEIPT_NODE_KEYS = [ + "cacheRoot", + "containerId", + "discoveryBindingDigest", + "discoveryStatePath", + "nodeId", + "sshBinding", +] as const; +function exactBindingKeys( + keys: Keys & (Exclude extends never ? unknown : never), +): Keys { + return keys; +} + +const EXACT_BINDING_KEYS = exactBindingKeys([ + "schemaVersion", + "peerTarget", + "resolvedHost", + "sshUser", + "port", + "lookupHost", + "hostKeyDigest", + "dockerCliFile", + "dockerShimFile", + "dockerShimSha256", + "knownHostsFile", + "knownHostsSha256", + "bindingFile", + "sshWrapperDirectory", + "sshWrapperFile", + "sshWrapperSha256", +] as const); + +interface PersistedReceipt { + readonly schemaVersion: 1; + readonly plan: ManagedClusterVllmPlan; + readonly planDigest: string; + readonly nodes: readonly PersistedReceiptNode[]; + readonly apiKeyFingerprint: string; +} + +interface PersistedReceiptNode { + readonly nodeId: string; + readonly cacheRoot: string; + readonly containerId: string; + readonly sshBinding: string | null; + readonly discoveryStatePath: string | null; + readonly discoveryBindingDigest: string | null; +} + +export interface PersistManagedClusterVllmRuntimeReceiptInput { + readonly plan: ManagedClusterVllmPlan; + readonly nodes: readonly PersistManagedClusterVllmRuntimeReceiptNode[]; + readonly apiKeyFingerprint: string; +} + +export interface PersistManagedClusterVllmRuntimeReceiptNode { + readonly nodeId: string; + readonly cacheRoot: string; + readonly containerId: string; + readonly sshBinding?: ManagedVllmSshBinding; + readonly discoveryStatePath?: string; +} + +export interface LoadedManagedClusterVllmRuntime { + readonly plan: ManagedClusterVllmPlan; + readonly nodes: readonly LoadedManagedClusterVllmRuntimeNode[]; + readonly apiKeyFingerprint: string; +} + +export interface LoadedManagedClusterVllmRuntimeNode extends PersistedReceiptNode { + readonly binding?: ManagedVllmSshBinding; +} + +type CleanupDeps = Pick< + ManagedClusterVllmLifecycleDeps, + "inspectNode" | "removeContainer" | "withLifecycleLock" +>; + +export interface ManagedClusterVllmRuntimeReceiptOptions { + readonly stateDir?: string; + /** @internal Test seam. */ + readonly loadApiKey?: () => string | null; + /** @internal Test seam. */ + readonly createLifecycleDeps?: (runtime: LoadedManagedClusterVllmRuntime) => CleanupDeps; + /** @internal Test seam. */ + readonly inspectNodesSync?: ( + runtime: LoadedManagedClusterVllmRuntime, + ) => ManagedClusterVllmNodeSnapshots; +} + +export interface RecoveredManagedClusterVllmEndpoint { + readonly baseUrl: string; + readonly apiKey: string; + readonly apiKeyFingerprint: string; + readonly plan: ManagedClusterVllmPlan; +} + +export type ManagedClusterVllmRuntimeCleanupResult = + | { readonly kind: "not-installed" } + | { readonly kind: "removed"; readonly removedContainerIds: readonly string[] }; + +export function managedClusterVllmRuntimeReceiptPath(stateDir = managedVllmStateDir()): string { + return path.join(stateDir, MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function exactKeys( + value: Record, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const sorted = [...expected].sort(); + if (actual.length !== sorted.length || actual.some((key, index) => key !== sorted[index])) { + throw new Error(`${label} fields are invalid`); + } +} + +function requireString(value: unknown, label: string, pattern: RegExp, maximum = 8192): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maximum || + value !== value.trim() || + !pattern.test(value) + ) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function requireAbsolutePath(value: unknown, label: string): string { + const candidate = requireString(value, label, /^[^\u0000-\u001f\u007f]+$/, 4096); + if (!path.posix.isAbsolute(candidate) || path.posix.normalize(candidate) !== candidate) { + throw new Error(`${label} must be a normalized absolute POSIX path`); + } + return candidate; +} + +function planDigest(plan: ManagedClusterVllmPlan): string { + return managedInferenceHexDigest(plan); +} + +function workerTarget(plan: ManagedClusterVllmPlan, nodeId: string): string { + const roles = Array.isArray(plan.roles) ? plan.roles : []; + const worker = roles.find((role) => isRecord(role) && role.nodeId === nodeId); + const execution = worker && isRecord(worker.execution) ? worker.execution : null; + if (!execution || execution.kind !== "ssh") { + throw new Error("Managed cluster worker SSH execution is invalid"); + } + return requireString( + execution.expectedTarget, + "managed cluster worker SSH target", + /^(?:[A-Za-z_][A-Za-z0-9._-]*@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}$/, + 286, + ); +} + +function assertExecutorContract( + plan: ManagedClusterVllmPlan, + nodes: readonly LoadedManagedClusterVllmRuntimeNode[], +): void { + assertManagedClusterVllmExecutorConfig({ + plan, + nodes: nodes.map(({ nodeId, cacheRoot, binding }) => ({ + nodeId, + modelCacheRoot: cacheRoot, + ...(binding ? { sshBinding: binding } : {}), + })), + }); +} + +function parseReceipt(value: unknown): PersistedReceipt { + if (!isRecord(value)) throw new Error("Managed cluster vLLM runtime receipt is invalid"); + exactKeys(value, RECEIPT_KEYS, "Managed cluster vLLM runtime receipt"); + if (value.schemaVersion !== 1 || !isRecord(value.plan)) { + throw new Error("Managed cluster vLLM runtime receipt schema is unsupported"); + } + const plan = value.plan as unknown as ManagedClusterVllmPlan; + if (!Array.isArray(plan.roles)) { + throw new Error("Managed cluster runtime receipt node ownership is incomplete"); + } + const digest = requireString(value.planDigest, "managed cluster plan digest", SHA256, 64); + if (planDigest(plan) !== digest) + throw new Error("Managed cluster vLLM runtime plan digest changed"); + if (!Array.isArray(value.nodes) || value.nodes.length !== plan.roles.length) { + throw new Error("Managed cluster runtime receipt node ownership is incomplete"); + } + const nodes = value.nodes.map((entry, index): PersistedReceiptNode => { + if (!isRecord(entry)) throw new Error("Managed cluster runtime receipt node is invalid"); + exactKeys(entry, RECEIPT_NODE_KEYS, "Managed cluster runtime receipt node"); + const nodeId = requireString( + entry.nodeId, + "managed cluster node ID", + MANAGED_CLUSTER_ID_PATTERN, + 128, + ); + const rolePlan = plan.roles.find((role) => role.nodeId === nodeId); + if (!rolePlan || rolePlan.rank !== index) { + throw new Error("Managed cluster runtime receipt node order does not match the plan"); + } + const remote = rolePlan.execution.kind === "ssh"; + return { + nodeId, + cacheRoot: requireAbsolutePath(entry.cacheRoot, "managed cluster cache root"), + containerId: requireString( + entry.containerId, + "managed cluster container ID", + CONTAINER_ID, + 64, + ), + sshBinding: remote + ? requireString(entry.sshBinding, "managed cluster SSH binding", BINDING_HANDLE) + : entry.sshBinding === null + ? null + : (() => { + throw new Error("Managed cluster local node cannot contain an SSH binding"); + })(), + discoveryStatePath: remote + ? requireAbsolutePath(entry.discoveryStatePath, "managed cluster discovery state path") + : entry.discoveryStatePath === null + ? null + : (() => { + throw new Error("Managed cluster local node cannot contain discovery state"); + })(), + discoveryBindingDigest: remote + ? requireString( + entry.discoveryBindingDigest, + "managed cluster discovery SSH binding digest", + SHA256, + 64, + ) + : entry.discoveryBindingDigest === null + ? null + : (() => { + throw new Error("Managed cluster local node cannot contain a binding digest"); + })(), + }; + }); + if ( + new Set(nodes.map(({ nodeId }) => nodeId)).size !== nodes.length || + new Set(nodes.map(({ containerId }) => containerId)).size !== nodes.length + ) { + throw new Error("Managed cluster container identities are ambiguous"); + } + return { + schemaVersion: 1, + plan, + planDigest: digest, + nodes, + apiKeyFingerprint: requireString( + value.apiKeyFingerprint, + "managed cluster API key fingerprint", + SHA256, + 64, + ), + }; +} + +function assertPrivateReceipt(stat: fs.Stats, filePath: string): void { + if (!stat.isFile() || (stat.mode & 0o777) !== 0o600) { + throw new Error( + `Managed cluster vLLM runtime receipt must be a private regular file: ${filePath}`, + ); + } + if (typeof process.getuid === "function" && stat.uid !== process.getuid()) { + throw new Error(`Managed cluster vLLM runtime receipt has the wrong owner: ${filePath}`); + } +} + +function loadPersistedReceipt(stateDir: string): PersistedReceipt | null { + const filePath = managedClusterVllmRuntimeReceiptPath(stateDir); + if (typeof fs.constants.O_NOFOLLOW !== "number") { + throw new Error("Secure no-follow file opens are unavailable on this platform"); + } + let fd: number | undefined; + try { + try { + fd = fs.openSync( + filePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | (fs.constants.O_NONBLOCK ?? 0), + ); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return null; + if (code === "ELOOP") { + throw new Error(`Refusing to read managed cluster runtime receipt through a symbolic link`); + } + throw error; + } + const stat = fs.fstatSync(fd); + assertPrivateReceipt(stat, filePath); + if (stat.size < 2 || stat.size > MAX_RECEIPT_BYTES) { + throw new Error(`Managed cluster vLLM runtime receipt is malformed: ${filePath}`); + } + return parseReceipt(JSON.parse(fs.readFileSync(fd, "utf8"))); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Managed cluster vLLM runtime receipt is malformed: ${filePath}`); + } + throw error; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function fsyncDirectory(directory: string): void { + const fd = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function writeReceipt(receipt: PersistedReceipt, stateDir: string): void { + ensureLocalAdapterStateDir(stateDir); + const filePath = managedClusterVllmRuntimeReceiptPath(stateDir); + const temporary = `${filePath}.tmp-${String(process.pid)}-${Date.now().toString(16)}`; + let fd: number | undefined; + try { + fd = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + 0o600, + ); + assertPrivateReceipt(fs.fstatSync(fd), temporary); + fs.writeFileSync(fd, `${JSON.stringify(receipt)}\n`, "utf8"); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = undefined; + fs.renameSync(temporary, filePath); + fsyncDirectory(stateDir); + } catch (error) { + if (fd !== undefined) { + try { + fs.closeSync(fd); + } catch { + // Preserve the receipt-write failure. + } + } + try { + fs.unlinkSync(temporary); + } catch { + // A leftover unique temporary file must not mask the receipt-write failure. + } + throw error; + } +} + +function durablePlan( + plan: ManagedClusterVllmPlan, + sshBindings: ReadonlyMap, +): ManagedClusterVllmPlan { + return { + ...plan, + roles: plan.roles.map((rolePlan) => + rolePlan.execution.kind === "ssh" + ? { + ...rolePlan, + execution: { + ...rolePlan.execution, + bindingHandle: + sshBindings.get(rolePlan.nodeId) ?? + (() => { + throw new Error(`Managed cluster worker ${rolePlan.nodeId} has no SSH binding`); + })(), + }, + } + : rolePlan, + ), + }; +} + +function loadedRuntime(receipt: PersistedReceipt): LoadedManagedClusterVllmRuntime { + const nodes = receipt.nodes.map((node): LoadedManagedClusterVllmRuntimeNode => { + if (node.sshBinding === null) return node; + const binding = loadManagedVllmSshBindingHandoff( + node.sshBinding, + workerTarget(receipt.plan, node.nodeId), + ); + if (encodeManagedVllmSshBindingHandoff(binding) !== node.sshBinding) { + throw new Error("Managed cluster runtime SSH binding identity changed"); + } + return { ...node, binding }; + }); + assertExecutorContract(receipt.plan, nodes); + return { ...receipt, nodes }; +} + +function bindingDigest(binding: ManagedVllmSshBinding): string { + return managedInferenceHexDigest( + Object.fromEntries(EXACT_BINDING_KEYS.map((field) => [field, binding[field]])), + ); +} + +function loadReceiptOwnedDiscoveryBinding( + runtime: LoadedManagedClusterVllmRuntime, +): readonly { readonly nodeId: string; readonly binding: ManagedVllmSshBinding | null }[] { + return runtime.nodes + .filter((node) => node.binding) + .map((node) => { + const binding = loadManagedVllmSshBindingForStatePath( + node.discoveryStatePath!, + node.binding!.peerTarget, + node.binding!.hostKeyDigest, + ); + if (binding && bindingDigest(binding) !== node.discoveryBindingDigest) { + throw new Error( + "Managed cluster discovery SSH binding does not match the runtime receipt identity", + ); + } + return { nodeId: node.nodeId, binding }; + }); +} + +export function loadManagedClusterVllmRuntimeReceipt( + options: Pick = {}, +): LoadedManagedClusterVllmRuntime | null { + const receipt = loadPersistedReceipt(options.stateDir ?? managedVllmStateDir()); + return receipt ? loadedRuntime(receipt) : null; +} + +function sameInput( + existing: LoadedManagedClusterVllmRuntime, + input: PersistManagedClusterVllmRuntimeReceiptInput, +): boolean { + const durable = durablePlan( + input.plan, + new Map( + existing.nodes.flatMap((node) => + node.sshBinding ? [[node.nodeId, node.sshBinding] as const] : [], + ), + ), + ); + const existingNodes = new Map(existing.nodes.map((node) => [node.nodeId, node])); + return ( + planDigest(durable) === planDigest(existing.plan) && + input.apiKeyFingerprint === existing.apiKeyFingerprint && + input.nodes.length === existing.nodes.length && + input.nodes.every((node) => { + const current = existingNodes.get(node.nodeId); + return ( + current?.nodeId === node.nodeId && + current.cacheRoot === node.cacheRoot && + current.containerId === node.containerId && + current.discoveryStatePath === (node.discoveryStatePath ?? null) && + current.discoveryBindingDigest === (node.sshBinding ? bindingDigest(node.sshBinding) : null) + ); + }) + ); +} + +/** Persist immutable ownership and pinned transport state for recovery/uninstall. */ +export function persistManagedClusterVllmRuntimeReceipt( + input: PersistManagedClusterVllmRuntimeReceiptInput, + options: Pick = {}, +): LoadedManagedClusterVllmRuntime { + const stateDir = options.stateDir ?? managedVllmStateDir(); + ensureLocalAdapterStateDir(stateDir); + requireString(input.apiKeyFingerprint, "managed cluster API key fingerprint", SHA256, 64); + if ( + input.nodes.length !== input.plan.roles.length || + new Set(input.nodes.map(({ nodeId }) => nodeId)).size !== input.nodes.length || + new Set(input.nodes.map(({ containerId }) => containerId)).size !== input.nodes.length + ) { + throw new Error("Managed cluster runtime receipt node ownership is incomplete or ambiguous"); + } + const inputNodes = input.plan.roles.map( + (rolePlan, index): LoadedManagedClusterVllmRuntimeNode => { + const node = input.nodes.find((candidate) => candidate.nodeId === rolePlan.nodeId); + if (!node) throw new Error(`Managed cluster rank ${String(index)} has no receipt input`); + requireString(node.containerId, "managed cluster container ID", CONTAINER_ID, 64); + const cacheRoot = requireAbsolutePath(node.cacheRoot, "managed cluster cache root"); + if ( + (rolePlan.execution.kind === "local" && + (node.sshBinding !== undefined || node.discoveryStatePath !== undefined)) || + (rolePlan.execution.kind === "ssh" && (!node.sshBinding || !node.discoveryStatePath)) + ) { + throw new Error(`Managed cluster node ${node.nodeId} transport ownership is invalid`); + } + return { + nodeId: node.nodeId, + cacheRoot, + containerId: node.containerId, + sshBinding: node.sshBinding ? encodeManagedVllmSshBindingHandoff(node.sshBinding) : null, + discoveryStatePath: node.discoveryStatePath ?? null, + discoveryBindingDigest: node.sshBinding ? bindingDigest(node.sshBinding) : null, + ...(node.sshBinding ? { binding: node.sshBinding } : {}), + }; + }, + ); + assertExecutorContract(input.plan, inputNodes); + + const existing = loadManagedClusterVllmRuntimeReceipt({ stateDir }); + if (existing) { + if (!sameInput(existing, input)) { + throw new Error("A different managed cluster runtime receipt already owns recovery state"); + } + return existing; + } + + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const createdBindingStatePaths: string[] = []; + try { + const runtimeBindings = new Map(); + const persistedNodes = inputNodes.map((node, index): PersistedReceiptNode => { + const { binding, ...persisted } = node; + if (!binding) return persisted; + const rolePlan = input.plan.roles[index]!; + const runtimeStatePath = `${receiptPath}.rank-${String(rolePlan.rank)}`; + const bindingPath = `${runtimeStatePath}.ssh-binding`; + try { + fs.mkdirSync(bindingPath, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new Error(`Managed cluster SSH binding state already exists: ${bindingPath}`); + } + throw error; + } + createdBindingStatePaths.push(runtimeStatePath); + const runtimeBinding = copyManagedVllmSshBinding(runtimeStatePath, binding); + runtimeBindings.set(node.nodeId, runtimeBinding); + return { ...persisted, sshBinding: encodeManagedVllmSshBindingHandoff(runtimeBinding) }; + }); + fsyncDirectory(stateDir); + const plan = durablePlan( + input.plan, + new Map( + persistedNodes.flatMap((node) => + node.sshBinding ? [[node.nodeId, node.sshBinding] as const] : [], + ), + ), + ); + const loadedNodes = persistedNodes.map((node) => ({ + ...node, + ...(runtimeBindings.has(node.nodeId) ? { binding: runtimeBindings.get(node.nodeId)! } : {}), + })); + assertExecutorContract(plan, loadedNodes); + const receipt: PersistedReceipt = { + schemaVersion: 1, + plan, + planDigest: planDigest(plan), + nodes: persistedNodes, + apiKeyFingerprint: input.apiKeyFingerprint, + }; + writeReceipt(receipt, stateDir); + return loadedRuntime(receipt); + } catch (error) { + try { + for (const statePath of createdBindingStatePaths.reverse()) { + clearManagedVllmSshBinding(statePath); + } + } catch { + // Preserve the receipt persistence error. + } + throw error; + } +} + +function clearReceipt(stateDir: string, runtime: LoadedManagedClusterVllmRuntime): void { + const filePath = managedClusterVllmRuntimeReceiptPath(stateDir); + fs.unlinkSync(filePath); + for (const rolePlan of runtime.plan.roles) { + if (rolePlan.execution.kind === "ssh") { + clearManagedVllmSshBinding(`${filePath}.rank-${String(rolePlan.rank)}`); + } + } + fsyncDirectory(stateDir); +} + +function defaultCreateLifecycleDeps(runtime: LoadedManagedClusterVllmRuntime): CleanupDeps { + return createManagedClusterVllmExecutor({ + plan: runtime.plan, + nodes: runtime.nodes.map(({ nodeId, cacheRoot, binding }) => ({ + nodeId, + modelCacheRoot: cacheRoot, + ...(binding ? { sshBinding: binding } : {}), + })), + }); +} + +function defaultInspectNodesSync( + runtime: LoadedManagedClusterVllmRuntime, +): ManagedClusterVllmNodeSnapshots { + return inspectManagedClusterVllmNodesSync({ + plan: runtime.plan, + nodes: runtime.nodes.map(({ nodeId, cacheRoot, binding }) => ({ + nodeId, + modelCacheRoot: cacheRoot, + ...(binding ? { sshBinding: binding } : {}), + })), + }); +} + +/** Recover only the exact healthy receipt-owned cluster; unsafe managed state is explicit. */ +export function recoverInstalledManagedClusterVllmEndpoint( + options: Pick< + ManagedClusterVllmRuntimeReceiptOptions, + "inspectNodesSync" | "loadApiKey" | "stateDir" + > = {}, +): RecoveredManagedClusterVllmEndpoint | null { + const runtime = loadManagedClusterVllmRuntimeReceipt({ stateDir: options.stateDir }); + if (!runtime) return null; + const apiKey = (options.loadApiKey ?? loadManagedVllmApiKey)(); + if (!apiKey || managedClusterVllmApiKeyFingerprint(apiKey) !== runtime.apiKeyFingerprint) { + throw new Error("Managed cluster API key no longer matches the runtime receipt"); + } + let snapshots: ManagedClusterVllmNodeSnapshots; + try { + snapshots = (options.inspectNodesSync ?? defaultInspectNodesSync)(runtime); + } catch (error) { + throw new Error( + `Could not inspect the managed cluster: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const state = classifyManagedClusterExistingState( + runtime.plan, + runtime.apiKeyFingerprint, + snapshots.nodes, + ); + const expectedContainers = new Map( + runtime.nodes.map(({ nodeId, containerId }) => [nodeId, containerId]), + ); + if ( + state.outcome !== "reuse" || + state.containers.length !== expectedContainers.size || + state.containers.some( + ({ nodeId, containerId }) => expectedContainers.get(nodeId) !== containerId, + ) + ) { + const reason = "reason" in state ? state.reason : "receipt-owned container IDs changed"; + throw new Error(`Managed cluster runtime is not recoverable: ${reason}`); + } + const baseUrl = managedClusterHeadRole(runtime.plan).endpoint; + if (!baseUrl) throw new Error("Managed cluster head endpoint is invalid"); + return { + baseUrl, + apiKey, + apiKeyFingerprint: runtime.apiKeyFingerprint, + plan: runtime.plan, + }; +} + +/** Remove only receipt-ID-owned containers, then retire fully accounted ownership state. */ +export async function cleanupInstalledManagedClusterVllmRuntime( + options: ManagedClusterVllmRuntimeReceiptOptions = {}, +): Promise { + const stateDir = options.stateDir ?? managedVllmStateDir(); + const runtime = loadManagedClusterVllmRuntimeReceipt({ stateDir }); + if (!runtime) return { kind: "not-installed" }; + + const apiKey = (options.loadApiKey ?? loadManagedVllmApiKey)(); + if (!apiKey || managedClusterVllmApiKeyFingerprint(apiKey) !== runtime.apiKeyFingerprint) { + throw new Error("Managed cluster API key no longer matches the runtime receipt"); + } + const discoveryBindings = loadReceiptOwnedDiscoveryBinding(runtime); + const deps = (options.createLifecycleDeps ?? defaultCreateLifecycleDeps)(runtime); + const cleanup = await cleanupManagedClusterManagedVllm(runtime.plan, apiKey, deps, { + containers: runtime.nodes.map(({ nodeId, containerId }) => ({ nodeId, containerId })), + }); + if (!cleanup.ok) throw new Error(cleanup.reason); + const expected = new Set(runtime.nodes.map(({ containerId }) => containerId)); + const accounted = [...cleanup.removedContainerIds, ...(cleanup.alreadyAbsentContainerIds ?? [])]; + if ( + accounted.length !== expected.size || + new Set(accounted).size !== expected.size || + accounted.some((id) => !expected.has(id)) + ) { + throw new Error("Managed cluster cleanup returned unexpected container identities"); + } + const currentDiscoveryBindings = loadReceiptOwnedDiscoveryBinding(runtime); + if ( + discoveryBindings.length !== currentDiscoveryBindings.length || + discoveryBindings.some( + (entry, index) => + entry.nodeId !== currentDiscoveryBindings[index]?.nodeId || + (entry.binding === null) !== (currentDiscoveryBindings[index]?.binding === null), + ) + ) { + throw new Error("Managed cluster discovery SSH binding changed during runtime cleanup"); + } + for (const entry of currentDiscoveryBindings) { + if (entry.binding) { + const node = runtime.nodes.find(({ nodeId }) => nodeId === entry.nodeId)!; + clearManagedVllmSshBinding(node.discoveryStatePath!); + } + } + clearReceipt(stateDir, runtime); + return { kind: "removed", removedContainerIds: cleanup.removedContainerIds }; +} diff --git a/src/lib/inference/serving/managed-cluster-ssh-binding.test-support.ts b/src/lib/inference/serving/managed-cluster-ssh-binding.test-support.ts new file mode 100644 index 0000000000..a97f967026 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-ssh-binding.test-support.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type { DualStationSshBindingFixture as ManagedVllmSshBindingFixture } from "../vllm-station-ssh-binding.test-support.js"; +export { + createDualStationSshBindingFixture as createManagedVllmSshBindingFixture, + retargetDualStationSshBindingFixture as retargetManagedVllmSshBindingFixture, +} from "../vllm-station-ssh-binding.test-support.js"; diff --git a/src/lib/inference/serving/managed-cluster-ssh-binding.ts b/src/lib/inference/serving/managed-cluster-ssh-binding.ts new file mode 100644 index 0000000000..e0e143008e --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-ssh-binding.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type { + DualStationSshBinding as ManagedVllmSshBinding, + QualifiedStationSshIdentity as QualifiedManagedVllmSshIdentity, + WriteDualStationSshBindingOptions as WriteManagedVllmSshBindingOptions, +} from "../vllm-station-ssh-binding.js"; +/** + * Cardinality-neutral names for the pinned SSH transport shared by managed + * vLLM clusters. The legacy Station implementation remains the compatibility + * provider until that existing path is migrated to the serving catalog. + */ +export { + assertDualStationSshBindingFiles as assertManagedVllmSshBindingFiles, + clearDualStationSshBinding as clearManagedVllmSshBinding, + copyDualStationSshBinding as copyManagedVllmSshBinding, + dualStationDockerSshUri as managedVllmDockerSshUri, + dualStationPinnedSshArgs as managedVllmPinnedSshArgs, + dualStationSshBindingDirectory as managedVllmSshBindingDirectory, + encodeDualStationSshBindingHandoff as encodeManagedVllmSshBindingHandoff, + loadDualStationSshBinding as loadManagedVllmSshBinding, + loadDualStationSshBindingForStatePath as loadManagedVllmSshBindingForStatePath, + loadDualStationSshBindingHandoff as loadManagedVllmSshBindingHandoff, + stationKnownHostsDigest as managedVllmKnownHostsDigest, + strictStationSshTransportArgs as strictManagedVllmSshTransportArgs, + writeDualStationSshBinding as writeManagedVllmSshBinding, +} from "../vllm-station-ssh-binding.js"; diff --git a/src/lib/inference/serving/managed-cluster-topology.test.ts b/src/lib/inference/serving/managed-cluster-topology.test.ts new file mode 100644 index 0000000000..3ba75b7ba0 --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-topology.test.ts @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { + MANAGED_CLUSTER_TOPOLOGY_ID, + MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + type ManagedClusterNodeObservation, + type ManagedClusterPeerObservation, + type ManagedClusterRailObservation, + type ManagedClusterTopologyQualificationInput, + qualifyManagedClusterTopology, +} from "./managed-cluster-topology.js"; + +const EVALUATED_AT = "2026-08-02T18:00:00.000Z"; +const READINESS_OBSERVED_AT = "2026-08-02T17:59:30.000Z"; +const REQUIRED_CAPABILITIES = [ + "host.platform.supported", + "host.platform.dgx_spark", + "host.docker.available", + "host.docker.daemon_reachable", + "host.docker.runtime_supported", + "host.docker.storage_compatible", + "host.gpu.nvidia_available", + "host.gpu.container_toolkit_available", + "host.gpu.cdi_healthy", +] as const; + +function readiness(overrides: Partial = {}): SystemReadinessReport { + const base = { + schemaVersion: "1.1.0", + mutated: false, + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: "1d6948d89b46eab739728215f9a19ef40b8f6121", + observedAt: READINESS_OBSERVED_AT, + }, + observations: [], + capabilities: REQUIRED_CAPABILITIES.map((id) => ({ + id, + state: "present" as const, + })), + qualifications: [ + { + id: "host.platform.dgx_spark", + status: "qualified" as const, + capabilityIds: ["host.platform.dgx_spark"], + }, + ], + findings: [], + evidence: [], + status: "supported" as const, + exitCode: 0 as const, + } satisfies SystemReadinessReport; + return { ...base, ...overrides } as SystemReadinessReport; +} + +function rail( + node: "head" | "worker", + index: 0 | 1, + overrides: Partial = {}, +): ManagedClusterRailObservation { + const headAddress = `192.168.${100 + index}.10`; + const workerAddress = `192.168.${100 + index}.11`; + const isHead = node === "head"; + return { + adapter: "connectx-7", + path: "direct", + physicalPortId: isHead ? "cx7-left-head" : "cx7-left-worker", + netdev: index === 0 ? "enp1s0f0np0" : "enP2p1s0f0np0", + hcaDevice: index === 0 ? "rocep1s0f0" : "roceP2p1s0f0", + hcaPort: 1, + address: isHead ? headAddress : workerAddress, + prefixLength: 24, + peerNodeId: isHead ? "spark-worker" : "spark-head", + peerAddress: isHead ? workerAddress : headAddress, + linkState: "up", + connectivity: "reachable", + roceGid: { + state: "resolved", + index: index === 0 ? 5 : 7, + value: isHead ? `fe80::${10 + index}` : `fe80::${20 + index}`, + }, + ...overrides, + }; +} + +function localNode(): ManagedClusterNodeObservation { + return { + nodeId: "spark-head", + gpuIds: ["GPU-head"], + readiness: readiness(), + runtimeState: "clear", + rails: [rail("head", 1), rail("head", 0)], + }; +} + +function peerNode(index = 0): ManagedClusterPeerObservation { + const suffix = index === 0 ? "" : `-${index}`; + return { + nodeId: `spark-worker${suffix}`, + gpuIds: [`GPU-worker${suffix}`], + readiness: readiness(), + runtimeState: "clear", + rails: + index === 0 + ? [rail("worker", 0), rail("worker", 1)] + : [ + rail("worker", 0, { peerNodeId: "spark-head" }), + rail("worker", 1, { peerNodeId: "spark-head" }), + ], + sshBinding: { + state: "pretrusted", + fromNodeId: "spark-head", + toNodeId: `spark-worker${suffix}`, + peerTarget: `spark-worker${suffix}.local`, + handle: `ssh-binding:worker${suffix || "-0"}`, + }, + }; +} + +function qualificationInput( + peers: readonly ManagedClusterPeerObservation[] = [peerNode()], +): ManagedClusterTopologyQualificationInput { + return { + intent: "automatic", + evaluatedAt: EVALUATED_AT, + maxReadinessAgeMs: 60_000, + local: localNode(), + peers, + }; +} + +function ringRail( + nodeId: string, + peerNodeId: string, + subnet: number, + addressHost: number, + peerHost: number, + index: number, +): ManagedClusterRailObservation { + return { + adapter: "connectx-7", + path: "direct", + physicalPortId: `cx7-${nodeId}`, + netdev: `eth${String(index)}`, + hcaDevice: `roce${String(index)}`, + hcaPort: 1, + address: `192.168.${String(subnet)}.${String(addressHost)}`, + prefixLength: 30, + peerNodeId, + peerAddress: `192.168.${String(subnet)}.${String(peerHost)}`, + linkState: "up", + connectivity: "reachable", + roceGid: { + state: "resolved", + index: 3 + index, + value: `fe80::${String(subnet)}:${String(addressHost)}`, + }, + }; +} + +function threeNodeQualificationInput(): ManagedClusterTopologyQualificationInput { + const input = qualificationInput(); + input.local.rails = [ + ringRail("spark-head", "spark-worker-a", 100, 1, 2, 0), + ringRail("spark-head", "spark-worker-b", 102, 1, 2, 1), + ]; + input.peers = [ + { + ...peerNode(), + nodeId: "spark-worker-a", + gpuIds: ["GPU-worker-a"], + rails: [ + ringRail("spark-worker-a", "spark-head", 100, 2, 1, 0), + ringRail("spark-worker-a", "spark-worker-b", 101, 1, 2, 1), + ], + sshBinding: { + state: "pretrusted", + fromNodeId: "spark-head", + toNodeId: "spark-worker-a", + peerTarget: "spark-worker-a.local", + handle: "ssh-binding:worker-a", + }, + }, + { + ...peerNode(), + nodeId: "spark-worker-b", + gpuIds: ["GPU-worker-b"], + rails: [ + ringRail("spark-worker-b", "spark-worker-a", 101, 2, 1, 0), + ringRail("spark-worker-b", "spark-head", 102, 2, 1, 1), + ], + sshBinding: { + state: "pretrusted", + fromNodeId: "spark-head", + toNodeId: "spark-worker-b", + peerTarget: "spark-worker-b.local", + handle: "ssh-binding:worker-b", + }, + }, + ]; + return input; +} + +function clone(value: T): T { + return structuredClone(value); +} + +describe("managed DGX Spark cluster topology qualification", () => { + it("returns no match when discovery finds no peer", () => { + expect(qualifyManagedClusterTopology(qualificationInput([]))).toMatchObject({ + outcome: "no-match", + code: "peer-count", + }); + }); + + it("qualifies the profile-declared node set as a direct ConnectX-7 topology", () => { + const result = qualifyManagedClusterTopology(qualificationInput()); + + expect(result).toMatchObject({ outcome: "qualified" }); + const qualified = result as Extract; + expect(qualified.artifact).toMatchObject({ + id: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + status: "qualified", + subjectNodeIds: ["spark-head", "spark-worker"], + output: { + controllerNodeId: "spark-head", + peers: [ + { + nodeId: "spark-worker", + target: "spark-worker.local", + sshBindingHandle: "ssh-binding:worker-0", + }, + ], + }, + }); + expect(qualified.artifact.subjectDigest).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(qualified.artifact.outputDigest).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(qualified.artifact.output.nodes).toEqual([ + { nodeId: "spark-head", gpuId: "GPU-head", rank: 0, role: "head" }, + { nodeId: "spark-worker", gpuId: "GPU-worker", rank: 1, role: "worker" }, + ]); + expect(qualified.artifact.output.rails).toHaveLength(2); + expect( + qualified.artifact.output.rails.flatMap(({ endpoints }) => + endpoints.map(({ nodeId }) => nodeId), + ), + ).toEqual(expect.arrayContaining(["spark-head", "spark-worker"])); + }); + + it("reports a fabric mismatch when two peers claim the same rail addresses", () => { + expect( + qualifyManagedClusterTopology(qualificationInput([peerNode(), peerNode(1)])), + ).toMatchObject({ + outcome: "no-match", + code: "fabric-mismatch", + }); + }); + + it("qualifies a three-node ring and assigns deterministic ranks", () => { + const result = qualifyManagedClusterTopology(threeNodeQualificationInput()); + expect(result).toMatchObject({ outcome: "qualified" }); + const artifact = (result as Extract).artifact; + expect(artifact.output.nodes).toEqual([ + { nodeId: "spark-head", gpuId: "GPU-head", rank: 0, role: "head" }, + { nodeId: "spark-worker-a", gpuId: "GPU-worker-a", rank: 1, role: "worker" }, + { nodeId: "spark-worker-b", gpuId: "GPU-worker-b", rank: 2, role: "worker" }, + ]); + expect(artifact.output.rails).toHaveLength(3); + expect(artifact.output.peers.map(({ nodeId }) => nodeId)).toEqual([ + "spark-worker-a", + "spark-worker-b", + ]); + }); + + it("reports strict fabric errors without a fixed peer-count branch", () => { + const input = qualificationInput([peerNode(), peerNode(1), peerNode(2)]); + input.intent = "explicit"; + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "error", + code: "fabric-mismatch", + }); + }); + + it("returns a strict error for a resume topology mismatch", () => { + const input = qualificationInput([]); + input.intent = "resume"; + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "error", + code: "peer-count", + }); + }); + + it.each([ + { + name: "an incompatible schema", + mutate: (report: SystemReadinessReport) => { + report.schemaVersion = "2.0.0"; + }, + code: "readiness-schema-incompatible", + }, + { + name: "a stale observation", + mutate: (report: SystemReadinessReport) => { + report.provenance.observedAt = "2026-08-02T17:58:00.000Z"; + }, + code: "readiness-stale", + }, + { + name: "an inconclusive result", + mutate: (report: SystemReadinessReport) => { + Object.assign(report, { status: "inconclusive", exitCode: 3 }); + }, + code: "readiness-inconclusive", + }, + { + name: "an incompatible result", + mutate: (report: SystemReadinessReport) => { + Object.assign(report, { status: "incompatible", exitCode: 2 }); + }, + code: "readiness-incompatible", + }, + { + name: "no Spark qualification", + mutate: (report: SystemReadinessReport) => { + report.qualifications = []; + }, + code: "spark-qualification-unavailable", + }, + { + name: "an unknown Spark qualification", + mutate: (report: SystemReadinessReport) => { + report.qualifications = [ + { + id: "host.platform.dgx_spark", + status: "unknown", + capabilityIds: [], + }, + ]; + }, + code: "spark-qualification-unavailable", + }, + ])("fails closed for $name", ({ mutate, code }) => { + const input = qualificationInput(); + mutate(input.peers[0]!.readiness); + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code, + }); + }); + + it("leaves serving runtime capability policy to preset resolution", () => { + const input = qualificationInput(); + input.peers[0]!.readiness.capabilities = input.peers[0]!.readiness.capabilities.map( + (capability) => + capability.id === "host.docker.runtime_supported" + ? { ...capability, state: "unknown" } + : capability, + ); + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "qualified", + }); + }); + + it.each([ + { + name: "the same node identity", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.peers[0]!.nodeId = input.local.nodeId; + input.peers[0]!.sshBinding.toNodeId = input.local.nodeId; + }, + code: "duplicate-node-identity", + }, + { + name: "the same GPU identity", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.peers[0]!.gpuIds = input.local.gpuIds; + }, + code: "duplicate-gpu-identity", + }, + { + name: "more than one local GPU identity", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.gpuIds = ["GPU-head", "GPU-extra"]; + }, + code: "gpu-identity-unavailable", + }, + ])("rejects $name", ({ mutate, code }) => { + const input = qualificationInput(); + mutate(input); + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code, + }); + }); + + it.each([ + { state: "conflict" as const, code: "runtime-conflict" }, + { state: "unknown" as const, code: "runtime-state-unknown" }, + ])("does not replace a peer runtime in the $state state", ({ state, code }) => { + const input = qualificationInput(); + input.peers[0]!.runtimeState = state; + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code, + }); + }); + + it.each([ + { + name: "one logical rail", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.rails = input.local.rails.slice(0, 1); + }, + code: "fabric-degraded", + }, + { + name: "three logical rails", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.rails = [...input.local.rails, rail("head", 0, { netdev: "extra0" })]; + }, + code: "fabric-multiple", + }, + { + name: "two physical ports", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.rails[1]!.physicalPortId = "cx7-right-head"; + }, + code: "fabric-multiple", + }, + { + name: "a switched path", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.rails[0]!.path = "switched"; + }, + code: "fabric-degraded", + }, + { + name: "an unreachable peer address", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.peers[0]!.rails[0]!.connectivity = "unreachable"; + }, + code: "fabric-degraded", + }, + { + name: "nonreciprocal addresses", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.peers[0]!.rails[0]!.peerAddress = "192.168.100.99"; + }, + code: "fabric-mismatch", + }, + { + name: "an unresolved RoCE GID", + mutate: (input: ManagedClusterTopologyQualificationInput) => { + input.local.rails[0]!.roceGid = { state: "unknown" }; + }, + code: "fabric-degraded", + }, + ])("rejects a fabric with $name", ({ mutate, code }) => { + const input = qualificationInput(); + mutate(input); + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code, + }); + }); + + it("rejects a peer whose SSH binding state is not trusted", () => { + const input = qualificationInput(); + input.peers[0]!.sshBinding.state = "untrusted"; + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code: "ssh-binding-unavailable", + }); + }); + + it("rejects a peer whose trusted SSH binding handle is empty", () => { + const input = qualificationInput(); + input.peers[0]!.sshBinding.handle = " "; + + expect(qualifyManagedClusterTopology(input)).toMatchObject({ + outcome: "no-match", + code: "ssh-binding-unavailable", + }); + }); + + it("produces the same artifact for either injected rail order", () => { + const firstInput = qualificationInput(); + const secondInput = clone(firstInput); + secondInput.local.rails = [...secondInput.local.rails].reverse(); + secondInput.peers[0]!.rails = [...secondInput.peers[0]!.rails].reverse(); + + const first = qualifyManagedClusterTopology(firstInput); + const second = qualifyManagedClusterTopology(secondInput); + expect(first.outcome).toBe("qualified"); + expect(second.outcome).toBe("qualified"); + const firstQualified = first as Extract; + const secondQualified = second as Extract; + expect(secondQualified.artifact).toEqual(firstQualified.artifact); + }); +}); diff --git a/src/lib/inference/serving/managed-cluster-topology.ts b/src/lib/inference/serving/managed-cluster-topology.ts new file mode 100644 index 0000000000..568d62d20a --- /dev/null +++ b/src/lib/inference/serving/managed-cluster-topology.ts @@ -0,0 +1,872 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; + +import { checkSystemReadinessSchemaVersion } from "../../readiness/compatibility.js"; +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { managedInferenceDigest } from "./catalog-integrity.js"; +import type { ManagedInferenceTopologyQualification } from "./types.js"; +import { MANAGED_CLUSTER_ID_PATTERN } from "./managed-cluster-identifiers.js"; + +export const MANAGED_CLUSTER_TOPOLOGY_ID = "host-cluster.direct-cx7" as const; +export const MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION = 1 as const; + +const SAFE_INTERFACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const SAFE_BINDING_HANDLE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,8191}$/; +const SSH_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const SSH_HOST_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; + +export type ManagedClusterQualificationIntent = "automatic" | "explicit" | "resume"; +export type ManagedClusterRuntimeState = "clear" | "conflict" | "unknown"; +export type ManagedClusterObservationState = "up" | "down" | "unknown"; +export type ManagedClusterConnectivityState = "reachable" | "unreachable" | "unknown"; +export type ManagedClusterRoceGidState = "resolved" | "missing" | "unknown"; + +export interface ManagedClusterRoceGidObservation { + state: ManagedClusterRoceGidState; + index?: number; + value?: string; +} + +export interface ManagedClusterRailObservation { + adapter: "connectx-7" | "other" | "unknown"; + path: "direct" | "switched" | "unknown"; + physicalPortId: string; + netdev: string; + hcaDevice: string; + hcaPort: number; + address: string; + prefixLength: number; + peerNodeId: string; + peerAddress: string; + linkState: ManagedClusterObservationState; + connectivity: ManagedClusterConnectivityState; + roceGid: ManagedClusterRoceGidObservation; +} + +export interface ManagedClusterNodeObservation { + nodeId: string; + gpuIds: readonly string[]; + readiness: SystemReadinessReport; + runtimeState: ManagedClusterRuntimeState; + rails: readonly ManagedClusterRailObservation[]; +} + +export interface ManagedClusterSshBindingObservation { + state: "pretrusted" | "untrusted" | "unknown"; + fromNodeId: string; + toNodeId: string; + peerTarget: string; + handle: string; +} + +export interface ManagedClusterPeerObservation extends ManagedClusterNodeObservation { + sshBinding: ManagedClusterSshBindingObservation; +} + +export interface ManagedClusterTopologyQualificationInput { + intent: ManagedClusterQualificationIntent; + evaluatedAt: string; + maxReadinessAgeMs: number; + local: ManagedClusterNodeObservation; + peers: readonly ManagedClusterPeerObservation[]; +} + +export interface ManagedClusterTopologyNode { + nodeId: string; + gpuId: string; + rank: number; + role: "head" | "worker"; +} + +export interface ManagedClusterTopologyRoceGid { + index: number; + value: string; +} + +export interface ManagedClusterTopologyRailEndpoint { + nodeId: string; + netdev: string; + hcaDevice: string; + hcaPort: number; + address: string; + prefixLength: number; + peerAddress: string; + roceGid: ManagedClusterTopologyRoceGid; +} + +export interface ManagedClusterTopologyRail { + index: number; + endpoints: readonly [ManagedClusterTopologyRailEndpoint, ManagedClusterTopologyRailEndpoint]; +} + +export interface ManagedClusterTopologyOutput { + controllerNodeId: string; + nodes: readonly ManagedClusterTopologyNode[]; + rails: readonly ManagedClusterTopologyRail[]; + masterAddress: string; + peers: readonly { + nodeId: string; + target: string; + sshBindingHandle: string; + }[]; +} + +export type ManagedClusterTopologyArtifact = + ManagedInferenceTopologyQualification & { + id: typeof MANAGED_CLUSTER_TOPOLOGY_ID; + schemaVersion: typeof MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION; + status: "qualified"; + }; + +export type ManagedClusterTopologyFailureCode = + | "peer-count" + | "qualification-policy-invalid" + | "readiness-schema-incompatible" + | "readiness-stale" + | "readiness-incompatible" + | "readiness-inconclusive" + | "spark-qualification-unavailable" + | "runtime-qualification-unavailable" + | "node-identity-unavailable" + | "duplicate-node-identity" + | "gpu-identity-unavailable" + | "duplicate-gpu-identity" + | "runtime-conflict" + | "runtime-state-unknown" + | "ssh-binding-unavailable" + | "fabric-degraded" + | "fabric-multiple" + | "fabric-mismatch"; + +export type ManagedClusterTopologyQualificationResult = + | { outcome: "qualified"; artifact: ManagedClusterTopologyArtifact } + | { outcome: "no-match"; code: ManagedClusterTopologyFailureCode; message: string } + | { outcome: "error"; code: ManagedClusterTopologyFailureCode; message: string }; + +interface QualificationFailure { + code: ManagedClusterTopologyFailureCode; + message: string; +} + +interface QualifiedNode { + observation: ManagedClusterNodeObservation; + gpuId: string; +} + +interface ValidatedRail { + observation: ManagedClusterRailObservation; + gid: ManagedClusterTopologyRoceGid; +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function failure( + intent: ManagedClusterQualificationIntent, + result: QualificationFailure, +): ManagedClusterTopologyQualificationResult { + return intent === "automatic" + ? { outcome: "no-match", ...result } + : { outcome: "error", ...result }; +} + +function validateQualificationPolicy( + evaluatedAt: string, + maxReadinessAgeMs: number, +): number | QualificationFailure { + const evaluatedAtMs = Date.parse(evaluatedAt); + if ( + !Number.isFinite(evaluatedAtMs) || + !Number.isSafeInteger(maxReadinessAgeMs) || + maxReadinessAgeMs <= 0 + ) { + return { + code: "qualification-policy-invalid", + message: "The topology qualification time or readiness age limit is invalid.", + }; + } + return evaluatedAtMs; +} + +function validateReadiness( + report: SystemReadinessReport, + evaluatedAtMs: number, + maxReadinessAgeMs: number, +): QualificationFailure | undefined { + const compatibility = checkSystemReadinessSchemaVersion(report.schemaVersion); + if (!compatibility.compatible || report.mutated !== false) { + return { + code: "readiness-schema-incompatible", + message: "A node readiness report has an incompatible schema.", + }; + } + + const observedAtMs = Date.parse(report.provenance.observedAt); + const readinessAgeMs = evaluatedAtMs - observedAtMs; + if (!Number.isFinite(observedAtMs) || readinessAgeMs < 0 || readinessAgeMs > maxReadinessAgeMs) { + return { + code: "readiness-stale", + message: "A node readiness report is outside the accepted observation window.", + }; + } + + if (report.status === "inconclusive") { + return { + code: "readiness-inconclusive", + message: "A node readiness report is inconclusive.", + }; + } + if ( + report.status !== "supported" || + report.exitCode !== 0 || + report.findings.some(({ severity }) => severity === "blocking" || severity === "fatal") + ) { + return { + code: "readiness-incompatible", + message: "A node readiness report is incompatible with this topology.", + }; + } + + const sparkQualifications = report.qualifications.filter( + (qualification) => qualification.id === "host.platform.dgx_spark", + ); + if (sparkQualifications.length !== 1 || sparkQualifications[0]?.status !== "qualified") { + return { + code: "spark-qualification-unavailable", + message: "A node does not have one qualified DGX Spark readiness result.", + }; + } + + return undefined; +} + +function validateNode( + node: ManagedClusterNodeObservation, + evaluatedAtMs: number, + maxReadinessAgeMs: number, +): QualifiedNode | QualificationFailure { + if (!MANAGED_CLUSTER_ID_PATTERN.test(node.nodeId)) { + return { + code: "node-identity-unavailable", + message: "A node identity is missing or invalid.", + }; + } + if (node.gpuIds.length !== 1 || !MANAGED_CLUSTER_ID_PATTERN.test(node.gpuIds[0] ?? "")) { + return { + code: "gpu-identity-unavailable", + message: "Each DGX Spark node must have exactly one valid GPU identity.", + }; + } + const readinessFailure = validateReadiness(node.readiness, evaluatedAtMs, maxReadinessAgeMs); + if (readinessFailure) return readinessFailure; + if (node.runtimeState === "conflict") { + return { + code: "runtime-conflict", + message: "An existing runtime conflicts with automatic managed cluster activation.", + }; + } + if (node.runtimeState !== "clear") { + return { + code: "runtime-state-unknown", + message: "The existing runtime state is unknown.", + }; + } + return { observation: node, gpuId: node.gpuIds[0]! }; +} + +function validAddress(address: string, prefixLength: number): boolean { + const version = net.isIP(address); + const maxPrefixLength = version === 4 ? 32 : version === 6 ? 128 : 0; + return ( + maxPrefixLength > 0 && + address !== "0.0.0.0" && + address !== "::" && + Number.isInteger(prefixLength) && + prefixLength > 0 && + prefixLength <= maxPrefixLength + ); +} + +function validateRail(rail: ManagedClusterRailObservation): ValidatedRail | QualificationFailure { + if ( + rail.adapter !== "connectx-7" || + rail.path !== "direct" || + rail.linkState !== "up" || + rail.connectivity !== "reachable" + ) { + return { + code: "fabric-degraded", + message: "The direct ConnectX-7 fabric is incomplete or degraded.", + }; + } + if ( + !MANAGED_CLUSTER_ID_PATTERN.test(rail.physicalPortId) || + !SAFE_INTERFACE_PATTERN.test(rail.netdev) || + !SAFE_INTERFACE_PATTERN.test(rail.hcaDevice) || + !Number.isInteger(rail.hcaPort) || + rail.hcaPort <= 0 || + rail.hcaPort > 255 || + !validAddress(rail.address, rail.prefixLength) || + net.isIP(rail.peerAddress) === 0 || + rail.address === rail.peerAddress || + !MANAGED_CLUSTER_ID_PATTERN.test(rail.peerNodeId) + ) { + return { + code: "fabric-mismatch", + message: "The ConnectX-7 rail identity or peer address is invalid.", + }; + } + const gid = rail.roceGid; + if ( + gid.state !== "resolved" || + !Number.isInteger(gid.index) || + (gid.index ?? -1) < 0 || + (gid.index ?? 4096) > 4095 || + typeof gid.value !== "string" || + net.isIP(gid.value) !== 6 || + gid.value === "::" + ) { + return { + code: "fabric-degraded", + message: "A ConnectX-7 rail does not have a resolved RoCE GID.", + }; + } + return { observation: rail, gid: { index: gid.index!, value: gid.value } }; +} + +function validateNodeRails( + node: ManagedClusterNodeObservation, +): readonly ValidatedRail[] | QualificationFailure { + if (node.rails.length < 2) { + return { + code: "fabric-degraded", + message: "One direct ConnectX-7 cable must expose two logical rails on each node.", + }; + } + if (node.rails.length > 2) { + return { + code: "fabric-multiple", + message: "More than one candidate ConnectX-7 cable topology was observed.", + }; + } + + const physicalPorts = new Set(node.rails.map(({ physicalPortId }) => physicalPortId)); + if (physicalPorts.size !== 1) { + return { + code: "fabric-multiple", + message: "The candidate logical rails belong to more than one physical port.", + }; + } + + const validated = node.rails.map((rail) => validateRail(rail)); + const validationFailure = validated.find((rail): rail is QualificationFailure => "code" in rail); + if (validationFailure) return validationFailure; + const rails = validated as ValidatedRail[]; + const uniqueNetdevs = new Set(rails.map(({ observation }) => observation.netdev)); + const uniqueHcas = new Set( + rails.map(({ observation }) => `${observation.hcaDevice}:${observation.hcaPort}`), + ); + const uniqueAddresses = new Set(rails.map(({ observation }) => observation.address)); + const uniqueGids = new Set( + rails.map(({ gid, observation }) => `${observation.hcaDevice}:${gid.index}:${gid.value}`), + ); + if ( + uniqueNetdevs.size !== 2 || + uniqueHcas.size !== 2 || + uniqueAddresses.size !== 2 || + uniqueGids.size !== 2 + ) { + return { + code: "fabric-mismatch", + message: "The two logical ConnectX-7 rails must have distinct interfaces and addresses.", + }; + } + return rails; +} + +function endpoint(nodeId: string, rail: ValidatedRail): ManagedClusterTopologyRailEndpoint { + const observation = rail.observation; + return { + nodeId, + netdev: observation.netdev, + hcaDevice: observation.hcaDevice, + hcaPort: observation.hcaPort, + address: observation.address, + prefixLength: observation.prefixLength, + peerAddress: observation.peerAddress, + roceGid: rail.gid, + }; +} + +function matchRails( + orderedNodes: readonly ManagedClusterNodeObservation[], +): readonly ManagedClusterTopologyRail[] | QualificationFailure { + const ranks = new Map(orderedNodes.map(({ nodeId }, rank) => [nodeId, rank])); + const validatedByNode = new Map(); + for (const node of orderedNodes) { + const validated = validateNodeRails(node); + if ("code" in validated) return validated; + validatedByNode.set(node.nodeId, validated); + } + + const candidates = orderedNodes.flatMap((node) => + validatedByNode.get(node.nodeId)!.map((rail) => ({ nodeId: node.nodeId, rail })), + ); + const used = new Set(); + const matches: Array<{ + leftNodeId: string; + left: ValidatedRail; + rightNodeId: string; + right: ValidatedRail; + }> = []; + const candidateKey = ({ nodeId, rail }: (typeof candidates)[number]) => + `${nodeId}\0${rail.observation.netdev}`; + + for (const candidate of candidates) { + if (used.has(candidateKey(candidate))) continue; + const observation = candidate.rail.observation; + const reciprocal = candidates.filter( + (other) => + other.nodeId === observation.peerNodeId && + other.rail.observation.peerNodeId === candidate.nodeId && + other.rail.observation.address === observation.peerAddress && + other.rail.observation.peerAddress === observation.address && + other.rail.observation.prefixLength === observation.prefixLength && + net.isIP(other.rail.observation.address) === net.isIP(observation.address), + ); + if (reciprocal.length !== 1) { + return { + code: "fabric-mismatch", + message: "Every ConnectX-7 rail must have one reciprocal endpoint in the cluster.", + }; + } + const peer = reciprocal[0]!; + const peerKey = candidateKey(peer); + if (used.has(peerKey)) { + return { + code: "fabric-mismatch", + message: "The ConnectX-7 rail pairing is ambiguous.", + }; + } + used.add(candidateKey(candidate)); + used.add(peerKey); + const candidateRank = ranks.get(candidate.nodeId)!; + const peerRank = ranks.get(peer.nodeId)!; + matches.push( + candidateRank < peerRank + ? { + leftNodeId: candidate.nodeId, + left: candidate.rail, + rightNodeId: peer.nodeId, + right: peer.rail, + } + : { + leftNodeId: peer.nodeId, + left: peer.rail, + rightNodeId: candidate.nodeId, + right: candidate.rail, + }, + ); + } + + if (used.size !== candidates.length) { + return { code: "fabric-mismatch", message: "The ConnectX-7 cluster graph is incomplete." }; + } + const endpointAddresses = new Set( + matches.flatMap(({ left, right }) => [left.observation.address, right.observation.address]), + ); + if (endpointAddresses.size !== candidates.length) { + return { + code: "fabric-mismatch", + message: "The ConnectX-7 rail endpoints must have distinct addresses.", + }; + } + + const adjacency = new Map(orderedNodes.map(({ nodeId }) => [nodeId, new Set()])); + for (const { leftNodeId, rightNodeId } of matches) { + adjacency.get(leftNodeId)!.add(rightNodeId); + adjacency.get(rightNodeId)!.add(leftNodeId); + } + const visited = new Set(); + const queue = [orderedNodes[0]!.nodeId]; + while (queue.length > 0) { + const nodeId = queue.shift()!; + if (visited.has(nodeId)) continue; + visited.add(nodeId); + queue.push(...adjacency.get(nodeId)!); + } + if (visited.size !== orderedNodes.length) { + return { code: "fabric-mismatch", message: "The ConnectX-7 cluster graph is disconnected." }; + } + + matches.sort( + (left, right) => + ranks.get(left.leftNodeId)! - ranks.get(right.leftNodeId)! || + compareStrings(left.left.observation.netdev, right.left.observation.netdev) || + ranks.get(left.rightNodeId)! - ranks.get(right.rightNodeId)!, + ); + return matches.map(({ leftNodeId, left, rightNodeId, right }, index) => ({ + index, + endpoints: [endpoint(leftNodeId, left), endpoint(rightNodeId, right)], + })); +} + +function validSshTarget(target: string): boolean { + if (target.length === 0 || target.length > 286 || target !== target.trim()) return false; + const parts = target.split("@"); + if (parts.length > 2) return false; + const host = parts.at(-1) ?? ""; + const username = parts.length === 2 ? parts[0] : undefined; + return ( + (username === undefined || SSH_USERNAME_PATTERN.test(username)) && + (net.isIP(host) === 4 || SSH_HOST_PATTERN.test(host)) + ); +} + +function validateSshBinding( + binding: ManagedClusterSshBindingObservation, + localNodeId: string, + peerNodeId: string, +): QualificationFailure | undefined { + if ( + binding.state !== "pretrusted" || + binding.fromNodeId !== localNodeId || + binding.toNodeId !== peerNodeId || + !validSshTarget(binding.peerTarget) || + !SAFE_BINDING_HANDLE_PATTERN.test(binding.handle) + ) { + return { + code: "ssh-binding-unavailable", + message: "The peer does not have a valid pretrusted SSH binding.", + }; + } + return undefined; +} + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function artifactEndpointError(value: unknown, expectedNodeId: string): string | undefined { + const candidate = record(value); + const gid = record(candidate?.roceGid); + if ( + !candidate || + candidate.nodeId !== expectedNodeId || + typeof candidate.netdev !== "string" || + !SAFE_INTERFACE_PATTERN.test(candidate.netdev) || + typeof candidate.hcaDevice !== "string" || + !SAFE_INTERFACE_PATTERN.test(candidate.hcaDevice) || + !Number.isInteger(candidate.hcaPort) || + (candidate.hcaPort as number) <= 0 || + (candidate.hcaPort as number) > 255 || + typeof candidate.address !== "string" || + typeof candidate.prefixLength !== "number" || + !validAddress(candidate.address, candidate.prefixLength) || + typeof candidate.peerAddress !== "string" || + net.isIP(candidate.peerAddress) !== net.isIP(candidate.address) || + candidate.peerAddress === candidate.address || + !gid || + !Number.isInteger(gid.index) || + (gid.index as number) < 0 || + (gid.index as number) > 4095 || + typeof gid.value !== "string" || + net.isIP(gid.value) !== 6 || + gid.value === "::" + ) { + return `the ${expectedNodeId} rail endpoint is invalid`; + } + return undefined; +} + +function topologyOutputError( + value: unknown, + subjectNodeIds: readonly string[], +): string | undefined { + const output = record(value); + if (!output) return "topology qualification output is invalid"; + const controllerNodeId = output.controllerNodeId; + if (typeof controllerNodeId !== "string" || !MANAGED_CLUSTER_ID_PATTERN.test(controllerNodeId)) { + return "topology controller node is invalid"; + } + if (!Array.isArray(output.nodes) || output.nodes.length !== subjectNodeIds.length) { + return "topology nodes are invalid"; + } + const nodes = output.nodes.map(record); + if (nodes.some((node) => !node)) return "topology nodes are invalid"; + const typedNodes = nodes as Record[]; + const nodeIds = typedNodes.map(({ nodeId }) => nodeId); + const gpuIds = typedNodes.map(({ gpuId }) => gpuId); + if ( + typedNodes.some( + (node, rank) => + node.rank !== rank || + node.role !== (rank === 0 ? "head" : "worker") || + typeof node.nodeId !== "string" || + !MANAGED_CLUSTER_ID_PATTERN.test(node.nodeId) || + typeof node.gpuId !== "string" || + !MANAGED_CLUSTER_ID_PATTERN.test(node.gpuId), + ) || + nodeIds[0] !== controllerNodeId || + new Set(nodeIds).size !== nodeIds.length || + new Set(gpuIds).size !== gpuIds.length || + [...(nodeIds as string[])] + .sort(compareStrings) + .some((nodeId, index) => nodeId !== subjectNodeIds[index]) + ) { + return "topology node roles or GPU identities are invalid"; + } + + if (!Array.isArray(output.rails) || output.rails.length !== typedNodes.length) { + return "topology rails are invalid"; + } + const rails: ManagedClusterTopologyRail[] = []; + const endpointCounts = new Map((nodeIds as string[]).map((nodeId) => [nodeId, 0])); + const endpointNetdevs = new Map( + (nodeIds as string[]).map((nodeId) => [nodeId, new Set()]), + ); + const endpointAddresses = new Set(); + const adjacency = new Map((nodeIds as string[]).map((nodeId) => [nodeId, new Set()])); + for (const [index, candidate] of output.rails.entries()) { + const rail = record(candidate); + if ( + !rail || + rail.index !== index || + !Array.isArray(rail.endpoints) || + rail.endpoints.length !== 2 + ) { + return "topology rail indexes or endpoints are invalid"; + } + const endpoints = rail.endpoints.map(record); + const leftNodeId = endpoints[0]?.nodeId; + const rightNodeId = endpoints[1]?.nodeId; + if ( + typeof leftNodeId !== "string" || + typeof rightNodeId !== "string" || + leftNodeId === rightNodeId || + !endpointCounts.has(leftNodeId) || + !endpointCounts.has(rightNodeId) + ) { + return "topology rail node identities are invalid"; + } + const leftError = artifactEndpointError(rail.endpoints[0], leftNodeId); + const rightError = artifactEndpointError(rail.endpoints[1], rightNodeId); + if (leftError || rightError) return leftError ?? rightError; + const typedRail = rail as unknown as ManagedClusterTopologyRail; + const [left, right] = typedRail.endpoints; + if ( + left.address !== right.peerAddress || + right.address !== left.peerAddress || + left.prefixLength !== right.prefixLength + ) { + return "topology rail addresses are not reciprocal"; + } + endpointCounts.set(leftNodeId, endpointCounts.get(leftNodeId)! + 1); + endpointCounts.set(rightNodeId, endpointCounts.get(rightNodeId)! + 1); + endpointNetdevs.get(leftNodeId)!.add(left.netdev); + endpointNetdevs.get(rightNodeId)!.add(right.netdev); + endpointAddresses.add(left.address); + endpointAddresses.add(right.address); + adjacency.get(leftNodeId)!.add(rightNodeId); + adjacency.get(rightNodeId)!.add(leftNodeId); + rails.push(typedRail); + } + if ( + [...endpointCounts.values()].some((count) => count !== 2) || + [...endpointNetdevs.values()].some((netdevs) => netdevs.size !== 2) || + endpointAddresses.size !== rails.length * 2 + ) { + return "topology rail identities are not distinct"; + } + const visited = new Set(); + const queue = [controllerNodeId]; + while (queue.length > 0) { + const nodeId = queue.shift()!; + if (visited.has(nodeId)) continue; + visited.add(nodeId); + queue.push(...adjacency.get(nodeId)!); + } + if (visited.size !== typedNodes.length) return "topology rail graph is disconnected"; + const controllerAddresses = rails.flatMap(({ endpoints }) => + endpoints.filter(({ nodeId }) => nodeId === controllerNodeId).map(({ address }) => address), + ); + if ( + typeof output.masterAddress !== "string" || + !controllerAddresses.includes(output.masterAddress) + ) { + return "topology master address does not belong to the controller node"; + } + + if (!Array.isArray(output.peers) || output.peers.length !== typedNodes.length - 1) { + return "topology peer binding is invalid"; + } + const workerNodeIds = new Set((nodeIds as string[]).slice(1)); + const boundNodeIds = new Set(); + for (const candidate of output.peers) { + const peer = record(candidate); + if ( + !peer || + typeof peer.nodeId !== "string" || + !workerNodeIds.has(peer.nodeId) || + boundNodeIds.has(peer.nodeId) || + typeof peer.target !== "string" || + !validSshTarget(peer.target) || + typeof peer.sshBindingHandle !== "string" || + !SAFE_BINDING_HANDLE_PATTERN.test(peer.sshBindingHandle) + ) { + return "topology peer binding is invalid"; + } + boundNodeIds.add(peer.nodeId); + } + return undefined; +} + +export function managedClusterTopologySubjectDigest(subjectNodeIds: readonly string[]): string { + return managedInferenceDigest([...subjectNodeIds].sort(compareStrings)); +} + +export function managedClusterTopologyOutputDigest(output: ManagedClusterTopologyOutput): string { + return managedInferenceDigest(output); +} + +export function getManagedClusterTopologyArtifactError( + artifact: ManagedInferenceTopologyQualification, + expectedSubjectNodeIds?: readonly string[], +): string | undefined { + if ( + artifact.id !== MANAGED_CLUSTER_TOPOLOGY_ID || + artifact.schemaVersion !== MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION + ) { + return "topology qualification identity is incompatible"; + } + if (artifact.status !== "qualified") return "topology qualification is not qualified"; + if (!Array.isArray(artifact.subjectNodeIds)) return "topology qualification subject is invalid"; + const subjectNodeIds = [...artifact.subjectNodeIds]; + if ( + subjectNodeIds.length < 2 || + subjectNodeIds.length > 1024 || + new Set(subjectNodeIds).size !== subjectNodeIds.length || + subjectNodeIds.some((nodeId) => !MANAGED_CLUSTER_ID_PATTERN.test(nodeId)) || + subjectNodeIds.some((nodeId, index) => index > 0 && subjectNodeIds[index - 1]! >= nodeId) + ) { + return "topology qualification subject is invalid"; + } + if ( + expectedSubjectNodeIds && + subjectNodeIds.some((nodeId, index) => nodeId !== expectedSubjectNodeIds[index]) + ) { + return "topology qualification subject does not match the readiness reports"; + } + + const outputError = topologyOutputError(artifact.output, subjectNodeIds); + if (outputError) return outputError; + try { + if (artifact.subjectDigest !== managedClusterTopologySubjectDigest(subjectNodeIds)) { + return "topology qualification subject digest does not match its subject"; + } + if ( + artifact.outputDigest !== + managedClusterTopologyOutputDigest(artifact.output as ManagedClusterTopologyOutput) + ) { + return "topology qualification output digest does not match its output"; + } + } catch { + return "topology qualification digest is invalid"; + } + return undefined; +} + +export function qualifyManagedClusterTopology( + input: Readonly, +): ManagedClusterTopologyQualificationResult { + if (input.peers.length < 1 || input.peers.length > 1023) { + return failure(input.intent, { + code: "peer-count", + message: "Managed cluster activation requires between one and 1023 discovered peers.", + }); + } + + const evaluatedAtMs = validateQualificationPolicy(input.evaluatedAt, input.maxReadinessAgeMs); + if (typeof evaluatedAtMs !== "number") return failure(input.intent, evaluatedAtMs); + + const orderedObservations = [ + input.local, + ...[...input.peers].sort((left, right) => compareStrings(left.nodeId, right.nodeId)), + ]; + const qualifiedNodes: QualifiedNode[] = []; + for (const observation of orderedObservations) { + const qualified = validateNode(observation, evaluatedAtMs, input.maxReadinessAgeMs); + if ("code" in qualified) return failure(input.intent, qualified); + qualifiedNodes.push(qualified); + } + const nodeIds = orderedObservations.map(({ nodeId }) => nodeId); + if (new Set(nodeIds).size !== nodeIds.length) { + return failure(input.intent, { + code: "duplicate-node-identity", + message: "The managed cluster observations contain a duplicate node identity.", + }); + } + const gpuIds = qualifiedNodes.map(({ gpuId }) => gpuId); + if (new Set(gpuIds).size !== gpuIds.length) { + return failure(input.intent, { + code: "duplicate-gpu-identity", + message: "The managed cluster observations contain a duplicate GPU identity.", + }); + } + + for (const peer of input.peers) { + const sshFailure = validateSshBinding(peer.sshBinding, input.local.nodeId, peer.nodeId); + if (sshFailure) return failure(input.intent, sshFailure); + } + const rails = matchRails(orderedObservations); + if ("code" in rails) return failure(input.intent, rails); + const controllerEndpoint = rails + .flatMap(({ endpoints }) => endpoints) + .filter(({ nodeId }) => nodeId === input.local.nodeId) + .sort((left, right) => compareStrings(left.netdev, right.netdev))[0]; + if (!controllerEndpoint) { + return failure(input.intent, { + code: "fabric-mismatch", + message: "The managed cluster controller has no qualified fabric endpoint.", + }); + } + + const output: ManagedClusterTopologyOutput = { + controllerNodeId: input.local.nodeId, + nodes: qualifiedNodes.map(({ observation, gpuId }, rank) => ({ + nodeId: observation.nodeId, + gpuId, + rank, + role: rank === 0 ? "head" : "worker", + })), + rails, + masterAddress: controllerEndpoint.address, + peers: orderedObservations.slice(1).map((observation) => { + const peer = observation as ManagedClusterPeerObservation; + return { + nodeId: peer.nodeId, + target: peer.sshBinding.peerTarget, + sshBindingHandle: peer.sshBinding.handle, + }; + }), + }; + const subjectNodeIds = [...nodeIds].sort(compareStrings); + return { + outcome: "qualified", + artifact: { + id: MANAGED_CLUSTER_TOPOLOGY_ID, + schemaVersion: MANAGED_CLUSTER_TOPOLOGY_SCHEMA_VERSION, + status: "qualified", + subjectNodeIds, + subjectDigest: managedClusterTopologySubjectDigest(subjectNodeIds), + outputDigest: managedClusterTopologyOutputDigest(output), + output, + }, + }; +} diff --git a/src/lib/inference/serving/managed-runtime-receipts.ts b/src/lib/inference/serving/managed-runtime-receipts.ts new file mode 100644 index 0000000000..c720da8341 --- /dev/null +++ b/src/lib/inference/serving/managed-runtime-receipts.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { listGatewayStateRoots } from "../../state/gateway-registry"; +import { managedVllmStateDir } from "../vllm-api-key"; +import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "../vllm-station-runtime-receipt-path"; +import { + isManagedClusterDiscoveryBindingStateEntry, + isManagedClusterRuntimeBindingStateEntry, + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, +} from "./managed-cluster-runtime-receipt-path"; + +export { MCP_LIFECYCLE_LOCK_DIRNAME } from "../../state/mcp-lifecycle-lock-storage"; +export { MANAGED_VLLM_API_KEY_FILE } from "../vllm-api-key"; +export { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "../vllm-station-runtime-receipt-path"; +export { + isManagedClusterDiscoveryBindingStateEntry, + isManagedClusterRuntimeBindingStateEntry, + MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE, + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, +} from "./managed-cluster-runtime-receipt-path"; + +export interface ManagedDistributedVllmRuntimeReceipts { + readonly managedClusterBindingPaths: readonly string[]; + readonly managedClusterDiscoveryBindingPaths: readonly string[]; + readonly managedClusterPath: string | null; + readonly stationBindingPaths: readonly string[]; + readonly stationPaths: readonly string[]; +} + +function pathExistsNoFollow(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +function matchingStateEntries( + stateRoots: readonly { readonly root: string }[], + predicate: (entry: string) => boolean, +): readonly string[] { + return stateRoots + .flatMap(({ root }) => { + try { + return fs + .readdirSync(root) + .filter(predicate) + .map((entry) => path.join(root, entry)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + }) + .sort(); +} + +/** Locate durable distributed-runtime ownership without parsing or following receipt paths. */ +export function findManagedDistributedVllmRuntimeReceipts( + options: { readonly homeDir?: string } = {}, +): ManagedDistributedVllmRuntimeReceipts { + const homeDir = options.homeDir ?? os.homedir(); + const stateRoots = listGatewayStateRoots(homeDir); + const managedClusterPath = path.join( + managedVllmStateDir(homeDir), + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, + ); + const stationPaths = stateRoots + .map(({ root }) => path.join(root, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE)) + .filter(pathExistsNoFollow); + const managedClusterBindingPaths = matchingStateEntries( + stateRoots, + isManagedClusterRuntimeBindingStateEntry, + ); + const managedClusterDiscoveryBindingPaths = matchingStateEntries( + stateRoots, + isManagedClusterDiscoveryBindingStateEntry, + ); + const stationBindingPaths = stateRoots + .map(({ root }) => `${path.join(root, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE)}.ssh-binding`) + .filter(pathExistsNoFollow); + return { + managedClusterBindingPaths, + managedClusterDiscoveryBindingPaths, + managedClusterPath: pathExistsNoFollow(managedClusterPath) ? managedClusterPath : null, + stationBindingPaths, + stationPaths, + }; +} + +/** Stop a new install before it can mutate state already owned by another managed runtime. */ +export function assertNoManagedDistributedVllmRuntimeReceipts( + options: { readonly homeDir?: string } = {}, +): void { + const receipts = findManagedDistributedVllmRuntimeReceipts(options); + const paths = [ + ...(receipts.managedClusterPath ? [receipts.managedClusterPath] : []), + ...receipts.managedClusterBindingPaths, + ...receipts.managedClusterDiscoveryBindingPaths, + ...receipts.stationPaths, + ...receipts.stationBindingPaths, + ]; + if (paths.length === 0) return; + throw new Error( + `Managed vLLM runtime state already exists at ${paths.join( + ", ", + )}; recover it through Local vLLM or uninstall it before starting a new managed install.`, + ); +} diff --git a/src/lib/inference/serving/resolver.test.ts b/src/lib/inference/serving/resolver.test.ts new file mode 100644 index 0000000000..068512fcb9 --- /dev/null +++ b/src/lib/inference/serving/resolver.test.ts @@ -0,0 +1,637 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import type { SystemReadinessReport } from "../../readiness/types.js"; +import { managedInferenceDigest } from "./catalog-integrity.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { + FIXTURE_MANAGED_CLUSTER_PRESET_ID, + fixtureManagedClusterSelection, +} from "./managed-cluster-fixture.test-support.js"; +import { + type ManagedClusterTopologyOutput, + managedClusterTopologyOutputDigest, +} from "./managed-cluster-topology.js"; +import { resolveManagedInferenceServing } from "./resolver.js"; +import type { + CompiledManagedInferenceCatalog, + ManagedInferencePresetRequirement, + ManagedInferenceReadinessSource, + ManagedInferenceResolverInput, + ManagedInferenceServingPreset, + ManagedInferenceServingRecipe, + ManagedInferenceTopologyQualification, +} from "./types.js"; + +const NOW = new Date("2026-08-02T18:00:00.000Z"); +const SOURCE_REVISION = "a".repeat(40); + +function shippedCatalog(): CompiledManagedInferenceCatalog { + return structuredClone(loadManagedInferenceCatalog()); +} + +function shippedCompiledPreset( + catalog = shippedCatalog(), +): CompiledManagedInferenceCatalog["presets"][number] { + const preset = catalog.presets.find( + ({ metadata }) => metadata.id === FIXTURE_MANAGED_CLUSTER_PRESET_ID, + ); + expect(preset).toBeDefined(); + return preset as CompiledManagedInferenceCatalog["presets"][number]; +} + +function shippedPreset(catalog = shippedCatalog()): ManagedInferenceServingPreset { + return shippedCompiledPreset(catalog); +} + +function shippedCompiledRecipe( + catalog = shippedCatalog(), +): CompiledManagedInferenceCatalog["recipes"][number] { + const recipeRef = shippedCompiledPreset(catalog).spec.plan.recipeRef; + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === recipeRef); + expect(recipe).toBeDefined(); + return recipe as CompiledManagedInferenceCatalog["recipes"][number]; +} + +function shippedRecipe(catalog = shippedCatalog()): ManagedInferenceServingRecipe { + return shippedCompiledRecipe(catalog); +} + +function shippedFixtureCatalog(): CompiledManagedInferenceCatalog { + const catalog = shippedCatalog(); + return { + ...catalog, + presets: [shippedCompiledPreset(catalog)], + recipes: [shippedCompiledRecipe(catalog)], + }; +} + +function catalogReadinessEntities(): Pick< + SystemReadinessReport, + "observations" | "capabilities" | "qualifications" +> { + const readinessRequirements = shippedPreset().spec.requirements.all.flatMap((requirement) => + "readiness" in requirement ? [requirement.readiness] : [], + ); + return { + observations: readinessRequirements.flatMap((readiness) => + readiness.kind === "observation" && "state" in readiness + ? [ + { + id: readiness.id, + state: readiness.state as SystemReadinessReport["observations"][number]["state"], + }, + ] + : [], + ), + capabilities: readinessRequirements.flatMap((readiness) => + readiness.kind === "capability" + ? [ + { + id: readiness.id, + state: readiness.state as SystemReadinessReport["capabilities"][number]["state"], + }, + ] + : [], + ), + qualifications: readinessRequirements.flatMap((readiness) => + readiness.kind === "qualification" + ? [ + { + id: readiness.id, + status: readiness.status as SystemReadinessReport["qualifications"][number]["status"], + }, + ] + : [], + ), + }; +} + +function readinessReport(overrides: Partial = {}): SystemReadinessReport { + const entities = catalogReadinessEntities(); + return { + schemaVersion: "1.1.0", + mutated: false, + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, + observedAt: "2026-08-02T17:59:50.000Z", + }, + ...entities, + findings: [], + evidence: [], + status: "supported", + exitCode: 0, + ...overrides, + } as SystemReadinessReport; +} + +function readinessSources(): ManagedInferenceReadinessSource[] { + return [ + { nodeId: "spark-head", report: readinessReport() }, + { nodeId: "spark-worker", report: readinessReport() }, + ]; +} + +function topology( + overrides: Partial> = {}, +): ManagedInferenceTopologyQualification { + const artifact = structuredClone(fixtureManagedClusterSelection().topologyQualification); + return { ...artifact, ...overrides }; +} + +function resolverInput( + overrides: Partial> = {}, +): ManagedInferenceResolverInput { + return { + readinessReports: readinessSources(), + topologyQualifications: [topology()], + now: NOW, + ...overrides, + }; +} + +function catalogWithSecondProfile(options: { + readonly firstPriority: number; + readonly secondPriority: number; + readonly secondSelection?: "automatic" | "explicit-only" | "disabled"; +}): { + readonly catalog: CompiledManagedInferenceCatalog; + readonly secondPresetId: string; + readonly secondRecipeId: string; +} { + const catalog = shippedCatalog(); + const firstCompiledRecipe = shippedCompiledRecipe(catalog); + const firstPreset = shippedPreset(catalog); + const firstRecipe = shippedRecipe(catalog); + const secondPresetId = "vllm.synthetic.dual-second"; + const secondRecipeId = "vllm.synthetic.second-recipe"; + const normalizedFirst = { + ...firstPreset, + spec: { ...firstPreset.spec, priority: options.firstPriority }, + } as ManagedInferenceServingPreset; + const secondRecipe = { + ...firstRecipe, + metadata: { + ...firstRecipe.metadata, + id: secondRecipeId, + displayName: "Synthetic model", + }, + spec: { + ...firstRecipe.spec, + model: { + ...firstRecipe.spec.model, + id: "example/AnotherModel", + revision: "b".repeat(40), + servedName: "another-model", + }, + readiness: { + ...firstRecipe.spec.readiness, + expectedModel: "another-model", + }, + }, + } as ManagedInferenceServingRecipe; + const secondPreset = { + ...firstPreset, + metadata: { + ...firstPreset.metadata, + id: secondPresetId, + displayName: "Synthetic preset", + }, + spec: { + ...firstPreset.spec, + selection: options.secondSelection ?? "automatic", + priority: options.secondPriority, + plan: { ...firstPreset.spec.plan, recipeRef: secondRecipeId }, + }, + } as ManagedInferenceServingPreset; + return { + catalog: { + ...catalog, + presets: [normalizedFirst, secondPreset], + recipes: [firstCompiledRecipe, secondRecipe], + }, + secondPresetId, + secondRecipeId, + }; +} + +describe("managed inference resolver", () => { + it("selects the shipped automatic preset from catalog data", () => { + const catalog = shippedFixtureCatalog(); + const compiledPreset = shippedCompiledPreset(catalog); + const compiledRecipe = shippedCompiledRecipe(catalog); + const result = resolveManagedInferenceServing(resolverInput(), catalog); + + expect(result).toMatchObject({ + outcome: "selected", + selection: "automatic", + presetDigest: managedInferenceDigest(compiledPreset), + recipeDigest: managedInferenceDigest(compiledRecipe), + preset: { metadata: { id: shippedPreset(catalog).metadata.id } }, + recipe: { metadata: { id: shippedRecipe(catalog).metadata.id } }, + topologyQualification: { output: { masterAddress: "192.168.100.10" } }, + }); + }); + + it("looks up and resolves an arbitrary explicit-only preset by ID", () => { + const { catalog, secondPresetId, secondRecipeId } = catalogWithSecondProfile({ + firstPriority: 100, + secondPriority: 1, + secondSelection: "explicit-only", + }); + const result = resolveManagedInferenceServing( + resolverInput({ + intent: { preset: secondPresetId, vllmModel: "another-model" }, + }), + catalog, + ); + + expect(result).toMatchObject({ + outcome: "selected", + selection: "explicit", + preset: { metadata: { id: secondPresetId } }, + recipe: { metadata: { id: secondRecipeId } }, + }); + }); + + it("selects the highest-priority matching automatic preset", () => { + const { catalog, secondPresetId, secondRecipeId } = catalogWithSecondProfile({ + firstPriority: 100, + secondPriority: 200, + }); + const result = resolveManagedInferenceServing(resolverInput(), catalog); + + expect(result).toMatchObject({ + outcome: "selected", + preset: { metadata: { id: secondPresetId } }, + recipe: { metadata: { id: secondRecipeId } }, + }); + }); + + it("selects a lower-priority profile when higher-priority requirements do not match", () => { + const { + catalog: baseCatalog, + secondPresetId, + secondRecipeId, + } = catalogWithSecondProfile({ + firstPriority: 200, + secondPriority: 100, + }); + const highCompiledPreset = shippedCompiledPreset(baseCatalog); + const secondCompiledPreset = baseCatalog.presets.find( + ({ metadata }) => metadata.id === secondPresetId, + ); + expect(secondCompiledPreset).toBeDefined(); + const highPreset = highCompiledPreset; + const unavailableHighPreset = { + ...highPreset, + spec: { + ...highPreset.spec, + requirements: { + all: [ + { + readiness: { + scope: "everyNode", + kind: "capability", + id: "host.synthetic.unavailable", + state: "present", + }, + }, + ...highPreset.spec.requirements.all, + ], + }, + }, + } as ManagedInferenceServingPreset; + const catalog: CompiledManagedInferenceCatalog = { + ...baseCatalog, + presets: [ + unavailableHighPreset, + secondCompiledPreset as CompiledManagedInferenceCatalog["presets"][number], + ], + }; + + expect(resolveManagedInferenceServing(resolverInput(), catalog)).toMatchObject({ + outcome: "selected", + preset: { metadata: { id: secondPresetId } }, + recipe: { metadata: { id: secondRecipeId } }, + }); + }); + + it("rejects equal-priority automatic matches as ambiguous", () => { + const { catalog, secondPresetId } = catalogWithSecondProfile({ + firstPriority: 100, + secondPriority: 100, + }); + const result = resolveManagedInferenceServing(resolverInput(), catalog); + + expect(result).toMatchObject({ + outcome: "rejected", + code: "ambiguous-selection", + }); + const rejected = result as Extract; + expect(rejected.message).toContain(shippedPreset(catalog).metadata.id); + expect(rejected.message).toContain(secondPresetId); + }); + + it("evaluates registered readiness entities and numeric facts without profile branches", () => { + const catalog = shippedCatalog(); + const preset = shippedPreset(catalog); + const topologyRequirement = preset.spec.requirements.all.find( + (requirement) => "topologyQualification" in requirement, + ); + expect(topologyRequirement).toBeDefined(); + const genericRequirements: ManagedInferencePresetRequirement[] = [ + { + readiness: { + scope: "everyNode", + kind: "capability", + id: "host.docker.available", + state: "present", + }, + }, + { + fact: "cluster.nodeCount", + state: "present", + operator: "between", + value: [2, 2], + }, + topologyRequirement as ManagedInferencePresetRequirement, + ]; + const customizedPreset = { + ...preset, + spec: { + ...preset.spec, + requirements: { all: genericRequirements }, + }, + } as ManagedInferenceServingPreset; + const customizedCatalog: CompiledManagedInferenceCatalog = { + ...catalog, + presets: [customizedPreset], + }; + + expect(resolveManagedInferenceServing(resolverInput(), customizedCatalog)).toMatchObject({ + outcome: "selected", + }); + const missingCapability = readinessSources(); + missingCapability[1] = { + nodeId: "spark-worker", + report: readinessReport({ + capabilities: readinessReport().capabilities.map((capability) => + capability.id === "host.docker.available" + ? { ...capability, state: "absent" } + : capability, + ), + }), + }; + expect( + resolveManagedInferenceServing( + resolverInput({ readinessReports: missingCapability }), + customizedCatalog, + ), + ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); + }); + + it("applies any-node readiness requirements as an existential match", () => { + const catalog = shippedCatalog(); + const preset = shippedPreset(catalog); + const topologyRequirement = preset.spec.requirements.all.find( + (requirement) => "topologyQualification" in requirement, + ); + expect(topologyRequirement).toBeDefined(); + const customizedPreset = { + ...preset, + spec: { + ...preset.spec, + requirements: { + all: [ + { + readiness: { + scope: "anyNode", + kind: "capability", + id: "host.docker.available", + state: "present", + }, + }, + topologyRequirement as ManagedInferencePresetRequirement, + ], + }, + }, + } as ManagedInferenceServingPreset; + const customizedCatalog: CompiledManagedInferenceCatalog = { + ...catalog, + presets: [customizedPreset], + }; + const reports = readinessSources(); + reports[1] = { + nodeId: "spark-worker", + report: readinessReport({ + capabilities: readinessReport().capabilities.map((capability) => + capability.id === "host.docker.available" + ? { ...capability, state: "absent" } + : capability, + ), + }), + }; + + expect( + resolveManagedInferenceServing( + resolverInput({ readinessReports: reports }), + customizedCatalog, + ), + ).toMatchObject({ outcome: "selected" }); + + reports[0] = { + nodeId: "spark-head", + report: readinessReport({ + capabilities: readinessReport().capabilities.map((capability) => + capability.id === "host.docker.available" + ? { ...capability, state: "absent" } + : capability, + ), + }), + }; + expect( + resolveManagedInferenceServing( + resolverInput({ readinessReports: reports }), + customizedCatalog, + ), + ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); + }); + + it("returns an immutable topology snapshot", () => { + const artifact = topology(); + const result = resolveManagedInferenceServing( + resolverInput({ topologyQualifications: [artifact] }), + ); + + expect(result.outcome).toBe("selected"); + const selected = result as Extract; + (artifact.output as { masterAddress: string }).masterAddress = "192.168.100.99"; + expect(selected.topologyQualification.output.masterAddress).toBe("192.168.100.10"); + expect(Object.isFrozen(selected.topologyQualification.output)).toBe(true); + }); + + it.each([ + { name: "provider", intent: { provider: "vllm" } }, + { name: "model", intent: { vllmModel: "another/model" } }, + { name: "extra arguments", intent: { vllmExtraArguments: ["--another-option"] } }, + ])("leaves existing $name intent authoritative for automatic selection", ({ intent }) => { + expect( + resolveManagedInferenceServing({ + readinessReports: [], + topologyQualifications: [], + intent, + now: NOW, + }), + ).toMatchObject({ outcome: "no-match", code: "explicit-intent" }); + }); + + it("rejects an unknown explicit preset", () => { + expect( + resolveManagedInferenceServing(resolverInput({ intent: { preset: "vllm.unknown" } })), + ).toMatchObject({ outcome: "rejected", code: "unknown-preset" }); + }); + + it("rejects a disabled explicit preset", () => { + const { catalog, secondPresetId } = catalogWithSecondProfile({ + firstPriority: 100, + secondPriority: 200, + secondSelection: "disabled", + }); + expect( + resolveManagedInferenceServing( + resolverInput({ intent: { preset: secondPresetId } }), + catalog, + ), + ).toMatchObject({ outcome: "rejected", code: "requirements-not-met" }); + }); + + it("rejects explicit preset intent that conflicts with its recipe", () => { + const presetId = shippedPreset().metadata.id; + expect( + resolveManagedInferenceServing( + resolverInput({ + intent: { + preset: presetId, + vllmExtraArguments: ["--max-model-len", "1"], + }, + }), + ), + ).toMatchObject({ outcome: "rejected", code: "incompatible-intent" }); + }); + + it.each([ + { + name: "stale provenance", + report: readinessReport({ + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, + observedAt: "2026-08-02T17:00:00.000Z", + }, + }), + }, + { + name: "incompatible report", + report: readinessReport({ status: "incompatible", exitCode: 2 }), + }, + { + name: "blocking finding", + report: readinessReport({ + findings: [{ id: "host.blocked", severity: "blocking", summary: "Blocked." }], + }), + }, + ])("rejects $name before selecting a recipe", ({ report }) => { + const sources = readinessSources(); + sources[1] = { nodeId: "spark-worker", report }; + + expect( + resolveManagedInferenceServing(resolverInput({ readinessReports: sources })), + ).toMatchObject({ outcome: "rejected", code: "invalid-readiness" }); + }); + + it("rejects a non-finite resolution time", () => { + expect( + resolveManagedInferenceServing(resolverInput({ now: new Date(Number.NaN) })), + ).toMatchObject({ outcome: "rejected", code: "invalid-readiness" }); + }); + + it.each([1, 3])("does not activate automatically for %i readiness reports", (count) => { + const reports = [ + ...readinessSources(), + { nodeId: "spark-third", report: readinessReport() }, + ].slice(0, count); + + expect( + resolveManagedInferenceServing(resolverInput({ readinessReports: reports })), + ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); + }); + + it("does not activate automatically without the required topology artifact", () => { + expect( + resolveManagedInferenceServing(resolverInput({ topologyQualifications: [] })), + ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); + }); + + it("rejects a topology artifact for different physical subjects", () => { + expect( + resolveManagedInferenceServing( + resolverInput({ + topologyQualifications: [topology({ subjectNodeIds: ["spark-head", "spark-third"] })], + }), + ), + ).toMatchObject({ outcome: "rejected", code: "invalid-topology" }); + }); + + it("rejects topology output mutated without a new digest", () => { + const artifact = topology(); + (artifact.output as { masterAddress: string }).masterAddress = "192.168.100.99"; + + expect( + resolveManagedInferenceServing(resolverInput({ topologyQualifications: [artifact] })), + ).toMatchObject({ outcome: "rejected", code: "invalid-topology" }); + }); + + it("rejects a stale topology subject digest", () => { + expect( + resolveManagedInferenceServing( + resolverInput({ + topologyQualifications: [topology({ subjectDigest: `sha256:${"f".repeat(64)}` })], + }), + ), + ).toMatchObject({ outcome: "rejected", code: "invalid-topology" }); + }); + + it("rejects an internally inconsistent topology with a recomputed output digest", () => { + const artifact = topology(); + (artifact.output as { masterAddress: string }).masterAddress = "192.168.100.99"; + (artifact as { outputDigest: string }).outputDigest = managedClusterTopologyOutputDigest( + artifact.output, + ); + + expect( + resolveManagedInferenceServing(resolverInput({ topologyQualifications: [artifact] })), + ).toMatchObject({ outcome: "rejected", code: "invalid-topology" }); + }); + + it("rejects ambiguous topology artifacts", () => { + expect( + resolveManagedInferenceServing( + resolverInput({ topologyQualifications: [topology(), topology()] }), + ), + ).toMatchObject({ outcome: "rejected", code: "invalid-topology" }); + }); + + it("rejects missing requirements for an explicit preset instead of falling back", () => { + expect( + resolveManagedInferenceServing( + resolverInput({ + readinessReports: readinessSources().slice(0, 1), + intent: { preset: shippedPreset().metadata.id }, + }), + ), + ).toMatchObject({ outcome: "rejected", code: "requirements-not-met" }); + }); +}); diff --git a/src/lib/inference/serving/resolver.ts b/src/lib/inference/serving/resolver.ts new file mode 100644 index 0000000000..ff727c662e --- /dev/null +++ b/src/lib/inference/serving/resolver.ts @@ -0,0 +1,508 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { checkSystemReadinessSchemaVersion } from "../../readiness/compatibility.js"; +import { getSystemReadinessReferenceErrors } from "../../readiness/references.js"; +import { + getManagedInferenceRecipeRegistrationError, + getManagedInferenceTopologyQualificationDescriptor, +} from "./adapter-registry.js"; +import { loadManagedInferenceCatalog } from "./catalog-loader.js"; +import { immutableManagedInferenceCopy, managedInferenceDigest } from "./catalog-integrity.js"; +import type { + CompiledManagedInferenceCatalog, + ManagedInferenceFactRequirement, + ManagedInferencePresetRequirement, + ManagedInferenceReadinessRequirement, + ManagedInferenceReadinessSource, + ManagedInferenceResolution, + ManagedInferenceResolverInput, + ManagedInferenceSelectionIntent, + ManagedInferenceServingPreset, + ManagedInferenceServingRecipe, + ManagedInferenceTopologyQualification, + ManagedInferenceTopologyRequirement, +} from "./types.js"; + +export const MANAGED_INFERENCE_READINESS_MAX_AGE_MS = 30_000; +const MAX_FUTURE_CLOCK_SKEW_MS = 5_000; +const SOURCE_REVISION = /^[0-9a-f]{40,64}$/u; +const PUBLIC_VERSION = + /^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?(?:\+[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/u; + +type SelectionOperator = ManagedInferenceFactRequirement["operator"]; + +type RequirementEvaluation = + | { + readonly outcome: "matched"; + readonly topologyQualifications: readonly ManagedInferenceTopologyQualification[]; + } + | { readonly outcome: "unmet"; readonly message: string } + | { readonly outcome: "invalid-topology"; readonly message: string }; + +interface MatchingCandidate { + readonly preset: ManagedInferenceServingPreset; + readonly presetDigest: string; + readonly recipe: ManagedInferenceServingRecipe; + readonly recipeDigest: string; + readonly priority: number; + readonly topologyQualification: ManagedInferenceTopologyQualification; +} + +function hasText(value: string | undefined): value is string { + return typeof value === "string" && value.trim().length > 0; +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function explicitIntentWithoutPreset(intent: ManagedInferenceSelectionIntent): boolean { + return ( + hasText(intent.provider) || + hasText(intent.vllmModel) || + (intent.vllmExtraArguments?.length ?? 0) > 0 + ); +} + +function readinessError( + source: ManagedInferenceReadinessSource, + nowMs: number, + maxAgeMs: number, +): string | undefined { + const { nodeId, report } = source; + if (!hasText(nodeId)) return "readiness node ID is empty"; + const compatibility = checkSystemReadinessSchemaVersion(report.schemaVersion); + if (!compatibility.compatible) return `${nodeId}: ${compatibility.reason}`; + if (report.mutated !== false) return `${nodeId}: readiness report is not read-only`; + if (!PUBLIC_VERSION.test(report.provenance.nemoclawVersion)) { + return `${nodeId}: readiness producer version is invalid`; + } + if (!SOURCE_REVISION.test(report.provenance.sourceRevision)) { + return `${nodeId}: readiness source revision is invalid`; + } + const observedAt = Date.parse(report.provenance.observedAt); + const ageMs = nowMs - observedAt; + if (!Number.isFinite(observedAt) || ageMs > maxAgeMs || ageMs < -MAX_FUTURE_CLOCK_SKEW_MS) { + return `${nodeId}: readiness report is stale or has an invalid observation time`; + } + const referenceErrors = getSystemReadinessReferenceErrors(report); + if (referenceErrors.length > 0) return `${nodeId}: ${referenceErrors[0]}`; + if (report.status !== "supported" || report.exitCode !== 0) { + return `${nodeId}: readiness status is ${report.status}`; + } + if (report.findings.some(({ severity }) => severity === "fatal" || severity === "blocking")) { + return `${nodeId}: readiness report contains a blocking finding`; + } + return undefined; +} + +function readinessReportsError( + sources: readonly ManagedInferenceReadinessSource[], + nowMs: number, + maxAgeMs: number, +): string | undefined { + const nodeIds = sources.map(({ nodeId }) => nodeId); + if (new Set(nodeIds).size !== nodeIds.length) { + return "readiness reports contain duplicate node IDs"; + } + for (const source of sources) { + const error = readinessError(source, nowMs, maxAgeMs); + if (error) return error; + } + return undefined; +} + +function scalarEquals(actual: unknown, expected: unknown): boolean { + return ( + (actual === null || ["string", "number", "boolean"].includes(typeof actual)) && + actual === expected + ); +} + +function matchesOperator(actual: unknown, operator: SelectionOperator, expected: unknown): boolean { + switch (operator) { + case "equals": + return scalarEquals(actual, expected); + case "oneOf": + return ( + Array.isArray(expected) && expected.some((candidate) => scalarEquals(actual, candidate)) + ); + case "atLeast": + return typeof actual === "number" && typeof expected === "number" && actual >= expected; + case "atMost": + return typeof actual === "number" && typeof expected === "number" && actual <= expected; + case "between": + return ( + typeof actual === "number" && + Array.isArray(expected) && + expected.length === 2 && + typeof expected[0] === "number" && + typeof expected[1] === "number" && + actual >= expected[0] && + actual <= expected[1] + ); + } +} + +function readinessScopeMatches( + scope: string, + reports: readonly ManagedInferenceReadinessSource[], + predicate: (source: ManagedInferenceReadinessSource) => boolean, +): boolean { + if (reports.length === 0) return false; + if (scope === "everyNode") return reports.every(predicate); + if (scope === "anyNode") return reports.some(predicate); + return false; +} + +function readinessRequirementMatches( + requirement: ManagedInferenceReadinessRequirement["readiness"], + reports: readonly ManagedInferenceReadinessSource[], +): boolean { + return readinessScopeMatches(requirement.scope, reports, ({ report }) => { + if (requirement.kind === "qualification") { + const matches = report.qualifications.filter(({ id }) => id === requirement.id); + return matches.length === 1 && matches[0]!.status === requirement.status; + } + if ("comparison" in requirement) return false; + const collection = + requirement.kind === "observation" ? report.observations : report.capabilities; + const matches = collection.filter(({ id }) => id === requirement.id); + return matches.length === 1 && matches[0]!.state === requirement.state; + }); +} + +function selectionFact( + path: string, + reports: readonly ManagedInferenceReadinessSource[], +): + | { + readonly state: "present"; + readonly value: ManagedInferenceFactRequirement["value"]; + } + | undefined { + if (path === "cluster.nodeCount") return { state: "present", value: reports.length }; + return undefined; +} + +function factRequirementMatches( + requirement: ManagedInferenceFactRequirement, + reports: readonly ManagedInferenceReadinessSource[], +): boolean { + const fact = selectionFact(requirement.fact, reports); + if (!fact) return false; + return ( + requirement.state === fact.state && + matchesOperator(fact.value, requirement.operator, requirement.value) + ); +} + +function evaluateTopologyRequirement( + requirement: ManagedInferenceTopologyRequirement["topologyQualification"], + artifacts: readonly ManagedInferenceTopologyQualification[], + expectedSubjectNodeIds: readonly string[], +): + | { + readonly outcome: "matched"; + readonly artifact: ManagedInferenceTopologyQualification; + } + | { readonly outcome: "unmet"; readonly message: string } + | { readonly outcome: "invalid-topology"; readonly message: string } { + const descriptor = getManagedInferenceTopologyQualificationDescriptor( + requirement.id, + requirement.schemaVersion, + ); + if (!descriptor) { + return { + outcome: "invalid-topology", + message: `Topology qualification ${requirement.id}@${String(requirement.schemaVersion)} is not registered.`, + }; + } + const matching = artifacts.filter( + ({ id, schemaVersion }) => id === requirement.id && schemaVersion === requirement.schemaVersion, + ); + if (matching.length === 0) { + return { + outcome: "unmet", + message: `Topology requirement ${requirement.id} did not match.`, + }; + } + if (matching.length !== 1) { + return { + outcome: "invalid-topology", + message: `Topology qualification ${requirement.id} has more than one candidate.`, + }; + } + const artifact = matching[0]!; + if (artifact.status !== requirement.status) { + return { + outcome: "unmet", + message: `Topology requirement ${requirement.id} did not match.`, + }; + } + const error = descriptor.validateArtifact(artifact, expectedSubjectNodeIds); + return error ? { outcome: "invalid-topology", message: error } : { outcome: "matched", artifact }; +} + +function evaluateRequirements( + preset: ManagedInferenceServingPreset, + reports: readonly ManagedInferenceReadinessSource[], + topologyQualifications: readonly ManagedInferenceTopologyQualification[], +): RequirementEvaluation { + const matchedTopologies: ManagedInferenceTopologyQualification[] = []; + const expectedSubjectNodeIds = reports.map(({ nodeId }) => nodeId).sort(compareStrings); + for (const requirement of preset.spec.requirements.all) { + if ("readiness" in requirement) { + if (!readinessRequirementMatches(requirement.readiness, reports)) { + return { + outcome: "unmet", + message: `Readiness requirement ${requirement.readiness.id} did not match.`, + }; + } + continue; + } + if ("fact" in requirement) { + if (!factRequirementMatches(requirement, reports)) { + return { + outcome: "unmet", + message: `Selection fact ${requirement.fact} did not match.`, + }; + } + continue; + } + const topology = evaluateTopologyRequirement( + requirement.topologyQualification, + topologyQualifications, + expectedSubjectNodeIds, + ); + if (topology.outcome !== "matched") return topology; + matchedTopologies.push(topology.artifact); + } + return { outcome: "matched", topologyQualifications: matchedTopologies }; +} + +function intentCompatibilityError( + intent: ManagedInferenceSelectionIntent, + preset: ManagedInferenceServingPreset, + recipe: ManagedInferenceServingRecipe, +): string | undefined { + if (hasText(intent.provider) && intent.provider !== recipe.spec.backend) { + return `provider ${intent.provider} conflicts with preset ${preset.metadata.id}`; + } + if ( + hasText(intent.vllmModel) && + intent.vllmModel !== recipe.spec.model.id && + intent.vllmModel !== recipe.spec.model.servedName + ) { + return `model ${intent.vllmModel} conflicts with preset ${preset.metadata.id}`; + } + if ((intent.vllmExtraArguments?.length ?? 0) > 0) { + return `extra vLLM arguments conflict with preset ${preset.metadata.id}`; + } + return undefined; +} + +function presetPriority(preset: ManagedInferenceServingPreset): number { + const priority = (preset.spec as { readonly priority?: unknown }).priority; + if (!Number.isSafeInteger(priority)) { + throw new Error(`managed inference preset ${preset.metadata.id} has an invalid priority`); + } + return priority as number; +} + +function recipeForPreset( + catalog: CompiledManagedInferenceCatalog, + preset: ManagedInferenceServingPreset, +): CompiledManagedInferenceCatalog["recipes"][number] { + const matches = catalog.recipes.filter( + ({ metadata }) => metadata.id === preset.spec.plan.recipeRef, + ); + if (matches.length !== 1) { + throw new Error( + `managed inference preset ${preset.metadata.id} does not resolve exactly one recipe ${preset.spec.plan.recipeRef}`, + ); + } + const compiledRecipe = matches[0]!; + const recipe = compiledRecipe; + if (recipe.spec.backend !== preset.spec.plan.backend) { + throw new Error( + `managed inference preset ${preset.metadata.id} backend does not match its recipe`, + ); + } + const registrationError = getManagedInferenceRecipeRegistrationError(recipe); + if (registrationError) { + throw new Error(`managed inference recipe ${recipe.metadata.id}: ${registrationError}`); + } + return compiledRecipe; +} + +function matchingCandidate( + catalog: CompiledManagedInferenceCatalog, + compiledPreset: CompiledManagedInferenceCatalog["presets"][number], + input: ManagedInferenceResolverInput, +): + | { readonly outcome: "matched"; readonly candidate: MatchingCandidate } + | { readonly outcome: "unmet"; readonly message: string } + | { readonly outcome: "invalid-topology"; readonly message: string } + | { readonly outcome: "incompatible-intent"; readonly message: string } { + const preset = compiledPreset; + const compiledRecipe = recipeForPreset(catalog, preset); + const recipe = compiledRecipe; + const intentError = intentCompatibilityError(input.intent ?? {}, preset, recipe); + if (intentError) return { outcome: "incompatible-intent", message: intentError }; + const requirements = evaluateRequirements( + preset, + input.readinessReports, + input.topologyQualifications, + ); + if (requirements.outcome !== "matched") return requirements; + if (requirements.topologyQualifications.length !== 1) { + return { + outcome: "invalid-topology", + message: `Preset ${preset.metadata.id} must resolve exactly one topology qualification.`, + }; + } + return { + outcome: "matched", + candidate: { + preset, + presetDigest: managedInferenceDigest(compiledPreset), + recipe, + recipeDigest: managedInferenceDigest(compiledRecipe), + priority: presetPriority(preset), + topologyQualification: requirements.topologyQualifications[0]!, + }, + }; +} + +function selectedResolution( + catalog: CompiledManagedInferenceCatalog, + candidate: MatchingCandidate, + selection: "automatic" | "explicit", +): ManagedInferenceResolution { + let topologyQualification: ManagedInferenceTopologyQualification; + try { + topologyQualification = immutableManagedInferenceCopy(candidate.topologyQualification); + } catch { + return { + outcome: "rejected", + code: "invalid-topology", + message: "Topology qualification is not immutable JSON data.", + }; + } + return { + outcome: "selected", + selection, + catalogDigest: catalog.catalogDigest, + presetDigest: candidate.presetDigest, + recipeDigest: candidate.recipeDigest, + preset: candidate.preset, + recipe: candidate.recipe, + topologyQualification, + }; +} + +export function resolveManagedInferenceServing( + input: ManagedInferenceResolverInput, + catalog: CompiledManagedInferenceCatalog = loadManagedInferenceCatalog(), +): ManagedInferenceResolution { + const intent = input.intent ?? {}; + const explicitPresetId = hasText(intent.preset) ? intent.preset : undefined; + if (!explicitPresetId && explicitIntentWithoutPreset(intent)) { + return { + outcome: "no-match", + code: "explicit-intent", + message: "Existing inference intent remains authoritative.", + }; + } + + const maxAgeMs = input.maxReadinessAgeMs ?? MANAGED_INFERENCE_READINESS_MAX_AGE_MS; + const nowMs = (input.now ?? new Date()).getTime(); + if (!Number.isFinite(maxAgeMs) || maxAgeMs < 0 || !Number.isFinite(nowMs)) { + return { + outcome: "rejected", + code: "invalid-readiness", + message: "Readiness freshness policy is invalid.", + }; + } + const reportsError = readinessReportsError(input.readinessReports, nowMs, maxAgeMs); + if (reportsError) { + return { outcome: "rejected", code: "invalid-readiness", message: reportsError }; + } + + if (explicitPresetId) { + const matches = catalog.presets.filter(({ metadata }) => metadata.id === explicitPresetId); + if (matches.length !== 1) { + return { + outcome: "rejected", + code: "unknown-preset", + message: `Unknown managed inference preset ${explicitPresetId}.`, + }; + } + const compiledPreset = matches[0]!; + const preset = compiledPreset; + if (preset.spec.selection === "disabled") { + return { + outcome: "rejected", + code: "requirements-not-met", + message: `Managed inference preset ${explicitPresetId} is disabled.`, + }; + } + const evaluated = matchingCandidate(catalog, compiledPreset, input); + if (evaluated.outcome === "matched") { + return selectedResolution(catalog, evaluated.candidate, "explicit"); + } + return { + outcome: "rejected", + code: + evaluated.outcome === "invalid-topology" + ? "invalid-topology" + : evaluated.outcome === "incompatible-intent" + ? "incompatible-intent" + : "requirements-not-met", + message: evaluated.message, + }; + } + + const matching: MatchingCandidate[] = []; + let firstInvalidTopology: string | undefined; + for (const compiledPreset of catalog.presets) { + const preset = compiledPreset; + if (preset.spec.selection !== "automatic") continue; + const evaluated = matchingCandidate(catalog, compiledPreset, input); + if (evaluated.outcome === "matched") matching.push(evaluated.candidate); + else if (evaluated.outcome === "invalid-topology") firstInvalidTopology ??= evaluated.message; + } + if (firstInvalidTopology) { + return { + outcome: "rejected", + code: "invalid-topology", + message: firstInvalidTopology, + }; + } + if (matching.length === 0) { + return { + outcome: "no-match", + code: "requirements-not-met", + message: "No automatic managed inference preset matched.", + }; + } + matching.sort( + (left, right) => + right.priority - left.priority || + compareStrings(left.preset.metadata.id, right.preset.metadata.id), + ); + const highestPriority = matching[0]!.priority; + const tied = matching.filter(({ priority }) => priority === highestPriority); + if (tied.length !== 1) { + return { + outcome: "rejected", + code: "ambiguous-selection", + message: `Automatic managed inference selection is ambiguous at priority ${String( + highestPriority, + )}: ${tied.map(({ preset }) => preset.metadata.id).join(", ")}.`, + }; + } + return selectedResolution(catalog, tied[0]!, "automatic"); +} diff --git a/src/lib/inference/serving/types.ts b/src/lib/inference/serving/types.ts index c2d5af58e5..d666260fb3 100644 --- a/src/lib/inference/serving/types.ts +++ b/src/lib/inference/serving/types.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { SystemReadinessReport } from "../../readiness/types.js"; + export type ServingDefinitionKind = "ServingRecipe" | "ServingPreset"; export type ServingSelectionPolicy = "automatic" | "explicit-only" | "disabled"; export type ReadinessEntityKind = "observation" | "capability" | "qualification"; @@ -13,128 +15,251 @@ export type ServingReadinessObservationRole = | "driver-version"; export interface ServingMetadata { - id: string; - displayName?: string; + readonly id: string; + readonly displayName?: string; +} + +export interface ServingArgument { + readonly name: string; + readonly value?: string | number | boolean; +} + +export interface ServingTopologyBinding { + readonly type: "topologyQualificationOutput"; + readonly qualificationId: string; + readonly schemaVersion: number; + readonly outputSchema: string; +} + +export type ServingModelPreparation = + | { readonly ref: "none/v1" } + | { + readonly ref: "snapshot-copy-and-exact-text-replacement/v1"; + readonly snapshotCopy: { + readonly sourcePath: string; + readonly digest: string; + readonly targetPath: string; + }; + readonly exactTextReplacement: { + readonly targetPath: string; + readonly expectedText: string; + readonly replacementText: string; + }; + }; + +export interface ServingTemporaryFilesystem { + readonly target: string; + readonly sizeBytes: number; + readonly mode: string; + readonly options: readonly string[]; +} + +export interface ManagedInferenceServingRecipe { + readonly apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; + readonly kind: "ServingRecipe"; + readonly metadata: ServingMetadata; + readonly spec: { + readonly backend: string; + readonly bindings: Readonly>; + readonly model: { + readonly id: string; + readonly revision: string; + readonly servedName: string; + readonly files?: readonly { readonly path: string; readonly digest: string }[]; + readonly downloadSizeBytes: number; + readonly gated: boolean; + readonly installFastSafetensors: boolean; + readonly preparation: ServingModelPreparation; + }; + readonly runtime: { + readonly image: string; + readonly imageDownloadSizeBytes: number; + readonly pullTimeoutSeconds: number; + readonly architecture: string; + readonly networkMode: string; + readonly ipcMode: string; + readonly sharedMemoryBytes: number; + readonly gpuRequest: string; + readonly devices: readonly string[]; + readonly ulimits: { + readonly memlock: number | string; + readonly stackBytes: number; + }; + readonly modelCache: { + readonly source: string; + readonly target: string; + }; + readonly temporaryFilesystems: readonly ServingTemporaryFilesystem[]; + readonly environment: Readonly>; + readonly components?: Readonly>; + }; + readonly execution: { + readonly materializerRef: string; + readonly lifecycleRef: string; + readonly topologyBinding: string; + readonly nodeCount: number; + readonly tensorParallelSize: number; + readonly pipelineParallelSize: number; + readonly distributedExecutorBackend: string; + readonly rendezvousPort: number; + }; + readonly serve: { + readonly authentication: string; + readonly executable: string; + readonly arguments: readonly ServingArgument[]; + }; + readonly readiness: { + readonly timeoutSeconds: number; + readonly expectedModel: string; + }; + }; } interface ServingRecipeEnvelope { - apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; - kind: "ServingRecipe"; - metadata: ServingMetadata; + readonly apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; + readonly kind: "ServingRecipe"; + readonly metadata: ServingMetadata; } interface GenericServingRecipe extends ServingRecipeEnvelope { - spec: { - backend: string; - providerId?: never; - server?: never; - model: { - id: string; - revision: string; - servedName?: string; - files?: Array<{ path: string; digest: string }>; - }; - runtime?: { - image?: string; - architecture?: "amd64" | "arm64"; - components?: Record; - }; - execution: { - receiptRef?: string; - materializerRef: string; - lifecycleRef: string; - }; - serve?: { - arguments?: Array<{ name: string; value?: string | number | boolean }>; - }; - readiness?: { - contractRef?: string; - timeoutSeconds?: number; - expectedModel?: string; + readonly spec: { + readonly backend: string; + readonly providerId?: never; + readonly server?: never; + readonly bindings?: Readonly>; + readonly model: { + readonly id: string; + readonly revision: string; + readonly servedName?: string; + readonly files?: readonly { readonly path: string; readonly digest: string }[]; + readonly downloadSizeBytes?: number; + readonly gated?: boolean; + readonly installFastSafetensors?: boolean; + readonly preparation?: ServingModelPreparation; + }; + readonly runtime?: Partial & { + readonly components?: Readonly>; + }; + readonly execution: { + readonly receiptRef?: string; + readonly materializerRef: string; + readonly lifecycleRef: string; + readonly topologyBinding?: string; + readonly nodeCount?: number; + readonly tensorParallelSize?: number; + readonly pipelineParallelSize?: number; + readonly distributedExecutorBackend?: string; + readonly rendezvousPort?: number; + }; + readonly serve?: { + readonly authentication?: string; + readonly executable?: string; + readonly arguments?: readonly ServingArgument[]; + }; + readonly readiness?: { + readonly contractRef?: string; + readonly timeoutSeconds?: number; + readonly expectedModel?: string; }; }; } interface LlamaCppServingRecipe extends ServingRecipeEnvelope { - spec: { - backend: "install-llama-cpp"; - providerId: "llama-cpp-local"; - server: { - technology: "llama.cpp"; - source: { repository: string; revision: string }; - }; - model: { - id: string; - revision: string; - servedName: string; - files: Array<{ - path: string; - digest: string; - sizeBytes: number; - format: "gguf"; - quantization: string; - license: string; - }>; - }; - runtime: { - image: string; - platforms: Array<"linux/amd64" | "linux/arm64">; - containerRuntime: "docker"; - hosts: 1; - cuda: { baseImage: string; minimumDriverVersion: string }; - gpu: { vendor: "nvidia"; count: 1; offload: "full"; cpuFallback: "reject" }; - resources: { memoryBytes: number; writableStorageBytes: number; pidsLimit: number }; - }; - execution: { - receiptRef: string; - materializerRef: string; - lifecycleRef: string; - }; - serve: { - protocol: "openai-completions"; - port: 8081; - chatTemplate: string; - contextSize: number; - slots: 1; - idleSleepSeconds: -1; - limits: { - maxRequestBodyBytes: number; - maxPromptTokens: number; - maxCompletionTokens: number; - requestTimeoutSeconds: number; + readonly spec: { + readonly backend: "install-llama-cpp"; + readonly providerId: "llama-cpp-local"; + readonly bindings?: never; + readonly server: { + readonly technology: "llama.cpp"; + readonly source: { readonly repository: string; readonly revision: string }; + }; + readonly model: { + readonly id: string; + readonly revision: string; + readonly servedName: string; + readonly files: readonly { + readonly path: string; + readonly digest: string; + readonly sizeBytes: number; + readonly format: "gguf"; + readonly quantization: string; + readonly license: string; + }[]; + }; + readonly runtime: { + readonly image: string; + readonly platforms: readonly ("linux/amd64" | "linux/arm64")[]; + readonly containerRuntime: "docker"; + readonly hosts: 1; + readonly cuda: { readonly baseImage: string; readonly minimumDriverVersion: string }; + readonly gpu: { + readonly vendor: "nvidia"; + readonly count: 1; + readonly offload: "full"; + readonly cpuFallback: "reject"; + }; + readonly resources: { + readonly memoryBytes: number; + readonly writableStorageBytes: number; + readonly pidsLimit: number; }; }; - readiness: { - contractRef: string; - timeoutSeconds: number; - expectedModel: string; - probes: { models: true; health: true; properties: true; metrics: true }; - }; - policy: { - egress: "disabled"; - modelSource: "verified-local"; - modelDownloads: "disabled"; - }; - surfaces: { - ui: "disabled"; - slotInspection: "disabled"; - router: "disabled"; - mcpProxy: "disabled"; - serverTools: "disabled"; - agentMode: "disabled"; - multimodalProjection: "disabled"; - }; - capabilities: { - agents: Array<{ id: string; qualificationRef: string }>; - protocols: ["openai-completions"]; - streaming: boolean; - toolCalls: boolean; - structuredOutputs: boolean; - parallelToolCalls: false; - responsesApi: false; - embeddings: false; - reranking: false; - multimodal: false; + readonly execution: { + readonly receiptRef: string; + readonly materializerRef: string; + readonly lifecycleRef: string; + readonly nodeCount?: never; + }; + readonly serve: { + readonly protocol: "openai-completions"; + readonly port: 8081; + readonly chatTemplate: string; + readonly contextSize: number; + readonly slots: 1; + readonly idleSleepSeconds: -1; + readonly limits: { + readonly maxRequestBodyBytes: number; + readonly maxPromptTokens: number; + readonly maxCompletionTokens: number; + readonly requestTimeoutSeconds: number; + }; + }; + readonly readiness: { + readonly contractRef: string; + readonly timeoutSeconds: number; + readonly expectedModel: string; + readonly probes: { + readonly models: true; + readonly health: true; + readonly properties: true; + readonly metrics: true; + }; + }; + readonly policy: { + readonly egress: "disabled"; + readonly modelSource: "verified-local"; + readonly modelDownloads: "disabled"; + }; + readonly surfaces: { + readonly ui: "disabled"; + readonly slotInspection: "disabled"; + readonly router: "disabled"; + readonly mcpProxy: "disabled"; + readonly serverTools: "disabled"; + readonly agentMode: "disabled"; + readonly multimodalProjection: "disabled"; + }; + readonly capabilities: { + readonly agents: readonly { readonly id: string; readonly qualificationRef: string }[]; + readonly protocols: readonly ["openai-completions"]; + readonly streaming: boolean; + readonly toolCalls: boolean; + readonly structuredOutputs: boolean; + readonly parallelToolCalls: false; + readonly responsesApi: false; + readonly embeddings: false; + readonly reranking: false; + readonly multimodal: false; }; }; } @@ -142,92 +267,249 @@ interface LlamaCppServingRecipe extends ServingRecipeEnvelope { export type ServingRecipe = GenericServingRecipe | LlamaCppServingRecipe; export type ServingReadinessComparison = - | { operator: "equals"; value: string | number | boolean } - | { operator: "one-of"; values: Array } - | { operator: "at-least"; value: number } - | { operator: "version-at-least"; value: string }; - -export interface ServingReadinessRequirement { - readiness: - | { - scope: "controller" | "everyNode"; - kind: "observation" | "capability"; - id: string; - state: "present" | "absent"; - } - | { - scope: "controller" | "everyNode"; - kind: "observation"; - id: string; - comparison: ServingReadinessComparison; - } - | { - scope: "controller" | "everyNode"; - kind: "qualification"; - id: string; - status: "qualified" | "unqualified"; + | { readonly operator: "equals"; readonly value: string | number | boolean } + | { readonly operator: "one-of"; readonly values: readonly (string | number | boolean)[] } + | { readonly operator: "at-least"; readonly value: number } + | { readonly operator: "version-at-least"; readonly value: string }; + +export type ServingReadinessRequirement = + | { + readonly readiness: { + readonly scope: "controller" | "everyNode" | "anyNode"; + readonly kind: "qualification"; + readonly id: string; + readonly status: string; + }; + } + | { + readonly readiness: { + readonly scope: "controller" | "everyNode" | "anyNode"; + readonly kind: "observation" | "capability"; + readonly id: string; + readonly state: string; }; + } + | { + readonly readiness: { + readonly scope: "controller" | "everyNode" | "anyNode"; + readonly kind: "observation"; + readonly id: string; + readonly comparison: ServingReadinessComparison; + }; + }; + +export type ServingFactValue = string | number | boolean | readonly (string | number | boolean)[]; + +export interface ServingFactRequirement { + readonly fact: string; + readonly state: "present" | "absent"; + readonly operator: "equals" | "oneOf" | "atLeast" | "atMost" | "between"; + readonly value: ServingFactValue; +} + +export interface ServingTopologyRequirement { + readonly topologyQualification: { + readonly id: string; + readonly schemaVersion: number; + readonly status: string; + }; +} + +export type ServingPresetRequirement = + | ServingReadinessRequirement + | ServingFactRequirement + | ServingTopologyRequirement; + +export interface ServingPresetTopologyBinding { + readonly valueFromTopologyQualification: { + readonly id: string; + readonly schemaVersion: number; + readonly output: string; + }; +} + +export interface ManagedInferenceServingPreset { + readonly apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; + readonly kind: "ServingPreset"; + readonly metadata: ServingMetadata; + readonly spec: { + readonly selection: ServingSelectionPolicy; + readonly priority: number; + readonly requirements: { readonly all: readonly ServingPresetRequirement[] }; + readonly plan: { + readonly backend: string; + readonly recipeRef: string; + readonly bindings: Readonly>; + }; + }; } export interface ServingPreset { - apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; - kind: "ServingPreset"; - metadata: ServingMetadata; - spec: { - selection: ServingSelectionPolicy; - priority: number; - requirements?: { all: ServingReadinessRequirement[] }; - plan: { - backend: string; - recipeRef: string; + readonly apiVersion: "nemoclaw.nvidia.com/managed-inference/v1"; + readonly kind: "ServingPreset"; + readonly metadata: ServingMetadata; + readonly spec: { + readonly selection: ServingSelectionPolicy; + readonly priority: number; + readonly requirements?: { readonly all: readonly ServingPresetRequirement[] }; + readonly plan: { + readonly backend: string; + readonly recipeRef: string; + readonly bindings?: Readonly>; }; }; } export interface ServingCatalogSourceProvenance { - path: string; - kind: ServingDefinitionKind; - id: string; - digest: string; + readonly path: string; + readonly kind: ServingDefinitionKind; + readonly id: string; + readonly digest: string; } export interface CompiledServingCatalogPayload { - schemaVersion: "1.0.0"; - compilerVersion: "1.1.0"; - sourceRevision: string; - readinessSchemaRef: "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json"; - recipes: ServingRecipe[]; - presets: ServingPreset[]; - sources: ServingCatalogSourceProvenance[]; + readonly schemaVersion: "1.0.0"; + readonly compilerVersion: "1.1.0"; + readonly sourceRevision: string; + readonly readinessSchemaRef: "https://github.com/NVIDIA/NemoClaw/schemas/system-readiness.schema.json"; + readonly recipes: readonly ServingRecipe[]; + readonly presets: readonly ServingPreset[]; + readonly sources: readonly ServingCatalogSourceProvenance[]; } export interface CompiledServingCatalog extends CompiledServingCatalogPayload { - catalogDigest: string; + readonly catalogDigest: string; } export interface ServingCatalogSource { - path: string; - contents: string; + readonly path: string; + readonly contents: string; } export interface ServingCatalogSchemas { - catalog: object; - preset: object; - recipe: object; + readonly catalog: object; + readonly preset: object; + readonly recipe: object; +} + +export interface ServingTopologyRegistryEntry { + readonly bindingOutput: string; + readonly outputSchema: string; +} + +export interface ServingReadinessRegistryEntry { + readonly kind: ReadinessEntityKind; + readonly valueType?: ReadinessValueType; + readonly role?: ServingReadinessObservationRole; } +export type ServingReadinessRegistryValue = + | ReadinessEntityKind + | ReadonlySet + | ServingReadinessRegistryEntry; + export interface ServingCatalogRegistries { - receipts: ReadonlySet; - materializers: ReadonlySet; - lifecycles: ReadonlySet; - readinessContracts: ReadonlySet; - readiness: ReadonlyMap< - string, - | { - kind: "observation"; - valueType?: ReadinessValueType; - role?: ServingReadinessObservationRole; - } - | { kind: "capability" | "qualification" } - >; + readonly receipts: ReadonlySet; + readonly materializers: ReadonlySet; + readonly lifecycles: ReadonlySet; + readonly readinessContracts: ReadonlySet; + readonly readiness: ReadonlyMap; + readonly facts?: ReadonlySet; + readonly topologyQualifications?: ReadonlyMap; + readonly validateRecipe?: (recipe: ServingRecipe) => string | undefined; +} + +export type ManagedInferenceServingArgument = ServingArgument; +export type ManagedInferenceTopologyBinding = ServingTopologyBinding; +export type ManagedInferenceModelPreparation = ServingModelPreparation; +export type ManagedInferenceTemporaryFilesystem = ServingTemporaryFilesystem; +export type ManagedInferenceReadinessRequirement = ServingReadinessRequirement; +export type ManagedInferenceFactValue = ServingFactValue; +export type ManagedInferenceFactRequirement = ServingFactRequirement; +export type ManagedInferenceTopologyRequirement = ServingTopologyRequirement; +export type ManagedInferencePresetRequirement = ServingPresetRequirement; +export type ManagedInferencePresetTopologyBinding = ServingPresetTopologyBinding; +export interface CompiledManagedInferenceCatalog + extends Omit { + readonly presets: readonly ManagedInferenceServingPreset[]; + readonly recipes: readonly ManagedInferenceServingRecipe[]; +} + +const MATERIALIZER_OWNED_SERVE_ARGUMENTS = new Set([ + "--api-key", + "--distributed-executor-backend", + "--headless", + "--host", + "--master-addr", + "--master-port", + "--nnodes", + "--node-rank", + "--pipeline-parallel-size", + "--revision", + "--served-model-name", + "--tensor-parallel-size", +]); + +export function isManagedInferenceMaterializerOwnedArgument(name: string): boolean { + return MATERIALIZER_OWNED_SERVE_ARGUMENTS.has(name); +} + +export interface ManagedInferenceTopologyQualification { + readonly id: string; + readonly schemaVersion: number; + readonly status: "qualified" | "unqualified" | "unknown"; + readonly subjectNodeIds: readonly string[]; + readonly subjectDigest: string; + readonly outputDigest: string; + readonly output: TOutput; +} + +export interface ManagedInferenceReadinessSource { + readonly nodeId: string; + readonly report: SystemReadinessReport; +} + +export interface ManagedInferenceSelectionIntent { + readonly provider?: string; + readonly vllmModel?: string; + readonly vllmExtraArguments?: readonly string[]; + readonly preset?: string; +} + +export interface ManagedInferenceResolverInput { + readonly readinessReports: readonly ManagedInferenceReadinessSource[]; + readonly topologyQualifications: readonly ManagedInferenceTopologyQualification[]; + readonly intent?: ManagedInferenceSelectionIntent; + readonly now?: Date; + readonly maxReadinessAgeMs?: number; } + +export interface ResolvedManagedInferenceSelection { + readonly outcome: "selected"; + readonly selection: "automatic" | "explicit"; + readonly catalogDigest: string; + readonly presetDigest: string; + readonly recipeDigest: string; + readonly preset: ManagedInferenceServingPreset; + readonly recipe: ManagedInferenceServingRecipe; + readonly topologyQualification: ManagedInferenceTopologyQualification; +} + +export type ManagedInferenceResolution = + | ResolvedManagedInferenceSelection + | { + readonly outcome: "no-match"; + readonly code: "explicit-intent" | "requirements-not-met"; + readonly message: string; + } + | { + readonly outcome: "rejected"; + readonly code: + | "unknown-preset" + | "incompatible-intent" + | "invalid-readiness" + | "invalid-topology" + | "ambiguous-selection" + | "requirements-not-met"; + readonly message: string; + }; diff --git a/src/lib/inference/serving/vllm-managed-support.ts b/src/lib/inference/serving/vllm-managed-support.ts new file mode 100644 index 0000000000..a384ccdd3e --- /dev/null +++ b/src/lib/inference/serving/vllm-managed-support.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { ensureDualStationVllmApiKey, loadDualStationVllmApiKey } from "../vllm-api-key.js"; +export { + buildLocalDualStationDockerEnv, + buildRemoteVllmDockerEnv, + buildVllmDockerEnv, +} from "../vllm-docker-env.js"; +export { resolveVllmInstallModel } from "../vllm-prompt.js"; +export { tryInstallManagedClusterManagedVllm } from "./managed-cluster-installer.js"; +export { recoverInstalledManagedClusterVllmEndpoint } from "./managed-cluster-runtime-receipt.js"; diff --git a/src/lib/inference/vllm-api-key.ts b/src/lib/inference/vllm-api-key.ts index afdcec37ee..d60b1bad98 100644 --- a/src/lib/inference/vllm-api-key.ts +++ b/src/lib/inference/vllm-api-key.ts @@ -12,34 +12,41 @@ import { ensureLocalAdapterStateDir } from "./local-adapter-lifecycle"; import { DUAL_STATION_VLLM_API_KEY_FILE } from "./vllm-station-runtime-receipt-path"; export const DUAL_STATION_VLLM_API_KEY_PATTERN = /^[a-f0-9]{64}$/; +// The original managed endpoint key was introduced with dual-Station serving. +// Keep its path stable while exposing topology-neutral names for later managed +// serving profiles. +export const MANAGED_VLLM_API_KEY_FILE = DUAL_STATION_VLLM_API_KEY_FILE; +export const MANAGED_VLLM_API_KEY_PATTERN = DUAL_STATION_VLLM_API_KEY_PATTERN; export interface DualStationVllmApiKeyOptions { stateDir?: string; randomBytes?: (size: number) => Buffer; } -function defaultStateDir(): string { +export function managedVllmStateDir(homeDir = os.homedir()): string { // The managed vLLM service is host-global rather than gateway-scoped. Every // gateway therefore reads the same key even when NEMOCLAW_GATEWAY_PORT is // changed for a second sandbox. - return nemoclawStateRoot(os.homedir(), DEFAULT_GATEWAY_PORT); + return nemoclawStateRoot(homeDir, DEFAULT_GATEWAY_PORT); } -export function dualStationVllmApiKeyPath(stateDir = defaultStateDir()): string { +export function dualStationVllmApiKeyPath(stateDir = managedVllmStateDir()): string { return path.join(stateDir, DUAL_STATION_VLLM_API_KEY_FILE); } +export const managedVllmApiKeyPath = dualStationVllmApiKeyPath; + function assertPrivateRegularFile(stat: fs.Stats, filePath: string): void { if (!stat.isFile()) { - throw new Error(`Refusing to read dual-Station vLLM API key from non-file path: ${filePath}`); + throw new Error(`Refusing to read managed vLLM API key from non-file path: ${filePath}`); } if ((stat.mode & 0o077) !== 0) { throw new Error( - `Dual-Station vLLM API key file must not be accessible by group or others: ${filePath}`, + `Managed vLLM API key file must not be accessible by group or others: ${filePath}`, ); } if (typeof process.getuid === "function" && stat.uid !== process.getuid()) { - throw new Error(`Dual-Station vLLM API key file is not owned by the current user: ${filePath}`); + throw new Error(`Managed vLLM API key file is not owned by the current user: ${filePath}`); } } @@ -47,7 +54,7 @@ function assertPrivateRegularFile(stat: fs.Stats, filePath: string): void { export function loadDualStationVllmApiKey( options: Pick = {}, ): string | null { - const filePath = dualStationVllmApiKeyPath(options.stateDir ?? defaultStateDir()); + const filePath = dualStationVllmApiKeyPath(options.stateDir ?? managedVllmStateDir()); const noFollow = fs.constants.O_NOFOLLOW; if (typeof noFollow !== "number") { throw new Error("Secure no-follow file opens are unavailable on this platform"); @@ -62,7 +69,7 @@ export function loadDualStationVllmApiKey( if (code === "ENOENT") return null; if (code === "ELOOP") { throw new Error( - `Refusing to read dual-Station vLLM API key through a symbolic link: ${filePath}`, + `Refusing to read managed vLLM API key through a symbolic link: ${filePath}`, ); } throw error; @@ -70,11 +77,11 @@ export function loadDualStationVllmApiKey( const opened = fs.fstatSync(fd); assertPrivateRegularFile(opened, filePath); if (opened.size < 64 || opened.size > 65) { - throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); + throw new Error(`Managed vLLM API key file is malformed: ${filePath}`); } const value = fs.readFileSync(fd, "utf8").trim(); if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(value)) { - throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); + throw new Error(`Managed vLLM API key file is malformed: ${filePath}`); } return value; } finally { @@ -84,7 +91,7 @@ export function loadDualStationVllmApiKey( /** Create the managed endpoint key once, or reuse the existing private key. */ export function ensureDualStationVllmApiKey(options: DualStationVllmApiKeyOptions = {}): string { - const stateDir = options.stateDir ?? defaultStateDir(); + const stateDir = options.stateDir ?? managedVllmStateDir(); ensureLocalAdapterStateDir(stateDir); const existing = loadDualStationVllmApiKey({ stateDir }); if (existing) return existing; @@ -92,7 +99,7 @@ export function ensureDualStationVllmApiKey(options: DualStationVllmApiKeyOption const randomBytes = options.randomBytes ?? crypto.randomBytes; const value = randomBytes(32).toString("hex"); if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(value)) { - throw new Error("Could not generate a valid dual-Station vLLM API key"); + throw new Error("Could not generate a valid managed vLLM API key"); } const filePath = dualStationVllmApiKeyPath(stateDir); @@ -123,7 +130,10 @@ export function ensureDualStationVllmApiKey(options: DualStationVllmApiKeyOption const persisted = loadDualStationVllmApiKey({ stateDir }); if (persisted !== value) { - throw new Error("Could not verify the persisted dual-Station vLLM API key"); + throw new Error("Could not verify the persisted managed vLLM API key"); } return value; } + +export const loadManagedVllmApiKey = loadDualStationVllmApiKey; +export const ensureManagedVllmApiKey = ensureDualStationVllmApiKey; diff --git a/src/lib/inference/vllm-docker-env.ts b/src/lib/inference/vllm-docker-env.ts index 6dcf3f14f9..77302fc793 100644 --- a/src/lib/inference/vllm-docker-env.ts +++ b/src/lib/inference/vllm-docker-env.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import { isIP } from "node:net"; import { buildSubprocessEnv } from "../subprocess-env"; @@ -8,6 +9,7 @@ import { assertDualStationSshBindingFiles, type DualStationSshBinding, dualStationDockerSshUri, + dualStationPinnedSshArgs, } from "./vllm-station-ssh-binding"; const DOCKER_CLIENT_ENV_NAMES = [ @@ -43,6 +45,30 @@ const SSH_TRANSPORT_ENV_NAMES = [ const CANONICAL_SSH_HOST_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; const CANONICAL_SSH_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const LISTENER_PROBE_MAX_BUFFER_BYTES = 1024 * 1024; + +function captureListenerCommand( + argv: readonly [string, ...string[]], + env: Readonly>, + timeoutMs: number, +): string { + const [file, ...args] = argv; + const result = spawnSync(file, args, { + encoding: "utf8", + env, + killSignal: "SIGKILL", + maxBuffer: LISTENER_PROBE_MAX_BUFFER_BYTES, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + windowsHide: true, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`managed vLLM listener probe exited with status ${String(result.status)}`); + } + return (result.stdout ?? "").trim(); +} function validateRemoteDockerSshUri(value: string): string { const invalid = () => @@ -122,6 +148,9 @@ export function buildLocalDualStationDockerEnv( return env; } +/** Cardinality-neutral name for managed-cluster consumers. */ +export const buildLocalManagedVllmDockerEnv = buildLocalDualStationDockerEnv; + /** Minimal environment shared by strict SSH probes and Docker's SSH helper. */ export function buildVllmSshTransportEnv( extra: Record = {}, @@ -135,6 +164,35 @@ export function buildVllmSshTransportEnv( return { ...env, ...extra }; } +/** Capture the listening TCP sockets from one node in a pinned managed-vLLM cluster. */ +export function captureManagedVllmTcpListeners( + role: "head" | "worker", + binding: DualStationSshBinding, + timeoutMs: number, +): string { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error("managed vLLM listener probe timeout must be a positive integer"); + } + if (role === "head") { + return captureListenerCommand( + ["ss", "-H", "-lnt"], + buildSubprocessEnv({ LC_ALL: "C" }), + timeoutMs, + ); + } + return captureListenerCommand( + [ + "/usr/bin/ssh", + ...dualStationPinnedSshArgs(binding), + "--", + binding.peerTarget, + "LC_ALL=C ss -H -lnt", + ], + buildVllmSshTransportEnv({ LC_ALL: "C" }), + timeoutMs, + ); +} + /** * Select one explicitly configured Docker-over-SSH daemon without allowing an * ambient context, client config, API pin, or TCP/TLS settings to influence it. diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 462b7b88ba..ba137c3a0f 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -99,14 +99,13 @@ vi.mock("./vllm-station-runtime-receipt", () => ({ vi.mock("./vllm-api-key", () => ({ ensureDualStationVllmApiKey: mocks.ensureApiKey, + ensureManagedVllmApiKey: mocks.ensureApiKey, loadDualStationVllmApiKey: mocks.loadApiKey, + loadManagedVllmApiKey: mocks.loadApiKey, + managedVllmStateDir: () => path.join(os.homedir(), ".nemoclaw"), })); -import { - detectVllmProfile, - installVllm, - persistConfiguredDualStationVllmRuntimeReceipt, -} from "./vllm"; +import { detectVllmProfile, installVllm, persistConfiguredManagedVllmRuntimeReceipt } from "./vllm"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { createDualStationSshBindingFixture, @@ -307,7 +306,7 @@ afterEach(() => { describe("dual DGX Station running-runtime receipt adoption", () => { it("persists cleanup ownership for the exact configured running pair", async () => { - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: true, persisted: true, }); @@ -325,7 +324,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { reason: "worker ownership changed", }); - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: false, reason: "worker ownership changed", }); @@ -335,7 +334,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { it("fails closed when the managed peer configuration is missing", async () => { delete process.env.NEMOCLAW_DGX_STATION_PEER; - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: false, reason: "the managed dual-Station peer configuration is missing", }); @@ -370,7 +369,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { return true; }); - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: true, persisted: true, }); @@ -399,7 +398,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { } }); - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: false, reason: "the managed dual-Station peer configuration is missing", }); @@ -416,7 +415,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { reason: "could not revalidate the managed pair: managed runtime identity changed", }); - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: false, reason: "the managed dual-Station cleanup receipt is unsafe: could not revalidate the managed pair: managed runtime identity changed", @@ -432,7 +431,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => { mocks.recoverRuntime.mockReturnValue({ kind: "ready", plan: recoveredPlan }); mocks.areContainersRunning.mockReturnValue(false); - await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ ok: false, reason: "the managed dual-Station containers changed before cleanup ownership validation", }); diff --git a/src/lib/inference/vllm-managed-cluster-runtime-receipt.test.ts b/src/lib/inference/vllm-managed-cluster-runtime-receipt.test.ts new file mode 100644 index 0000000000..4ac23bd972 --- /dev/null +++ b/src/lib/inference/vllm-managed-cluster-runtime-receipt.test.ts @@ -0,0 +1,624 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { managedInferenceDigest, managedInferenceHexDigest } from "./serving/catalog-integrity"; +import { loadManagedInferenceCatalog } from "./serving/catalog-loader"; +import { fixtureManagedClusterSelection } from "./serving/managed-cluster-fixture.test-support"; +import { + type ManagedClusterNodeSnapshot, + type ManagedClusterVllmLifecycleDeps, + managedClusterVllmApiKeyFingerprint, +} from "./serving/managed-cluster-lifecycle"; +import { + MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL, + MANAGED_CLUSTER_TRANSACTION_LABEL, + type ManagedClusterVllmPlan, + type ManagedClusterVllmRolePlan, + materializeManagedClusterVllmPlan, +} from "./serving/managed-cluster-materialize"; +import { + cleanupInstalledManagedClusterVllmRuntime, + loadManagedClusterVllmRuntimeReceipt, + managedClusterVllmRuntimeReceiptPath, + type PersistManagedClusterVllmRuntimeReceiptInput, + persistManagedClusterVllmRuntimeReceipt, + recoverInstalledManagedClusterVllmEndpoint, +} from "./serving/managed-cluster-runtime-receipt"; +import { MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE } from "./serving/managed-cluster-runtime-receipt-path"; +import { + clearManagedVllmSshBinding, + copyManagedVllmSshBinding, + encodeManagedVllmSshBindingHandoff, + type ManagedVllmSshBinding, +} from "./serving/managed-cluster-ssh-binding"; +import { + createManagedVllmSshBindingFixture, + type ManagedVllmSshBindingFixture, +} from "./serving/managed-cluster-ssh-binding.test-support"; +import { managedClusterTopologyOutputDigest } from "./serving/managed-cluster-topology"; +import type { CompiledManagedInferenceCatalog } from "./serving/types"; + +const API_KEY = "a".repeat(64); +const HEAD_ID = "b".repeat(64); +const WORKER_ID = "c".repeat(64); +const TRANSACTION_ID = "d".repeat(32); + +let root: string; +let stateDir: string; +let sshFixture: ManagedVllmSshBindingFixture; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-cluster-runtime-receipt-")); + stateDir = path.join(root, ".nemoclaw"); + sshFixture = createManagedVllmSshBindingFixture("nvidia@spark-worker.local"); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.doUnmock("./serving/catalog-loader"); + vi.resetModules(); + sshFixture.cleanup(); + fs.rmSync(root, { recursive: true, force: true }); +}); + +function plan(sshBinding: ManagedVllmSshBinding = sshFixture.binding): ManagedClusterVllmPlan { + const selection = fixtureManagedClusterSelection(); + const sourceTopology = selection.topologyQualification; + const output = { + ...sourceTopology.output, + peers: sourceTopology.output.peers.map((peer) => ({ + ...peer, + target: sshBinding.peerTarget, + sshBindingHandle: encodeManagedVllmSshBindingHandoff(sshBinding), + })), + }; + return materializeManagedClusterVllmPlan({ + ...selection, + topologyQualification: { + ...sourceTopology, + outputDigest: managedClusterTopologyOutputDigest(output), + output, + }, + }); +} + +function input( + sshBinding: ManagedVllmSshBinding = sshFixture.binding, +): PersistManagedClusterVllmRuntimeReceiptInput { + const runtimePlan = plan(sshBinding); + return { + plan: runtimePlan, + nodes: runtimePlan.roles.map((rolePlan) => ({ + nodeId: rolePlan.nodeId, + cacheRoot: "/home/nvidia/.cache/huggingface", + containerId: rolePlan.rank === 0 ? HEAD_ID : WORKER_ID, + ...(rolePlan.execution.kind === "ssh" + ? { + sshBinding, + discoveryStatePath: path.join(stateDir, MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE), + } + : {}), + })), + apiKeyFingerprint: managedClusterVllmApiKeyFingerprint(API_KEY), + }; +} + +function catalogWithUnrelatedProfile(): CompiledManagedInferenceCatalog { + const current = loadManagedInferenceCatalog(); + const selectedPreset = current.presets[0]!; + const selectedRecipe = current.recipes[0]!; + const unrelatedRecipe = { + ...selectedRecipe, + metadata: { ...selectedRecipe.metadata, id: "vllm.unrelated.managed-cluster.v1" }, + }; + const unrelatedPreset = { + ...selectedPreset, + metadata: { ...selectedPreset.metadata, id: "vllm.unrelated.managed-cluster" }, + spec: { + ...selectedPreset.spec, + plan: { + ...selectedPreset.spec.plan, + recipeRef: unrelatedRecipe.metadata.id, + }, + }, + }; + const sources = [ + ...current.sources, + { + path: "managed-inference/presets/vllm.unrelated.managed-cluster.yaml", + kind: "ServingPreset" as const, + id: unrelatedPreset.metadata.id, + digest: managedInferenceDigest(unrelatedPreset), + }, + { + path: "managed-inference/recipes/vllm.unrelated.managed-cluster.v1.yaml", + kind: "ServingRecipe" as const, + id: unrelatedRecipe.metadata.id, + digest: managedInferenceDigest(unrelatedRecipe), + }, + ] as const; + const contents = { + compilerVersion: current.compilerVersion, + presets: [...current.presets, unrelatedPreset], + recipes: [...current.recipes, unrelatedRecipe], + readinessSchemaRef: current.readinessSchemaRef, + schemaVersion: current.schemaVersion, + sources, + sourceRevision: current.sourceRevision, + } as const; + return { ...contents, catalogDigest: managedInferenceDigest(contents) }; +} + +function persistWithDiscoveryBinding(): { + discoveryBindingPath: string; + discoveryStatePath: string; + runtime: ReturnType; +} { + fs.mkdirSync(stateDir, { mode: 0o700 }); + const discoveryStatePath = path.join(stateDir, MANAGED_CLUSTER_MANAGED_SERVING_STATE_FILE); + const binding = copyManagedVllmSshBinding(discoveryStatePath, sshFixture.binding); + return { + discoveryBindingPath: `${discoveryStatePath}.ssh-binding`, + discoveryStatePath, + runtime: persistManagedClusterVllmRuntimeReceipt(input(binding), { stateDir }), + }; +} + +function snapshot( + rolePlan: ManagedClusterVllmRolePlan, + containerId: string, +): ManagedClusterNodeSnapshot { + return { + containers: [ + { + id: containerId, + name: rolePlan.containerName, + image: rolePlan.image, + running: true, + healthy: true, + labels: { + ...rolePlan.baseLabels, + [MANAGED_CLUSTER_API_KEY_FINGERPRINT_LABEL]: managedClusterVllmApiKeyFingerprint(API_KEY), + [MANAGED_CLUSTER_TRANSACTION_LABEL]: TRANSACTION_ID, + }, + }, + ], + listeningPorts: + rolePlan.role === "head" ? [rolePlan.endpoint ? 8000 : 0, 25000].filter(Boolean) : [25000], + }; +} + +function cleanupDeps( + runtimePlan: ManagedClusterVllmPlan, + ids: { head: string; worker: string } = { head: HEAD_ID, worker: WORKER_ID }, +): { + deps: Pick< + ManagedClusterVllmLifecycleDeps, + "inspectNode" | "removeContainer" | "withLifecycleLock" + >; + removeContainer: ReturnType; +} { + const removeContainer = vi.fn(async () => ({ ok: true as const })); + return { + deps: { + inspectNode: async (rolePlan) => + rolePlan.role === "head" + ? snapshot(runtimePlan.roles[0], ids.head) + : snapshot(runtimePlan.roles[1], ids.worker), + removeContainer, + withLifecycleLock: async (_plan, operation) => await operation(), + }, + removeContainer, + }; +} + +describe("managed cluster vLLM runtime receipt", () => { + it("uses the host-global default gateway state root", async () => { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + vi.resetModules(); + const { managedClusterVllmRuntimeReceiptPath: selectedReceiptPath } = await import( + "./serving/managed-cluster-runtime-receipt" + ); + expect(selectedReceiptPath()).toBe( + path.join(os.homedir(), ".nemoclaw", "managed-cluster-vllm-runtime.json"), + ); + }); + + it("persists a private exact runtime plan with a promoted SSH binding and no bearer key", () => { + const source = input(); + const runtime = persistManagedClusterVllmRuntimeReceipt(source, { stateDir }); + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const raw = fs.readFileSync(receiptPath, "utf8"); + const worker = runtime.nodes.find((node) => node.binding)!; + + expect(fs.statSync(receiptPath).mode & 0o777).toBe(0o600); + expect(fs.lstatSync(`${receiptPath}.rank-1.ssh-binding`).isDirectory()).toBe(true); + expect(raw).not.toContain(API_KEY); + expect(worker.sshBinding).not.toBe(sshFixture.token); + expect(runtime.plan.roles[1].execution).toMatchObject({ + kind: "ssh", + expectedTarget: sshFixture.binding.peerTarget, + bindingHandle: worker.sshBinding, + }); + expect(worker.binding?.bindingFile).toContain(`${receiptPath}.rank-1.ssh-binding/`); + expect(loadManagedClusterVllmRuntimeReceipt({ stateDir })).toEqual(runtime); + }); + + it("is idempotent only for the same committed runtime", () => { + const source = input(); + const first = persistManagedClusterVllmRuntimeReceipt(source, { stateDir }); + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const original = fs.readFileSync(receiptPath, "utf8"); + + expect(persistManagedClusterVllmRuntimeReceipt(source, { stateDir })).toEqual(first); + expect( + persistManagedClusterVllmRuntimeReceipt( + { ...source, nodes: [...source.nodes].reverse() }, + { stateDir }, + ), + ).toEqual(first); + expect(() => + persistManagedClusterVllmRuntimeReceipt( + { + ...source, + nodes: source.nodes.map((node) => + node.nodeId === source.plan.roles[1]?.nodeId + ? { ...node, containerId: "e".repeat(64) } + : node, + ), + }, + { stateDir }, + ), + ).toThrow("different managed cluster runtime receipt"); + expect(fs.readFileSync(receiptPath, "utf8")).toBe(original); + }); + + it("keeps a selected profile receipt valid when unrelated catalog entries change", async () => { + const source = input(); + const currentCatalog = loadManagedInferenceCatalog(); + const changedCatalog = catalogWithUnrelatedProfile(); + persistManagedClusterVllmRuntimeReceipt(source, { stateDir }); + + expect(changedCatalog.catalogDigest).not.toBe(currentCatalog.catalogDigest); + vi.doMock("./serving/catalog-loader", () => ({ + loadManagedInferenceCatalog: () => changedCatalog, + getManagedInferenceCompiledPreset: (id: string) => + changedCatalog.presets.find(({ metadata }) => metadata.id === id), + getManagedInferenceCompiledRecipe: (id: string) => + changedCatalog.recipes.find(({ metadata }) => metadata.id === id), + })); + vi.resetModules(); + const { loadManagedClusterVllmRuntimeReceipt: loadAgainstChangedCatalog } = await import( + "./serving/managed-cluster-runtime-receipt" + ); + const loaded = loadAgainstChangedCatalog({ stateDir }); + const currentPreset = changedCatalog.presets.find( + ({ metadata }) => metadata.id === source.plan.presetId, + ); + const currentRecipe = changedCatalog.recipes.find( + ({ metadata }) => metadata.id === source.plan.recipeId, + ); + + expect(currentPreset).toBeDefined(); + expect(currentRecipe).toBeDefined(); + expect(loaded?.plan.catalogDigest).toBe(currentCatalog.catalogDigest); + expect(loaded?.plan.presetDigest).toBe(managedInferenceDigest(currentPreset)); + expect(loaded?.plan.recipeDigest).toBe(managedInferenceDigest(currentRecipe)); + }); + + it("preserves a receipt-write failure when temporary-file cleanup also fails", () => { + const rename = vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw new Error("receipt rename failed"); + }); + const unlink = vi.spyOn(fs, "unlinkSync").mockImplementationOnce(() => { + throw new Error("temporary cleanup failed"); + }); + + expect(() => persistManagedClusterVllmRuntimeReceipt(input(), { stateDir })).toThrow( + "receipt rename failed", + ); + expect(rename).toHaveBeenCalledOnce(); + expect(unlink).toHaveBeenCalledOnce(); + }); + + it("refuses a pre-existing SSH binding tree without mutating it", () => { + fs.mkdirSync(stateDir, { mode: 0o700 }); + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const bindingPath = `${receiptPath}.rank-1.ssh-binding`; + const markerPath = path.join(bindingPath, "foreign-state"); + fs.mkdirSync(bindingPath, { mode: 0o700 }); + fs.writeFileSync(markerPath, "leave intact\n", { mode: 0o600 }); + + expect(() => persistManagedClusterVllmRuntimeReceipt(input(), { stateDir })).toThrow( + "Managed cluster SSH binding state already exists", + ); + expect(fs.readFileSync(markerPath, "utf8")).toBe("leave intact\n"); + expect(fs.existsSync(receiptPath)).toBe(false); + }); + + it("refuses a symbolic-link receipt", () => { + fs.mkdirSync(stateDir, { mode: 0o700 }); + const target = path.join(root, "redirected.json"); + fs.writeFileSync(target, "{}\n", { mode: 0o600 }); + fs.symlinkSync(target, managedClusterVllmRuntimeReceiptPath(stateDir)); + + expect(() => loadManagedClusterVllmRuntimeReceipt({ stateDir })).toThrow("symbolic link"); + }); + + it("rejects changed plan contents before using the persisted binding", () => { + persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const receipt = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + receipt.plan.model.id = "foreign/model"; + fs.writeFileSync(receiptPath, `${JSON.stringify(receipt)}\n`, { + mode: 0o600, + }); + + expect(() => loadManagedClusterVllmRuntimeReceipt({ stateDir })).toThrow("plan digest changed"); + }); + + it("rejects a digest-valid receipt whose plan omits role ownership", () => { + persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + const receiptPath = managedClusterVllmRuntimeReceiptPath(stateDir); + const receipt = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + receipt.plan.roles = {}; + receipt.planDigest = managedInferenceHexDigest(receipt.plan); + fs.writeFileSync(receiptPath, `${JSON.stringify(receipt)}\n`, { mode: 0o600 }); + + expect(() => loadManagedClusterVllmRuntimeReceipt({ stateDir })).toThrow( + "node ownership is incomplete", + ); + }); + + it("recovers only the exact healthy receipt-owned endpoint synchronously", () => { + const runtime = persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + expect( + recoverInstalledManagedClusterVllmEndpoint({ + stateDir, + loadApiKey: () => API_KEY, + inspectNodesSync: () => ({ + nodes: runtime.plan.roles.map((rolePlan) => ({ + nodeId: rolePlan.nodeId, + snapshot: snapshot(rolePlan, rolePlan.rank === 0 ? HEAD_ID : WORKER_ID), + })), + }), + }), + ).toEqual({ + baseUrl: runtime.plan.roles[0].endpoint, + apiKey: API_KEY, + apiKeyFingerprint: runtime.apiKeyFingerprint, + plan: runtime.plan, + }); + }); + + it("signals a managed endpoint whose exact receipt-owned IDs changed", () => { + const runtime = persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + expect(() => + recoverInstalledManagedClusterVllmEndpoint({ + stateDir, + loadApiKey: () => API_KEY, + inspectNodesSync: () => ({ + nodes: runtime.plan.roles.map((rolePlan) => ({ + nodeId: rolePlan.nodeId, + snapshot: snapshot(rolePlan, rolePlan.rank === 0 ? "e".repeat(64) : WORKER_ID), + })), + }), + }), + ).toThrow("not recoverable"); + }); + + it("signals a persisted managed endpoint whose key no longer matches", () => { + persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + expect(() => + recoverInstalledManagedClusterVllmEndpoint({ + stateDir, + loadApiKey: () => "f".repeat(64), + inspectNodesSync: vi.fn(), + }), + ).toThrow("API key no longer matches"); + }); + + it("removes only both exact receipt-owned containers before retiring state", async () => { + const { discoveryStatePath, runtime } = persistWithDiscoveryBinding(); + const { deps, removeContainer } = cleanupDeps(runtime.plan); + await expect( + cleanupInstalledManagedClusterVllmRuntime({ + stateDir, + loadApiKey: () => API_KEY, + createLifecycleDeps: () => deps, + }), + ).resolves.toEqual({ + kind: "removed", + removedContainerIds: [HEAD_ID, WORKER_ID], + }); + expect(removeContainer).toHaveBeenNthCalledWith(1, runtime.plan.roles[0], HEAD_ID); + expect(removeContainer).toHaveBeenNthCalledWith(2, runtime.plan.roles[1], WORKER_ID); + expect(fs.existsSync(`${discoveryStatePath}.ssh-binding`)).toBe(false); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(false); + expect( + fs.existsSync(`${managedClusterVllmRuntimeReceiptPath(stateDir)}.rank-1.ssh-binding`), + ).toBe(false); + }); + + it("preserves a replaced canonical discovery binding before container cleanup", async () => { + const { discoveryBindingPath, discoveryStatePath, runtime } = persistWithDiscoveryBinding(); + clearManagedVllmSshBinding(discoveryStatePath); + copyManagedVllmSshBinding(discoveryStatePath, sshFixture.binding); + const { deps, removeContainer } = cleanupDeps(runtime.plan); + const createLifecycleDeps = vi.fn(() => deps); + + await expect( + cleanupInstalledManagedClusterVllmRuntime({ + stateDir, + loadApiKey: () => API_KEY, + createLifecycleDeps, + }), + ).rejects.toThrow("does not match the runtime receipt identity"); + + expect(createLifecycleDeps).not.toHaveBeenCalled(); + expect(removeContainer).not.toHaveBeenCalled(); + expect(fs.existsSync(discoveryBindingPath)).toBe(true); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(true); + expect( + fs.existsSync(`${managedClusterVllmRuntimeReceiptPath(stateDir)}.rank-1.ssh-binding`), + ).toBe(true); + }); + + it("retains durable ownership when a leftover discovery binding is unsafe to retire", async () => { + const { discoveryBindingPath, runtime } = persistWithDiscoveryBinding(); + const snapshots: Record<"head" | "worker", ManagedClusterNodeSnapshot> = { + head: snapshot(runtime.plan.roles[0], HEAD_ID), + worker: snapshot(runtime.plan.roles[1], WORKER_ID), + }; + const discoveryBindingModeByRole: Record = { + head: 0o700, + worker: 0o755, + }; + const removeContainer = vi.fn(async (rolePlan: ManagedClusterVllmRolePlan, id: string) => { + snapshots[rolePlan.role] = { + ...snapshots[rolePlan.role], + containers: snapshots[rolePlan.role].containers.filter((container) => container.id !== id), + }; + fs.chmodSync(discoveryBindingPath, discoveryBindingModeByRole[rolePlan.role]); + return { ok: true as const }; + }); + const options = { + stateDir, + loadApiKey: () => API_KEY, + createLifecycleDeps: () => ({ + inspectNode: async (rolePlan: ManagedClusterVllmRolePlan) => snapshots[rolePlan.role], + removeContainer, + withLifecycleLock: async (_plan: ManagedClusterVllmPlan, operation: () => Promise) => + await operation(), + }), + }; + + await expect(cleanupInstalledManagedClusterVllmRuntime(options)).rejects.toThrow( + "must be an owner-only directory", + ); + + expect(removeContainer).toHaveBeenCalledTimes(2); + expect(fs.existsSync(discoveryBindingPath)).toBe(true); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(true); + expect( + fs.existsSync(`${managedClusterVllmRuntimeReceiptPath(stateDir)}.rank-1.ssh-binding`), + ).toBe(true); + + fs.chmodSync(discoveryBindingPath, 0o700); + await expect(cleanupInstalledManagedClusterVllmRuntime(options)).resolves.toEqual({ + kind: "removed", + removedContainerIds: [], + }); + expect(removeContainer).toHaveBeenCalledTimes(2); + expect(fs.existsSync(discoveryBindingPath)).toBe(false); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(false); + expect( + fs.existsSync(`${managedClusterVllmRuntimeReceiptPath(stateDir)}.rank-1.ssh-binding`), + ).toBe(false); + }); + + it("retains the receipt and resumes after one exact container removal fails", async () => { + const { discoveryStatePath, runtime } = persistWithDiscoveryBinding(); + const snapshots: Record<"head" | "worker", ManagedClusterNodeSnapshot> = { + head: snapshot(runtime.plan.roles[0], HEAD_ID), + worker: snapshot(runtime.plan.roles[1], WORKER_ID), + }; + let workerAttempts = 0; + const removeContainer = vi.fn(async (rolePlan: ManagedClusterVllmRolePlan, id: string) => { + const shouldFailWorker = rolePlan.role === "worker" && workerAttempts === 0; + workerAttempts += Number(rolePlan.role === "worker"); + const removeOwnedContainer = () => { + snapshots[rolePlan.role] = { + ...snapshots[rolePlan.role], + containers: snapshots[rolePlan.role].containers.filter( + (container) => container.id !== id, + ), + }; + return { ok: true as const }; + }; + return shouldFailWorker + ? ({ + ok: false as const, + reason: "worker daemon unavailable", + } as const) + : removeOwnedContainer(); + }); + const deps = { + inspectNode: async (rolePlan: ManagedClusterVllmRolePlan) => snapshots[rolePlan.role], + removeContainer, + withLifecycleLock: async (_plan: ManagedClusterVllmPlan, operation: () => Promise) => + await operation(), + }; + const options = { + stateDir, + loadApiKey: () => API_KEY, + createLifecycleDeps: () => deps, + }; + + await expect(cleanupInstalledManagedClusterVllmRuntime(options)).rejects.toThrow( + "worker daemon unavailable", + ); + expect(fs.existsSync(`${discoveryStatePath}.ssh-binding`)).toBe(true); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(true); + await expect(cleanupInstalledManagedClusterVllmRuntime(options)).resolves.toEqual({ + kind: "removed", + removedContainerIds: [WORKER_ID], + }); + expect(removeContainer.mock.calls.map((call) => call[1])).toEqual([ + HEAD_ID, + WORKER_ID, + WORKER_ID, + ]); + expect(fs.existsSync(`${discoveryStatePath}.ssh-binding`)).toBe(false); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(false); + }); + + it("preserves recovery state when the observed container IDs changed", async () => { + const runtime = persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + const { deps, removeContainer } = cleanupDeps(runtime.plan, { + head: "e".repeat(64), + worker: WORKER_ID, + }); + await expect( + cleanupInstalledManagedClusterVllmRuntime({ + stateDir, + loadApiKey: () => API_KEY, + createLifecycleDeps: () => deps, + }), + ).rejects.toThrow("head receipt-owned container is absent but related runtime state exists"); + expect(removeContainer).not.toHaveBeenCalled(); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(true); + expect( + fs.existsSync(`${managedClusterVllmRuntimeReceiptPath(stateDir)}.rank-1.ssh-binding`), + ).toBe(true); + }); + + it("preserves recovery state when the managed API key changed", async () => { + persistManagedClusterVllmRuntimeReceipt(input(), { stateDir }); + const createLifecycleDeps = vi.fn(); + await expect( + cleanupInstalledManagedClusterVllmRuntime({ + stateDir, + loadApiKey: () => "f".repeat(64), + createLifecycleDeps, + }), + ).rejects.toThrow("API key no longer matches"); + expect(createLifecycleDeps).not.toHaveBeenCalled(); + expect(fs.existsSync(managedClusterVllmRuntimeReceiptPath(stateDir))).toBe(true); + }); + + it("does nothing when no runtime receipt exists", async () => { + await expect( + cleanupInstalledManagedClusterVllmRuntime({ + stateDir, + loadApiKey: vi.fn(), + createLifecycleDeps: vi.fn(), + }), + ).resolves.toEqual({ kind: "not-installed" }); + }); +}); diff --git a/src/lib/inference/vllm-managed-recovery.test.ts b/src/lib/inference/vllm-managed-recovery.test.ts new file mode 100644 index 0000000000..691a52b05a --- /dev/null +++ b/src/lib/inference/vllm-managed-recovery.test.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const managedClusterRecovery = vi.hoisted(() => ({ endpoint: vi.fn() })); + +vi.mock("./serving/managed-cluster-runtime-receipt", async (importOriginal) => ({ + ...(await importOriginal()), + recoverInstalledManagedClusterVllmEndpoint: managedClusterRecovery.endpoint, +})); + +import { isNemoClawManagedVllmRunning, persistConfiguredManagedVllmRuntimeReceipt } from "./vllm"; + +describe("managed vLLM cluster recovery", () => { + beforeEach(() => { + managedClusterRecovery.endpoint.mockReset(); + }); + + it("recognizes and confirms an installer-owned managed cluster receipt", async () => { + managedClusterRecovery.endpoint.mockReturnValue({ + baseUrl: "http://10.40.0.1:8000", + apiKey: "a".repeat(64), + }); + + expect(isNemoClawManagedVllmRunning()).toBe(true); + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ + ok: true, + persisted: true, + }); + }); + + it("fails closed instead of falling through when managed cluster recovery is unsafe", async () => { + managedClusterRecovery.endpoint.mockImplementation(() => { + throw new Error("receipt-owned container IDs changed"); + }); + + expect(isNemoClawManagedVllmRunning()).toBe(false); + await expect(persistConfiguredManagedVllmRuntimeReceipt()).resolves.toEqual({ + ok: false, + reason: "managed vLLM recovery failed: receipt-owned container IDs changed", + }); + }); +}); diff --git a/src/lib/inference/vllm-managed-runtime-receipts.test.ts b/src/lib/inference/vllm-managed-runtime-receipts.test.ts new file mode 100644 index 0000000000..478d5e75db --- /dev/null +++ b/src/lib/inference/vllm-managed-runtime-receipts.test.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { DEFAULT_GATEWAY_PORT } from "../core/ports"; +import { nemoclawStateRoot } from "../state/state-root"; +import { createProductionManagedClusterDiscoveryDeps } from "./serving/managed-cluster-discovery-production"; +import { managedClusterVllmRuntimeReceiptPath } from "./serving/managed-cluster-runtime-receipt"; +import { + assertNoManagedDistributedVllmRuntimeReceipts, + findManagedDistributedVllmRuntimeReceipts, +} from "./serving/managed-runtime-receipts"; +import { dualStationVllmRuntimeReceiptPath } from "./vllm-station-runtime-receipt"; + +const temporaryHomes: string[] = []; + +function temporaryHome(): string { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-receipts-")); + temporaryHomes.push(homeDir); + return homeDir; +} + +function touch(filePath: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, "stale\n", { mode: 0o600 }); +} + +afterEach(() => { + vi.unstubAllEnvs(); + for (const homeDir of temporaryHomes.splice(0)) fs.rmSync(homeDir, { recursive: true }); +}); + +describe("managed distributed vLLM receipt preflight", () => { + it("allows installation when no durable distributed receipt exists", () => { + expect(() => + assertNoManagedDistributedVllmRuntimeReceipts({ homeDir: temporaryHome() }), + ).not.toThrow(); + }); + + it("blocks a host-global managed cluster receipt without parsing stale contents", () => { + const homeDir = temporaryHome(); + const receiptPath = managedClusterVllmRuntimeReceiptPath( + nemoclawStateRoot(homeDir, DEFAULT_GATEWAY_PORT), + ); + touch(receiptPath); + + expect(findManagedDistributedVllmRuntimeReceipts({ homeDir })).toEqual({ + managedClusterBindingPaths: [], + managedClusterDiscoveryBindingPaths: [], + managedClusterPath: receiptPath, + stationBindingPaths: [], + stationPaths: [], + }); + expect(() => assertNoManagedDistributedVllmRuntimeReceipts({ homeDir })).toThrow( + "recover it through Local vLLM", + ); + }); + + it("blocks Station receipts across safely enumerated gateway roots", () => { + const homeDir = temporaryHome(); + const receiptPath = dualStationVllmRuntimeReceiptPath(nemoclawStateRoot(homeDir, 18080)); + touch(receiptPath); + + expect(findManagedDistributedVllmRuntimeReceipts({ homeDir }).stationPaths).toEqual([ + receiptPath, + ]); + }); + + it.each([ + { topology: "Spark", gatewayPort: DEFAULT_GATEWAY_PORT }, + { topology: "Station", gatewayPort: 18080 }, + ])("blocks an orphaned $topology SSH binding tree", ({ topology, gatewayPort }) => { + const homeDir = temporaryHome(); + const stateRoot = nemoclawStateRoot(homeDir, gatewayPort); + const receiptPath = + topology === "Spark" + ? managedClusterVllmRuntimeReceiptPath(stateRoot) + : dualStationVllmRuntimeReceiptPath(stateRoot); + const bindingPath = + topology === "Spark" ? `${receiptPath}.rank-1.ssh-binding` : `${receiptPath}.ssh-binding`; + fs.mkdirSync(bindingPath, { recursive: true, mode: 0o700 }); + + expect(() => assertNoManagedDistributedVllmRuntimeReceipts({ homeDir })).toThrow(bindingPath); + }); + + it("blocks an orphaned managed cluster discovery binding claim", () => { + const homeDir = temporaryHome(); + const bindingPath = path.join( + nemoclawStateRoot(homeDir), + "managed-cluster-managed-serving.json.spark-worker.ssh-binding", + ); + fs.mkdirSync(bindingPath, { recursive: true, mode: 0o700 }); + + expect( + findManagedDistributedVllmRuntimeReceipts({ homeDir }).managedClusterDiscoveryBindingPaths, + ).toEqual([bindingPath]); + expect(() => assertNoManagedDistributedVllmRuntimeReceipts({ homeDir })).toThrow(bindingPath); + }); + + it("enumerates every ranked runtime binding and per-node discovery claim", () => { + const homeDir = temporaryHome(); + const stateRoot = nemoclawStateRoot(homeDir); + const runtimeBindings = [ + "managed-cluster-vllm-runtime.json.rank-1.ssh-binding", + "managed-cluster-vllm-runtime.json.rank-2.ssh-binding", + ].map((entry) => path.join(stateRoot, entry)); + const discoveryBindings = [ + "managed-cluster-managed-serving.json.node-a.ssh-binding", + "managed-cluster-managed-serving.json.node-b.ssh-binding", + ].map((entry) => path.join(stateRoot, entry)); + for (const bindingPath of [...runtimeBindings, ...discoveryBindings]) { + fs.mkdirSync(bindingPath, { recursive: true, mode: 0o700 }); + } + + const receipts = findManagedDistributedVllmRuntimeReceipts({ homeDir }); + expect(receipts.managedClusterBindingPaths).toEqual(runtimeBindings); + expect(receipts.managedClusterDiscoveryBindingPaths).toEqual(discoveryBindings); + }); + + it("places each production cluster discovery claim in the scanner-visible gateway root", () => { + const homeDir = temporaryHome(); + vi.stubEnv("HOME", homeDir); + const deps = createProductionManagedClusterDiscoveryDeps(() => { + throw new Error("unexpected host probe"); + }); + + expect(deps.resolveBindingStatePath("spark-worker")).toBe( + path.join(nemoclawStateRoot(homeDir), "managed-cluster-managed-serving.json.spark-worker"), + ); + }); + + it("treats a receipt symlink as existing without following it", () => { + const homeDir = temporaryHome(); + const receiptPath = managedClusterVllmRuntimeReceiptPath(nemoclawStateRoot(homeDir)); + fs.mkdirSync(path.dirname(receiptPath), { recursive: true }); + fs.symlinkSync(path.join(homeDir, "missing-target"), receiptPath); + + expect(findManagedDistributedVllmRuntimeReceipts({ homeDir }).managedClusterPath).toBe( + receiptPath, + ); + expect(() => assertNoManagedDistributedVllmRuntimeReceipts({ homeDir })).toThrow( + "Managed vLLM runtime state already exists", + ); + }); +}); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 97d7ea907a..32433e579d 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -469,24 +469,24 @@ describe("dual-Station managed vLLM lifecycle", () => { expect(fake.operations).toEqual([]); }); - it("anchors the default lock under the effective account home instead of mutable HOME", async () => { + it("anchors the default lock under the host-global managed state home", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-lock-home-")); const accountHome = path.join(root, "account-home"); - const ambientHome = path.join(root, "ambient-home"); + const managedHome = path.join(root, "managed-home"); fs.mkdirSync(accountHome, { mode: 0o700 }); const userInfo = os.userInfo(); const userInfoSpy = vi.spyOn(os, "userInfo").mockReturnValue({ ...userInfo, homedir: accountHome, }); - vi.stubEnv("HOME", ambientHome); + vi.stubEnv("HOME", managedHome); try { await withDualStationVllmLifecycleLock( () => { expect( - fs.existsSync(path.join(accountHome, ".nemoclaw", "state", "mcp-lifecycle-locks")), + fs.existsSync(path.join(managedHome, ".nemoclaw", "state", "mcp-lifecycle-locks")), ).toBe(true); - expect(fs.existsSync(ambientHome)).toBe(false); + expect(fs.existsSync(path.join(accountHome, ".nemoclaw"))).toBe(false); }, { pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, { diff --git a/src/lib/inference/vllm-station-lifecycle-lock.test.ts b/src/lib/inference/vllm-station-lifecycle-lock.test.ts index f654e12cef..0b2120a79e 100644 --- a/src/lib/inference/vllm-station-lifecycle-lock.test.ts +++ b/src/lib/inference/vllm-station-lifecycle-lock.test.ts @@ -10,7 +10,9 @@ import { type DualStationControllerUidFileStat, readDualStationControllerUid, withDualStationVllmLifecycleLock, + withHostGlobalVllmLifecycleLock, } from "./vllm-station-lifecycle-lock"; +import { managedVllmStateDir } from "./vllm-api-key"; function controllerUidStat( kind: "directory" | "file", @@ -113,4 +115,35 @@ describe("dual-Station controller UID binding", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + it("shares the managed-state home even when passwd and HOME directories differ", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-vllm-lock-")); + const passwdHome = path.join(root, "passwd-home"); + const managedHome = path.join(root, "managed-home"); + fs.mkdirSync(passwdHome, { mode: 0o700 }); + fs.mkdirSync(managedHome, { mode: 0o700 }); + const userInfo = os.userInfo(); + const userInfoSpy = vi.spyOn(os, "userInfo").mockReturnValue({ + ...userInfo, + homedir: passwdHome, + }); + vi.stubEnv("HOME", managedHome); + + try { + expect(managedVllmStateDir()).toBe(path.join(managedHome, ".nemoclaw")); + await withHostGlobalVllmLifecycleLock( + () => { + expect( + fs.existsSync(path.join(managedHome, ".nemoclaw", "state", "mcp-lifecycle-locks")), + ).toBe(true); + expect(fs.existsSync(path.join(passwdHome, ".nemoclaw"))).toBe(false); + }, + { pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, + ); + } finally { + vi.unstubAllEnvs(); + userInfoSpy.mockRestore(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/src/lib/inference/vllm-station-lifecycle-lock.ts b/src/lib/inference/vllm-station-lifecycle-lock.ts index 1db4076f21..ab25ec9120 100644 --- a/src/lib/inference/vllm-station-lifecycle-lock.ts +++ b/src/lib/inference/vllm-station-lifecycle-lock.ts @@ -2,13 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { type McpLifecycleLockOptions, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; -import { STATE_DIR_NAME } from "../state/state-root"; +import { managedVllmStateDir } from "./vllm-api-key"; -const DUAL_STATION_VLLM_LIFECYCLE_LOCK = "dual-station-vllm:host-global"; +const HOST_GLOBAL_VLLM_LIFECYCLE_LOCK = "dual-station-vllm:host-global"; const DUAL_STATION_CONTROLLER_CONFIG_DIR = "/etc/nemoclaw"; export const DUAL_STATION_CONTROLLER_UID_FILE = path.join( DUAL_STATION_CONTROLLER_CONFIG_DIR, @@ -125,13 +124,25 @@ export function assertDualStationControllerAccount( return controllerUid; } +/** Serialize every host-managed vLLM profile under the effective account home. */ +export function withHostGlobalVllmLifecycleLock( + operation: () => Promise | T, + options: McpLifecycleLockOptions = {}, +): Promise { + const stateDir = options.stateDir ?? path.join(managedVllmStateDir(), "state"); + return withMcpLifecycleLock(HOST_GLOBAL_VLLM_LIFECYCLE_LOCK, operation, { + ...options, + stateDir, + }); +} + /** * Serialize the host-managed dual-Station service across gateway instances. * * Dual-Station lifecycle supports one effective controller account per host. - * This anchors every supported caller at that account's passwd home instead of - * mutable HOME or a gateway-specific root. Host preparation binds that account - * in root-owned state before the lease can be acquired. + * This anchors every supported caller at the same host-global state root used + * by managed vLLM receipts and credentials. Host preparation binds the + * controller account in root-owned state before the lease can be acquired. */ export function withDualStationVllmLifecycleLock( operation: () => Promise | T, @@ -146,9 +157,5 @@ export function withDualStationVllmLifecycleLock( identityDeps.readControllerUid, identityDeps.effectiveControllerUid, ); - const stateDir = options.stateDir ?? path.join(os.userInfo().homedir, STATE_DIR_NAME, "state"); - return withMcpLifecycleLock(DUAL_STATION_VLLM_LIFECYCLE_LOCK, operation, { - ...options, - stateDir, - }); + return withHostGlobalVllmLifecycleLock(operation, options); } diff --git a/src/lib/inference/vllm-station-runtime-cleanup-entry.ts b/src/lib/inference/vllm-station-runtime-cleanup-entry.ts index c0f0821b41..eab3535829 100644 --- a/src/lib/inference/vllm-station-runtime-cleanup-entry.ts +++ b/src/lib/inference/vllm-station-runtime-cleanup-entry.ts @@ -1,19 +1,95 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { cleanupInstalledDualStationVllmRuntime } from "./vllm-station-runtime-receipt"; +import fs from "node:fs"; +import path from "node:path"; + +import { + cleanupInstalledManagedClusterVllmRuntime, + managedClusterVllmRuntimeReceiptPath, +} from "./serving/managed-cluster-runtime-receipt"; +import { + cleanupInstalledDualStationVllmRuntime, + dualStationVllmRuntimeReceiptPath, +} from "./vllm-station-runtime-receipt"; + +function pathExistsNoFollow(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +function requestedReceipt(): { + readonly filePath: string; + readonly stateDir: string; + readonly topology: "managed-cluster" | "station"; +} { + const args = process.argv.slice(2); + if (args.length !== 1) { + throw new Error("managed distributed vLLM cleanup requires one exact receipt path"); + } + const filePath = args[0]!; + if (!path.isAbsolute(filePath) || path.normalize(filePath) !== filePath) { + throw new Error("managed distributed vLLM cleanup receipt path is invalid"); + } + const stateDir = path.dirname(filePath); + if (filePath === managedClusterVllmRuntimeReceiptPath(stateDir)) { + return { filePath, stateDir, topology: "managed-cluster" }; + } + if (filePath === dualStationVllmRuntimeReceiptPath(stateDir)) { + return { filePath, stateDir, topology: "station" }; + } + throw new Error("managed distributed vLLM cleanup receipt name is unsupported"); +} async function main(): Promise { - const result = await cleanupInstalledDualStationVllmRuntime(); - if (result.kind === "not-installed") return; + const receipt = requestedReceipt(); + if (!pathExistsNoFollow(receipt.filePath)) { + throw new Error(`managed distributed vLLM receipt disappeared: ${receipt.filePath}`); + } + const otherReceipt = + receipt.topology === "managed-cluster" + ? dualStationVllmRuntimeReceiptPath(receipt.stateDir) + : managedClusterVllmRuntimeReceiptPath(receipt.stateDir); + if (pathExistsNoFollow(otherReceipt)) { + throw new Error( + "both managed cluster and dual-Station runtime receipts exist; refusing ambiguous cleanup", + ); + } + + if (receipt.topology === "managed-cluster") { + const managedCluster = await cleanupInstalledManagedClusterVllmRuntime({ + stateDir: receipt.stateDir, + }); + if (managedCluster.kind !== "removed" || pathExistsNoFollow(receipt.filePath)) { + throw new Error("the requested managed cluster runtime receipt was not removed"); + } + console.log( + managedCluster.removedContainerIds.length > 0 + ? `Removed managed cluster vLLM containers: ${managedCluster.removedContainerIds.join(", ")}` + : "Removed managed cluster vLLM state; no receipt-owned containers remained.", + ); + return; + } + + const station = await cleanupInstalledDualStationVllmRuntime({ stateDir: receipt.stateDir }); + if (station.kind !== "removed" || pathExistsNoFollow(receipt.filePath)) { + throw new Error("the requested dual-Station runtime receipt was not removed"); + } console.log( - `Removed managed dual-Station vLLM containers: ${result.removedContainerIds.join(", ")}`, + station.removedContainerIds.length > 0 + ? `Removed managed dual-Station vLLM containers: ${station.removedContainerIds.join(", ")}` + : "Removed managed dual-Station vLLM state; no receipt-owned containers remained.", ); } main().catch((error: unknown) => { console.error( - `Refusing uninstall before managed dual-Station cleanup: ${ + `Refusing uninstall before managed distributed vLLM cleanup: ${ error instanceof Error ? error.message : String(error) }`, ); diff --git a/src/lib/inference/vllm-station-ssh-binding.ts b/src/lib/inference/vllm-station-ssh-binding.ts index bfce00f32c..8b98c8abb6 100644 --- a/src/lib/inference/vllm-station-ssh-binding.ts +++ b/src/lib/inference/vllm-station-ssh-binding.ts @@ -728,6 +728,34 @@ export function loadDualStationSshBinding( return validateDualStationSshBindingFiles(binding); } +/** Load the sole validated binding owned by one resume-state path, if present. */ +export function loadDualStationSshBindingForStatePath( + statePath: string, + expectedPeerTarget: string, + expectedHostKeyDigest: string, +): DualStationSshBinding | null { + const runtimeDirectory = dualStationSshBindingDirectory(statePath); + try { + assertDirectory(runtimeDirectory, 0o700, "Station SSH binding root"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + const entries = fs.readdirSync(runtimeDirectory, { withFileTypes: true }); + if ( + entries.length !== 1 || + !entries[0]?.isDirectory() || + !VERSION_DIRECTORY_PATTERN.test(entries[0].name) + ) { + throw new Error("Station SSH binding root does not contain one exact binding"); + } + return loadDualStationSshBinding( + path.join(runtimeDirectory, entries[0].name, "binding.json"), + expectedPeerTarget, + expectedHostKeyDigest, + ); +} + export function encodeDualStationSshBindingHandoff(binding: DualStationSshBinding): string { const canonical = validateDualStationSshBindingFiles(binding); const handoff: BindingHandoff = { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 75effadb70..66d8c72818 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -32,12 +32,16 @@ import { runCapture } from "../runner"; import { redactFull } from "../security/redact"; import { isSafeModelId } from "../validation"; import { getGpuIndicesByName } from "./nim"; -import { ensureDualStationVllmApiKey, loadDualStationVllmApiKey } from "./vllm-api-key"; import { buildLocalDualStationDockerEnv, buildRemoteVllmDockerEnv, buildVllmDockerEnv, -} from "./vllm-docker-env"; + ensureDualStationVllmApiKey, + loadDualStationVllmApiKey, + recoverInstalledManagedClusterVllmEndpoint, + resolveVllmInstallModel, + tryInstallManagedClusterManagedVllm, +} from "./serving/vllm-managed-support"; import { buildVllmServeCommand, NEMOTRON_ULTRA_DUAL_STATION_IMAGE, @@ -48,7 +52,6 @@ import { type VllmModelDef, type VllmPlatform, } from "./vllm-models"; -import { resolveVllmInstallModel } from "./vllm-prompt"; import { type DualStationVllmPlan, NEMOCLAW_DGX_STATION_PEER_ENV, @@ -199,8 +202,16 @@ function hfCacheMount(): string { return `${hostHfCacheDir()}:${HF_CACHE_CONTAINER_DIR}`; } -function hfDownloadCacheMount(): string { - return `${hostHfCacheDir()}:${HF_DOWNLOAD_CACHE_CONTAINER_DIR}`; +function hfDownloadCacheMount(hostCacheDir = hostHfCacheDir()): string { + const normalized = path.posix.normalize(hostCacheDir); + if ( + !path.posix.isAbsolute(hostCacheDir) || + normalized !== hostCacheDir || + hostCacheDir.includes(":") + ) { + throw new Error("vLLM model cache must be a normalized absolute path without ':'"); + } + return `${normalized}:${HF_DOWNLOAD_CACHE_CONTAINER_DIR}`; } function hfModelCacheKey(model: VllmModelDef): string | null { @@ -209,7 +220,7 @@ function hfModelCacheKey(model: VllmModelDef): string | null { return `models--${modelParts.join("--")}`; } -function hfModelSnapshotDir(model: VllmModelDef): string | null { +export function hfModelSnapshotDir(model: VllmModelDef): string | null { const revision = model.revision; const modelCacheKey = hfModelCacheKey(model); if (!revision || !modelCacheKey || !HF_CACHE_COMPONENT_PATTERN.test(revision)) { @@ -231,8 +242,10 @@ function hostUserIdentity(): string | null { return `${String(process.getuid())}:${String(process.getgid())}`; } -function hostUserDockerArgs(): string[] { - const identity = hostUserIdentity(); +function hostUserDockerArgs(identity = hostUserIdentity()): string[] { + if (identity !== null && !/^[1-9][0-9]*:(?:0|[1-9][0-9]*)$/.test(identity)) { + throw new Error("vLLM model download user must be one non-root numeric uid and numeric gid"); + } return identity ? ["--user", identity] : []; } @@ -514,10 +527,11 @@ export async function pullImage( } // Run `hf download ` inside a one-shot container of the same image. -function downloadModel( +export function downloadModel( profile: VllmProfile, model: VllmModelDef, dockerEnv: Record = buildVllmDockerEnv(), + target: { hostCacheDir?: string; userIdentity?: string } = {}, ): Promise<{ ok: boolean; reason?: string }> { emit(`Pre-downloading model with hf: ${model.id}`); return new Promise((resolve) => { @@ -528,11 +542,11 @@ function downloadModel( "-t", "--rm", "--pull=never", - ...hostUserDockerArgs(), + ...hostUserDockerArgs(target.userIdentity), "--entrypoint", "hf", "-v", - hfDownloadCacheMount(), + hfDownloadCacheMount(target.hostCacheDir), "-e", `HF_HOME=${HF_DOWNLOAD_CACHE_CONTAINER_DIR}`, ...buildHfTokenDockerArgs(), @@ -883,19 +897,30 @@ function vllmContainerReplacementTarget( } export function isNemoClawManagedVllmRunning(): boolean { + try { + if (recoverInstalledManagedClusterVllmEndpoint()) return true; + } catch { + return false; + } const ownership = inspectVllmContainerOwnership(NEMOCLAW_VLLM_CONTAINER_NAME); return (ownership.kind === "managed" || ownership.kind === "dual-managed") && ownership.running; } -export type PersistConfiguredDualStationVllmRuntimeResult = +export type PersistConfiguredManagedVllmRuntimeResult = | { ok: true; persisted: boolean } | { ok: false; reason: string }; /** - * Adopt an already-running installer-qualified pair into durable uninstall - * ownership after onboarding has authenticated and validated its endpoint. + * Confirm an installer-owned receipt or adopt an already-running Station pair + * after onboarding has authenticated and validated its endpoint. */ -export async function persistConfiguredDualStationVllmRuntimeReceipt(): Promise { +export async function persistConfiguredManagedVllmRuntimeReceipt(): Promise { + try { + if (recoverInstalledManagedClusterVllmEndpoint()) return { ok: true, persisted: true }; + } catch (error) { + return { ok: false, reason: `managed vLLM recovery failed: ${(error as Error).message}` }; + } + const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim(); let configuredPlan: DualStationVllmPlan | null = null; if (configuredPeer) { @@ -1519,7 +1544,7 @@ interface InstallVllmOptions { beforeInstall?: (modelId: string) => void; } -function imageIsCached( +export function imageIsCached( profile: VllmProfile, dockerEnv: Record = buildVllmDockerEnv(), ): boolean { @@ -1574,6 +1599,22 @@ async function runVllmInstall( profile: VllmProfile, opts: InstallVllmOptions, ): Promise<{ ok: boolean }> { + const managedCluster = await tryInstallManagedClusterManagedVllm( + { + platform: profile.platform, + nonInteractive: opts.nonInteractive, + promptFn: opts.promptFn, + beforeInstall: opts.beforeInstall, + }, + { + prerequisites: dockerPrereqsOk, + pullImage, + downloadModel, + printDownloadAuthentication: printHfDownloadAuthentication, + }, + ); + if (managedCluster.kind === "handled") return managedCluster.result; + let dualStationPlan: DualStationVllmPlan | null = null; let peerModelSnapshot: "ready" | "staging-required" | null = null; const explicitModel = String(process.env.NEMOCLAW_VLLM_MODEL ?? "").trim(); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 13c2dd0e7f..24f8e5a79e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1086,8 +1086,8 @@ const { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const handleVllmSelection = createSetupNimVllmHandler({ VLLM_PORT, runCapture, getLocalProviderBaseUrl, getLocalProviderValidationBaseUrl, - getManagedVllmProviderBinding: localInference.getManagedDualStationVllmProviderBinding, queryVllmModels: (baseUrl, apiKey) => { const result = localInference.probeVllmModels(baseUrl, apiKey); return result.ok ? result.body : ""; }, isSafeModelId, requireValue, validateOpenAiLikeSelection, - applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning: vllmInference.isNemoClawManagedVllmRunning, persistConfiguredDualStationVllmRuntimeReceipt: vllmInference.persistConfiguredDualStationVllmRuntimeReceipt, + getManagedVllmProviderBinding: localInference.getManagedVllmProviderBinding, queryVllmModels: (baseUrl, apiKey) => { const result = localInference.probeVllmModels(baseUrl, apiKey); return result.ok ? result.body : ""; }, isSafeModelId, requireValue, validateOpenAiLikeSelection, + applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning: vllmInference.isNemoClawManagedVllmRunning, persistConfiguredManagedVllmRuntimeReceipt: vllmInference.persistConfiguredManagedVllmRuntimeReceipt, exitProcess: (code) => process.exit(code), }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 82c995869a..ffbfcac411 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -36,6 +36,7 @@ function buildDeps( })), getWindowsHostOllamaDockerRequirement: vi.fn(() => SUPPORTED_WINDOWS_OLLAMA), detectVllmProfile: vi.fn(() => null), + getLocalProviderAvailabilityEndpoint: vi.fn(() => "http://127.0.0.1:8000/v1/models"), ...overrides, }; } @@ -160,6 +161,24 @@ describe("detectInferenceProviderHostState", () => { expect(state.vllmRunning).toBe(false); }); + it("fails the vLLM running probe closed when managed endpoint resolution fails", () => { + const runCapture = vi.fn(() => "200"); + const state = detectWithDeps( + buildDeps({ + runCapture, + getLocalProviderAvailabilityEndpoint: () => { + throw new Error("managed state unavailable"); + }, + }), + ); + + expect(state.vllmRunning).toBe(false); + expect(runCapture).not.toHaveBeenCalledWith( + expect.arrayContaining([expect.stringContaining("8000")]), + expect.anything(), + ); + }); + it("detects a reachable Windows-host Ollama beside WSL-local Ollama and warns outside mirrored networking", () => { const logs: string[] = []; const deps = buildDeps({ diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index 4a41d5a8f8..5134774bb4 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -85,6 +85,7 @@ export interface DetectInferenceProviderHostStateDeps { runtime: ContainerRuntime | null, ) => WindowsHostOllamaDockerRequirement; detectVllmProfile: (gpu: InferenceProviderHostGpu | null | undefined) => VllmProfile | null; + getLocalProviderAvailabilityEndpoint: (provider: string) => string | null; } const LOCAL_PROVIDER_PROBE_CURL_ARGS = ["--connect-timeout", "2", "--max-time", "5"] as const; @@ -113,16 +114,23 @@ function buildDeps( detectVllmProfile: overrides.detectVllmProfile ?? ((gpu) => detectVllmProfile(gpu as Parameters[0])), + getLocalProviderAvailabilityEndpoint: + overrides.getLocalProviderAvailabilityEndpoint ?? getLocalProviderAvailabilityEndpoint, }; } -function probeVllmRunning(runCapture: RunCapture): boolean { - const endpoint = getLocalProviderAvailabilityEndpoint("vllm-local"); +function probeVllmRunning(deps: DetectInferenceProviderHostStateDeps): boolean { + let endpoint: string | null; + try { + endpoint = deps.getLocalProviderAvailabilityEndpoint("vllm-local"); + } catch { + return false; + } if (!endpoint) return false; const writeOut = endpoint.endsWith("/health") ? ["--noproxy", "*", "--write-out", "%{http_code}"] : []; - const output = runCapture( + const output = deps.runCapture( ["curl", "-sf", ...LOCAL_PROVIDER_PROBE_CURL_ARGS, ...writeOut, endpoint], { ignoreError: true, @@ -179,7 +187,7 @@ export function detectInferenceProviderHostState( const ollamaHost = input.probeOllama === false ? null : deps.findReachableOllamaHost(); const ollamaRunning = ollamaHost !== null; const isWindowsHostOllama = ollamaHost === OLLAMA_HOST_DOCKER_INTERNAL; - const vllmRunning = input.probeVllm === false ? false : probeVllmRunning(deps.runCapture); + const vllmRunning = input.probeVllm === false ? false : probeVllmRunning(deps); const vllmProfile = deps.detectVllmProfile(input.gpu); const hasVllmImage = !!( vllmProfile && diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index ca0fffb653..28abd05289 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { isBedrockRuntimeEndpoint } from "../inference/bedrock-runtime"; import { canonicalEndpoint } from "../core/url-utils"; +import { isBedrockRuntimeEndpoint } from "../inference/bedrock-runtime"; import { assertEndpointResolvesPublic, type EndpointDnsLookupFn, @@ -15,7 +15,7 @@ import { isAdvisoryGatewayRouteConflict, } from "../inference/gateway-route-compatibility"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; -import { getManagedDualStationVllmProviderBinding } from "../inference/local"; +import { getManagedVllmProviderBinding } from "../inference/local"; import { assertNoExplicitOpenShellGatewayEndpoint, assertNoOpenShellGatewayEndpointOverride, @@ -37,6 +37,7 @@ function matchesOnboardEndpoint( const selected = canonicalEndpoint(endpointUrl, flavor); return selected !== null && selected === canonicalEndpoint(onboardEndpointUrl, flavor); } + import type { CommonDeps, HermesDeps, @@ -442,7 +443,7 @@ export function createSetupInference( run: deps.run, VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, getManagedVllmProviderBinding: - deps.getManagedVllmProviderBinding ?? getManagedDualStationVllmProviderBinding, + deps.getManagedVllmProviderBinding ?? getManagedVllmProviderBinding, }, ); if (outcome.done) return outcome.result; diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 822e0a2cf0..5041cfe56b 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -40,7 +40,7 @@ function deps(overrides: Partial = {}): SetupNimVllmDeps { applyVllmRuntimeContextWindow: vi.fn(), isDgxSparkHost: () => false, isNemoClawManagedVllmRunning: () => false, - persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + persistConfiguredManagedVllmRuntimeReceipt: async () => ({ ok: true, persisted: true, }), @@ -130,7 +130,7 @@ describe("setupNim vLLM route containment", () => { .flat() .join("\n"); expect(renderedOutput).not.toContain(apiKey); - expect(renderedOutput).toContain("Using managed dual-Station vLLM endpoint"); + expect(renderedOutput).toContain("Using managed vLLM endpoint"); expect(renderedOutput).not.toContain("localhost:8000"); }); @@ -154,7 +154,7 @@ describe("setupNim vLLM route containment", () => { await expect(handler(selection)).rejects.toThrow("exit 1"); expect(validateOpenAiLikeSelection).not.toHaveBeenCalled(); expect(console.error).toHaveBeenCalledWith( - " To install 'required/model', stop the managed dual-Station vLLM deployment, then rerun the original install/onboard command.", + " To install 'required/model', stop the managed vLLM deployment, then rerun the original install/onboard command.", ); expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain("localhost"); }); @@ -166,7 +166,7 @@ describe("setupNim vLLM route containment", () => { deps({ runCapture, getManagedVllmProviderBinding: () => { - throw new Error("Managed dual-Station vLLM authentication is missing."); + throw new Error("Managed vLLM authentication is missing."); }, queryVllmModels, }), @@ -181,7 +181,7 @@ describe("setupNim vLLM route containment", () => { }); it("persists cleanup ownership after validating a managed dual endpoint", async () => { - const persistConfiguredDualStationVllmRuntimeReceipt = vi.fn(async () => ({ + const persistConfiguredManagedVllmRuntimeReceipt = vi.fn(async () => ({ ok: true as const, persisted: true, })); @@ -192,12 +192,12 @@ describe("setupNim vLLM route containment", () => { apiKey: "a".repeat(64), }), queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), - persistConfiguredDualStationVllmRuntimeReceipt, + persistConfiguredManagedVllmRuntimeReceipt, }), ); await expect(handler(state(null))).resolves.toBe("selected"); - expect(persistConfiguredDualStationVllmRuntimeReceipt).toHaveBeenCalledOnce(); + expect(persistConfiguredManagedVllmRuntimeReceipt).toHaveBeenCalledOnce(); }); it("fails closed when managed dual cleanup ownership cannot be persisted", async () => { @@ -208,7 +208,7 @@ describe("setupNim vLLM route containment", () => { apiKey: "a".repeat(64), }), queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), - persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + persistConfiguredManagedVllmRuntimeReceipt: async () => ({ ok: false, reason: "pair identity changed", }), @@ -217,7 +217,7 @@ describe("setupNim vLLM route containment", () => { await expect(handler(state(null))).rejects.toThrow("exit 1"); expect(console.error).toHaveBeenCalledWith( - " Managed dual-Station cleanup ownership could not be persisted: pair identity changed", + " Managed vLLM cleanup ownership could not be persisted: pair identity changed", ); }); @@ -229,7 +229,7 @@ describe("setupNim vLLM route containment", () => { apiKey: "a".repeat(64), }), queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), - persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + persistConfiguredManagedVllmRuntimeReceipt: async () => ({ ok: true, persisted: false, }), @@ -238,7 +238,7 @@ describe("setupNim vLLM route containment", () => { await expect(handler(state(null))).rejects.toThrow("exit 1"); expect(console.error).toHaveBeenCalledWith( - " Managed dual-Station cleanup ownership could not be persisted: the managed dual-Station cleanup receipt was not written", + " Managed vLLM cleanup ownership could not be persisted: the managed cleanup receipt was not written", ); }); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index 58fbce3c5d..4459c25214 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -50,7 +50,7 @@ export interface SetupNimVllmDeps { applyVllmRuntimeContextWindow(models: VllmModels, model: string): void; isDgxSparkHost?: () => boolean; isNemoClawManagedVllmRunning?: () => boolean; - persistConfiguredDualStationVllmRuntimeReceipt(): Promise< + persistConfiguredManagedVllmRuntimeReceipt(): Promise< | { ok: true; persisted: boolean; @@ -78,7 +78,7 @@ async function managedVllmValidationOptions(baseUrl: string, apiKey: string) { trustedPrivateHosts: [hostname], }); if (!preflight.ok || !preflight.trustedPrivateCapability) { - throw new Error("Managed dual-Station vLLM endpoint authorization failed."); + throw new Error("Managed vLLM endpoint authorization failed."); } return { apiKey, @@ -257,10 +257,10 @@ export function createSetupNimVllmHandler( } const apiKey = managedBinding?.apiKey ?? null; - const managedDualEndpoint = managedBinding != null; + const managedEndpoint = managedBinding != null; console.log( - managedDualEndpoint - ? " ✓ Using managed dual-Station vLLM endpoint" + managedEndpoint + ? " ✓ Using managed vLLM endpoint" : ` ✓ Using existing vLLM on localhost:${deps.VLLM_PORT}`, ); const raw = apiKey @@ -273,8 +273,8 @@ export function createSetupNimVllmHandler( models = JSON.parse(raw); } catch { console.error( - managedDualEndpoint - ? " Could not query the managed dual-Station vLLM models endpoint. Is the deployment running and reachable?" + managedEndpoint + ? " Could not query the managed vLLM models endpoint. Is the deployment running and reachable?" : ` Could not query vLLM models endpoint. Is vLLM running on localhost:${deps.VLLM_PORT}?`, ); deps.exitProcess(1); @@ -295,15 +295,15 @@ export function createSetupNimVllmHandler( requiredModel && detectedModel !== requiredModel && (options.managedInstall === true || - managedDualEndpoint || + managedEndpoint || !reportedModelMatchesRequest(models, detectedModel, requiredModel)) ) { console.error( ` Detected vLLM model '${detectedModel}' does not match the shared gateway route '${requiredModel}'.`, ); console.error( - managedDualEndpoint - ? ` To install '${requiredModel}', stop the managed dual-Station vLLM deployment, then rerun the original install/onboard command.` + managedEndpoint + ? ` To install '${requiredModel}', stop the managed vLLM deployment, then rerun the original install/onboard command.` : ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, ); console.error(` To keep '${detectedModel}' instead, start detailed setup:`); @@ -363,13 +363,11 @@ export function createSetupNimVllmHandler( return "retry-selection"; } - if (managedDualEndpoint) { - const receipt = await deps.persistConfiguredDualStationVllmRuntimeReceipt(); + if (managedEndpoint) { + const receipt = await deps.persistConfiguredManagedVllmRuntimeReceipt(); if (!receipt.ok || !receipt.persisted) { - const reason = receipt.ok - ? "the managed dual-Station cleanup receipt was not written" - : receipt.reason; - console.error(` Managed dual-Station cleanup ownership could not be persisted: ${reason}`); + const reason = receipt.ok ? "the managed cleanup receipt was not written" : receipt.reason; + console.error(` Managed vLLM cleanup ownership could not be persisted: ${reason}`); deps.exitProcess(1); } } diff --git a/test/helpers/vitest-npm-script.ts b/test/helpers/vitest-npm-script.ts index 3e983d256a..0b984df17f 100644 --- a/test/helpers/vitest-npm-script.ts +++ b/test/helpers/vitest-npm-script.ts @@ -24,6 +24,7 @@ export function runVitestNpmScript( '#!/bin/sh\nprintf \'vitest %s\\n\' "$*" > "$COMMAND_LOG"\n', { mode: 0o755 }, ); + fs.writeFileSync(path.join(fakeBin, "tsx"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); fs.writeFileSync( scriptShell, `#!/bin/sh\nPATH="$FAKE_BIN:${path.dirname(process.execPath)}:/usr/bin:/bin"\nexport PATH\nexec /bin/sh "$@"\n`, diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 35f1e4d199..7919fc45ff 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -46,6 +46,14 @@ function runTests(...tests: string[]): () => string[] { } export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ + { + pattern: /(?:^|\/)managed-inference\/(?:presets|recipes|schemas)\/[^/]+\.(?:json|yaml)$/, + testsToRun: runTests( + "src/lib/inference/serving/catalog.test.ts", + "src/lib/inference/serving/resolver.test.ts", + "test/managed-inference-catalog-compiler.test.ts", + ), + }, { pattern: /(?:^|\/)(?:Dockerfile|agents\/(?:hermes|langchain-deepagents-code)\/Dockerfile)$/, testsToRun: runTests("src/lib/onboard/managed-startup-profile.test.ts"), diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index e48b9ed7c2..35ca7236a5 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -182,11 +182,11 @@ DGX_COMMIT_ID="d0e99cc"\nDGX_PLATFORM="DGX Server for GALAXY-GB300" expect(result.status, output).toBe(0); expect(output).toMatch(/Detected DGX Spark/); expect(output).toMatch( - /Express install will configure managed local vLLM using the DGX Spark profile default model/, - ); - expect(output).toMatch( - /Managed vLLM pulls the configured vLLM image\/model and runs a local vLLM inference container/, + /Express install will configure managed vLLM with automatic DGX Spark serving-profile selection/, ); + expect(output).toMatch(/one exactly qualified pretrusted managed cluster topology/); + expect(output).toMatch(/related or ambiguous setup remains untouched and stops installation/); + expect(output).toMatch(/experimental pending physical end-to-end validation/); expect(output).toMatch(/Sandbox name: my-assistant/); expect(output).toMatch(/Sandbox policy: suggested mode, tier 'balanced'/); expect(output).toMatch(/Run express install/); @@ -207,9 +207,8 @@ DGX_COMMIT_ID="d0e99cc"\nDGX_PLATFORM="DGX Server for GALAXY-GB300" expect(output).toMatch( /Express install will configure managed local vLLM with model custom-qwen3\.6/, ); - expect(output).toMatch( - /Managed vLLM pulls the configured vLLM image\/model and runs a local vLLM inference container/, - ); + expect(output).toMatch(/The explicit model remains authoritative/); + expect(output).toMatch(/keeps the existing single-host DGX Spark profile/); expect(output).toMatch( /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-qwen3\.6 POLICY=suggested YES=1 SANDBOX=my-assistant/, ); diff --git a/test/managed-inference-catalog-compiler.test.ts b/test/managed-inference-catalog-compiler.test.ts new file mode 100644 index 0000000000..907c002041 --- /dev/null +++ b/test/managed-inference-catalog-compiler.test.ts @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import catalogSchema from "../managed-inference/schemas/catalog.schema.json" with { type: "json" }; +import presetSchema from "../managed-inference/schemas/preset.schema.json" with { type: "json" }; +import recipeSchema from "../managed-inference/schemas/recipe.schema.json" with { type: "json" }; +import { getManagedInferenceServingCatalogRegistries } from "../src/lib/inference/serving/adapter-registry.js"; +import { compileTrustedServingCatalog } from "../src/lib/inference/serving/catalog.js"; +import type { ServingCatalogSource } from "../src/lib/inference/serving/types.js"; + +const REPOSITORY_ROOT = path.join(import.meta.dirname, ".."); +const PROFILE_ID = "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731"; +const RECIPE_ID = "vllm.deepseek-v4-flash-0731.spark-dual.v1"; + +function catalogSources(): ServingCatalogSource[] { + return (["presets", "recipes"] as const).flatMap((kind) => { + const directory = path.join(REPOSITORY_ROOT, "managed-inference", kind); + return readdirSync(directory) + .filter((name) => name.endsWith(".yaml")) + .map((name) => ({ + path: `managed-inference/${kind}/${name}`, + contents: readFileSync(path.join(directory, name), "utf8"), + })); + }); +} + +function compile(sources: readonly ServingCatalogSource[]) { + return compileTrustedServingCatalog({ + sources, + sourceRevision: "a".repeat(40), + schemas: { + catalog: catalogSchema, + preset: presetSchema, + recipe: recipeSchema, + }, + registries: getManagedInferenceServingCatalogRegistries(), + }); +} + +describe("managed inference YAML profile contract", () => { + it("compiles the shipped managed-cluster profile through the canonical catalog (#8129)", () => { + const catalog = compile(catalogSources()); + const preset = catalog.presets.find(({ metadata }) => metadata.id === PROFILE_ID); + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === RECIPE_ID); + + expect(preset?.spec.plan.recipeRef).toBe(RECIPE_ID); + expect(recipe?.spec.execution.nodeCount).toBe(2); + expect(preset?.spec.requirements?.all).toContainEqual({ + fact: "cluster.nodeCount", + state: "present", + operator: "equals", + value: 2, + }); + }); + + it("accepts another compatible profile as YAML-only catalog additions (#8129)", () => { + const sources = catalogSources(); + const recipeSource = sources.find(({ path: sourcePath }) => sourcePath.includes("/recipes/"))!; + const presetSource = sources.find(({ path: sourcePath }) => sourcePath.includes("/presets/"))!; + const syntheticRecipeId = "vllm.synthetic.managed-cluster.v1"; + const syntheticPresetId = "vllm.synthetic.managed-cluster"; + const catalog = compile([ + ...sources, + { + path: "managed-inference/recipes/vllm.synthetic.managed-cluster.v1.yaml", + contents: recipeSource.contents.replace(RECIPE_ID, syntheticRecipeId), + }, + { + path: "managed-inference/presets/vllm.synthetic.managed-cluster.yaml", + contents: presetSource.contents + .replace(PROFILE_ID, syntheticPresetId) + .replace(RECIPE_ID, syntheticRecipeId) + .replace("priority: 400", "priority: 399"), + }, + ]); + + expect(catalog.recipes.some(({ metadata }) => metadata.id === syntheticRecipeId)).toBe(true); + expect(catalog.presets.some(({ metadata }) => metadata.id === syntheticPresetId)).toBe(true); + }); + + it("does not enable arbitrary remote model code in shipped managed recipes (#8129)", () => { + const unsafeRecipes = compile(catalogSources()) + .recipes.filter((recipe) => { + const { serve } = recipe.spec; + return ( + serve !== undefined && + "arguments" in serve && + serve.arguments?.some(({ name }) => name === "--trust-remote-code") + ); + }) + .map(({ metadata }) => metadata.id); + + expect(unsafeRecipes).toEqual([]); + }); + + it("keeps shipped profile identities out of production TypeScript (#8129)", () => { + const servingRoot = path.join(REPOSITORY_ROOT, "src", "lib", "inference", "serving"); + const productionSources = readdirSync(servingRoot) + .filter((name) => name.endsWith(".ts") && !name.endsWith(".test.ts")) + .map((name) => readFileSync(path.join(servingRoot, name), "utf8")) + .join("\n"); + + expect(productionSources).not.toContain(PROFILE_ID); + expect(productionSources).not.toContain(RECIPE_ID); + }); +}); diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 6d9556c85b..e9cca2cd0a 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -48,6 +48,7 @@ const E2E_WORKFLOW_CONTRACTS = [ ] as const; const OPAQUE_INPUTS = [ + "managed-inference/recipes/vllm.example.managed-cluster.v1.yaml", "Dockerfile", "agents/hermes/Dockerfile", "agents/langchain-deepagents-code/Dockerfile", @@ -87,6 +88,11 @@ describe("Vitest opaque-input watch triggers", () => { }); it("maps current opaque inputs to their direct contract tests (#6692)", () => { + expect(triggeredBy("managed-inference/recipes/vllm.example.managed-cluster.v1.yaml")).toEqual([ + "src/lib/inference/serving/catalog.test.ts", + "src/lib/inference/serving/resolver.test.ts", + "test/managed-inference-catalog-compiler.test.ts", + ]); expect(triggeredBy("Dockerfile")).toEqual(["src/lib/onboard/managed-startup-profile.test.ts"]); expect(triggeredBy("agents/hermes/Dockerfile")).toEqual([ "src/lib/onboard/managed-startup-profile.test.ts",