diff --git a/.lycheeignore b/.lycheeignore index d2437c912db9..7344ac1d7f4c 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -76,6 +76,11 @@ ^https://github\.com/ai-dynamo/dynamo/tree/main/recipes/deepseek-v4/deepseek-v4-pro-0813/perf$ ^https://github\.com/ai-dynamo/dynamo/blob/main/recipes/deepseek-v4/deepseek-v4-pro-0813/perf/README\.md$ +# The Solar Open2 250B recipe page is introduced by the same PR as its docs link, +# so the rendered docs URL 404s from CI until the page is published. Remove this +# entry once the page is live. +^https://docs\.nvidia\.com/dynamo/latest/recipes/model-recipes/solar-open2-250b$ + # The DeepSeek-V4.1-Flash docs page is published by the same PR that adds this # link, so the URL returns 404 until the docs deploy. Remove this entry after # the page is live. diff --git a/docs/fern/index.yml b/docs/fern/index.yml index c51466d0200b..2dc1cfe234ef 100644 --- a/docs/fern/index.yml +++ b/docs/fern/index.yml @@ -795,6 +795,9 @@ navigation: - page: Kimi-K3 path: pages/recipes/model-recipes/kimi-k3.mdx slug: kimi-k3 + - page: Solar Open2 250B + path: pages/recipes/model-recipes/solar-open2-250b.mdx + slug: solar-open2-250b - page: Qwen3.8-Flash-Next path: pages/recipes/model-recipes/qwen-3-8-flash-next.mdx slug: qwen-3-8-flash-next diff --git a/docs/fern/pages/recipes/model-recipes/solar-open2-250b.mdx b/docs/fern/pages/recipes/model-recipes/solar-open2-250b.mdx new file mode 100644 index 000000000000..d7516236fe9e --- /dev/null +++ b/docs/fern/pages/recipes/model-recipes/solar-open2-250b.mdx @@ -0,0 +1,261 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Solar Open2 250B" +subtitle: "Serve Solar-Open2-250B-NVFP4 with Dynamo on B200, aggregated or disaggregated." +--- + +## Profiles + +| Profile | Serving mode | GPUs | Context | +|---|---|---:|---| +| `agg-b200-chat` | Aggregated, two replicas, KV-aware routing | 4 | 1M | +| `disagg-b200-chat` | Disaggregated 1P:2D, round-robin routing | 8 | 1M | + +## Prerequisites + +- Kubernetes cluster with B200 nodes and the Dynamo operator installed +- A `ReadWriteMany` storage class. `model-cache.yaml` requests 1200Gi, which holds + the checkpoint alongside benchmark traces and artifacts; the checkpoint itself is + about 153 GB. +- A Hugging Face token, as a secret named `hf-token-secret`, used by the download + Job to fetch the checkpoint: + +```bash +kubectl create secret generic hf-token-secret --from-literal=HF_TOKEN= +``` +- For the disaggregated profile, an RDMA/InfiniBand device plugin exposing `rdma/ib` + on the GPU nodes. If your plugin advertises a different resource name, substitute it + in the worker resource requests. + +## Prepare the model cache + +Edit `storageClassName` in `model-cache.yaml` first. + +```bash +kubectl apply -f recipes/solar-open2-250b/model-cache/model-cache.yaml +kubectl apply -f recipes/solar-open2-250b/model-cache/model-download.yaml +kubectl wait --for=condition=complete job/model-download --timeout=2h +``` + +The checkpoint is approximately 153 GB. Both profiles share this cache. + +## Deploy + +Aggregated: + +```bash +kubectl apply -f recipes/solar-open2-250b/vllm/agg-b200-chat/deploy.yaml +kubectl wait --for=condition=Ready \ + dynamographdeployment/solar-open2-250b-vllm-b200-agg-chat --timeout=30m +``` + +Disaggregated: + +```bash +kubectl apply -f recipes/solar-open2-250b/vllm/disagg-b200-chat/deploy.yaml +kubectl wait --for=condition=Ready \ + dynamographdeployment/solar-open2-250b-vllm-b200-disagg-chat --timeout=30m +``` + +## Smoke test + +Port-forward the frontend of the profile you deployed. For the aggregated profile: + +```bash +kubectl port-forward svc/solar-open2-250b-vllm-b200-agg-chat-frontend 8000:8000 & +``` + +For the disaggregated profile: + +```bash +kubectl port-forward svc/solar-open2-250b-vllm-b200-disagg-chat-frontend 8000:8000 & +``` + +Then, against either: + +```bash +curl -s http://localhost:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"solar-open2-250b", + "messages":[{"role":"user","content":"What is 12 times 7?"}], + "max_tokens":128}' +``` + +Solar Open2 is a reasoning model. Short generations may return the answer in +`reasoning_content` with `content` empty; allow a larger `max_tokens` to see a +final answer in `content`. + +## Benchmark + +Both performance Jobs read a mooncake trace from the `shared-model-cache` PVC. The +trace is not part of the model download, so stage it first. The published figures use +`nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl` (1805 requests, mean +ISL 37.4k, mean OSL 1008), which ships in this repository under +`recipes/nemotron-3-ultra/perf/traces/`. + +Copy it in through any running pod that mounts the PVC. Note that the worker pods +mount that volume at `/shared-model-cache`, while the benchmark Job mounts the same +volume at `/model-cache`, so the file staged below is what `TRACE_FILE` refers to: + +```bash +TRACE=recipes/nemotron-3-ultra/perf/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl + +# The traces are stored in Git LFS. A plain clone leaves a small pointer file in +# place of the data, so fetch the real contents first and check the size: the file +# is roughly 900 KB, not a few hundred bytes. +git lfs install +git lfs pull --include="$TRACE" +ls -l "$TRACE" + +# Whichever profile you deployed: +DGD=solar-open2-250b-vllm-b200-agg-chat # or solar-open2-250b-vllm-b200-disagg-chat + +# The workers mount the cache PVC; the frontend does not. Match the worker +# components by name rather than excluding the frontend, so that pods carrying no +# component-type label are not selected either. +POD=$(kubectl get pods \ + -l "nvidia.com/dynamo-graph-deployment-name=$DGD,nvidia.com/dynamo-component-type in (worker,prefill,decode)" \ + --field-selector status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}') + +kubectl exec "$POD" -- mkdir -p /shared-model-cache/traces +kubectl cp "$TRACE" "$POD:/shared-model-cache/traces/" +``` + +Set `TRACE_FILE` in the Job to match whichever trace you stage. + +Then run the Job for the profile under test: + +```bash +kubectl apply -f recipes/solar-open2-250b/perf/perf.yaml # aggregated +kubectl apply -f recipes/solar-open2-250b/perf/perf-disagg.yaml # disaggregated +``` + +Each Job runs the trace once and writes its summary to `ARTIFACT_DIR`. To run it +again, delete the Job first and re-apply; Jobs are immutable, so re-applying over an +existing one fails. + +## Runtime image + +Both manifests pin a purpose-built runtime image by digest. It extends the +Dynamo 1.4.1 vLLM runtime with the Solar Open2 architecture and parser support +and a Dynamo frontend compatibility patch. The changes are Python-only. + +The stock Dynamo 1.4.1 vLLM runtime does not serve this model — it has no Solar +Open2 model definition — so the pinned image is required rather than optional. +Pin by digest, not by tag. + +## Configuration notes + +### Both profiles + +- **NVFP4 weights, FP8 KV cache.** Both manifests set `--kv-cache-dtype fp8`; + the disaggregated profile additionally sets `--calculate-kv-scales`. FP8 halves + KV bytes relative to BF16, which roughly doubles the KV pool and raises the + realised prefix-cache hit rate. +- **MoE backend.** `--moe-backend flashinfer_trtllm` with + `--enable-flashinfer-autotune`. +- **Attention.** FlashInfer is selected automatically. Note that FlashAttention + is not compatible with an FP8 KV cache on this model and will fail at startup + if forced. +- **Hybrid KV pages.** Solar Open2 has 12 grouped-query attention layers and 36 + KDA linear-attention layers. The effective KV page is a hybrid page of roughly + 1072 tokens at `--block-size 16`, not the block size itself. Tools that assume + block size equals page size need adjusting. +- Speculative decoding is not supported on this checkpoint and is not enabled. + +### Aggregated profile + +- Two worker replicas at tensor parallel size 2, fronted by a KV-aware router + (`--router-mode kv`, `--router-kv-events`) with a ZMQ KV-event publisher on + each worker. KV-aware routing outperforms round-robin on prefix-reuse + workloads. +- **Expert parallelism is left off.** It measured as neutral for aggregated + serving on this checkpoint, so the simpler configuration is used. + +### Disaggregated profile + +- Separate prefill and decode workers connected by `NixlConnector`, with + `kv_role: kv_producer` on prefill and `kv_role: kv_consumer` on decode. +- **The topology is one prefill worker and two decode workers** — TP4 with + expert parallelism for prefill (4 GPUs), TP2 for each decode worker (4 GPUs), + 8 GPUs in total. +- **`VLLM_SSM_CONV_STATE_LAYOUT=DS` is required.** Solar Open2's KDA layers + register as SSM layers, and their convolution state must cross the KV-transfer + boundary. Without this variable the engine asserts during initialization and + the workers will not start. +- **Each worker requests `rdma/ib` equal to its GPU count**, four on prefill and two + on each decode worker. This is required, not optional: without it the device plugin + does not inject `/dev/infiniband`, UCX falls back to TCP, and nothing logs an error. + Verify inside a running worker with `kubectl exec -c main -- ls /dev/infiniband` + and expect `uverbsN`, `umadN` and `rdma_cm`. One HCA per GPU is correct; requesting + more measured worse. If your RDMA device plugin advertises a different resource name + than `rdma/ib`, substitute it in both the requests and the limits. +- Transport is configured with `NCCL_IB_DISABLE=0`, `UCX_TLS=^cuda_ipc`, + `UCX_RNDV_SCHEME=get_zcopy` and `UCX_MAX_RNDV_RAILS=4`. `UCX_MAX_RNDV_RAILS` tracks + the number of HCAs per worker; adjust it if your nodes differ. `UCX_NET_DEVICES` is + deliberately left unset; pinning it produced `NIXL_ERR_BACKEND`. +- **Per-user output throughput runs close to the floor.** The disaggregated profile + clears the 50 tok/s target with little margin, and individual runs on contended + hardware may land at or below it. +- **The frontend uses round-robin routing, not KV-aware routing.** This is + deliberate. KV-aware routing improves the aggregated profile, but with a single + prefill worker there is nothing for a prefix-aware router to choose between. + KV-aware and load-aware routing were both measured on this topology and neither + improved on round-robin. + +## Known limitations + +- The disaggregated profile delivers slightly lower per-GPU throughput than the + 4-GPU aggregated profile, at roughly an order of magnitude higher + time-to-first-token. Prefer the aggregated profile unless independent + prefill/decode scaling is the priority. +- The 1M context length requires sufficient KV capacity; reduce + `--max-model-len` if deploying on a smaller GPU allocation. +- Speculative decoding is not available on this checkpoint. + +### API compatibility + +Some OpenAI-compatible surfaces do not behave as advertised. These originate in the +Dynamo frontend, are not specific to this model, and no recipe setting affects them. + +- **Logprobs.** Requesting `logprobs` on `/v1/chat/completions` returns HTTP 500. +- **Responses API.** `store=true` succeeds but the response is not retrievable + afterwards (`GET /v1/responses/{id}` returns 404). Unsupported parameters such as + `frequency_penalty` are accepted and silently ignored. +- **Completions.** `stop_token_ids` with more than three IDs returns HTTP 500 rather + than a 400. + +### Reasoning budget and structured output + +The recipe sets `reasoning_effort: high`. Generations are long: on GPQA Diamond the +median output is roughly 6,400 tokens, the 99th percentile 131,000, and the longest +observed 209,000. A small `max_tokens` therefore truncates the model mid-reasoning +and returns `finish_reason=length` with no final content, which shows up most often +on structured-output requests where the JSON never arrives in `message.content`. + +Omit `max_tokens` and let the 1M context window bound the request, or set it well +above the reasoning length the workload needs. Values in the low thousands are not +sufficient for this model. + +## Measured performance + +Chat workload, 15% subset of the internal `nim_turbo` 8k/1k 70kv trace +(1805 requests, mean ISL 37.4k tokens, mean OSL 1008 tokens). SLA: p50 TTFT +below 5 s and p50 user output throughput at or above 50 tok/s. + +| Workload | Recipe | Framework | SKU | Concurrency | System output tok/s/GPU | User output tok/s (P50) | TTFT P50 (ms) | +|---|---|---|---|---:|---:|---:|---:| +| Chat (15% subset) | Aggregated, 2 replicas, KV-aware routing | vLLM | B200 | 20 | 216.88 | 52.02 | 308 | +| Chat (15% subset) | Disaggregated 1P:2D, round-robin | vLLM | B200 | 44 | 206.20 | 51.69 | 2277 | + +Figures are measured with the profile's benchmark manifest (`perf.yaml` for +aggregated, `perf-disagg.yaml` for disaggregated). That configuration isolates the +warmup phase from profiling; it does not clear server-side KV cache. For runs that +must start from a cold cache, restart the worker pods between them. + +## Source + +- Manifests: `recipes/solar-open2-250b/` +- Model: `nota-ai/Solar-Open2-250B-Nota-NVFP4` diff --git a/recipes/README.md b/recipes/README.md index 53a226feb4b5..f137230b1bfc 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -98,6 +98,8 @@ These recipes are under active development and may require additional setup step | **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/)** | vLLM | Aggregated | 8x GB200 (2 NVL4 trays) | ✅ | Text only — same model as B200 agg; TP=8 + EP cross-node via NVLink72 (MNNVL) + ComputeDomain. Requires [custom container build](deepseek-v4/container/). | | **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/)** | vLLM | Disaggregated | 16x GB200 (4 NVL4 trays) | ✅ | Text only — DP=8 + EP per worker, 1P + 1D, NVLink72 (MNNVL) + ComputeDomain. Requires [custom container build](deepseek-v4/container/). | | **[DeepSeek-V4-Pro](deepseek-v4/deepseek-v4-pro/sglang/agg/)** | SGLang | Aggregated | 8x B200 | ✅ | Text only — MoE model (1.6T / 49B active, 1M context), TP=8, MXFP4 MoE via FlashInfer, EAGLE MTP (3 steps / 4 draft tokens), reasoning + tool calling. Prebuilt image available (shared with [DeepSeek-V4-Flash](deepseek-v4/deepseek-v4-flash/sglang/agg/)). | +| **[Solar-Open2-250B-NVFP4](solar-open2-250b/vllm/agg-b200-chat/)** | vLLM | Aggregated | 4x B200 | ✅ | Text only — NVFP4 checkpoint, 2 replicas × TP2, reasoning + tool calling. Prebuilt image available. | +| **[Solar-Open2-250B-NVFP4](solar-open2-250b/vllm/disagg-b200-chat/)** | vLLM | Disaggregated | 8x B200 | ✅ | Text only — NVFP4 checkpoint, 1× prefill TP4 + EP and 2× decode TP2, NIXL/RDMA KV transfer, reasoning + tool calling. Prebuilt image available. | ## Recipe Structure diff --git a/recipes/solar-open2-250b/README.md b/recipes/solar-open2-250b/README.md new file mode 100644 index 000000000000..c62ae9b026e3 --- /dev/null +++ b/recipes/solar-open2-250b/README.md @@ -0,0 +1,9 @@ + + +# Solar Open2 250B NVFP4 Recipes + +Documentation for these recipes is published at +. diff --git a/recipes/solar-open2-250b/model-cache/model-cache.yaml b/recipes/solar-open2-250b/model-cache/model-cache.yaml new file mode 100644 index 000000000000..c491a6b1c9b5 --- /dev/null +++ b/recipes/solar-open2-250b/model-cache/model-cache.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shared-model-cache +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1200Gi + # Edit this + storageClassName: "your-storage-class-name" diff --git a/recipes/solar-open2-250b/model-cache/model-download.yaml b/recipes/solar-open2-250b/model-cache/model-download.yaml new file mode 100644 index 000000000000..9ac9ce913e72 --- /dev/null +++ b/recipes/solar-open2-250b/model-cache/model-download.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: batch/v1 +kind: Job +metadata: + name: model-download +spec: + backoffLimit: 3 + completions: 1 + parallelism: 1 + template: + metadata: + labels: + app: model-download + spec: + restartPolicy: Never + containers: + - name: model-download + image: python:3.10-slim + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + command: ["sh", "-c"] + envFrom: + - secretRef: + name: hf-token-secret + env: + - name: MODEL_NAME + value: nota-ai/Solar-Open2-250B-Nota-NVFP4 + # The deployments load this snapshot directory explicitly, so the + # download must fetch the same revision. Without it the job follows the + # repository's current default and the workers fail on a directory that + # was never downloaded. + - name: MODEL_REVISION + value: 3cc673335f4c2f41181e05a98fc2b8324d586a64 + - name: HF_HOME + value: /shared-model-cache + - name: HF_XET_HIGH_PERFORMANCE + value: "1" + args: + - | + set -eux + pip install --no-cache-dir huggingface_hub==1.16.4 + hf download $MODEL_NAME --revision $MODEL_REVISION + resources: + requests: + cpu: "8" + memory: "128Gi" + volumeMounts: + - name: shared-model-cache + mountPath: /shared-model-cache + volumes: + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache diff --git a/recipes/solar-open2-250b/perf/perf-disagg.yaml b/recipes/solar-open2-250b/perf/perf-disagg.yaml new file mode 100644 index 000000000000..8d512e29067c --- /dev/null +++ b/recipes/solar-open2-250b/perf/perf-disagg.yaml @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mooncake-aiperf-config-disagg +data: + perf.yaml: | + schemaVersion: "2.0" + + benchmark: + model: ${TARGET_MODEL} + + tokenizer: + name: ${TOKENIZER} + trust_remote_code: true + + endpoint: + url: ${INFERENCE_URL} + type: chat + streaming: true + timeout: 7200 + use_server_token_count: true + extra: + ignore_eos: true + + dataset: + type: file + path: ${TRACE_FILE} + format: mooncake_trace + sampling: sequential + + cache_bust: + target: warmup_isolation_system + + synthesis: + max_isl: ${MAX_ISL:1048576} + max_osl: ${CAP_OSL:12000} + + warmup: + type: concurrency + concurrency: 16 + requests: 16 + + profiling: + type: concurrency + concurrency: ${CONCURRENCY} + requests: ${REQUESTS} + + runtime: + ui: none + + artifacts: + dir: ${ARTIFACT_DIR} + summary: [json] + records: [jsonl] +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mooncake-aiperf-job-disagg +spec: + backoffLimit: 1 + + template: + spec: + restartPolicy: Never + + + containers: + - name: aiperf + image: nvcr.io/nvstaging/ai-dynamo/aiperf:0.13.0rc2 + imagePullPolicy: IfNotPresent + + command: ["/bin/bash", "-lc"] + args: + - | + set -euo pipefail + export REQUESTS="$(wc -l < "$TRACE_FILE" | tr -d '[:space:]')" + exec aiperf profile --config /etc/aiperf/perf.yaml + + env: + # Served model name advertised by the deployment in + # vllm/disagg-b200-chat/deploy.yaml. + - name: TARGET_MODEL + value: "solar-open2-250b" + + - name: TOKENIZER + value: "nota-ai/Solar-Open2-250B-Nota-NVFP4" + + # Frontend service of the aggregated deployment. Must match + # metadata.name in deploy.yaml, suffixed with -frontend. + - name: INFERENCE_URL + value: "http://solar-open2-250b-vllm-b200-disagg-chat-frontend:8000/v1/chat/completions" + + # 15% subset of the nim_turbo 8k/1k 70% KV-reuse chat trace: 1805 + # requests, mean ISL 37.4k, mean OSL 1008. This is the trace behind + # the published result for this profile. + - name: TRACE_FILE + value: "/model-cache/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl" + + # Concurrency 44 is the operating point the published numbers were + # measured at for the 1P:2D disaggregated topology. + - name: CONCURRENCY + value: "44" + + - name: MAX_ISL + value: "1048576" + + - name: CAP_OSL + value: "12000" + + - name: ARTIFACT_DIR + value: "/model-cache/aiperf-artifacts" + + volumeMounts: + - name: aiperf-config + mountPath: /etc/aiperf + readOnly: true + + # Read the trace and write artifacts on the shared PVC. + - name: shared-model-cache + mountPath: /model-cache + + volumes: + - name: aiperf-config + configMap: + name: mooncake-aiperf-config-disagg + + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache \ No newline at end of file diff --git a/recipes/solar-open2-250b/perf/perf.yaml b/recipes/solar-open2-250b/perf/perf.yaml new file mode 100644 index 000000000000..60be9d6b5678 --- /dev/null +++ b/recipes/solar-open2-250b/perf/perf.yaml @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mooncake-aiperf-config +data: + perf.yaml: | + schemaVersion: "2.0" + + benchmark: + model: ${TARGET_MODEL} + + tokenizer: + name: ${TOKENIZER} + trust_remote_code: true + + endpoint: + url: ${INFERENCE_URL} + type: chat + streaming: true + timeout: 7200 + use_server_token_count: true + extra: + ignore_eos: true + + dataset: + type: file + path: ${TRACE_FILE} + format: mooncake_trace + sampling: sequential + + cache_bust: + target: warmup_isolation_system + + synthesis: + max_isl: ${MAX_ISL:1048576} + max_osl: ${CAP_OSL:12000} + + warmup: + type: concurrency + concurrency: 16 + requests: 16 + + profiling: + type: concurrency + concurrency: ${CONCURRENCY} + requests: ${REQUESTS} + + runtime: + ui: none + + artifacts: + dir: ${ARTIFACT_DIR} + summary: [json] + records: [jsonl] +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mooncake-aiperf-job +spec: + backoffLimit: 1 + + template: + spec: + restartPolicy: Never + + + containers: + - name: aiperf + image: nvcr.io/nvstaging/ai-dynamo/aiperf:0.13.0rc2 + imagePullPolicy: IfNotPresent + + command: ["/bin/bash", "-lc"] + args: + - | + set -euo pipefail + export REQUESTS="$(wc -l < "$TRACE_FILE" | tr -d '[:space:]')" + exec aiperf profile --config /etc/aiperf/perf.yaml + + env: + # Served model name advertised by the deployment in + # vllm/agg-b200-chat/deploy.yaml. + - name: TARGET_MODEL + value: "solar-open2-250b" + + - name: TOKENIZER + value: "nota-ai/Solar-Open2-250B-Nota-NVFP4" + + # Frontend service of the aggregated deployment. Must match + # metadata.name in deploy.yaml, suffixed with -frontend. + - name: INFERENCE_URL + value: "http://solar-open2-250b-vllm-b200-agg-chat-frontend:8000/v1/chat/completions" + + # 15% subset of the nim_turbo 8k/1k 70% KV-reuse chat trace: 1805 + # requests, mean ISL 37.4k, mean OSL 1008. This is the trace behind + # the published result for this profile. + - name: TRACE_FILE + value: "/model-cache/traces/nim_turbo_8k_1k_70kv_chat_new_noschedule_short_15perc.jsonl" + + # Concurrency 20 is the operating point the published numbers were + # measured at for the 2-replica aggregated topology. + - name: CONCURRENCY + value: "20" + + - name: MAX_ISL + value: "1048576" + + - name: CAP_OSL + value: "12000" + + - name: ARTIFACT_DIR + value: "/model-cache/aiperf-artifacts" + + volumeMounts: + - name: aiperf-config + mountPath: /etc/aiperf + readOnly: true + + # Read the trace and write artifacts on the shared PVC. + - name: shared-model-cache + mountPath: /model-cache + + volumes: + - name: aiperf-config + configMap: + name: mooncake-aiperf-config + + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache \ No newline at end of file diff --git a/recipes/solar-open2-250b/vllm/agg-b200-chat/deploy.yaml b/recipes/solar-open2-250b/vllm/agg-b200-chat/deploy.yaml new file mode 100644 index 000000000000..2b9a938b8153 --- /dev/null +++ b/recipes/solar-open2-250b/vllm/agg-b200-chat/deploy.yaml @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: solar-open2-250b-vllm-b200-agg-chat +spec: + backendFramework: vllm + env: + - name: DYN_ROUTER_MODE + value: kv + - name: HF_HOME + value: /shared-model-cache + - name: HF_HUB_CACHE + value: /shared-model-cache/hub + - name: HF_HUB_OFFLINE + value: '1' + - name: TRANSFORMERS_OFFLINE + value: '1' + - name: VLLM_WORKER_MULTIPROC_METHOD + value: spawn + - name: PYTHONHASHSEED + value: '0' + - name: NCCL_IB_DISABLE + value: '1' + - name: NCCL_CUMEM_ENABLE + value: '1' + - name: NCCL_NVLS_ENABLE + value: '1' + - name: VLLM_ALLREDUCE_USE_SYMM_MEM + value: '0' + - name: VLLM_ALLREDUCE_USE_FLASHINFER + value: '1' + - name: VLLM_USE_NCCL_SYMM_MEM + value: '0' + - name: VLLM_FLASHINFER_ALLREDUCE_BACKEND + value: mnnvl + components: + - name: Frontend + type: frontend + replicas: 1 + minAvailable: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.1-solar-open2-250b-post.1@sha256:42fd3ed0607aa19b7d4c1b0a8f8aaaefefc8ec6292294ea2b5e6757353891f49 + imagePullPolicy: IfNotPresent + command: + - python3 + - -m + - dynamo.frontend + args: + - --router-kv-events + - --event-plane=zmq + - --router-track-prefill-tokens + - --router-kv-overlap-score-credit-decay=0.1 + - --router-min-initial-workers=2 + - --dyn-chat-processor=vllm + - --tool-call-parser=solar_open2 + - --reasoning-parser=solar_open2 + - --enable-auto-tool-choice + - --default-chat-template-kwargs={"think_render_option":"preserved","reasoning_effort":"high"} + - --max-model-len=1048576 + - --http-host=0.0.0.0 + - --http-port=8000 + ports: + - name: http + containerPort: 8000 + resources: + requests: + cpu: '16' + memory: 128Gi + volumeMounts: + - name: runtime-tmp + mountPath: /tmp + volumes: + - name: runtime-tmp + emptyDir: + sizeLimit: 20Gi + - name: Worker + type: worker + replicas: 2 + minAvailable: 2 + sharedMemorySize: 64Gi + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.1-solar-open2-250b-post.1@sha256:42fd3ed0607aa19b7d4c1b0a8f8aaaefefc8ec6292294ea2b5e6757353891f49 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + add: + - IPC_LOCK + - SYS_PTRACE + - SYS_RESOURCE + command: + - python3 + - -m + - dynamo.vllm + args: + - --model=/shared-model-cache/hub/models--nota-ai--Solar-Open2-250B-Nota-NVFP4/snapshots/3cc673335f4c2f41181e05a98fc2b8324d586a64 + - --served-model-name=solar-open2-250b + - --tensor-parallel-size=2 + - --dtype=bfloat16 + - --kv-cache-dtype=fp8 + - --max-model-len=1048576 + - --max-num-seqs=512 + - --max-num-batched-tokens=32768 + - --enable-chunked-prefill + - --no-async-scheduling + - --gpu-memory-utilization=0.95 + - --disable-custom-all-reduce + - --moe-backend=flashinfer_trtllm + - --enable-flashinfer-autotune + - --compilation-config={"pass_config":{"fuse_allreduce_rms":true}} + - --enable-prefix-caching + - --reasoning-parser=solar_open2 + - --logits-processors=vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor + - --kv-events-config={"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5571","enable_kv_cache_events":true} + resources: + limits: + nvidia.com/gpu: '2' + requests: + cpu: '16' + ephemeral-storage: 64Gi + memory: 544Gi + nvidia.com/gpu: '2' + volumeMounts: + - name: shared-model-cache + mountPath: /shared-model-cache + - name: runtime-tmp + mountPath: /tmp + volumes: + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache + - name: runtime-tmp + emptyDir: + sizeLimit: 100Gi diff --git a/recipes/solar-open2-250b/vllm/disagg-b200-chat/deploy.yaml b/recipes/solar-open2-250b/vllm/disagg-b200-chat/deploy.yaml new file mode 100644 index 000000000000..1a755b839115 --- /dev/null +++ b/recipes/solar-open2-250b/vllm/disagg-b200-chat/deploy.yaml @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta1 +kind: DynamoGraphDeployment +metadata: + name: solar-open2-250b-vllm-b200-disagg-chat +spec: + backendFramework: vllm + env: + - name: DYN_ROUTER_MODE + value: round_robin + - name: HF_HOME + value: /shared-model-cache + - name: HF_HUB_CACHE + value: /shared-model-cache/hub + - name: HF_HUB_OFFLINE + value: '1' + - name: TRANSFORMERS_OFFLINE + value: '1' + - name: VLLM_WORKER_MULTIPROC_METHOD + value: spawn + - name: PYTHONHASHSEED + value: '0' + - name: NCCL_IB_DISABLE + value: '0' + - name: NCCL_CUMEM_ENABLE + value: '1' + - name: NCCL_NVLS_ENABLE + value: '1' + - name: UCX_TLS + value: ^cuda_ipc + - name: UCX_RNDV_SCHEME + value: get_zcopy + - name: UCX_MAX_RNDV_RAILS + value: '4' + - name: VLLM_SSM_CONV_STATE_LAYOUT + value: DS + - name: VLLM_ALLREDUCE_USE_SYMM_MEM + value: '0' + - name: VLLM_ALLREDUCE_USE_FLASHINFER + value: '1' + - name: VLLM_USE_NCCL_SYMM_MEM + value: '0' + - name: VLLM_FLASHINFER_ALLREDUCE_BACKEND + value: mnnvl + components: + - name: Frontend + type: frontend + replicas: 1 + minAvailable: 1 + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.1-solar-open2-250b-post.1@sha256:42fd3ed0607aa19b7d4c1b0a8f8aaaefefc8ec6292294ea2b5e6757353891f49 + imagePullPolicy: IfNotPresent + command: + - python3 + - -m + - dynamo.frontend + args: + - --dyn-chat-processor=vllm + - --tool-call-parser=solar_open2 + - --reasoning-parser=solar_open2 + - --enable-auto-tool-choice + - --default-chat-template-kwargs={"think_render_option":"preserved","reasoning_effort":"high"} + - --max-model-len=1048576 + - --http-host=0.0.0.0 + - --http-port=8000 + ports: + - name: http + containerPort: 8000 + resources: + requests: + cpu: '16' + memory: 128Gi + volumeMounts: + - name: runtime-tmp + mountPath: /tmp + volumes: + - name: runtime-tmp + emptyDir: + sizeLimit: 20Gi + - name: PrefillWorker + type: prefill + replicas: 1 + minAvailable: 1 + sharedMemorySize: 64Gi + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.1-solar-open2-250b-post.1@sha256:42fd3ed0607aa19b7d4c1b0a8f8aaaefefc8ec6292294ea2b5e6757353891f49 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + add: + - IPC_LOCK + - SYS_PTRACE + - SYS_RESOURCE + command: + - python3 + - -m + - dynamo.vllm + args: + - --model=/shared-model-cache/hub/models--nota-ai--Solar-Open2-250B-Nota-NVFP4/snapshots/3cc673335f4c2f41181e05a98fc2b8324d586a64 + - --served-model-name=solar-open2-250b + - --tensor-parallel-size=4 + - --enable-expert-parallel + - --dtype=bfloat16 + - --kv-cache-dtype=fp8 + - --calculate-kv-scales + - --max-model-len=1048576 + - --max-num-seqs=512 + - --max-num-batched-tokens=65536 + - --enable-chunked-prefill + - --no-async-scheduling + - --gpu-memory-utilization=0.90 + - --disable-custom-all-reduce + - --moe-backend=flashinfer_trtllm + - --enable-flashinfer-autotune + - --compilation-config={"pass_config":{"fuse_allreduce_rms":true}} + - --enable-prefix-caching + - --block-size=16 + - --reasoning-parser=solar_open2 + - --logits-processors=vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor + - --kv-transfer-config={"kv_connector":"NixlConnector","kv_role":"kv_producer","kv_connector_extra_config":{"num_threads":8}} + - --disaggregation-mode=prefill + - --kv-events-config={"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5571","enable_kv_cache_events":true} + resources: + limits: + nvidia.com/gpu: '4' + rdma/ib: '4' + requests: + cpu: '16' + ephemeral-storage: 64Gi + memory: 640Gi + nvidia.com/gpu: '4' + rdma/ib: '4' + volumeMounts: + - name: shared-model-cache + mountPath: /shared-model-cache + - name: runtime-tmp + mountPath: /tmp + volumes: + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache + - name: runtime-tmp + emptyDir: + sizeLimit: 100Gi + - name: DecodeWorker + type: decode + replicas: 2 + minAvailable: 2 + sharedMemorySize: 64Gi + podTemplate: + spec: + containers: + - name: main + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.1-solar-open2-250b-post.1@sha256:42fd3ed0607aa19b7d4c1b0a8f8aaaefefc8ec6292294ea2b5e6757353891f49 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + runAsGroup: 0 + capabilities: + add: + - IPC_LOCK + - SYS_PTRACE + - SYS_RESOURCE + command: + - python3 + - -m + - dynamo.vllm + args: + - --model=/shared-model-cache/hub/models--nota-ai--Solar-Open2-250B-Nota-NVFP4/snapshots/3cc673335f4c2f41181e05a98fc2b8324d586a64 + - --served-model-name=solar-open2-250b + - --tensor-parallel-size=2 + - --dtype=bfloat16 + - --kv-cache-dtype=fp8 + - --calculate-kv-scales + - --max-model-len=1048576 + - --max-num-seqs=512 + - --max-num-batched-tokens=65536 + - --enable-chunked-prefill + - --gpu-memory-utilization=0.90 + - --disable-custom-all-reduce + - --moe-backend=flashinfer_trtllm + - --enable-flashinfer-autotune + - --compilation-config={"pass_config":{"fuse_allreduce_rms":true}} + - --enable-prefix-caching + - --block-size=16 + - --reasoning-parser=solar_open2 + - --logits-processors=vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor + - --kv-transfer-config={"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"num_threads":8}} + - --disaggregation-mode=decode + resources: + limits: + nvidia.com/gpu: '2' + rdma/ib: '2' + requests: + cpu: '16' + ephemeral-storage: 64Gi + memory: 320Gi + nvidia.com/gpu: '2' + rdma/ib: '2' + volumeMounts: + - name: shared-model-cache + mountPath: /shared-model-cache + - name: runtime-tmp + mountPath: /tmp + volumes: + - name: shared-model-cache + persistentVolumeClaim: + claimName: shared-model-cache + - name: runtime-tmp + emptyDir: + sizeLimit: 100Gi