From c460dc82c68be3f77269ef825720016ed169c323 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 26 Aug 2026 23:52:46 -0500 Subject: [PATCH 01/11] port GB200 DeepSeek V4 disaggregation to srt-slurm --- benchmarks/llm-d/Dockerfile | 53 -- benchmarks/llm-d/README.md | 16 - benchmarks/llm-d/binaries.env | 32 - benchmarks/llm-d/envoy.yaml | 129 ---- benchmarks/llm-d/epp-config.yaml | 61 -- benchmarks/llm-d/extract-binaries.sh | 50 -- .../dsv4_fp4_gb200_llmd-vllm-disagg.sh | 60 -- .../dsv4-fp4-gb200-low-latency.yaml | 192 ------ .../dsv4-fp4-gb200-mid-curve-megamoe.yaml | 196 ------ benchmarks/multi_node/llm-d/README.md | 143 ---- benchmarks/multi_node/llm-d/job.slurm | 252 ------- benchmarks/multi_node/llm-d/server.sh | 638 ------------------ benchmarks/multi_node/llm-d/submit.sh | 127 ---- .../configs/install-vllm-router.sh | 11 + ...-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml | 164 +++++ .../8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml | 152 +++++ .../disagg-gb200-3p1d-dep8-dep8-c4096.yaml | 161 +++++ configs/nvidia-master.yaml | 47 +- docs/configuration-procedures.md | 21 +- docs/configuration-procedures_zh.md | 21 +- runners/launch_gb200-nv.sh | 135 ++-- 21 files changed, 564 insertions(+), 2097 deletions(-) delete mode 100644 benchmarks/llm-d/Dockerfile delete mode 100644 benchmarks/llm-d/README.md delete mode 100644 benchmarks/llm-d/binaries.env delete mode 100644 benchmarks/llm-d/envoy.yaml delete mode 100644 benchmarks/llm-d/epp-config.yaml delete mode 100755 benchmarks/llm-d/extract-binaries.sh delete mode 100755 benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh delete mode 100644 benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml delete mode 100644 benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml delete mode 100644 benchmarks/multi_node/llm-d/README.md delete mode 100644 benchmarks/multi_node/llm-d/job.slurm delete mode 100755 benchmarks/multi_node/llm-d/server.sh delete mode 100755 benchmarks/multi_node/llm-d/submit.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/install-vllm-router.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml diff --git a/benchmarks/llm-d/Dockerfile b/benchmarks/llm-d/Dockerfile deleted file mode 100644 index cfc04557c5..0000000000 --- a/benchmarks/llm-d/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Combined image for the InferenceX llmd-vllm framework. -# -# Base = vllm/vllm-openai (vLLM with the OpenAI-compatible API server). -# We add the EPP, the routing-sidecar, and Envoy on top so every node in -# a SLURM allocation can play any role (prefill, decode, or coordinator) -# from a single image. -# -# Configs (epp-config.yaml, envoy.yaml, per-topology recipes) are NOT -# baked in. They are mounted at runtime by job.slurm so config-only -# iteration does not require an image rebuild. See -# benchmarks/multi_node/llm-d/job.slurm for the expected mount layout. - -# All FROM-referenced images are declared as global-scope ARGs. BuildKit -# does NOT expand a build-arg inside `COPY --from=${VAR}` directly ("variable -# expansion is not supported for --from"); the supported pattern is to bind -# each ARG to a named FROM stage in the global scope and COPY --from that -# stage alias. Defaults MUST match benchmarks/llm-d/binaries.env, which is the -# single source of truth (it also drives the mounted-binary path via -# extract-binaries.sh). Override at build time from that file, e.g.: -# source benchmarks/llm-d/binaries.env -# docker build \ -# --build-arg EPP_FROM_IMAGE="$EPP_FROM_IMAGE" \ -# --build-arg ROUTING_SIDECAR_IMAGE="$ROUTING_SIDECAR_IMAGE" \ -# --build-arg ENVOY_FROM_IMAGE="$ENVOY_FROM_IMAGE" ... -ARG VLLM_BASE=vllm/vllm-openai:v0.26.0 -ARG EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 -ARG ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0 -ARG ENVOY_FROM_IMAGE=envoyproxy/envoy:distroless-v1.33.2 - -# Stage aliases for the three add-on binary sources (see note above). -FROM ${EPP_FROM_IMAGE} AS epp_src -FROM ${ROUTING_SIDECAR_IMAGE} AS sidecar_src -FROM ${ENVOY_FROM_IMAGE} AS envoy_src - -FROM ${VLLM_BASE} -# Force-upgrade nvidia-cutlass-dsl only when explicitly requested. The -# v0.22 base needed it because its bundled wheel lacked cute.arch.fmin -# for the DSV4 sparse-attention code; stock v0.23.0 (what upstream -# wide-ep-lws runs) ships a working cutlass, so we leave it untouched -# there. Build the v0.22 base with --build-arg UPGRADE_CUTLASS_DSL=1. -ARG UPGRADE_CUTLASS_DSL=0 -RUN if [ "$UPGRADE_CUTLASS_DSL" = "1" ]; then \ - pip install --no-cache-dir --upgrade nvidia-cutlass-dsl; \ - fi - -COPY --from=epp_src \ - /app/epp /usr/local/bin/epp - -COPY --from=sidecar_src \ - /app/pd-sidecar /usr/local/bin/pd-sidecar - -COPY --from=envoy_src \ - /usr/local/bin/envoy /usr/local/bin/ diff --git a/benchmarks/llm-d/README.md b/benchmarks/llm-d/README.md deleted file mode 100644 index 027a18eee5..0000000000 --- a/benchmarks/llm-d/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# llmd-vllm framework artifacts - -This directory holds the static, baked-into-the-image pieces of the -`llmd-vllm` benchmark framework. - -| File | Purpose | -|---|---| -| `Dockerfile` | Combined image: vLLM (DeepEP-enabled), EPP, routing-sidecar, Envoy. One image, every node uses what its role requires. | -| `epp-config.yaml` | Fallback EPP scheduling config. Used when no recipe overrides it via `CONFIG_FILE`. `disagg-profile-handler` + `kv-cache-utilization-scorer` + `random-picker` over the file-discovery endpoint set. | -| `envoy.yaml` | Static Envoy: listener `:8080`, ext_proc to `127.0.0.1:9002`, ORIGINAL_DST cluster reading `x-gateway-destination-endpoint`. | - -The runtime pieces (per-node `server.sh`, the SLURM job script, recipe -files, and the endpoint discovery mechanism) live under -`benchmarks/multi_node/llm-d/` and `benchmarks/multi_node/llm-d-recipes/`. -See the README in `benchmarks/multi_node/llm-d/` for the endpoints-file -generation flow. diff --git a/benchmarks/llm-d/binaries.env b/benchmarks/llm-d/binaries.env deleted file mode 100644 index 4782f489b8..0000000000 --- a/benchmarks/llm-d/binaries.env +++ /dev/null @@ -1,32 +0,0 @@ -# Single source of truth for the non-vLLM executables the llmd-vllm -# framework needs (EPP, P/D sidecar, Envoy). -# -# This file is sourced by benchmarks/multi_node/llm-d/job.slurm to -# optionally bind-mount extracted binaries (via $LLMD_BIN_DIR) in place -# of the combined image's baked-in copies. extract-binaries.sh pulls -# each image once and copies the binary out to $LLMD_BIN_DIR; the -# Dockerfile reads the same URLs when building the combined image. -# -# By default this is a no-op: we build and use an image that already -# contains all binaries, so the job.slurm mount loop skips them (its -# -x check fails until extract-binaries.sh has populated $LLMD_BIN_DIR). - -# --- Endpoint Picker (EPP / inference scheduler) --- -EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" -EPP_BIN_PATH="/app/epp" - -# --- P/D routing sidecar (pd-sidecar) --- -ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0" -ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar" - -# --- Envoy front proxy --- -ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" -ENVOY_BIN_PATH="/usr/local/bin/envoy" - -# Platform the binaries must target (GB200 = Grace = arm64). -LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/arm64}" - -# Shared-filesystem dir the extracted binaries live in, mounted into -# /usr/local/bin by job.slurm. Defaults to the same Lustre area the -# launcher already caches enroot squash images in (proven writable). -LLMD_BIN_DIR="${LLMD_BIN_DIR:-/mnt/lustre01/users-public/sa-shared/llm-d-bins}" diff --git a/benchmarks/llm-d/envoy.yaml b/benchmarks/llm-d/envoy.yaml deleted file mode 100644 index ccba51ba0b..0000000000 --- a/benchmarks/llm-d/envoy.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# Envoy front door for the llmd-vllm framework. -# -# Listener : 0.0.0.0:8080 (benchmark client target) -# ext_proc : EPP on 127.0.0.1:9002 -# Cluster : ORIGINAL_DST, picks the address from the -# x-gateway-destination-endpoint header that EPP sets. - -static_resources: - listeners: - - name: main - address: - socket_address: { address: 0.0.0.0, port_value: 8080 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - codec_type: AUTO - stream_idle_timeout: 0s - request_timeout: 0s - # Write per-request access logs to their own file so - # they survive an SIGTERM-on-failure: when envoy's - # stdout is captured via `> envoy.log`, glibc - # block-buffers it, and the buffer is lost when srun - # SIGTERMs envoy after a failed bench. FileAccessLog - # opens its own fd and flushes per record. - access_log: - - name: envoy.access_loggers.file - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: /benchmark_logs/envoy_access.log - log_format: - text_format_source: - inline_string: >- - "[ACCESS] %REQ(:METHOD)% %REQ(:PATH)% %PROTOCOL% - status=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% - upstream=%UPSTREAM_HOST% cluster=%UPSTREAM_CLUSTER% - dst=%REQ(x-gateway-destination-endpoint)% - duration=%DURATION%ms\n" - route_config: - name: route - virtual_hosts: - - name: vh - domains: ["*"] - routes: - - match: { prefix: "/" } - route: - cluster: original_dst - timeout: 0s - http_filters: - - name: envoy.filters.http.ext_proc - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - grpc_service: - envoy_grpc: - cluster_name: epp - timeout: 10s - # message_timeout caps how long Envoy will wait for any - # one ext_proc message ack from EPP. Generation can take - # many seconds; 1000s mirrors the upstream llm-d guide. - message_timeout: 1000s - # FULL_DUPLEX_STREAMED for both directions: the dev EPP - # (ghcr.io/llm-d/llm-d-router-endpoint-picker-dev:main) - # does not ack BUFFERED body mode and Envoy times out - # with 504. Trailer modes also have to be SEND for the - # request lifecycle to terminate cleanly. - processing_mode: - request_header_mode: SEND - response_header_mode: SEND - request_body_mode: FULL_DUPLEX_STREAMED - response_body_mode: FULL_DUPLEX_STREAMED - request_trailer_mode: SEND - response_trailer_mode: SEND - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: epp - type: STATIC - connect_timeout: 1s - # Without an explicit circuit_breakers block Envoy applies its DEFAULT - # per-cluster thresholds: max_connections/max_pending_requests/max_requests - # all = 1024. The ext_proc filter holds one long-lived HTTP/2 stream to - # this EPP cluster for the ENTIRE lifetime of every request (header -> - # FULL_DUPLEX_STREAMED body -> response -> trailers, up to message_timeout - # 1000s). At conc4096 that means only 1024 active + 1024 pending = 2048 - # requests ever reach EPP; the rest get a fast 500 and never see the - # engine. This capped every high-tpt run at exactly 1024 completions - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: epp - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: { address: 127.0.0.1, port_value: 9002 } - - name: original_dst - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - # Same default-1024 circuit-breaker trap as the epp cluster above: this is - # the cluster that actually carries request bodies to the picked - # prefill/decode endpoint, so its budget must also clear max-concurrency. - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - original_dst_lb_config: - use_http_header: true - http_header_name: x-gateway-destination-endpoint - -admin: - address: - socket_address: { address: 0.0.0.0, port_value: 9901 } diff --git a/benchmarks/llm-d/epp-config.yaml b/benchmarks/llm-d/epp-config.yaml deleted file mode 100644 index 3ff0eea87f..0000000000 --- a/benchmarks/llm-d/epp-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Default EPP scheduling config (fallback when CONFIG_FILE is unset). -# -# Mirrors the upstream llm-d well-lit-path P/D guide: -# guides/pd-disaggregation/router/pd-disaggregation.values.yaml -# in github.com/llm-d/llm-d. Plugins, scheduling profiles, and scorer -# weights are unchanged from upstream. -# -# Single delta vs upstream: file-discovery. The upstream guide assumes -# a Kubernetes control plane drives endpoint discovery; in our SLURM -# setup the coordinator node writes /tmp/endpoints.yaml at job start -# (see benchmarks/multi_node/llm-d/README.md) and EPP loads it via the -# file-discovery plugin instead. - -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - # Endpoint discovery (replaces upstream's K8s discovery). - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - # P/D routing - identical to upstream pd-disaggregation guide. - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: active-request-scorer - - type: max-score-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: max-score-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: max-score-picker - -dataLayer: - discovery: - pluginRef: file-disc diff --git a/benchmarks/llm-d/extract-binaries.sh b/benchmarks/llm-d/extract-binaries.sh deleted file mode 100755 index ed9f00e577..0000000000 --- a/benchmarks/llm-d/extract-binaries.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# -# Extract the EPP, pd-sidecar, and Envoy binaries from their source -# images into $LLMD_BIN_DIR, so a STOCK vllm/vllm-openai image can be run -# with these mounted in at runtime (see job.slurm) instead of rebuilding -# a combined image on every vLLM version bump. -# -# Run this ONCE on a host with docker (arm64, or x86 with --platform -# emulation), and re-run only when a binary's source image changes in -# benchmarks/llm-d/binaries.env. Idempotent: overwrites in place. -# -# Usage: -# ./extract-binaries.sh # uses binaries.env defaults -# LLMD_BIN_DIR=/some/dir ./extract-binaries.sh -# LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh # for an x86 test - -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "$HERE/binaries.env" - -echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)" -mkdir -p "$LLMD_BIN_DIR" - -# (source image, path inside image, output filename) -extract() { - local image="$1" src="$2" out="$3" - echo " $out <- $image:$src" - local cid - cid="$(docker create --platform "$LLMD_BIN_PLATFORM" "$image")" - # shellcheck disable=SC2064 - trap "docker rm -f '$cid' >/dev/null 2>&1 || true" RETURN - docker cp "$cid:$src" "$LLMD_BIN_DIR/$out" - chmod +x "$LLMD_BIN_DIR/$out" -} - -extract "$EPP_FROM_IMAGE" "$EPP_BIN_PATH" epp -extract "$ROUTING_SIDECAR_IMAGE" "$ROUTING_SIDECAR_BIN_PATH" pd-sidecar -extract "$ENVOY_FROM_IMAGE" "$ENVOY_BIN_PATH" envoy - -echo "Done. Contents of $LLMD_BIN_DIR:" -ls -la "$LLMD_BIN_DIR" - -# Linking sanity reminder: epp/pd-sidecar are Go (static); envoy is a -# dynamically-linked C++ binary. Verify inside the stock vLLM container -# that it resolves: ldd /usr/local/bin/envoy (no "not found" lines). -echo -echo "NOTE: verify 'ldd $LLMD_BIN_DIR/envoy' resolves cleanly inside the" -echo " target vLLM image before relying on the mounted-binary path." diff --git a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh b/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh deleted file mode 100755 index f3eca3f27b..0000000000 --- a/benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Wrapper for the DeepSeek-V4-Pro GB200 llmd-vllm P/D disagg benchmark -# (mid-curve 1P1D and high-tpt 2P1D). Sibling of gptoss_fp4_h200_llmd-vllm.sh - -# same shape, different topology (GB200 = 4 GPUs/node, role spans 2 nodes; -# H200 = 8 GPUs/node, role on a single node). The runner resolves this script via -# SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" -# from launch_gb200-nv.sh. - -set -euo pipefail - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1 - -# GB200 = 4 GPUs per node (Grace+Blackwell). The shared submit.sh -# defaults GPUS_PER_NODE to 8, which is wrong for this SKU and would -# overshoot DP_SIZE = nodes * 8. -export GPUS_PER_NODE="${GPUS_PER_NODE:-4}" - -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Worker count per role (Option B multi-engine). Prefer an explicit -# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else fall -# back to the matrix num-worker fields (PREFILL_NUM_WORKERS/DECODE_NUM_WORKERS); -# else 1 (single engine = unchanged 1P+1D / mid-curve). submit.sh reads these. -export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}" -export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}" - -JOB_ID=$(bash ./submit.sh \ - "$PREFILL_NODES" \ - "$DECODE_NODES" \ - "$ISL" "$OSL" "${CONC_LIST// /x}" inf \ - "$RANDOM_RANGE_RATIO") - -if [[ -z "$JOB_ID" ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml deleted file mode 100644 index e29106062d..0000000000 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml +++ /dev/null @@ -1,192 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, low-latency P/D disagg via llmd-vllm. -# -# Latency end of the curve. Engine config mirrors the upstream srt-slurm -# recipe disagg-gb200-low-latency.yaml (less the dynamo-only frontend keys), -# served via the InferenceX llmd-vllm path (no NATS/etcd infra node). It is -# the direct sibling of the dsv4-fp4-gb200-dynamo-vllm conc=1 low-latency -# point (same topology: 1 prefill DEP8 + 1 decode pure-TP8). -# -# Topology (set by the nvidia-master.yaml key via additional-settings): -# low-latency key (conc 1): 1P1D = 1 prefill + 1 decode engine, 4 GB200 -# nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# -# The ONLY difference from the mid-curve-megamoe recipe is the DECODE engine: -# 1. DECODE is PURE TP=8 (tensor-parallel-size 8, NO DP, NO EP) - ONE model -# replica across 8 GPUs / 2 GB200 nodes, launched headless multi-node TP -# (leader + follower). This is the latency-optimal decode: no all-to-all -# MoE routing, no DP-attention. `enable-expert-parallel: false` + -# decode ep:1/dp-attn:false in the master key is what selects the -# pure-TP path in server.sh. Decode knobs are latency-sized: max-num-seqs -# 256, max-num-batched-tokens 256, max-cudagraph-capture-size 256, -# FULL_DECODE_ONLY graphs, gpu-memory-utilization 0.9. -# 2. PREFILL is IDENTICAL to the mid-curve/high-tpt prefill (copied verbatim, -# DEP8 with enable-ep-weight-filter + moe-backend deep_gemm_mega_moe). An -# earlier srt-slurm-style low-latency prefill (weight-offload, no filter, -# moe-backend auto) warmed up ALL expert shapes (~4069-step DeepGEMM) and -# never became ready in the bring-up window; the mid-curve prefill's -# weight-filter collapses that warmup, and it is already proven on GB200. -# -# ---- EPP scheduling config (identical to the mid-curve recipe) ---- -# EPP routing is engine-config-agnostic, so the scheduling profiles match the -# upstream generic wide-ep-lws router config byte-for-byte: -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Prefill: DEP=8 (TP=1, DP=8, EP). IDENTICAL to the mid-curve/high-tpt prefill -# block - copied verbatim (all args + env). enable-ep-weight-filter + -# moe-backend deep_gemm_mega_moe keep only the served experts resident and -# collapse the DeepGEMM warmup, so bring-up is fast. A pure srt-slurm-style -# low-latency prefill (weight-offload, no filter, moe-backend auto) instead -# warmed up ALL expert shapes (~4069-step DeepGEMM) and never became ready -# inside the bring-up window (e2e runs 28742781217 / 28744015942). Only the -# DECODE below differs from mid-curve: pure TP=8 instead of DEP=8. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.95 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL resets under - # load (matters for the low-middle 1P4D point; harmless at conc=1). Matches - # the DEP8 recipe + srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# Decode: PURE TP=8 (one replica across 8 GPUs / 2 nodes). NO EP, NO DP. -# Mirrors the srt-slurm low-latency decode vllm_config. tensor-parallel-size -# is supplied by server.sh from the master key (tp:8); the recipe sets EP off. -decode: - tp: 8 - enable-expert-parallel: false - extra-args: >- - --kv-cache-dtype fp8 - --max-model-len 16384 - --max-num-seqs 256 - --max-num-batched-tokens 256 - --max-cudagraph-capture-size 256 - --gpu-memory-utilization 0.9 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --no-disable-hybrid-kv-cache-manager - --enable-sleep-mode - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml deleted file mode 100644 index 89292bc09c..0000000000 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml +++ /dev/null @@ -1,196 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, MegaMOE P/D disagg via llmd-vllm. -# -# This file defines only the COMMON per-engine shape + engine env. The -# multi-engine topology (how many prefill/decode engines, node counts, -# concurrency) is set by the nvidia-master.yaml key via additional-settings, -# NOT here - so the same recipe backs BOTH supported topologies: -# mid-curve key (conc 256/512/1024): 1P1D = 1 prefill + 1 decode engine, -# 4 GB200 nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# high-tpt key (conc 4096): 2P1D = 2 prefill + 1 decode engine, -# 6 GB200 nodes / 24 GPUs. PREFILL_NODES=4 PREFILL_WORKERS=2 DECODE_NODES=2. -# -# Per-engine shape (identical in every engine; GB200 = 4 GPUs/node): -# Each engine is DEP8 - TP=1 DP=8 EP=8, spanning 2 nodes (LWS_GROUP_SIZE=2), -# with DP-attention + deep_gemm_mega_moe + EP weight filter. Decode engines -# additionally use FULL_DECODE_ONLY graphs. -# -# Selected via additional-settings: CONFIG_FILE=, GPUS_PER_NODE=4, -# plus the PREFILL_NODES/PREFILL_WORKERS/DECODE_NODES for the chosen topology. -# -# -# ---- EPP scheduling config (matched to upstream wide-ep-lws) ---- -# Scorers match the upstream generic wide-ep-lws router config -# (llm-d guides/wide-ep-lws/router/wide-ep-lws.values.yaml): -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -# EPP routing is engine-config-agnostic, so this recipe file also backs the -# high-tpt (2P1D conc4096) key. -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Engine config mirrors the upstream srt-slurm mid-curve-megamoe recipe -# byte-for-byte (less the dynamo-only frontend keys). The MegaMOE shape -# differs from the low-latency 1P+1D in three ways: -# 1. Decode is DEP=8 (TP=1, DP=8, EP) instead of pure TP=8. This is -# what the "MegaMOE" name refers to: every MoE rank runs experts -# locally with DP-attention to hide cross-rank latency, instead of -# sharding tensors. -# 2. moe-backend = deep_gemm_mega_moe on both sides (the higher-perf -# path; the simple recipe doesn't override moe-backend). -# 3. enable-ep-weight-filter on both sides keeps only the experts a -# rank actually serves resident on GPU; without it, DEP=8 OOMs -# because every rank tries to hold all experts. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL connection - # resets under the conc4096 warmup burst. max-tpt in-sweep died with - # "gloo tcp Connection reset by peer" -> EngineDeadError (no OOM/CUDA/HW - # fault); a longer PG timeout keeps a momentarily-stalled peer from being - # dropped. Value matches srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -decode: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --max-num-seqs 512 - --max-num-batched-tokens 512 - --max-cudagraph-capture-size 512 - --gpu-memory-utilization 0.9 - --max-model-len 9280 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - # See prefill note: harden the decode's cross-node DP8/EP8 coordination - # against transient gloo resets under the conc4096 warmup burst. - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md deleted file mode 100644 index 81dbd51995..0000000000 --- a/benchmarks/multi_node/llm-d/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# llmd-vllm multi-node SLURM scaffolding - -This directory holds the SLURM-side orchestration for the `llmd-vllm` -benchmark framework. It mirrors the AMD `sglang-disagg` pattern under -`benchmarks/multi_node/amd_utils/` (NOT the Dynamo / srt-slurm pattern): -InferenceX itself owns the SLURM job, no vendor multi-node tool involved. - -| File | Role | -|---|---| -| `submit.sh` | sbatch wrapper. Validates env, exports tuning vars, returns `JOB_ID`. May read `slurm.time_limit` from the recipe to override `TIME_LIMIT`. | -| `job.slurm` | sbatch entrypoint. Allocates `PREFILL_NODES + DECODE_NODES` nodes, derives per-node IPs, runs one Docker container per node via `srun`, threads role assignment env into each. | -| `server.sh` | Per-node entry. Reads `NODE_RANK = SLURM_PROCID`, picks role, starts vLLM (with the wide-EP / DeepEP / NIXL flag set from the llm-d wide-EP-lws guide), starts the pd-sidecar on each leader, and on the decode leader additionally writes `endpoints.yaml`, starts EPP + Envoy, runs `benchmark_serving.py`, and `scancel`s the job. | - -## Topology - -For an `xP` prefill nodes / `yD` decode nodes run, total nodes = `xP + yD`. -There is **no dedicated coordinator node**. The decode leader doubles as -the coordinator (EPP + Envoy + bench), exactly like the AMD path's -"decode rank 0" coordinator role. - -| Rank | Role | -|---|---| -| `0` | prefill leader (`LWS_WORKER_INDEX=0`, DP rank 0) + pd-sidecar | -| `1 .. xP-1` | prefill workers | -| `xP` | decode leader + pd-sidecar + EPP + Envoy + benchmark client | -| `xP+1 .. xP+yD-1` | decode workers | - -Each instance (prefill or decode) is one vLLM engine spanning multiple -nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, -GPUS_PER_NODE=8` you get DP=16 prefill + DP=16 decode (the wide-EP -reference). Per-rank split: `--data-parallel-size 16 ---data-parallel-size-local 8 --data-parallel-start-rank -$((LWS_WORKER_INDEX * 8))`. - -## How `endpoints.yaml` is generated (file-discovery contract) - -The EPP runs in **no-Kubernetes mode**, using the `file-discovery` plugin -from `llm-d-inference-scheduler` (branch `filediscovery-4`). At startup, -it reads `/tmp/endpoints.yaml`. The file lists every backend the EPP can -route to, with role labels. - -The file is generated at runtime by `server.sh` on the decode leader -(rank `PREFILL_NODES`). Because all node IPs are only known after -`sbatch` allocates the job, the file cannot be baked into the image and -is not part of the repo. - -Generation flow: - -1. `submit.sh` calls `sbatch -N (xP+yD)`. `sbatch` allocates nodes. -2. `job.slurm` resolves each node's IP via `srun ip route get 1.1.1.1`, - slices them into `PREFILL_LEADER_IP` (= IPS[0]) and `DECODE_LEADER_IP` - (= IPS[PREFILL_NODES]), and passes both into the container as env - vars. -3. On the decode leader, `server.sh` writes `/tmp/endpoints.yaml` - inside the container with one entry per node: - - ```yaml - endpoints: - - name: prefill-0 - address: - port: "8200" # vLLM port (EPP hits prefill vLLM directly) - labels: - llm-d.ai/role: prefill - - name: prefill-1 # one entry per prefill node - address: - port: "8200" - labels: - llm-d.ai/role: prefill - - name: decode-0 - address: - port: "8000" # pd-sidecar port - labels: - llm-d.ai/role: decode - - name: decode-1 # one entry per decode node - address: - port: "8000" - labels: - llm-d.ai/role: decode - ``` - -4. The EPP (started immediately after) loads the file via - `dataLayer.discovery.pluginRef: file-disc` (see - `benchmarks/llm-d/epp-config.yaml`). The plugin enumerates the - endpoints into the EPP datastore before the EPP starts serving - `ext_proc`, so Envoy never gets a request before discovery is ready. -5. The `disagg-profile-handler` in the EPP config uses `prefill-filter` - and `decode-filter` to pick the right backend per request phase, - matching on the `llm-d.ai/role` label. - -### Why one entry per node (not per DP rank) - -Each instance is a vLLM engine that spans multiple nodes via -`--data-parallel-hybrid-lb`. With hybrid-lb, every node runs its own -api-server (prefill) or pd-sidecar (decode) on a fixed port -(`VLLM_PORT`=8200 / `SIDECAR_PORT`=8000) and internally load-balances -its own local DP ranks. So `server.sh` `add_role()` emits one endpoint -per node (`prefill-0..N-1`, `decode-0..M-1`) and lets EPP fan out -across nodes, while each node's hybrid-lb spreads work across its local -ranks. - -Per-node (rather than per-DP-rank) emission is what the multi-engine -high-tpt topology needs: with `PREFILL_WORKERS>1` the prefill nodes form -several independent DEP engines, and one endpoint per node is what -exposes every engine to EPP. Emitting per-DP-rank instead would address -ranks directly and bypass each node's internal hybrid-lb. - -### Live reload - -`watchFile: false` in `epp-config.yaml`. Endpoints are static for the -job lifetime - no reason to pay for `fsnotify` here. Set `watchFile: -true` (and rewrite `/tmp/endpoints.yaml` from the coordinator) only if -you want to drain or add an instance mid-run. - -### Validation rules (enforced by the plugin) - -- `address` must be a literal IPv4 address (no IPv6, no hostnames). -- `port` is a string in `1..65535`. -- File capped at 1 MiB. -- Names must be unique within their namespace (we use the default - namespace, so they must be globally unique in the file). - -The IPs we collect from `ip route get 1.1.1.1` are always IPv4 on the -H200 / B200 cluster's primary fabric. If you point at a different -interface and it returns an IPv6 address, EPP will reject the file at -startup. - -## Recipe files - -`benchmarks/multi_node/llm-d-recipes/.yaml` is selected via -`CONFIG_FILE=.yaml` in the master config's `additional-settings`. -Each recipe carries: - -- top-level `plugins:` / `schedulingProfiles:` / `dataLayer:` - fed into - the EPP via `--config-file`. Lets you change routing strategy without - rebuilding the image. -- `prefill:` / `decode:` blocks with `extra-args` (appended to the vLLM - launch command on each node of that role) and `env` (exported before - vLLM starts). -- `slurm.time_limit` - overrides `TIME_LIMIT` for that recipe. - -When `CONFIG_FILE` is unset or the file is missing, the EPP falls back -to `/etc/epp/config.yaml` baked into the image, and vLLM runs with no -extra flags beyond the wide-EP common set in `server.sh`. diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm deleted file mode 100644 index f260c1a6fb..0000000000 --- a/benchmarks/multi_node/llm-d/job.slurm +++ /dev/null @@ -1,252 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=llm-d-bench -#SBATCH --ntasks-per-node=1 -# --output, --error, -N, -n, --time set by submit.sh -# -# Allocates PREFILL_NODES + DECODE_NODES nodes, derives per-node IPs, then -# srun-runs server.sh inside one Docker container per node. NODE_RANK -# (= SLURM_PROCID) drives role selection inside server.sh. - -set -euo pipefail - -echo "=== llm-d job start ===" -echo "UTC: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" - -# Repo root. $(pwd) = the sbatch submit dir, which the wrapper sets to -# benchmarks/multi_node/llm-d/ before invoking submit.sh, so 3 up = -# repo root. Using $(dirname "$0") would resolve to a SLURM staging -# copy under /var/spool/... and miss the checkout entirely. -DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) -export DI_REPO_DIR -echo "REPO DIR: ${DI_REPO_DIR}" - -ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) -echo "Allocated nodes ($TOTAL_NODES): $(echo "$ALL_NODES" | tr '\n' ' ')" - -if [[ "$TOTAL_NODES" -ne "$NUM_NODES" ]]; then - echo "Error: SLURM allocated $TOTAL_NODES nodes, expected $NUM_NODES" >&2 - exit 1 -fi - -# Per-node IPs in rank order. -IPS=() -for NODE in $ALL_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --nodelist="$NODE" \ - bash -c 'ip route get 1.1.1.1 | awk "/src/ {print \$7}"') - IPS+=("$IP") -done -echo "Node IPs: ${IPS[*]}" - -if ! srun --nodes="$NUM_NODES" --ntasks-per-node=1 test -d "$MODEL_DIR"; then - echo "FATAL: model not found on all $NUM_NODES nodes: $MODEL_DIR" >&2 - exit 1 -fi - -# Rank slicing: -# prefill leader = rank 0 -# prefill workers = ranks 1 .. PREFILL_NODES-1 -# decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) -# decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 -PREFILL_LEADER_IP="${IPS[0]}" -DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" - -# DP leader addresses for vLLM --data-parallel-address (rank 0 of each instance). -PREFILL_DP_ADDR="$PREFILL_LEADER_IP" -DECODE_DP_ADDR="$DECODE_LEADER_IP" - -ALL_IP_LIST=$(IFS=,; echo "${IPS[*]}") - -SANITIZED_USER=$(echo "$USER" | tr -c 'a-zA-Z0-9_.-' '_') -DOCKER_CONT_NAME="llmd_bench_${SANITIZED_USER}_${SLURM_JOB_ID}" -export DOCKER_CONT_NAME -: "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" -DOCKER_MOUNT_PATH="/workspace" - -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. -BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! - -# Container engine: 'docker' (default) for clusters where the SLURM -# user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' -# for clusters that require enroot+pyxis srun (e.g. gb200-nv, where the -# SLURM user is not in the docker group). Same env contract on both -# paths; only the per-node container launch differs. -LLMD_CONTAINER_ENGINE="${LLMD_CONTAINER_ENGINE:-docker}" -echo "LLMD_CONTAINER_ENGINE=$LLMD_CONTAINER_ENGINE" - -if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then - # One docker run per node, one task per node. server.sh dispatches by NODE_RANK. - srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -euo pipefail -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f || true - -exec docker run --rm \ - --init \ - --entrypoint bash \ - --stop-timeout 10 \ - --user 0:0 \ - --network host \ - --ipc host \ - --gpus all \ - --ulimit memlock=-1 --ulimit stack=67108864 \ - --shm-size 32G \ - --cap-add SYS_PTRACE --cap-add IPC_LOCK --cap-add SYS_RAWIO \ - --device /dev/infiniband \ - --security-opt seccomp=unconfined \ - --privileged \ - -v ${MODEL_DIR}:/models:ro \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \ - -e SLURM_JOB_ID=\$SLURM_JOB_ID \ - -e NODE_RANK=\$SLURM_PROCID \ - -e NUM_NODES=$NUM_NODES \ - -e PREFILL_NODES=$PREFILL_NODES \ - -e DECODE_NODES=$DECODE_NODES \ - -e PREFILL_WORKERS=$PREFILL_WORKERS \ - -e DECODE_WORKERS=$DECODE_WORKERS \ - -e ALL_IPS=$ALL_IP_LIST \ - -e PREFILL_LEADER_IP=$PREFILL_LEADER_IP \ - -e DECODE_LEADER_IP=$DECODE_LEADER_IP \ - -e PREFILL_DP_ADDR=$PREFILL_DP_ADDR \ - -e DECODE_DP_ADDR=$DECODE_DP_ADDR \ - -e MODEL_DIR=/models \ - -e MODEL_NAME=$MODEL_NAME \ - -e GPUS_PER_NODE=$GPUS_PER_NODE \ - -e PREFILL_DP_SIZE=$PREFILL_DP_SIZE \ - -e DECODE_DP_SIZE=$DECODE_DP_SIZE \ - -e BENCH_INPUT_LEN=$BENCH_INPUT_LEN \ - -e BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN \ - -e BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY \ - -e BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE \ - -e BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO \ - -e BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER \ - -e BENCHMARK_LOGS_DIR=/benchmark_logs \ - -e RUN_EVAL=$RUN_EVAL \ - -e EVAL_ONLY=$EVAL_ONLY \ - -e EVAL_CONC=$EVAL_CONC \ - -e FRAMEWORK=$FRAMEWORK \ - -e PRECISION=$PRECISION \ - -e MODEL_PREFIX=$MODEL_PREFIX \ - -e RUNNER_TYPE=$RUNNER_TYPE \ - -e RESULT_FILENAME=$RESULT_FILENAME \ - -e SPEC_DECODING=$SPEC_DECODING \ - -e IS_MULTINODE=$IS_MULTINODE \ - -e CONFIG_FILE=$CONFIG_FILE \ - --name \"${DOCKER_CONT_NAME}_\$SLURM_PROCID\" \ - \"\$DOCKER_IMAGE_NAME\" -lc ' - set -o pipefail - ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log - ' -" - - srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true - -elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then - : "${LLMD_SQUASH_FILE:?LLMD_SQUASH_FILE must be set when LLMD_CONTAINER_ENGINE=pyxis}" - if [[ ! -s "$LLMD_SQUASH_FILE" ]]; then - echo "Error: LLMD_SQUASH_FILE does not exist or is empty: $LLMD_SQUASH_FILE" >&2 - exit 1 - fi - - # Pre-export non-SLURM static vars so pyxis --container-env= - # whitelisting can pick them up. Vars that already come from submit.sh - # via export are no-ops here; we re-state them for clarity. ALL_IPS - # is computed locally above as $ALL_IP_LIST. - export ALL_IPS="$ALL_IP_LIST" - export NUM_NODES PREFILL_NODES DECODE_NODES PREFILL_LEADER_IP DECODE_LEADER_IP - export PREFILL_WORKERS DECODE_WORKERS - export PREFILL_DP_ADDR DECODE_DP_ADDR MODEL_NAME GPUS_PER_NODE - export PREFILL_DP_SIZE DECODE_DP_SIZE - export BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY - export BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER - export RUN_EVAL EVAL_ONLY EVAL_CONC FRAMEWORK PRECISION MODEL_PREFIX - export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE - - PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" - PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS" - PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE" - PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE" - PYXIS_ENV_LIST+=",BENCH_INPUT_LEN,BENCH_OUTPUT_LEN,BENCH_MAX_CONCURRENCY" - PYXIS_ENV_LIST+=",BENCH_REQUEST_RATE,BENCH_RANDOM_RANGE_RATIO,BENCH_NUM_PROMPTS_MULTIPLIER" - PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,FRAMEWORK,PRECISION,MODEL_PREFIX" - PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" - - PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" - PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" - PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro" - - # Optional: mount the epp / pd-sidecar / envoy binaries from a shared - # filesystem instead of relying on them being baked into the image. - # This lets a STOCK vllm/vllm-openai image be used directly (no - # combined-image rebuild per vLLM version bump) - see - # benchmarks/llm-d/binaries.env + extract-binaries.sh. Each mount is - # gated on the file existing, so this is a no-op when the binaries - # have not been extracted (the baked-image path keeps working), and - # harmless when they have (mounting a binary over the identical one). - # shellcheck source=/dev/null - [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ - source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" - for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR:-}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then - PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" - echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" - fi - done - - # MODEL_DIR / BENCHMARK_LOGS_DIR / NODE_RANK are translated to their - # in-container values inside bash -lc (host MODEL_DIR is the source - # path of the bind mount, but server.sh expects /models inside). - srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - --container-image="$LLMD_SQUASH_FILE" \ - --container-name="${DOCKER_CONT_NAME}" \ - --container-mounts="$PYXIS_MOUNTS" \ - --container-remap-root \ - --container-writable \ - --container-env="$PYXIS_ENV_LIST" \ - bash -lc ' -set -o pipefail -echo "Rank $SLURM_PROCID on $(hostname)" -export NODE_RANK="$SLURM_PROCID" -export MODEL_DIR=/models -export BENCHMARK_LOGS_DIR=/benchmark_logs -'"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' - -else - echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 - exit 1 -fi diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh deleted file mode 100755 index 7e189767bc..0000000000 --- a/benchmarks/multi_node/llm-d/server.sh +++ /dev/null @@ -1,638 +0,0 @@ -#!/usr/bin/env bash -# -# Per-node entrypoint for the llmd-vllm wide-EP P/D disagg benchmark. -# NODE_RANK is set by srun (= $SLURM_PROCID) in job.slurm. -# -# Roles: -# Rank 0 -> prefill leader (DP rank 0) -# Ranks 1 .. PREFILL_NODES-1 -> prefill workers -# Rank PREFILL_NODES -> decode leader (DP rank 0) + pd-sidecar -# + EPP + Envoy + benchmark client (coordinator) -# Ranks PREFILL_NODES+1 .. -> decode workers -# -# Each instance (prefill or decode) is one vLLM engine spanning its role's nodes -# via --data-parallel-hybrid-lb; the leader accepts traffic, workers serve their -# local DP ranks. - -set -euo pipefail - -source /workspace/benchmarks/benchmark_lib.sh - -# ---------------------------------------------------------------- -# Config + service ports -# ---------------------------------------------------------------- -NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-0}}" -PREFILL_NODES="${PREFILL_NODES:-1}" -DECODE_NODES="${DECODE_NODES:-1}" -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" -VLLM_PORT=8200 -SIDECAR_PORT=8000 -ENVOY_PORT=8080 -EPP_GRPC_PORT=9002 -EPP_HEALTH_PORT=9003 -EPP_METRICS_PORT=9090 - -# Weights live at MODEL_DIR (/models, bind-mounted by job.slurm). MODEL_NAME is -# the served-model-name, not a filesystem path. -MODEL="${MODEL_DIR}" - -# ---------------------------------------------------------------- -# Host IP + default interface -# ---------------------------------------------------------------- -# Resolved without iproute2 (`ip` is absent on the arm64 vLLM base); python3's -# socket layer exposes the kernel's source-IP / iface choice. -_HOST_INFO=$(python3 -c ' -import socket -s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -try: - s.connect(("1.1.1.1", 80)) - ip = s.getsockname()[0] -finally: - s.close() -iface = "" -try: - with open("/proc/net/route") as f: - f.readline() # header - for line in f: - parts = line.split() - if parts[1] == "00000000": # default route dest - iface = parts[0]; break -except OSError: - pass -print(ip, iface) -' 2>/dev/null) || true -HOST_IP=$(echo "$_HOST_INFO" | awk '{print $1}') -DEFAULT_IFACE=$(echo "$_HOST_INFO" | awk '{print $2}') -DEFAULT_IFACE="${DEFAULT_IFACE:-eth0}" - -VLLM_LOG="/benchmark_logs/vllm_rank${NODE_RANK}.log" -SIDECAR_LOG="/benchmark_logs/sidecar_rank${NODE_RANK}.log" -EPP_LOG="/benchmark_logs/epp.log" -ENVOY_LOG="/benchmark_logs/envoy.log" - -echo "=== rank=$NODE_RANK host=$HOST_IP model=$MODEL ===" - -# ---------------------------------------------------------------- -# Role + topology (Option B engine grouping) -# ---------------------------------------------------------------- -# A role's nodes split into PREFILL_WORKERS / DECODE_WORKERS independent DP/EP -# engines, each spanning (role_nodes / role_workers) nodes with its own DP -# coordinator (leader IP) and rank range. workers=1 => one engine over all role -# nodes (1P+1D / mid-curve); >1 => high-tpt (e.g. 2 prefill : 1 decode, DEP8 each). -PREFILL_WORKERS="${PREFILL_WORKERS:-1}" -DECODE_WORKERS="${DECODE_WORKERS:-1}" -IFS=',' read -r -a _ALL_IPS <<< "${ALL_IPS:-}" - -if [[ "$NODE_RANK" -lt "$PREFILL_NODES" ]]; then - ROLE="prefill" - DP_SIZE="$PREFILL_DP_SIZE" - _local_rank="$NODE_RANK" - _nodes_per_worker=$(( PREFILL_NODES / PREFILL_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -elif [[ "$NODE_RANK" -lt $((PREFILL_NODES + DECODE_NODES)) ]]; then - ROLE="decode" - DP_SIZE="$DECODE_DP_SIZE" - _local_rank=$(( NODE_RANK - PREFILL_NODES )) - _nodes_per_worker=$(( DECODE_NODES / DECODE_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( PREFILL_NODES + (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -else - echo "ERROR: NODE_RANK=$NODE_RANK out of range" >&2 - exit 1 -fi - -# Each engine's DP coordinator = its leader node's IP (ALL_IPS[leader rank]); -# fall back to the role leader env when ALL_IPS is unset. -if [[ -n "${_ALL_IPS[${_group_leader_rank}]:-}" ]]; then - DP_ADDR="${_ALL_IPS[${_group_leader_rank}]}" -elif [[ "$ROLE" == "prefill" ]]; then - DP_ADDR="$PREFILL_DP_ADDR" -else - DP_ADDR="$DECODE_DP_ADDR" -fi - -DP_SIZE_LOCAL="$GPUS_PER_NODE" -START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) - -# Defaults: TP=1, DP=role_total, EP on (the H200 1P+1D shape). Recipe overrides below. -TP_SIZE=1 -ROLE_ENABLE_EP=true - -echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" - -# ---------------------------------------------------------------- -# Recipe: per-role serve args + env (/etc/llmd-recipes/$CONFIG_FILE) -# ---------------------------------------------------------------- -# Per-role keys: tp (int -> --tensor-parallel-size), enable-expert-parallel -# (bool -> --enable-expert-parallel + DP/wide-EP knobs), extra-args (appended -# verbatim), env (map, exported before vllm serve). Absent keys keep the -# defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. -ROLE_EXTRA_ARGS="" -if [[ -n "${CONFIG_FILE:-}" ]]; then - RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - echo "Loading $ROLE recipe from $RECIPE_PATH" - eval "$(python3 - <&2 - fi -fi -echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" - -# ---------------------------------------------------------------- -# Transport env (NCCL / UCX / NIXL), recipe-overridable -# ---------------------------------------------------------------- -export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-$DEFAULT_IFACE} -export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-$DEFAULT_IFACE} -export VLLM_SKIP_P2P_CHECK=1 -# Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes -# across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe -# may set this to 0. -export VLLM_RANDOMIZE_DP_DUMMY_INPUTS=${VLLM_RANDOMIZE_DP_DUMMY_INPUTS:-1} -export VLLM_USE_DEEP_GEMM=1 -# Cold-start budget for engine-core readiness. DSV4-Pro on GB200 cold-starts in -# ~9-11 min (weight load + DeepGEMM JIT warmup + cudagraph capture + NIXL/UCX -# handshake); the 600s vLLM default is too tight, so allow 30 min. -export VLLM_ENGINE_READY_TIMEOUT_S=${VLLM_ENGINE_READY_TIMEOUT_S:-1800} -# DeepGEMM JIT links -l:libcuda.so.1 at warmup; the compat dir is on -# LD_LIBRARY_PATH (runtime) but not LIBRARY_PATH (link time). Prepend it, plus -# the arch-specific toolkit lib dir resolved from `uname -m`. -case "$(uname -m)" in - aarch64|arm64) _NCT_LIB=/usr/lib/aarch64-linux-gnu ;; - *) _NCT_LIB=/usr/lib/x86_64-linux-gnu ;; -esac -export LIBRARY_PATH=/usr/local/cuda/compat:${_NCT_LIB}:${LIBRARY_PATH:-} -export VLLM_NIXL_SIDE_CHANNEL_HOST="$HOST_IP" -export VLLM_LOGGING_LEVEL=${VLLM_LOGGING_LEVEL:-INFO} - -# Pin NIXL/UCX to IB verbs (rc) so cross-node KV rides the IB HCAs (job.slurm -# exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. -export UCX_TLS=${UCX_TLS:-cuda_copy,cuda_ipc,rc} - -# ---------------------------------------------------------------- -# Wide-EP NVSHMEM / ibgda env (only when an engine spans >1 node) -# ---------------------------------------------------------------- -# Single-node-per-role recipes avoid DeepEP / NVSHMEM ibgda, so leave these off -# there to avoid triggering ibgda code paths that are not needed. -if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - export NVIDIA_GDRCOPY=enabled - # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override - # NVSHMEM_REMOTE_TRANSPORT to none. - export NVSHMEM_REMOTE_TRANSPORT=${NVSHMEM_REMOTE_TRANSPORT:-ibgda} - export NVSHMEM_IB_ENABLE_IBGDA=${NVSHMEM_IB_ENABLE_IBGDA:-true} - export NVSHMEM_SYMMETRIC_SIZE=${NVSHMEM_SYMMETRIC_SIZE:-16G} - export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME=${NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME:-$DEFAULT_IFACE} - # NVSHMEM ignores NVSHMEM_HCA_PE_MAPPING when NVSHMEM_HCA_LIST is set, so - # clear the latter when the recipe provides an explicit PE mapping. - if [[ -n "${NVSHMEM_HCA_PE_MAPPING:-}" ]]; then - unset NVSHMEM_HCA_LIST 2>/dev/null || true - fi -fi - -# ---------------------------------------------------------------- -# Bring up vLLM engine (every node) -# ---------------------------------------------------------------- -# KV role: prefill=producer, decode=consumer (override via KV_ROLE_OVERRIDE). -if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then - KV_ROLE="$KV_ROLE_OVERRIDE" -elif [[ "$ROLE" == "prefill" ]]; then - KV_ROLE="kv_producer" -else - KV_ROLE="kv_consumer" -fi -KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" - -COMMON_ARGS=( - --port "$VLLM_PORT" - --served-model-name "$MODEL_NAME" - --trust-remote-code - --disable-access-log-for-endpoints=/health,/metrics - --tensor-parallel-size "$TP_SIZE" - --kv_transfer_config "$KV_TRANSFER_CONFIG" -) -# A single frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps -# throughput, so run several. Incompatible with --headless, so it is the one -# flag the headless-worker branch below drops. Overridable via LLMD_API_SERVER_COUNT. -# LB is hybrid: --data-parallel-hybrid-lb; one api-server per node internally -# load-balances its local DP ranks -> ONE serving port (VLLM_PORT) per node, so -# the local rank-0 health port is always VLLM_PORT. -HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="${LLMD_API_SERVER_COUNT:-4}" -# Multiple frontends only help the DP (wide-EP) path, where they load-balance -# across the node's local DP ranks. A pure-TP engine has a single core with one -# frontend, so it keeps the default count (also avoids --api-server-count -# interacting with the --headless multi-node TP launch below). Every DEP8 node -# gets it; pure-TP nodes get none. -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=(--api-server-count "$API_SERVER_COUNT") -fi -# Set to 1 by the pure-TP multi-node branch below on --headless followers, which -# run no local api-server; gates the post-launch health wait. -IS_HEADLESS_FOLLOWER=0 -# --moe-backend is model-specific (DSR1-FP8 wants deep_gemm, gpt-oss-MXFP4 -# rejects it), so each recipe sets its own via extra-args. - -# EP/DP knobs only when the recipe enables EP. Pure tensor-parallel roles skip -# them (vLLM rejects --data-parallel-size combined with TP>1). -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=( - --enable-expert-parallel - --data-parallel-size "$DP_SIZE" - ) - if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - COMMON_ARGS+=( - --data-parallel-hybrid-lb - --data-parallel-size-local "$DP_SIZE_LOCAL" - --data-parallel-address "$DP_ADDR" - --data-parallel-rpc-port 5555 - --data-parallel-start-rank "$START_RANK" - ) - fi -elif [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - # Pure TP spanning >1 node (e.g. DSV4-Pro decode TP=8 on GB200's 4-GPU - # nodes): use vLLM's headless multi-node API - leader binds --master-addr, - # followers join --headless with matching --nnodes/--node-rank. - COMMON_ARGS+=( - --master-addr "$DP_ADDR" - --nnodes "$LWS_GROUP_SIZE" - --node-rank "$LWS_WORKER_INDEX" - ) - if [[ "$LWS_WORKER_INDEX" -gt 0 ]]; then - COMMON_ARGS+=(--headless) - IS_HEADLESS_FOLLOWER=1 - fi -fi - -echo "Starting vLLM ($ROLE) DP=$DP_SIZE local=$DP_SIZE_LOCAL start_rank=$START_RANK group_size=$LWS_GROUP_SIZE" -# shellcheck disable=SC2086 -vllm serve "$MODEL" "${COMMON_ARGS[@]}" $ROLE_EXTRA_ARGS \ - > "$VLLM_LOG" 2>&1 & -VLLM_PID=$! - -# Each rank waits for its own engine /health before continuing (for wide-EP this -# blocks the bench until worker DP shards are up; a no-op for single-node). A -# pure-TP --headless follower runs no local api-server (only the TP-group leader -# binds a health port, and its /health only reports ready once every TP worker -# has joined), so it skips the wait and stays alive via the final `wait`. -if [[ "$IS_HEADLESS_FOLLOWER" -eq 1 ]]; then - echo "vLLM headless TP follower on rank $NODE_RANK (worker_index=$LWS_WORKER_INDEX): no local api-server, skipping health wait" -else - wait_for_server_ready --port "$HEALTH_PORT" --server-log "$VLLM_LOG" --server-pid "$VLLM_PID" - echo "vLLM ready on rank $NODE_RANK ($ROLE worker_index=$LWS_WORKER_INDEX, health port $HEALTH_PORT)" -fi - -# ---------------------------------------------------------------- -# Bring up pd-sidecar (every decode node) -# ---------------------------------------------------------------- -# The sidecar forwards a prefill request, reads kv_transfer_params from vLLM's -# response, then hits its local decode vLLM, whose NIXLv2 connector pulls KV -# directly from prefill vLLM. -# -# DEP8 (EP on, hybrid-LB): every decode node runs an api-server for its local DP -# ranks, so every decode node runs a sidecar and endpoints.yaml lists one decode -# endpoint per node. Pure-TP: only the TP-group leader has an api-server -# (followers are --headless), so only the leader runs a sidecar and only leaders -# are listed as endpoints. -if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then - SIDECAR_CONNECTOR="nixlv2" - SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" - --kv-connector="$SIDECAR_CONNECTOR" --secure-proxy=false - --enable-prefiller-sampling) - SIDECAR_HEALTH_PORT="$SIDECAR_PORT" - echo "Starting pd-sidecar (decode node_rank=$NODE_RANK worker_index=$LWS_WORKER_INDEX): ${SIDECAR_FLAGS[*]}" - pd-sidecar "${SIDECAR_FLAGS[@]}" > "$SIDECAR_LOG" 2>&1 & - SIDECAR_PID=$! - wait_for_server_ready --port "$SIDECAR_HEALTH_PORT" --server-log "$SIDECAR_LOG" --server-pid "$SIDECAR_PID" - echo "pd-sidecar ready on $HOST_IP:$SIDECAR_HEALTH_PORT" -fi - -# ================================================================ -# Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval -# ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT - - # ---- Write endpoints.yaml (file-discovery) ---- - # namespace must match EPP's --pool-namespace (file-discovery filters by it; - # the schema default 'default' would drop every entry). See README.md. - python3 - < DEP8 hybrid-LB (an api-server per node); -# EP off => pure-TP (only each TP-group leader has an api-server). -decode_ep = ('$ROLE_ENABLE_EP' == 'true') -VLLM_PORT = int('$VLLM_PORT') -SIDECAR_PORT = int('$SIDECAR_PORT') -# ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. -prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -decode_ips = all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']] -endpoints = [] - -def add_role(role, ips, base_port, group_size=1): - # group_size == 1: one endpoint per node (DEP8 hybrid-LB: each node's - # api-server / sidecar load-balances its local DP ranks). - # group_size > 1: one endpoint per TP-group leader (pure-TP: followers are - # --headless with no api-server), i.e. every group_size-th node IP. - serving_ips = ips[::group_size] if group_size > 1 else ips - for i, ip in enumerate(serving_ips): - endpoints.append({'name': f'{role}-{i}', 'namespace': NS, 'address': ip, - 'port': str(base_port), 'labels': {'llm-d.ai/role': role}}) - -# Prefill (DEP8 in every current recipe): one endpoint per node, EPP hits vLLM -# directly (VLLM_PORT). Decode: EPP hits the pd-sidecar (SIDECAR_PORT); one -# endpoint per node for DEP8, or one per TP-group leader for pure-TP. -add_role('prefill', prefill_ips, VLLM_PORT) -decode_group = 1 if decode_ep else max(1, dn // decode_workers) -add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) -print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open('/tmp/endpoints.yaml').read()) -PY - - # ---- Bring up EPP ---- - # Config: when a recipe is set, project it down to the keys EPP's strict - # decoder accepts (it rejects the per-role vLLM / slurm keys); else use the - # default mounted at /etc/epp/config.yaml. - if [[ -n "$CONFIG_FILE" && -f "/etc/llmd-recipes/$CONFIG_FILE" ]]; then - EPP_CONFIG="/tmp/epp-config-from-recipe.yaml" - python3 - < ext_proc trips, Envoy 500s). - epp \ - --pool-name=epp \ - --pool-namespace=inferencex \ - --config-file="$EPP_CONFIG" \ - --grpc-port="$EPP_GRPC_PORT" \ - --grpc-health-port="$EPP_HEALTH_PORT" \ - --metrics-port="$EPP_METRICS_PORT" \ - --secure-serving=false \ - > "$EPP_LOG" 2>&1 & - EPP_PID=$! - - # Wait for EPP's gRPC listener before starting Envoy (Envoy's ext_proc dials - # it). gRPC has no plain HTTP /health, so probe the TCP listener directly. - echo "Waiting for EPP on 127.0.0.1:$EPP_GRPC_PORT" - EPP_WAIT_DEADLINE=$(( $(date +%s) + 60 )) - until (echo > "/dev/tcp/127.0.0.1/$EPP_GRPC_PORT") 2>/dev/null; do - if ! kill -0 "$EPP_PID" 2>/dev/null; then - echo "ERROR: EPP died before binding $EPP_GRPC_PORT" >&2 - exit 1 - fi - if [[ "$(date +%s)" -ge "$EPP_WAIT_DEADLINE" ]]; then - echo "ERROR: EPP did not bind $EPP_GRPC_PORT within 60s" >&2 - exit 1 - fi - sleep 1 - done - echo "EPP listening on $EPP_GRPC_PORT" - - # ---- Bring up Envoy ---- - envoy -c /etc/envoy/envoy.yaml > "$ENVOY_LOG" 2>&1 & - ENVOY_PID=$! - - # Probe admin /ready (9901); /health on :8080 routes through ext_proc -> EPP - # and needs request routing metadata, so it would 503 until traffic flows. - echo "Waiting for envoy admin on 127.0.0.1:9901/ready" - ENVOY_WAIT_DEADLINE=$(( $(date +%s) + 120 )) - until [[ "$(curl --output /dev/null --silent --write-out '%{http_code}' \ - "http://127.0.0.1:9901/ready" 2>/dev/null)" == "200" ]]; do - if ! kill -0 "$ENVOY_PID" 2>/dev/null; then - echo "ERROR: envoy died before admin /ready returned 200" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - if [[ "$(date +%s)" -ge "$ENVOY_WAIT_DEADLINE" ]]; then - echo "ERROR: envoy admin /ready did not return 200 within 120s" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - sleep 2 - done - echo "Envoy admin ready; listener should be on $ENVOY_PORT" - - # ---- Gate on ALL prefill vLLM /health endpoints (cross-node) ---- - # Prefill ranks wait on their own local /health; wait_for_server_ready only - # probes localhost, so the decode leader polls the prefill nodes here. - # endpoints.yaml lists one prefill endpoint per node, so with PREFILL_WORKERS>1 - # (multiple independent DP engines) EVERY prefill node must be probed, not just - # IPS[0]. curl gets an explicit connect/max timeout so a blackholed endpoint - # trips the deadline instead of hanging the whole run (a single timeout-less - # curl once wedged a 2P run for 7h before it was cancelled). - _prefill_ips=( "${_ALL_IPS[@]:0:${PREFILL_NODES}}" ) - [[ ${#_prefill_ips[@]} -gt 0 ]] || _prefill_ips=( "$PREFILL_LEADER_IP" ) - - # On failure, dump enough to tell a server-not-ready problem (TCP connects but - # /health is slow) apart from a network/subnet problem (TCP connect refused or - # times out, host unreachable). Uses only bash builtins + coreutils: the arm64 - # serving image has no iproute2 / nc. - _diag_prefill_endpoint() { - local ip="$1" port="$2" - { - echo "=== NET DIAG: decode -> prefill ${ip}:${port} ===" - echo "[diag] decode node: $(hostname -f 2>/dev/null || hostname) local-ips: $(hostname -I 2>/dev/null)" - echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE:-} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-}" - # Local source address the kernel would pick to reach ip: reveals which - # subnet/interface the route uses, without needing iproute2. - python3 - "$ip" <<'PY' 2>&1 || true -import socket, struct, sys -ip = sys.argv[1] -# Source address the kernel picks to reach ip (which local iface/subnet). -try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect((ip, 80)) - print(f"[diag] local source IP toward {ip}: {s.getsockname()[0]}") - s.close() -except Exception as e: - print(f"[diag] no route to {ip}: {e}") -# Subnet check via /proc/net/route (no iproute2 needed). Fields are little-endian -# hex. On-link route (gateway 0.0.0.0) == same subnet; a non-zero gateway means the -# target is reached across a router == DIFFERENT subnet. -def _ntoa(v): # little-endian int -> dotted quad - return socket.inet_ntoa(struct.pack(' best[0]: - best = (plen, iface, gw, dest) - if best is None: - print(f"[diag] {ip}: NO matching route -> unreachable") - else: - plen, iface, gw, dest = best - if gw == 0: - print(f"[diag] {ip}: ON-LINK via {iface}, subnet {_ntoa(dest)}/{plen} -> SAME subnet (no router hop)") - else: - print(f"[diag] {ip}: via GATEWAY {_ntoa(gw)} on {iface}, subnet {_ntoa(dest)}/{plen} -> DIFFERENT subnet (crosses a router)") -except Exception as e: - print(f"[diag] subnet classification failed: {e}") -PY - # L4: raw TCP connect (bash /dev/tcp, 5s cap) - port reachable-but-slow - # vs unreachable/filtered. - if timeout 5 bash -c "exec 3<>/dev/tcp/${ip}/${port}" 2>/dev/null; then - echo "[diag] TCP connect ${ip}:${port} OK -> port open; server up but /health slow/not-ready (NOT a network issue)" - else - echo "[diag] TCP connect ${ip}:${port} FAILED/timed out -> closed, filtered, or unreachable (LIKELY network/subnet/firewall issue)" - fi - # L3: ICMP reachability, if ping is present. - if command -v ping >/dev/null 2>&1; then - ping -c 2 -W 2 "$ip" 2>&1 || echo "[diag] ping ${ip} failed (ICMP blocked or host down)" - fi - # Verbose HTTP connect detail (DNS/connect/TLS timing, HTTP status). - curl -v --connect-timeout 5 --max-time 8 "http://${ip}:${port}/health" 2>&1 || true - echo "=== END NET DIAG ${ip}:${port} ===" - } >&2 - } - - # Log the decode->prefill target layout up front so a subnet/interface - # mismatch is visible even on a run that eventually succeeds. Every prefill - # node serves on VLLM_PORT (hybrid LB). - echo "[diag] decode-leader $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}" - echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s): ${_prefill_ips[*]}" - PREFILL_WAIT_DEADLINE=$(( $(date +%s) + 300 )) - for _pidx in "${!_prefill_ips[@]}"; do - _pip="${_prefill_ips[$_pidx]}" - _pport="$VLLM_PORT" - until curl --output /dev/null --silent --fail \ - --connect-timeout 5 --max-time 10 \ - "http://$_pip:$_pport/health"; do - if [[ "$(date +%s)" -ge "$PREFILL_WAIT_DEADLINE" ]]; then - echo "ERROR: prefill vLLM at $_pip:$_pport not ready within 5 min" >&2 - _diag_prefill_endpoint "$_pip" "$_pport" - exit 1 - fi - sleep 5 - done - echo "Prefill vLLM at $_pip:$_pport is ready" - done - echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" - - # ---- Benchmark sweep (one run per concurrency level) ---- - # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). - IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" - # GPU counts embedded in the result filename as _gpus_/_ctx_/_gen_ tokens so the - # CI "Process result" step (benchmark-multinode-tmpl.yml) can parse them and run - # process_result.py for llm-d -- same filename convention as amd_utils/bench.sh. - # ctx = prefill GPUs, gen = decode GPUs; nodes*GPUS_PER_NODE is correct for any - # PREFILL_WORKERS/DECODE_WORKERS split (e.g. high-tpt 2P -> 16 prefill GPUs). - _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) - _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) - _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) - for max_concurrency in "${CONCURRENCIES[@]}"; do - num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) - [[ "$num_prompts" -lt 16 ]] && num_prompts=16 - # Bench against Envoy (EPP routes to decode; the sidecar pulls from - # prefill via NIXL). --bench-serving-dir = the /workspace repo bind-mount; - # --tokenizer = /models (served-model-name is not a valid HF repo id). - # DSV4-Pro needs trust-remote-code + tokenizer-mode deepseek_v4 (the older - # transformers wheel does not register it) + chat template / --dsv4 to - # match the dynamo-vllm bench prompt formatting. - bench_extra_args=() - if [[ "${MODEL_NAME,,}" == *"deepseek-v4"* ]]; then - bench_extra_args+=( - --trust-remote-code - --tokenizer-mode deepseek_v4 - --use-chat-template - --dsv4 - ) - fi - - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. - run_benchmark_serving \ - --bench-serving-dir /workspace \ - --tokenizer /models \ - --model "$MODEL_NAME" \ - --port "$ENVOY_PORT" \ - --backend openai \ - --input-len "$BENCH_INPUT_LEN" \ - --output-len "$BENCH_OUTPUT_LEN" \ - --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ - --result-dir "$BENCHMARK_LOGS_DIR/" \ - "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" - done - - # ---- Eval (optional) ---- - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - # Concurrency for the eval and, crucially, for the concurrency stamped - # into meta_env.json. run_eval/append_lm_eval_summary read - # EVAL_CONCURRENT_REQUESTS and CONC (not EVAL_CONC), so mirror the AMD - # multi-node servers: use the workflow-provided EVAL_CONC when set, else - # fall back to the max of the (x-delimited) BENCH_MAX_CONCURRENCY list. - # Exporting CONC makes meta_env.json's "conc" match what - # utils/evals/validate_scores.py --expected-concs verifies; without it - # CONC is empty, the metadata records conc=1, and score verification - # fails ("eval metadata concurrency does not match workflow request") - # even when accuracy passes. - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(printf '%s' "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - export CONC="${EVAL_CONCURRENT_REQUESTS}" - # Run from /workspace (the repo bind-mount) so results*.json land where - # the host-side workflow checks look; the subshell keeps the cd local. - ( - cd /workspace - run_eval --framework lm-eval --port "$ENVOY_PORT" - append_lm_eval_summary - ) - fi - - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait -fi diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh deleted file mode 100755 index 11c34736f8..0000000000 --- a/benchmarks/multi_node/llm-d/submit.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -# -# Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Modeled after benchmarks/multi_node/amd_utils/submit.sh; prints JOB_ID on -# stdout so the runner can poll for completion. -# -# Topology (matches the llm-d wide-EP guide reference): -# 1 prefill instance with DP=PREFILL_NODES * GPUS_PER_NODE -# 1 decode instance with DP=DECODE_NODES * GPUS_PER_NODE -# each instance spans PREFILL_NODES / DECODE_NODES nodes via vLLM -# --data-parallel-hybrid-lb. Total nodes = PREFILL_NODES + DECODE_NODES. - -set -euo pipefail - -# Repo root resolved from this script's location, so paths below are -# independent of the caller's $PWD (the wrapper cd's into llm-d/ before -# invoking this script). -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -check_env() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - echo "Error: ${name} not set" >&2 - exit 1 - fi -} - -check_env SLURM_ACCOUNT -check_env SLURM_PARTITION -check_env TIME_LIMIT -check_env MODEL_PATH -check_env MODEL_NAME -check_env CONTAINER_IMAGE -check_env RUNNER_NAME -check_env BENCHMARK_LOGS_DIR - -PREFILL_NODES=$1 -DECODE_NODES=$2 -ISL=$3 -OSL=$4 -CONCURRENCIES=$5 -REQUEST_RATE=${6:-inf} -RANDOM_RANGE_RATIO=${7:-0.8} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export MODEL_DIR=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export NUM_NODES=$NUM_NODES -export PREFILL_NODES=$PREFILL_NODES -export DECODE_NODES=$DECODE_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -# Worker count per role (Option B): the role's nodes are split into this many -# INDEPENDENT DP/EP engines (default 1 = one engine over all role nodes). Each -# engine spans role_nodes/workers nodes, so DP_SIZE is PER-ENGINE. Matches how -# dynamo/AMD and upstream oci-high-tpt run 2P high-tpt (2 prefill : 1 decode). -export PREFILL_WORKERS="${PREFILL_WORKERS:-1}" -export DECODE_WORKERS="${DECODE_WORKERS:-1}" -if (( PREFILL_NODES % PREFILL_WORKERS != 0 )); then - echo "Error: PREFILL_NODES ($PREFILL_NODES) not divisible by PREFILL_WORKERS ($PREFILL_WORKERS)" >&2 - exit 1 -fi -if (( DECODE_NODES % DECODE_WORKERS != 0 )); then - echo "Error: DECODE_NODES ($DECODE_NODES) not divisible by DECODE_WORKERS ($DECODE_WORKERS)" >&2 - exit 1 -fi -export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE )) -export DECODE_DP_SIZE=$(( DECODE_NODES / DECODE_WORKERS * GPUS_PER_NODE )) -export BENCH_INPUT_LEN=$ISL -export BENCH_OUTPUT_LEN=$OSL -export BENCH_MAX_CONCURRENCY=$CONCURRENCIES -export BENCH_REQUEST_RATE=$REQUEST_RATE -export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO -# Match the AMD multinode default. -export BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" - -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-llmd-vllm}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-none}" -export IS_MULTINODE="${IS_MULTINODE:-true}" -export CONFIG_FILE="${CONFIG_FILE:-}" - -# Recipe may override SLURM time limit (longer topologies need more wall time). -if [[ -n "$CONFIG_FILE" ]]; then - RECIPE_PATH="${REPO_ROOT}/benchmarks/multi_node/llm-d-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - RECIPE_TIME=$(python3 -c " -import yaml, sys -r = yaml.safe_load(open('$RECIPE_PATH')) -t = r.get('slurm', {}).get('time_limit', '') -print(t) -" 2>/dev/null || true) - [[ -n "$RECIPE_TIME" ]] && TIME_LIMIT="$RECIPE_TIME" - fi -fi - -mkdir -p "$BENCHMARK_LOGS_DIR" - -JOB_ID=$(sbatch \ - --parsable \ - --exclusive \ - -N "$NUM_NODES" \ - -n "$NUM_NODES" \ - --ntasks-per-node=1 \ - --gres=gpu:"$GPUS_PER_NODE" \ - --time "$TIME_LIMIT" \ - --partition "$SLURM_PARTITION" \ - --account "$SLURM_ACCOUNT" \ - --job-name "$RUNNER_NAME" \ - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" \ - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" \ - "$(dirname "$0")/job.slurm") - -if [[ -z "$JOB_ID" ]]; then - echo "Error: sbatch failed" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/install-vllm-router.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/install-vllm-router.sh new file mode 100644 index 0000000000..622579120e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/install-vllm-router.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# The pinned backend image already carries vLLM and every pure-Python Router +# dependency. Install the released official Router wheel (and its compiled +# orjson dependency) without replacing the serving runtime. +python3 -m pip install --no-cache-dir "vllm-router==0.1.15" + +command -v vllm-router >/dev/null +vllm-router --help >/dev/null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml new file mode 100644 index 0000000000..0911fb4903 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml @@ -0,0 +1,164 @@ +name: "dsv4-gb200-vllm-router-1p1d-dep8-dep8-c256-c512-c1024" +setup_script: "install-vllm-router.sh" + +model: + path: "deepseek-v4-pro" + container: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + frameworks: + vllm: "0.26" + vllm-router: "0.1.15" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: nixl + dp_launch_mode: per_node + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + numa-bind: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 9280 + max-num-seqs: 512 + max-num-batched-tokens: 512 + max-cudagraph-capture-size: 512 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + prefill_environment: + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + decode_environment: {} + +environment: + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: "NVL" + NCCL_NET_GDR_C2C: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" + NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + TILELANG_CLEANUP_TEMP_FILES: "1" + NVIDIA_GDRCOPY: "enabled" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + container-writable: "" + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir=/logs/vllm_isl_8192_osl_1024 + mkdir -p "$result_dir" + for concurrency in 256 512 1024; do + num_prompts=$((concurrency * 10)) + num_warmups=$((concurrency * 2)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "$SRT_FRONTEND_URL" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --served-model-name deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer /model \ + --tokenizer-mode deepseek_v4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 96 \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --num-warmups "$num_warmups" \ + --ignore-eos \ + --trust-remote-code \ + --use-chat-template \ + --dsv4 \ + --seed 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" + done diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml new file mode 100644 index 0000000000..fc849ec467 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml @@ -0,0 +1,152 @@ +name: "dsv4-gb200-vllm-router-1p1d-tp8-tp8-c1" +setup_script: "install-vllm-router.sh" + +model: + path: "deepseek-v4-pro" + container: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + frameworks: + vllm: "0.26" + vllm-router: "0.1.15" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: nixl + dp_launch_mode: per_node + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + numa-bind: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 16384 + max-num-seqs: 256 + max-num-batched-tokens: 256 + max-cudagraph-capture-size: 256 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + prefill_environment: {} + decode_environment: {} + +environment: + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: "NVL" + NCCL_NET_GDR_C2C: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" + NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + TILELANG_CLEANUP_TEMP_FILES: "1" + NVIDIA_GDRCOPY: "enabled" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + container-writable: "" + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir=/logs/vllm_isl_8192_osl_1024 + mkdir -p "$result_dir" + concurrency=1 + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "$SRT_FRONTEND_URL" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --served-model-name deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer /model \ + --tokenizer-mode deepseek_v4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 96 \ + --num-prompts 16 \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --num-warmups 2 \ + --ignore-eos \ + --trust-remote-code \ + --use-chat-template \ + --dsv4 \ + --seed 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml new file mode 100644 index 0000000000..88b5895729 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml @@ -0,0 +1,161 @@ +name: "dsv4-gb200-vllm-router-3p1d-dep8-dep8-c4096" +setup_script: "install-vllm-router.sh" + +model: + path: "deepseek-v4-pro" + container: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + frameworks: + vllm: "0.26" + vllm-router: "0.1.15" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 6 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: nixl + dp_launch_mode: per_node + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + numa-bind: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 9280 + max-num-seqs: 512 + max-num-batched-tokens: 512 + max-cudagraph-capture-size: 512 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + prefill_environment: + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + decode_environment: {} + +environment: + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: "NVL" + NCCL_NET_GDR_C2C: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" + NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + TILELANG_CLEANUP_TEMP_FILES: "1" + NVIDIA_GDRCOPY: "enabled" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + container-writable: "" + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir=/logs/vllm_isl_8192_osl_1024 + mkdir -p "$result_dir" + concurrency=4096 + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "$SRT_FRONTEND_URL" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --served-model-name deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer /model \ + --tokenizer-mode deepseek_v4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 96 \ + --num-prompts 40960 \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --num-warmups 8192 \ + --ignore-eos \ + --trust-remote-code \ + --use-chat-template \ + --dsv4 \ + --seed 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_32_ctx_24_gen_8.json" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 61e9b7200a..7f451e4cd4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4420,16 +4420,17 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true -# TODO: change image to official llmd image. -# Build source: benchmarks/llm-d/Dockerfile. +# Keep the historical key so this refresh remains attached to the existing +# performance series. The implementation now uses srt-slurm's native vLLM +# backend and official vLLM Router frontend; no llm-d sidecars are launched. dsv4-fp4-gb200-llmd-vllm: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb200 precision: fp4 - framework: llmd-vllm - router: { name: llm-d-router, version: "0.9.0" } + framework: vllm + router: { name: vllm-router, version: "0.1.15" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4438,47 +4439,42 @@ dsv4-fp4-gb200-llmd-vllm: - isl: 8192 osl: 1024 search-space: - # Low latency: 1 prefill DEP8 + 1 decode TP8. + # Low latency: one TP8 prefill + one TP8 decode. vLLM Router has one + # DP-rank expansion factor for both pools, so mixed DEP8/TP8 cannot be + # represented without sending decode traffic to nonexistent DP ranks. - spec-decoding: "none" conc-list: [1] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" + additional-settings: [] # Mid curve: 1 prefill DEP8 + 1 decode DEP8. - spec-decoding: "none" - conc-list: [256, 512, 1024] + # The recipe deliberately runs all three points after one model load. + conc-list: [1] prefill: num-worker: 1 tp: 1 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml" decode: num-worker: 1 tp: 1 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" + additional-settings: [] # Max throughput: 3 prefill DEP8 + 1 decode DEP8. - spec-decoding: "none" @@ -4489,18 +4485,13 @@ dsv4-fp4-gb200-llmd-vllm: ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=6" - - "PREFILL_WORKERS=3" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml" decode: num-worker: 1 tp: 1 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" + additional-settings: [] # MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image # and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index e42027abb5..8d4426cb5d 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -126,22 +126,6 @@ Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchm Do not ship one side alone. `srtctl` reads the recipe, while matrix generation reads the master config. Recipe-only changes can mislabel results. Master-only changes do not alter the deployed recipe. -## Register an llm-d recipe - -Sources: [`benchmarks/llm-d/README.md`](../benchmarks/llm-d/README.md), [`benchmarks/multi_node/llm-d/README.md`](../benchmarks/multi_node/llm-d/README.md), [`llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/), and the current [`llmd-vllm` benchmark wrapper](../benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh). - -llm-d is not the srt-slurm path: InferenceX owns the Slurm allocation and starts one container per node. - -1. Copy the nearest YAML under [`benchmarks/multi_node/llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) and set EPP plugins/scheduling, role-specific `extra-args`/`env`, and optional `slurm.time_limit`. -2. Add/update the `llmd-vllm` master entry. Set `multinode: true`, `disagg: true`, router metadata, `kv-p2p-transfer`, prefill/decode worker topology, concurrency, and `CONFIG_FILE=.yaml` in `additional-settings`. -3. Keep `PREFILL_NODES`, `DECODE_NODES`, `GPUS_PER_NODE`, and worker counts consistent with the allocation and with each role's DP/TP/EP layout. -4. Confirm [`submit.sh`](../benchmarks/multi_node/llm-d/submit.sh) → [`job.slurm`](../benchmarks/multi_node/llm-d/job.slurm) → [`server.sh`](../benchmarks/multi_node/llm-d/server.sh) propagation and the selected wrapper/launcher route. -5. Verify file discovery. The decode leader creates `/tmp/endpoints.yaml`. Prefill endpoints use vLLM port 8200, while decode endpoints use sidecar port 8000. Names must be unique, addresses must be literal IPv4, and ports must be strings in `1..65535`. -6. Confirm EPP loads discovery before Envoy receives traffic and role labels select the proper prefill/decode backends. -7. Generate the key, inspect topology and `additional-settings`, then append the changelog. - -A missing/unset `CONFIG_FILE` silently selects the image's `/etc/epp/config.yaml` fallback and removes recipe-specific vLLM flags. Treat that as a validation failure unless fallback is explicitly intended. - ## Update an image Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants), the matching master configs, runtime scripts, and checked-in recipes. @@ -150,9 +134,8 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 2. Find every affected config key, runtime script, Dockerfile, and checked-in recipe. Do not assume the master YAML is the only image reference. 3. Update the master `image` and any required env vars, flags, package versions, or patches as one coherent change. 4. For srt-slurm, update `model.container` and keep it identical to master `image`. -5. For llm-d, distinguish the serving image selected by the master config from the build source in [`benchmarks/llm-d/Dockerfile`](../benchmarks/llm-d/Dockerfile). Update both only when the build contract changes. -6. Append a changelog entry selecting all affected keys (wildcards are allowed when intentional), including old/new versions and material runtime changes. -7. Generate each affected family and verify no stale tag survives in its runtime path. +5. Append a changelog entry selecting all affected keys (wildcards are allowed when intentional), including old/new versions and material runtime changes. +6. Generate each affected family and verify no stale tag survives in its runtime path. ## Add or change MTP diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0ebd17fc68..4af2c07441 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -126,22 +126,6 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 不得只提交一侧：`srtctl` 读取配方，而矩阵生成读取主配置。仅改配方可能给结果贴错标签；仅改主配置不会改变实际部署的配方。 -## 注册 llm-d 配方 - -来源：[`benchmarks/llm-d/README.md`](../benchmarks/llm-d/README.md)、[`benchmarks/multi_node/llm-d/README.md`](../benchmarks/multi_node/llm-d/README.md)、[`llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) 和当前 [`llmd-vllm` 基准 wrapper](../benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh)。 - -llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并在每个节点启动一个容器。 - -1. 复制 [`benchmarks/multi_node/llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) 下最接近的 YAML，设置 EPP plugin/scheduling、角色特定 `extra-args`/`env`，以及可选 `slurm.time_limit`。 -2. 添加/更新 `llmd-vllm` 主条目。设置 `multinode: true`、`disagg: true`、router 元数据、`kv-p2p-transfer`、prefill/decode worker 拓扑、并发，以及 `additional-settings` 中的 `CONFIG_FILE=.yaml`。 -3. 保持 `PREFILL_NODES`、`DECODE_NODES`、`GPUS_PER_NODE` 和 worker 数与 allocation 及各角色 DP/TP/EP 布局一致。 -4. 确认 [`submit.sh`](../benchmarks/multi_node/llm-d/submit.sh) → [`job.slurm`](../benchmarks/multi_node/llm-d/job.slurm) → [`server.sh`](../benchmarks/multi_node/llm-d/server.sh) 的传递，以及所选 wrapper/launcher 路由。 -5. 验证文件发现：decode leader 生成 `/tmp/endpoints.yaml`；prefill endpoint 使用 vLLM 端口 8200，decode endpoint 使用 sidecar 端口 8000；名称唯一；地址为 IPv4 字面量；端口是 `1..65535` 范围内的字符串。 -6. 确认 EPP 在 Envoy 收到流量前完成 discovery 加载，且角色标签为请求阶段选择正确的 prefill/decode backend。 -7. 生成 key，检查拓扑和 `additional-settings`，再追加 changelog。 - -`CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 - ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 @@ -150,9 +134,8 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并 2. 找出所有受影响的配置 key、运行时脚本、Dockerfile 和检入配方。不要假设主 YAML 是唯一镜像引用。 3. 将主配置 `image` 与所需 env、参数、软件包版本或补丁作为一个一致变更更新。 4. 对 srt-slurm，更新 `model.container` 并保持其与主配置 `image` 完全一致。 -5. 对 llm-d，区分主配置选择的服务镜像和 [`benchmarks/llm-d/Dockerfile`](../benchmarks/llm-d/Dockerfile) 中的构建来源；仅在构建契约变化时同时更新两者。 -6. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 -7. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 +5. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 +6. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 ## 添加或修改 MTP diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 4d55473ce6..4689d2794e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -99,59 +99,6 @@ import_squash() { ) || exit 1 } -if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then - export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" - export MODEL_NAME="deepseek-ai/DeepSeek-V4-Pro" - else - echo "Unsupported MODEL_PREFIX/PRECISION for llmd-vllm on GB200: $MODEL_PREFIX/$PRECISION" >&2 - exit 1 - fi - - SQUASH_FILE="${SQUASH_DIR}/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash "$SQUASH_FILE" "$IMAGE" - - export LLMD_CONTAINER_ENGINE=pyxis - export LLMD_SQUASH_FILE="$SQUASH_FILE" - - export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" - mkdir -p "$BENCHMARK_LOGS_DIR" - - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh" - BENCH_SCRIPT="benchmarks/multi_node/${SCRIPT_NAME}" - if [[ ! -f "$BENCH_SCRIPT" ]]; then - echo "Error: llm-d wrapper not found: $BENCH_SCRIPT" >&2 - exit 1 - fi - - JOB_ID=$(bash "$BENCH_SCRIPT") - if [[ -z "$JOB_ID" ]]; then - echo "Error: failed to submit llm-d job" >&2 - exit 1 - fi - echo "Submitted llm-d job: $JOB_ID" - - trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP - - LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 - - while IFS= read -r -d '' result_file; do - copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 - done < <(find "$BENCHMARK_LOGS_DIR" -name "${RESULT_FILENAME}*.json" -print0 2>/dev/null) - - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR" -type d -name eval_results -print -quit 2>/dev/null) - if [[ -z "$EVAL_DIR" ]]; then - EVAL_DIR="$BENCHMARK_LOGS_DIR/eval_results" - fi - copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 - fi - - scancel "$JOB_ID" 2>/dev/null || true - exit 0 -fi - # MODEL_PATH: Override with pre-downloaded paths on GB200 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared GB200 cluster. @@ -223,7 +170,7 @@ elif [[ $FRAMEWORK == "dynamo-trt" ]]; then echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: gptoss, dsr1, kimik2.5, or glm5" exit 1 fi -elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then +elif [[ $FRAMEWORK == "dynamo-vllm" || $FRAMEWORK == "vllm" ]]; then if [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" @@ -242,7 +189,13 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then # params (e.g. ffn.experts.w13_input_scale), which KeyErrors at load. # The lowercase Lustre sibling is the FP8 checkpoint, so name the # CamelCase FP4 path explicitly (Linux is case-sensitive). - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro" + if [[ "$FRAMEWORK" == "vllm" ]]; then + # Preserve the checkpoint used by the historical llm-d series. + # It is pre-staged at the same node-local path on every GB200 node. + export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" + else + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro" + fi export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" MODEL_PATHS_EXTRA=' "deepseek-v4-pro-mxfp4": "/mnt/lustre01/models/DeepSeek-V4-Pro"' elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then @@ -258,7 +211,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp4, minimaxm3/fp8" + echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for vllm paths: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp4, minimaxm3/fp8" exit 1 fi else @@ -271,10 +224,10 @@ uses_watchtower_shared_fs() { case "$MODEL_PREFIX" in minimaxm2.5|minimaxm3|kimik2.5|kimik3|qwen3.5|glm5.2) return 0 ;; esac - # dsv4 multinode runs only under dynamo-vllm on watchtower, which likewise + # DSV4 multinode runs under Dynamo-vLLM or direct vLLM Router on watchtower. # needs the srt-slurm workspace/outputs on a compute-visible shared FS # (the runner home is not cross-mounted to compute nodes). - [[ "$FRAMEWORK" == "dynamo-vllm" && "$MODEL_PREFIX" == "dsv4" ]] && return 0 + [[ ( "$FRAMEWORK" == "dynamo-vllm" || "$FRAMEWORK" == "vllm" ) && "$MODEL_PREFIX" == "dsv4" ]] && return 0 return 1 } @@ -471,6 +424,22 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then # `recipes/vllm/deepseek-v4/deepseek-v4/...` in that case). mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 +elif [[ $FRAMEWORK == "vllm" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then + SRT_SLURM_PIN="76e7d76961b2dcb27cb05c1e9e0910ceb75104ec" + git clone https://github.com/SemiAnalysisAI/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$SRT_SLURM_PIN" || { + echo "Error: srt-slurm HEAD does not match SRT_SLURM_PIN=$SRT_SLURM_PIN" >&2 + exit 1 + } + mkdir -p recipes/vllm/deepseek-v4-pro/GB200/8k1k configs + cp -rT \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k" \ + recipes/vllm/deepseek-v4-pro/GB200/8k1k + cp \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/install-vllm-router.sh" \ + configs/install-vllm-router.sh elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then if [[ "$USES_DCGM_POWER" == "1" ]]; then # Note (wenyao): on this cluster the DSV4-Pro checkpoint lives on the @@ -584,6 +553,29 @@ if uses_watchtower_shared_fs; then SRTCTL_ROOT="$SRT_REPO_DIR" fi +# Resolve a compute-visible InferenceX checkout before rendering mounts. The +# fixed-sequence custom benchmark invokes the checked-in benchmark_serving.py +# directly; it must therefore see the same checkout from every allocated node. +export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" +if uses_watchtower_shared_fs; then + WORKSPACE_FS_TYPE=$(findmnt -n -o FSTYPE -T "$GITHUB_WORKSPACE" 2>/dev/null || true) + if [[ "$WORKSPACE_FS_TYPE" == "lustre" ]]; then + echo "Using existing Lustre-backed INFMAX_WORKSPACE=$INFMAX_WORKSPACE" + else + SHARED_INFMAX_WORKSPACE="${SHARED_BASE}/infmax-workspace-${RUN_KEY}" + mkdir -p "$SHARED_INFMAX_WORKSPACE" || exit 1 + rsync -a --delete \ + --exclude='.git/' \ + --exclude='srt-slurm*/' \ + --exclude='outputs/' \ + --exclude='LOGS/' \ + --exclude='*.sqsh' \ + "${GITHUB_WORKSPACE}/" "${SHARED_INFMAX_WORKSPACE}/" || exit 1 + export INFMAX_WORKSPACE="$SHARED_INFMAX_WORKSPACE" + echo "Staged node-local workspace to INFMAX_WORKSPACE=$INFMAX_WORKSPACE" + fi +fi + # Agentic runs bind-mount two persistent caches into every worker container # (Lustre, shared across nodes): aiperf's content-addressed dataset mmap # cache (~65 GB per corpus, re-tokenized from scratch without it) and the @@ -606,6 +598,9 @@ if [[ "$IS_AGENTIC" == "1" ]]; then DEFAULT_MOUNTS_BLOCK+=" ${DYNAMO_WHEELS_CACHE_HOST_PATH}: /configs/dynamo-wheels" fi +elif [[ "$FRAMEWORK" == "vllm" && "$MODEL_PREFIX" == "dsv4" ]]; then + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${INFMAX_WORKSPACE}: /infmax-workspace" fi echo "Creating srtslurm.yaml configuration..." @@ -657,30 +652,6 @@ cat srtslurm.yaml echo "Running make setup..." make setup ARCH=aarch64 || exit 1 -# Export eval-related env vars for srt-slurm post-benchmark eval. Current -# Watchtower runners keep GITHUB_WORKSPACE on Lustre, so compute nodes can -# mount it directly; avoid copying the checkout from Lustre back to Lustre. -# Retain staging as a fallback for runners whose workspace is node-local. -export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" -if uses_watchtower_shared_fs; then - WORKSPACE_FS_TYPE=$(findmnt -n -o FSTYPE -T "$GITHUB_WORKSPACE" 2>/dev/null || true) - if [[ "$WORKSPACE_FS_TYPE" == "lustre" ]]; then - echo "Using existing Lustre-backed INFMAX_WORKSPACE=$INFMAX_WORKSPACE" - else - SHARED_INFMAX_WORKSPACE="${SHARED_BASE}/infmax-workspace-${RUN_KEY}" - mkdir -p "$SHARED_INFMAX_WORKSPACE" || exit 1 - rsync -a --delete \ - --exclude='.git/' \ - --exclude='srt-slurm*/' \ - --exclude='outputs/' \ - --exclude='LOGS/' \ - --exclude='*.sqsh' \ - "${GITHUB_WORKSPACE}/" "${SHARED_INFMAX_WORKSPACE}/" || exit 1 - export INFMAX_WORKSPACE="$SHARED_INFMAX_WORKSPACE" - echo "Staged node-local workspace to INFMAX_WORKSPACE=$INFMAX_WORKSPACE" - fi -fi - echo "Submitting job with srtctl..." # Resolve the recipe path before editing or submitting it. From 55e2f7a5e1fd8f0022245432a176e5ce017444b7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 26 Aug 2026 23:54:27 -0500 Subject: [PATCH 02/11] document GB200 vLLM Router migration --- perf-changelog.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7dfdec714b..0dde26245b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6537,3 +6537,14 @@ - "Use a float32 Mamba SSM state, as Hopper's flashinfer verify kernel requires, unlike the bfloat16 the Blackwell arms must use." - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2756 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - "Replace the manual llm-d, EPP, Envoy, pd-sidecar, and Slurm orchestration with srt-slurm's native vLLM backend and official vLLM Router 0.1.15 frontend, using NIXL for P/D KV transfer." + - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points while collapsing equal-topology points into one model load." + - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." + - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2757 From c0ea801ae9b44618c974af629ab5bea91588192b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 00:18:05 -0500 Subject: [PATCH 03/11] fix(gb200): target the resolved vLLM Router endpoint --- .../8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml | 5 ++--- .../GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml | 5 ++--- .../GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml | 5 ++--- perf-changelog.yaml | 1 + 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml index 0911fb4903..83ff322ea2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml @@ -12,8 +12,7 @@ identity: container: image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" frameworks: - vllm: "0.26" - vllm-router: "0.1.15" + vllm: "0.26.0" slurm: time_limit: "8:00:00" @@ -137,7 +136,7 @@ benchmark: num_warmups=$((concurrency * 2)) python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ --backend openai \ - --base-url "$SRT_FRONTEND_URL" \ + --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ --endpoint /v1/completions \ --model deepseek-ai/DeepSeek-V4-Pro \ --served-model-name deepseek-ai/DeepSeek-V4-Pro \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml index fc849ec467..d7c5df069d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml @@ -12,8 +12,7 @@ identity: container: image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" frameworks: - vllm: "0.26" - vllm-router: "0.1.15" + vllm: "0.26.0" slurm: time_limit: "8:00:00" @@ -126,7 +125,7 @@ benchmark: concurrency=1 python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ --backend openai \ - --base-url "$SRT_FRONTEND_URL" \ + --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ --endpoint /v1/completions \ --model deepseek-ai/DeepSeek-V4-Pro \ --served-model-name deepseek-ai/DeepSeek-V4-Pro \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml index 88b5895729..3ad5c23edb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml @@ -12,8 +12,7 @@ identity: container: image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" frameworks: - vllm: "0.26" - vllm-router: "0.1.15" + vllm: "0.26.0" slurm: time_limit: "8:00:00" @@ -135,7 +134,7 @@ benchmark: concurrency=4096 python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ --backend openai \ - --base-url "$SRT_FRONTEND_URL" \ + --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ --endpoint /v1/completions \ --model deepseek-ai/DeepSeek-V4-Pro \ --served-model-name deepseek-ai/DeepSeek-V4-Pro \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0dde26245b..9e62f1355f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6547,4 +6547,5 @@ - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points while collapsing equal-topology points into one model load." - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." + - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2757 From 8feaa11e0f5b03cf9c15c8efbb323a1c79945481 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 00:56:33 -0500 Subject: [PATCH 04/11] docs: keep migration notes English-only --- docs/configuration-procedures_zh.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 4af2c07441..0ebd17fc68 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -126,6 +126,22 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 不得只提交一侧：`srtctl` 读取配方，而矩阵生成读取主配置。仅改配方可能给结果贴错标签；仅改主配置不会改变实际部署的配方。 +## 注册 llm-d 配方 + +来源：[`benchmarks/llm-d/README.md`](../benchmarks/llm-d/README.md)、[`benchmarks/multi_node/llm-d/README.md`](../benchmarks/multi_node/llm-d/README.md)、[`llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) 和当前 [`llmd-vllm` 基准 wrapper](../benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh)。 + +llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并在每个节点启动一个容器。 + +1. 复制 [`benchmarks/multi_node/llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) 下最接近的 YAML，设置 EPP plugin/scheduling、角色特定 `extra-args`/`env`，以及可选 `slurm.time_limit`。 +2. 添加/更新 `llmd-vllm` 主条目。设置 `multinode: true`、`disagg: true`、router 元数据、`kv-p2p-transfer`、prefill/decode worker 拓扑、并发，以及 `additional-settings` 中的 `CONFIG_FILE=.yaml`。 +3. 保持 `PREFILL_NODES`、`DECODE_NODES`、`GPUS_PER_NODE` 和 worker 数与 allocation 及各角色 DP/TP/EP 布局一致。 +4. 确认 [`submit.sh`](../benchmarks/multi_node/llm-d/submit.sh) → [`job.slurm`](../benchmarks/multi_node/llm-d/job.slurm) → [`server.sh`](../benchmarks/multi_node/llm-d/server.sh) 的传递，以及所选 wrapper/launcher 路由。 +5. 验证文件发现：decode leader 生成 `/tmp/endpoints.yaml`；prefill endpoint 使用 vLLM 端口 8200，decode endpoint 使用 sidecar 端口 8000；名称唯一；地址为 IPv4 字面量；端口是 `1..65535` 范围内的字符串。 +6. 确认 EPP 在 Envoy 收到流量前完成 discovery 加载，且角色标签为请求阶段选择正确的 prefill/decode backend。 +7. 生成 key，检查拓扑和 `additional-settings`，再追加 changelog。 + +`CONFIG_FILE` 未设置或文件缺失时，会静默选择镜像内 `/etc/epp/config.yaml` fallback，并移除配方特定 vLLM 参数。除非明确打算使用 fallback，否则应将其视为验证失败。 + ## 更新镜像 来源：[`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-negotiable-benchmark-invariants)、对应主配置、运行时脚本与检入的 Recipe。 @@ -134,8 +150,9 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 2. 找出所有受影响的配置 key、运行时脚本、Dockerfile 和检入配方。不要假设主 YAML 是唯一镜像引用。 3. 将主配置 `image` 与所需 env、参数、软件包版本或补丁作为一个一致变更更新。 4. 对 srt-slurm，更新 `model.container` 并保持其与主配置 `image` 完全一致。 -5. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 -6. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 +5. 对 llm-d，区分主配置选择的服务镜像和 [`benchmarks/llm-d/Dockerfile`](../benchmarks/llm-d/Dockerfile) 中的构建来源；仅在构建契约变化时同时更新两者。 +6. 追加选择全部受影响 key 的 changelog 条目（有意覆盖多个 key 时可以使用通配符），并列出旧/新版本及实质运行时变更。 +7. 生成每个受影响的配置族，确认其运行时路径中没有残留旧 tag。 ## 添加或修改 MTP From 619e9acdb58853994866d3e7e2d1b140dd5a9798 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 01:07:15 -0500 Subject: [PATCH 05/11] chore(gb200): pin multinode DP routing fix --- perf-changelog.yaml | 1 + runners/launch_gb200-nv.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9e62f1355f..bbe0961b48 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6548,4 +6548,5 @@ - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." + - "Pin srt-slurm PR #7 at a2e458649f2d9de246954aaf0a900fc664811217 so multi-node hybrid-DP pools retain their nonzero global rank offsets instead of being re-expanded by vLLM Router into invalid local ranks that return HTTP 500." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2757 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 4689d2794e..584b825352 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -425,7 +425,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 elif [[ $FRAMEWORK == "vllm" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SRT_SLURM_PIN="76e7d76961b2dcb27cb05c1e9e0910ceb75104ec" + SRT_SLURM_PIN="a2e458649f2d9de246954aaf0a900fc664811217" git clone https://github.com/SemiAnalysisAI/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout "$SRT_SLURM_PIN" || exit 1 From 38ac93e398d08751d6c040f675c207c5c063b117 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 04:48:57 -0500 Subject: [PATCH 06/11] fix(gb200): isolate long disagg sweep points --- ...=> disagg-gb200-1p1d-dep8-dep8-c1024.yaml} | 13 +- .../disagg-gb200-1p1d-dep8-dep8-c256.yaml | 162 ++++++++++++++++++ .../disagg-gb200-1p1d-dep8-dep8-c512.yaml | 162 ++++++++++++++++++ configs/nvidia-master.yaml | 42 ++++- perf-changelog.yaml | 3 +- 5 files changed, 370 insertions(+), 12 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/{disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml => disagg-gb200-1p1d-dep8-dep8-c1024.yaml} (94%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml similarity index 94% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml index 83ff322ea2..7016d62faf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml @@ -1,4 +1,4 @@ -name: "dsv4-gb200-vllm-router-1p1d-dep8-dep8-c256-c512-c1024" +name: "dsv4-gb200-vllm-router-1p1d-dep8-dep8-c1024" setup_script: "install-vllm-router.sh" model: @@ -15,7 +15,7 @@ identity: vllm: "0.26.0" slurm: - time_limit: "8:00:00" + time_limit: "12:00:00" health_check: max_attempts: 2160 @@ -131,10 +131,10 @@ benchmark: set -euo pipefail result_dir=/logs/vllm_isl_8192_osl_1024 mkdir -p "$result_dir" - for concurrency in 256 512 1024; do - num_prompts=$((concurrency * 10)) - num_warmups=$((concurrency * 2)) - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + concurrency=1024 + num_prompts=$((concurrency * 10)) + num_warmups=$((concurrency * 2)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ --backend openai \ --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ --endpoint /v1/completions \ @@ -160,4 +160,3 @@ benchmark: --save-result \ --result-dir "$result_dir" \ --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" - done diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml new file mode 100644 index 0000000000..c65008b7b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml @@ -0,0 +1,162 @@ +name: "dsv4-gb200-vllm-router-1p1d-dep8-dep8-c256" +setup_script: "install-vllm-router.sh" + +model: + path: "deepseek-v4-pro" + container: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + frameworks: + vllm: "0.26.0" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: nixl + dp_launch_mode: per_node + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + numa-bind: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 9280 + max-num-seqs: 512 + max-num-batched-tokens: 512 + max-cudagraph-capture-size: 512 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + prefill_environment: + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + decode_environment: {} + +environment: + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: "NVL" + NCCL_NET_GDR_C2C: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" + NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + TILELANG_CLEANUP_TEMP_FILES: "1" + NVIDIA_GDRCOPY: "enabled" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + container-writable: "" + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir=/logs/vllm_isl_8192_osl_1024 + mkdir -p "$result_dir" + concurrency=256 + num_prompts=$((concurrency * 10)) + num_warmups=$((concurrency * 2)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --served-model-name deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer /model \ + --tokenizer-mode deepseek_v4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 96 \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --num-warmups "$num_warmups" \ + --ignore-eos \ + --trust-remote-code \ + --use-chat-template \ + --dsv4 \ + --seed 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml new file mode 100644 index 0000000000..b1df796e85 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml @@ -0,0 +1,162 @@ +name: "dsv4-gb200-vllm-router-1p1d-dep8-dep8-c512" +setup_script: "install-vllm-router.sh" + +model: + path: "deepseek-v4-pro" + container: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26@sha256:a9095d4c835935c4070be2040de0a5ef3b44098f603092ab66d743b0e731b7b4" + frameworks: + vllm: "0.26.0" + +slurm: + time_limit: "6:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: nixl + dp_launch_mode: per_node + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + numa-bind: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 9280 + max-num-seqs: 512 + max-num-batched-tokens: 512 + max-cudagraph-capture-size: 512 + trust-remote-code: true + enable-cumem-allocator: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + moe-backend: deep_gemm_mega_moe + prefill_environment: + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + decode_environment: {} + +environment: + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: "NVL" + NCCL_NET_GDR_C2C: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" + NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" + TILELANG_CLEANUP_TEMP_FILES: "1" + NVIDIA_GDRCOPY: "enabled" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + container-writable: "" + +benchmark: + type: custom + command: | + set -euo pipefail + result_dir=/logs/vllm_isl_8192_osl_1024 + mkdir -p "$result_dir" + concurrency=512 + num_prompts=$((concurrency * 10)) + num_warmups=$((concurrency * 2)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://$SRT_FRONTEND_HOST:$SRT_FRONTEND_PORT" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --served-model-name deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer /model \ + --tokenizer-mode deepseek_v4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 96 \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --num-warmups "$num_warmups" \ + --ignore-eos \ + --trust-remote-code \ + --use-chat-template \ + --dsv4 \ + --seed 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "results_concurrency_${concurrency}_gpus_16_ctx_8_gen_8.json" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7f451e4cd4..de930176b2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4458,17 +4458,51 @@ dsv4-fp4-gb200-llmd-vllm: dp-attn: false additional-settings: [] - # Mid curve: 1 prefill DEP8 + 1 decode DEP8. + # Mid curve: one DEP8 prefill + one DEP8 decode. Keep each measured + # concurrency in its own allocation: the full 8k/1k corpus takes hours + # per point at saturation, so chaining all three cannot fit the Slurm + # wall limit reliably. - spec-decoding: "none" - # The recipe deliberately runs all three points after one model load. - conc-list: [1] + conc-list: [256] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: [] + + - spec-decoding: "none" + conc-list: [512] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: [] + + - spec-decoding: "none" + conc-list: [1024] prefill: num-worker: 1 tp: 1 ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256-c512-c1024.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml" decode: num-worker: 1 tp: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bbe0961b48..2d35831479 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6544,7 +6544,8 @@ - fixed-seq-len description: - "Replace the manual llm-d, EPP, Envoy, pd-sidecar, and Slurm orchestration with srt-slurm's native vLLM backend and official vLLM Router 0.1.15 frontend, using NIXL for P/D KV transfer." - - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points while collapsing equal-topology points into one model load." + - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points." + - "Isolate c256, c512, and c1024 in separate equal-topology allocations with observed-runtime-sized 4h, 6h, and 12h limits; the first hardware sweep showed that chaining their complete 8k/1k corpora cannot fit a single eight-hour allocation." - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." From d3c7a86ab57d1864f904745f5ba59b40f474c3ce Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 05:01:41 -0500 Subject: [PATCH 07/11] fix(gb200): preserve saturated P/D requests --- .../GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml | 2 ++ .../GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml | 2 ++ .../GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml | 2 ++ .../GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml | 2 ++ .../GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml | 2 ++ perf-changelog.yaml | 1 + 6 files changed, 11 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml index 7016d62faf..6e1d1f0086 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c1024.yaml @@ -36,6 +36,8 @@ resources: frontend: type: vllm-router enable_multiple_frontends: false + args: + request-timeout-secs: 21600 backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml index c65008b7b6..f742c5a15d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c256.yaml @@ -36,6 +36,8 @@ resources: frontend: type: vllm-router enable_multiple_frontends: false + args: + request-timeout-secs: 21600 backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml index b1df796e85..1b4d1ee8a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-dep8-dep8-c512.yaml @@ -36,6 +36,8 @@ resources: frontend: type: vllm-router enable_multiple_frontends: false + args: + request-timeout-secs: 21600 backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml index d7c5df069d..3618e71d51 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-1p1d-tp8-tp8-c1.yaml @@ -36,6 +36,8 @@ resources: frontend: type: vllm-router enable_multiple_frontends: false + args: + request-timeout-secs: 21600 backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml index 3ad5c23edb..717a5939b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-3p1d-dep8-dep8-c4096.yaml @@ -36,6 +36,8 @@ resources: frontend: type: vllm-router enable_multiple_frontends: false + args: + request-timeout-secs: 21600 backend: type: vllm diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2d35831479..308172aac6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6546,6 +6546,7 @@ - "Replace the manual llm-d, EPP, Envoy, pd-sidecar, and Slurm orchestration with srt-slurm's native vLLM backend and official vLLM Router 0.1.15 frontend, using NIXL for P/D KV transfer." - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points." - "Isolate c256, c512, and c1024 in separate equal-topology allocations with observed-runtime-sized 4h, 6h, and 12h limits; the first hardware sweep showed that chaining their complete 8k/1k corpora cannot fit a single eight-hour allocation." + - "Raise vLLM Router's per-request timeout to six hours: the first hardware sweep showed saturated 8k prefills crossing the 1,800-second default, after which the router timed out a healthy prefill and the decode stage returned HTTP 500 for the invalid P/D handoff." - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." From cc0ed9cfbe47cdb611f031ccf7e0c3685f993524 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 06:32:33 -0500 Subject: [PATCH 08/11] fix(eval): bound multinode correctness load --- configs/nvidia-master.yaml | 4 + perf-changelog.yaml | 1 + utils/matrix_logic/generate_sweep_configs.py | 25 ++++++- .../test_generate_sweep_configs.py | 73 +++++++++++++++++++ utils/matrix_logic/test_validation.py | 40 ++++++++++ utils/matrix_logic/validation.py | 2 + 6 files changed, 144 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index de930176b2..db3fa3eb40 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4464,6 +4464,7 @@ dsv4-fp4-gb200-llmd-vllm: # wall limit reliably. - spec-decoding: "none" conc-list: [256] + eval-conc: 256 prefill: num-worker: 1 tp: 1 @@ -4480,6 +4481,7 @@ dsv4-fp4-gb200-llmd-vllm: - spec-decoding: "none" conc-list: [512] + eval-conc: 256 prefill: num-worker: 1 tp: 1 @@ -4496,6 +4498,7 @@ dsv4-fp4-gb200-llmd-vllm: - spec-decoding: "none" conc-list: [1024] + eval-conc: 256 prefill: num-worker: 1 tp: 1 @@ -4513,6 +4516,7 @@ dsv4-fp4-gb200-llmd-vllm: # Max throughput: 3 prefill DEP8 + 1 decode DEP8. - spec-decoding: "none" conc-list: [4096] + eval-conc: 256 prefill: num-worker: 3 tp: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 308172aac6..a15efe71ba 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6547,6 +6547,7 @@ - "Run InferenceX's checked-in benchmark_serving.py directly through srt-slurm's custom benchmark contract; preserve the c1, c256, c512, c1024, and c4096 search points." - "Isolate c256, c512, and c1024 in separate equal-topology allocations with observed-runtime-sized 4h, 6h, and 12h limits; the first hardware sweep showed that chaining their complete 8k/1k corpora cannot fit a single eight-hour allocation." - "Raise vLLM Router's per-request timeout to six hours: the first hardware sweep showed saturated 8k prefills crossing the 1,800-second default, after which the router timed out a healthy prefill and the decode stage returned HTTP 500 for the invalid P/D handoff." + - "Bound GSM8K correctness evaluation at concurrency 256 and teach the matrix generator to share one eval across split recipes with the same serving topology; the first hardware sweep's automatically selected concurrency 4096 exceeded the decode pool's live capacity and returned HTTP 500 before lm-eval could score the model." - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e3872c3d77..e9186b9006 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -440,8 +440,20 @@ def _multinode_parallelism_key(entry: dict) -> tuple: Fields.EVAL_ALL_CONCS.value, Fields.EXP_NAME.value, } + def eval_topology_value(key, value): + """Exclude recipe identity while retaining topology-affecting settings.""" + if key not in (Fields.PREFILL.value, Fields.DECODE.value): + return value + worker = dict(value) + settings = worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or [] + worker[Fields.ADDITIONAL_SETTINGS.value] = [ + setting for setting in settings + if not setting.startswith("CONFIG_FILE=") + ] + return worker + return tuple(sorted( - (key, _freeze_matrix_value(value)) + (key, _freeze_matrix_value(eval_topology_value(key, value))) for key, value in entry.items() if key not in ignored_fields )) @@ -475,6 +487,13 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) mn_eval_conc = {} # index -> chosen eval concurrency for multinode entries def _eligible_eval_concs(entry): + explicit_eval_conc = entry.get(Fields.EVAL_CONC.value) + if explicit_eval_conc is not None: + return ( + [explicit_eval_conc] + if explicit_eval_conc >= MIN_EVAL_CONC + else [] + ) conc = entry[Fields.CONC.value] conc_values = conc if isinstance(conc, list) else [conc] return sorted(c for c in conc_values if c >= MIN_EVAL_CONC) @@ -823,6 +842,8 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.RUN_EVAL.value: False, # Default, may be overridden by mark_eval_entries } entry.update(component_metadata(bmk, val)) + if Fields.EVAL_CONC.value in bmk: + entry[Fields.EVAL_CONC.value] = bmk[Fields.EVAL_CONC.value] add_multinode_node_count( entry, runner_data, @@ -1195,6 +1216,8 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.RUN_EVAL.value: False, } entry.update(component_metadata(bmk, val)) + if Fields.EVAL_CONC.value in bmk: + entry[Fields.EVAL_CONC.value] = bmk[Fields.EVAL_CONC.value] add_multinode_node_count( entry, runner_data, diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index cbde1afc4a..6f02475917 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -418,6 +418,79 @@ def test_marks_multinode_agentic_entry_at_highest_eligible_conc(self): assert marked[0]["conc"] == [32] assert marked[0]["eval-conc"] == 32 + def test_multinode_explicit_eval_concurrency_ignores_recipe_identity(self): + """Split recipes for one topology share one bounded correctness eval.""" + common = { + "model": "m", "runner": "cluster:gb200-nv", "framework": "vllm", + "precision": "fp4", "isl": 8192, "osl": 1024, + "spec-decoding": "none", "disagg": True, + "decode": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": [], + }, + "eval-conc": 256, + } + matrix_values = [ + { + **common, + "conc": [256], + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": ["CONFIG_FILE=recipes/c256.yaml"], + }, + }, + { + **common, + "conc": [1024], + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": ["CONFIG_FILE=recipes/c1024.yaml"], + }, + }, + ] + + result = mark_eval_entries(matrix_values) + + marked = [entry for entry in result if entry["run-eval"]] + assert len(marked) == 1 + assert marked[0]["conc"] == [256] + assert marked[0]["eval-conc"] == 256 + + def test_multinode_eval_group_preserves_topology_settings(self): + """Non-recipe additional settings still define distinct eval groups.""" + common = { + "model": "m", "runner": "cluster:gb200-nv", "framework": "vllm", + "precision": "fp4", "isl": 8192, "osl": 1024, + "spec-decoding": "none", "disagg": True, + "decode": {"num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True}, + "conc": [256], + "eval-conc": 128, + } + matrix_values = [ + { + **common, + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": [ + "CONFIG_FILE=recipes/a.yaml", "PREFILL_NODES=2", + ], + }, + }, + { + **common, + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": [ + "CONFIG_FILE=recipes/b.yaml", "PREFILL_NODES=4", + ], + }, + }, + ] + + result = mark_eval_entries(matrix_values) + + assert sum(entry["run-eval"] for entry in result) == 2 + def test_multinode_agentic_groups_are_independent_per_topology(self): """Two distinct multi-node agentic topologies (e.g. differing by prefill EP/DP) must each get their own eval row.""" diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index f20c36b3de..35c3ad682f 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -887,6 +887,46 @@ def test_valid_with_conc_list(self): assert entry.prefill.num_worker == 5 assert entry.decode.tp == 8 + def test_valid_with_explicit_eval_concurrency(self): + """A correctness eval may use a bounded load below a throughput point.""" + entry = MultiNodeSearchSpaceEntry(**{ + "prefill": { + "num-worker": 1, + "tp": 8, + "ep": 1, + "dp-attn": False, + }, + "decode": { + "num-worker": 1, + "tp": 8, + "ep": 1, + "dp-attn": False, + }, + "conc-list": [4096], + "eval-conc": 256, + }) + assert entry.eval_conc == 256 + + @pytest.mark.parametrize("invalid_eval_conc", [0, -1, 1.5, "256"]) + def test_explicit_eval_concurrency_is_strictly_positive(self, invalid_eval_conc): + with pytest.raises(Exception): + MultiNodeSearchSpaceEntry(**{ + "prefill": { + "num-worker": 1, + "tp": 8, + "ep": 1, + "dp-attn": False, + }, + "decode": { + "num-worker": 1, + "tp": 8, + "ep": 1, + "dp-attn": False, + }, + "conc-list": [4096], + "eval-conc": invalid_eval_conc, + }) + def test_valid_with_conc_range(self): """Valid multinode search space with range.""" entry = MultiNodeSearchSpaceEntry(**{ diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 436ea97b39..9e86c6ce80 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -581,6 +581,8 @@ class MultiNodeSearchSpaceEntry(BaseModel): default=None, alias=Fields.CONC_END.value) conc_list: Optional[List[int]] = Field( default=None, alias=Fields.CONC_LIST.value) + eval_conc: Optional[int] = Field( + default=None, alias=Fields.EVAL_CONC.value, gt=0, strict=True) @model_validator(mode='after') def validate_conc_fields(self): From ca9c54dedb2f28065154c48156a31048c77439a3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 06:42:58 -0500 Subject: [PATCH 09/11] chore(srt): repin native router stack --- perf-changelog.yaml | 2 +- runners/launch_gb200-nv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a15efe71ba..05a15160d9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6551,5 +6551,5 @@ - "Change the c1 latency topology from DEP8 prefill plus TP8 decode to TP8 plus TP8 because official vLLM Router exposes one intra-node DP expansion factor for both pools; the mixed topology would expand the TP8 decode URL into nonexistent DP ranks." - "Delete the now-unused llm-d container build, EPP/Envoy configuration, sidecars, wrapper, bespoke Slurm scripts, and their documentation." - "Use the custom-benchmark contract's injected SRT_FRONTEND_HOST and SRT_FRONTEND_PORT for the resolved vLLM Router endpoint, and verify the backend's exact runtime vLLM 0.26.0 identity." - - "Pin srt-slurm PR #7 at a2e458649f2d9de246954aaf0a900fc664811217 so multi-node hybrid-DP pools retain their nonzero global rank offsets instead of being re-expanded by vLLM Router into invalid local ranks that return HTTP 500." + - "Pin srt-slurm PR #7 at 10a58d2ebb4d756c423424049c736b909b72e14f so multi-node hybrid-DP pools retain their nonzero global rank offsets instead of being re-expanded by vLLM Router into invalid local ranks that return HTTP 500; this revision also normalizes ATOM recipe flags." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2757 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 584b825352..4f90e00dd4 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -425,7 +425,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 elif [[ $FRAMEWORK == "vllm" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SRT_SLURM_PIN="a2e458649f2d9de246954aaf0a900fc664811217" + SRT_SLURM_PIN="10a58d2ebb4d756c423424049c736b909b72e14f" git clone https://github.com/SemiAnalysisAI/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout "$SRT_SLURM_PIN" || exit 1 From 5d718e53be4e781017f7c9a640b2c68c8890d153 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 06:49:34 -0500 Subject: [PATCH 10/11] fix(eval): scope split-recipe grouping --- utils/matrix_logic/generate_sweep_configs.py | 9 ++++-- .../test_generate_sweep_configs.py | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e9186b9006..e0929d49f4 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -440,9 +440,14 @@ def _multinode_parallelism_key(entry: dict) -> tuple: Fields.EVAL_ALL_CONCS.value, Fields.EXP_NAME.value, } + ignore_recipe_identity = entry.get(Fields.EVAL_CONC.value) is not None + def eval_topology_value(key, value): - """Exclude recipe identity while retaining topology-affecting settings.""" - if key not in (Fields.PREFILL.value, Fields.DECODE.value): + """Group explicit bounded evals by topology, not split recipe path.""" + if ( + not ignore_recipe_identity + or key not in (Fields.PREFILL.value, Fields.DECODE.value) + ): return value worker = dict(value) settings = worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or [] diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 6f02475917..33c261f34d 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -491,6 +491,36 @@ def test_multinode_eval_group_preserves_topology_settings(self): assert sum(entry["run-eval"] for entry in result) == 2 + def test_default_eval_selection_keeps_recipe_identity(self): + """Existing recipes remain independent unless they opt into an eval bound.""" + common = { + "model": "m", "runner": "cluster:gb200-nv", "framework": "vllm", + "precision": "fp4", "isl": 8192, "osl": 1024, + "spec-decoding": "none", "disagg": True, + "decode": {"num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True}, + "conc": [256], + } + matrix_values = [ + { + **common, + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": ["CONFIG_FILE=recipes/a.yaml"], + }, + }, + { + **common, + "prefill": { + "num-worker": 1, "tp": 1, "ep": 8, "dp-attn": True, + "additional-settings": ["CONFIG_FILE=recipes/b.yaml"], + }, + }, + ] + + result = mark_eval_entries(matrix_values) + + assert sum(entry["run-eval"] for entry in result) == 2 + def test_multinode_agentic_groups_are_independent_per_topology(self): """Two distinct multi-node agentic topologies (e.g. differing by prefill EP/DP) must each get their own eval row.""" From 69330f34215814586989081ea4f56829a5e455fe Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 06:56:31 -0500 Subject: [PATCH 11/11] fix(eval): preserve absent worker settings --- utils/matrix_logic/generate_sweep_configs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e0929d49f4..8187c18102 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -450,11 +450,12 @@ def eval_topology_value(key, value): ): return value worker = dict(value) - settings = worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or [] - worker[Fields.ADDITIONAL_SETTINGS.value] = [ - setting for setting in settings - if not setting.startswith("CONFIG_FILE=") - ] + settings_key = Fields.ADDITIONAL_SETTINGS.value + if settings_key in worker: + worker[settings_key] = [ + setting for setting in (worker[settings_key] or []) + if not setting.startswith("CONFIG_FILE=") + ] return worker return tuple(sorted(