From 268ff96480ce94aae155dd3a1da4fe28f20c6dff Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:01:52 -0700 Subject: [PATCH 1/8] feat(config): add Kimi K2.6 B200 Dynamo vLLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add seven 8K/1K disaggregated configuration points and load their recipes from NVIDIA/srt-slurm. 中文：新增 Kimi K2.6 B200 Dynamo vLLM 配置，注册七个 8K/1K 分离式推理配置项，并从 NVIDIA/srt-slurm 加载相应配方。 --- configs/nvidia-master.yaml | 108 ++++++++++++++++++++++++++++++++++++ perf-changelog.yaml | 8 +++ runners/launch_b200-dgxc.sh | 6 ++ 3 files changed, 122 insertions(+) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..8f274b9190 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4866,6 +4866,114 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 16 dp-attn: true +kimik2.6-fp4-b200-dynamo-vllm: + image: vllm/vllm-openai:v0.25.1 + model: nvidia/Kimi-K2.6-NVFP4 + model-prefix: kimik2.6 + runner: b200-multinode + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260721" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [32] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [128] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [512] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [1024] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [2048] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [8192] + prefill: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + dsv4-fp4-b200-dynamo-vllm: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..b64d8fc508 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,11 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + description: + - "Add Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration." + - "Include seven 8K/1K prefill/decode topology and concurrency entries." + - "Image: vllm/vllm-openai:v0.25.1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..18880da324 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -54,6 +54,9 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" +elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" + export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" @@ -113,6 +116,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 0b3daaa5b46f490982496af90da07f1645fa0fd8 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:02:23 -0700 Subject: [PATCH 2/8] chore(changelog): add PR link for #2360 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在性能变更日志中补充 #2360 的 PR 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b64d8fc508..3686ed00d9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,4 +5123,4 @@ - "Add Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration." - "Include seven 8K/1K prefill/decode topology and concurrency entries." - "Image: vllm/vllm-openai:v0.25.1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360 From 63eb685598f0de8784efad7eb81cf7982d5a9f49 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:53:29 -0700 Subject: [PATCH 3/8] fix(config): pin Kimi K2.6 srt-slurm recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetch the public recipe PR ref and check out its immutable commit before launching the configuration. 中文：获取公开配方 PR 引用，并在启动配置前检出其不可变提交。 --- runners/launch_b200-dgxc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 18880da324..2d41975825 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -119,6 +119,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 + git fetch origin refs/pull/282/head + git checkout d94c8505136b1e021ade99f1e7cd3359fc491cb9 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 904b8b8e8f5dc30b7413353f0292f4e019edf111 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 16:00:25 -0700 Subject: [PATCH 4/8] fix(config): use staged Kimi K2.6 model path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the B200 launch script at the Kimi K2.6 NVFP4 checkpoint staged on cluster storage. 中文：将 B200 启动脚本指向集群存储中已准备好的 Kimi K2.6 NVFP4 模型路径。 --- runners/launch_b200-dgxc.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 2d41975825..56fc0bbcee 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -6,13 +6,11 @@ SLURM_ACCOUNT="benchmark" set -x -# MODEL_PATH: Override with pre-downloaded paths on the shared Lustre tree. +# MODEL_PATH: Override with pre-downloaded paths on cluster-accessible storage. # Bench scripts and srt-slurm yaml configs specify HuggingFace model IDs for -# portability, but we resolve to /lustre/fsw/models/* here to avoid repeated +# portability, but we resolve to pre-staged paths here to avoid repeated # downloading on every dgxc node. Runs for both single-node and multinode # launches. -# NOTE: per-node /raid/models/* would be faster but is only populated on a -# subset of dgxc nodes today, so we use Lustre for reliability. if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2" export SRT_SLURM_MODEL_PREFIX="dsr1" @@ -55,7 +53,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" + export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" From 3b7880d3252266d5c0e0740e1af51efa5212bedd Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 16:10:28 -0700 Subject: [PATCH 5/8] fix(config): fail fast on pinned recipe checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop the B200 launcher immediately when the pinned srt-slurm recipe commit cannot be checked out. 中文：当固定的 srt-slurm 配方提交无法检出时，立即终止 B200 启动流程。 --- runners/launch_b200-dgxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 56fc0bbcee..f3105c6431 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -118,7 +118,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 git fetch origin refs/pull/282/head - git checkout d94c8505136b1e021ade99f1e7cd3359fc491cb9 + git checkout d94c8505136b1e021ade99f1e7cd3359fc491cb9 || exit 1 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 6dfe532870598d676bbc4d48a8bccc18b15816bb Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 16:25:17 -0700 Subject: [PATCH 6/8] fix(config): skip Kimi K2.6 login-node preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass --no-preflight only for Kimi K2.6 FP4 Dynamo-vLLM so srtctl can submit recipes whose model path is available on Slurm compute nodes. 中文：仅为 Kimi K2.6 FP4 Dynamo-vLLM 传递 --no-preflight，使 srtctl 能够提交模型路径仅在 Slurm 计算节点可用的配方。 --- runners/launch_b200-dgxc.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index f3105c6431..d76d5a2a10 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -267,7 +267,14 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + + SRTCTL_PREFLIGHT_ARGS=() + # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. + if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) + fi + + SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output From 166bf64d25d76370785e23757a4a8fb7f1ad7e87 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 16:40:03 -0700 Subject: [PATCH 7/8] fix(config): use shared Kimi K2.6 B200 model path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point Kimi K2.6 NVFP4 at the shared Lustre path exposed on B200 Slurm nodes. 中文：将 Kimi K2.6 NVFP4 指向 B200 Slurm 节点可访问的共享 Lustre 路径。 --- runners/launch_b200-dgxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index d76d5a2a10..9670b4d9de 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -53,7 +53,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" + export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" From 48edc42434ea9c7720e191758f3e89bd8c3add9d Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 28 Jul 2026 20:03:15 -0700 Subject: [PATCH 8/8] fix(config): use unique Kimi K2.6 prefill identities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the pinned srt-slurm recipe revision so multi-prefill workers rely on vLLM-generated engine IDs. 中文：更新固定的 srt-slurm 配方版本，使多预填充工作进程使用 vLLM 自动生成的 engine ID。 --- runners/launch_b200-dgxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 9670b4d9de..a644f36cd5 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -118,7 +118,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 git fetch origin refs/pull/282/head - git checkout d94c8505136b1e021ade99f1e7cd3359fc491cb9 || exit 1 + git checkout 003cc5d5f350fbe8838e950ad197736fb85a5559 || exit 1 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1