From ba76967e9ac34ae0a44819bb0e0df9070452e01d Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:17:01 -0700 Subject: [PATCH 1/6] feat: add DSV4 FP4 B300 Dynamo-SGLang MTP config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:新增 DSV4 FP4 B300 Dynamo-SGLang MTP 配置,并接入启用聊天模板输入和 EAGLE 投机解码的共享 srt-slurm 配方矩阵。 --- configs/nvidia-master.yaml | 142 +++++++++++++++++++++++++++++++++++++ perf-changelog.yaml | 8 +++ runners/launch_b300-nv.sh | 25 +++++-- 3 files changed, 171 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..a404e94837 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6410,6 +6410,148 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 8 dp-attn: true +# DSV4 NVFP4 B300 disaggregated dynamo-sglang MTP configuration. +# Each search-space entry selects one variant from the shared srt-slurm recipe +# matrix. Runs on the b300 pool via runners/launch_b300-nv.sh. Prefill workers +# are TP4/DEP4 (two workers per 8-GPU B300 node); decode is DEP8 (one full node) +# except the TP4 topologies. EAGLE draft depth is tapered down as concurrency +# grows (3 steps / 4 draft tokens -> 2/3 at 6p1d -> 1/2 at 8p1d). +dsv4-fp4-b300-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b300 + precision: fp4 + framework: dynamo-sglang + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1p1d-tp4-tp4 (pure TP, no dp-attn). 2 nodes (1P + 1D). + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:override_tp4_tp4" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + # Scale-out: 1x DEP4 prefill + 6x TP4 decode. 4 nodes (1P + 3D). + - spec-decoding: "mtp" + conc-list: [8, 32, 64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:override_dep4_tp4" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # 1p1d-dep4-dep8: 1x DEP4 prefill + DEP8 decode. 2 nodes (1P + 1D). + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[0]" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 1p2d-dep4-dep8: 1x DEP4 prefill + 2x DEP8 decode. 3 nodes (1P + 2D). + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[1]" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + # 2p1d-dep4-dep8: 2x DEP4 prefill + DEP8 decode. 2 nodes (1P + 1D). + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[2]" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 4p1d-dep4-dep8: 4x DEP4 prefill + DEP8 decode. 3 nodes (2P + 1D). + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[3]" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 6p1d-dep4-dep8: 6x DEP4 prefill + DEP8 decode. 4 nodes (3P + 1D). + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[4]" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 8p1d-dep4-dep8. 5 nodes (4P + 1D). + - spec-decoding: "mtp" + conc-list: [3072] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[5]" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + kimik2.5-int4-h100-vllm: image: vllm/vllm-openai:v0.22.0 model: moonshotai/Kimi-K2.5 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..1b6bb2212a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,11 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang-mtp + description: + - "Add DSV4 NVFP4 B300 disaggregated Dynamo SGLang MTP configuration for 8k/1k" + - "Use the shared srt-slurm recipe matrix from NVIDIA/srt-slurm#281 with EAGLE speculative decoding and chat-template inputs" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 098e984be5..f4379afcd9 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -41,6 +41,9 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- fi export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" @@ -54,7 +57,7 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dy export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm or dynamo-sglang, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" exit 1 fi @@ -76,6 +79,9 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -184,10 +190,18 @@ if [[ -z "$CONFIG_FILE" ]]; then exit 1 fi -# Override the job name in the config file with the runner name -sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" +# Resolve the recipe path before editing it. CONFIG_FILE may include an +# srt-slurm matrix selector such as :zip_override_dep4_dep8[0]. +CONFIG_PATH="${CONFIG_FILE%%:*}" +if [[ ! -f "$CONFIG_PATH" ]]; then + echo "Error: CONFIG_FILE does not exist after srt-slurm setup: $CONFIG_PATH" >&2 + exit 1 +fi + +# Override the job name in the recipe with the runner name. +sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" if [[ "$MODEL_PREFIX" == "minimaxm3" && -n "$MINIMAX_M3_SLURM_EXCLUDED_NODELIST" ]]; then - sed -i "/^name:.*/a sbatch_directives:\n exclude: \"${MINIMAX_M3_SLURM_EXCLUDED_NODELIST}\"" "$CONFIG_FILE" + sed -i "/^name:.*/a sbatch_directives:\n exclude: \"${MINIMAX_M3_SLURM_EXCLUDED_NODELIST}\"" "$CONFIG_PATH" fi SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" @@ -201,6 +215,9 @@ SRTCTL_APPLY_ARGS=( if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && ( "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml || "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/*-tp1-*.yaml ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi +if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "$MODEL_PATH" == /scratch/models/* ]]; then + SRTCTL_APPLY_ARGS+=(--no-preflight) +fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi From 51c3739c2fbf0755ab66d4dc87745bab319a0747 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:18:29 -0700 Subject: [PATCH 2/6] chore: link perf changelog to #2363 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:将性能变更日志链接更新为 SemiAnalysisAI/InferenceX#2363。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1b6bb2212a..6c9fbabea3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,4 +5123,4 @@ - "Add DSV4 NVFP4 B300 disaggregated Dynamo SGLang MTP configuration for 8k/1k" - "Use the shared srt-slurm recipe matrix from NVIDIA/srt-slurm#281 with EAGLE speculative decoding and chat-template inputs" - "Image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2363 From 25eddd2c2e43dcde355b50ad518db758830dfc0b Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 15:33:12 -0700 Subject: [PATCH 3/6] fix: use B300 DSV4 NVFP4 model path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文:将 B300 DSV4 Dynamo-SGLang 配置指向服务器上预置的 DeepSeek-V4-Pro-NVFP4 模型路径。 --- runners/launch_b300-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f4379afcd9..904aa8f059 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -42,7 +42,7 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- export MODEL_PATH="${SELECTED_MODEL_PATH:-/data/models/dsv4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-sglang" ]]; then - export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro}" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/DeepSeek-V4-Pro-NVFP4}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M2.5-NVFP4" From d685ccfb4bb9b0fec711d8e8890de29851c62ce3 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 16:12:38 -0700 Subject: [PATCH 4/6] fix: pin srt-slurm recipe revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 固定 srt-slurm 配方版本,确保 DSV4 B300 Dynamo-SGLang 运行使用可复现的配置。 --- runners/launch_b300-nv.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 904aa8f059..b76b9a9609 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -82,6 +82,8 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 + git fetch origin 2065f6e7e798b48c0dc8d20a33c1f75427659dec || exit 1 + git checkout 2065f6e7e798b48c0dc8d20a33c1f75427659dec || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 2fda5e974357c752c947cca5842f584c3c9848bf Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 28 Jul 2026 01:02:46 -0700 Subject: [PATCH 5/6] fix(dsv4): pin UCX-enabled srt-slurm revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复(dsv4):固定包含 UCX CUDA 传输配置的 srt-slurm 版本。 --- runners/launch_b300-nv.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index b76b9a9609..43fa9c43b3 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -82,8 +82,8 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 - git fetch origin 2065f6e7e798b48c0dc8d20a33c1f75427659dec || exit 1 - git checkout 2065f6e7e798b48c0dc8d20a33c1f75427659dec || exit 1 + git fetch origin cb94d592dfd03a7f3f1ede9f0a3bbbb8a8452ab7 || exit 1 + git checkout cb94d592dfd03a7f3f1ede9f0a3bbbb8a8452ab7 || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From 6b1d6f15499c032be91bf47287c943d1ef05b47c Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 29 Jul 2026 00:09:08 -0700 Subject: [PATCH 6/6] fix: use checked-in DSV4 MTP recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin NVIDIA/srt-slurm main, overlay the checked-in recipes, and enable CUDA-aware UCX transport for every MTP topology. 修复:固定 NVIDIA/srt-slurm main 提交,覆盖仓库内置配方,并为所有 MTP 拓扑启用支持 CUDA 的 UCX 传输。 --- .../8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml | 134 +++++++++++++++++ .../8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml | 125 ++++++++++++++++ .../8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml | 135 ++++++++++++++++++ .../8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml | 129 +++++++++++++++++ .../8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml | 135 ++++++++++++++++++ .../8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml | 135 ++++++++++++++++++ .../8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml | 135 ++++++++++++++++++ .../8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml | 135 ++++++++++++++++++ configs/nvidia-master.yaml | 29 ++-- perf-changelog.yaml | 4 +- runners/launch_b300-nv.sh | 5 +- 11 files changed, 1083 insertions(+), 18 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..a07d5bb0a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,134 @@ +name: "disagg-b300-1p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..75c1815c8a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml @@ -0,0 +1,125 @@ +name: "disagg-b300-1p1d-tp4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 32768 + + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..fef5285ceb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml @@ -0,0 +1,135 @@ +name: "disagg-b300-1p2d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..d47b469945 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml @@ -0,0 +1,129 @@ +name: "disagg-b300-1p6d-dep4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 6 + gpus_per_decode: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..e796e2c680 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,135 @@ +name: "disagg-b300-2p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..681f350d90 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,135 @@ +name: "disagg-b300-4p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..4813ac793f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,135 @@ +name: "disagg-b300-6p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 3 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + mem-fraction-static: 0.9 + max-running-requests: 4096 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..ed3b27e906 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,135 @@ +name: "disagg-b300-8p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 4 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 1280 + swa-full-tokens-ratio: 0.1 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 47fb3918a8..d4becea465 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6441,12 +6441,13 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 8 dp-attn: true -# DSV4 FP4 B300 disaggregated dynamo-sglang MTP configuration. -# Each search-space entry selects one variant from the shared srt-slurm recipe -# matrix. Runs on the b300 pool via runners/launch_b300-nv.sh. Prefill workers -# are TP4/DEP4 (two workers per 8-GPU B300 node); decode is DEP8 (one full node) -# except the TP4 topologies. EAGLE draft depth is tapered down as concurrency -# grows (3 steps / 4 draft tokens -> 2/3 at 6p1d -> 1/2 at 8p1d). +# DSV4 B300 disaggregated dynamo-sglang MTP configuration. +# Each search-space entry maps one-to-one to a checked-in srt-slurm recipe under +# recipes/sglang/deepseek-v4/8k1k/. Runs on the b300 pool via +# runners/launch_b300-nv.sh. Prefill workers are TP4/DEP4 (two workers per +# 8-GPU B300 node); decode is DEP8 (one full node) except the TP4 topologies. +# EAGLE draft depth is tapered down as concurrency grows +# (3 steps / 4 draft tokens -> 2/3 at 6p1d -> 1/2 at 8p1d). dsv4-fp4-b300-dynamo-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4 model: deepseek-ai/DeepSeek-V4-Pro @@ -6471,7 +6472,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:override_tp4_tp4" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -6486,7 +6487,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:override_dep4_tp4" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -6501,7 +6502,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6516,7 +6517,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[1]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -6531,7 +6532,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[2]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6546,7 +6547,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[3]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6561,7 +6562,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[4]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6576,7 +6577,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml:zip_override_dep4_dep8[5]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 055317cc19..4d7d9b646c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5163,7 +5163,7 @@ - config-keys: - dsv4-fp4-b300-dynamo-sglang-mtp description: - - "Add DSV4 FP4 B300 disaggregated Dynamo SGLang MTP configuration for 8k/1k" - - "Use the shared srt-slurm recipe matrix from NVIDIA/srt-slurm#281 with EAGLE speculative decoding and chat-template inputs" + - "Add DSV4 B300 disaggregated Dynamo SGLang MTP configuration for 8k/1k" + - "Add 8 checked-in srt-slurm recipes with EAGLE speculative decoding and chat-template inputs" - "Image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2363 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 2c6a07d20b..f647446f02 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -82,8 +82,9 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 - git fetch origin cb94d592dfd03a7f3f1ede9f0a3bbbb8a8452ab7 || exit 1 - git checkout cb94d592dfd03a7f3f1ede9f0a3bbbb8a8452ab7 || exit 1 + git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && "$CONFIG_FILE" == recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/*.yaml ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1