diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml index 71a0ffb9f3..9619a3fd59 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 1P1D: TP4 Prefill + TP4 Decode +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode # Pure tensor parallel, no expert parallel (STP) # 8k1k sa-bench concurrency sweep on GB300 # @@ -83,7 +83,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -125,7 +125,7 @@ backend: log-level: "info" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml index 00e576439a..fa43632fbc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 5P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP # Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. @@ -93,7 +93,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -128,7 +128,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml index ecab285097..691d0d72cf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP # Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml index d35f444694..44aa3934d8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 7P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP # Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3c92a87b4f..263dec901e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6223,8 +6223,8 @@ qwen3.5-fp8-h100-sglang-mtp: # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- qwen3.5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + image: lmsysorg/sglang:v0.5.14-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb300 precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d00b0ce3a4..31791d9e9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5755,3 +5755,10 @@ - "8k1k: 9 STP configs spanning 1P13D DEP2+13D TP4 (conc 78), 1P4D DEP2+4D TEP16 (conc 24), 1P8D DEP2+8D TEP8 (conc 48), 2P15D DEP2+15D TP4 (conc 180/216), 3P1D DEP2+1D DEP4 (conc 1536), 7P12D DEP2+12D TEP4 (conc 1152), 8P9D DEP2+9D TEP4 (conc 1728), 10P3D DEP2+3D DEP8 (conc 4608)." - "Runner: launch_gb300-nv.sh updated to clone NVIDIA/srt-slurm@v1.0.29 and overlay local recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/ for the dynamo-trt+qwen3.5+fp4 path, and to skip srtctl preflight for this path." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2206 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang + description: + - "Bump SGLang image to v0.5.14-cu130 for the Qwen3.5 FP4 GB300 disaggregated dynamo-sglang config" + - "Switch to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint (model pre-staged on cluster as Qwen3.5-397B-A17B-NVFP4-V2)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2238 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index e90c8cad2e..27154c404e 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -83,7 +83,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 + export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4-V2 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our