From 00ecba32b6eb062f5be9b8914a3c8a5eb26710ef Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 14 Jul 2026 10:25:44 -0700 Subject: [PATCH 1/4] Switch Qwen3.5 FP4 configs to nvidia/Qwen3.5-397B-A17B-NVFP4-V2 Bump all 7 Qwen3.5 FP4 configs (b200 sglang/trt +/-mtp, b300 sglang +/-mtp, gb300 dynamo-sglang disagg) to the V2 checkpoint pre-staged on all clusters as Qwen3.5-397B-A17B-NVFP4-V2. - configs/nvidia-master.yaml: model -> V2 for all 7 keys - gb300 disagg recipes: served-model-name -> V2 (weights still resolve via the qwen3.5-fp4 alias in launch_gb300-nv.sh) - launch_b200-dgxc.sh / launch_gb300-nv.sh: MODEL_PATH filename -> V2 - launch_b300-nv.sh: V2 added to STAGED_MODELS Co-Authored-By: Claude Opus 4.7 --- .../8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml | 6 +++--- .../8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml | 6 +++--- .../8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml | 6 +++--- .../8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml | 6 +++--- configs/nvidia-master.yaml | 14 +++++++------- perf-changelog.yaml | 13 +++++++++++++ runners/launch_b200-dgxc.sh | 2 +- runners/launch_b300-nv.sh | 1 + runners/launch_gb300-nv.sh | 2 +- 9 files changed, 35 insertions(+), 21 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml index 71a0ffb9f3..9619a3fd59 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 1P1D: TP4 Prefill + TP4 Decode +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode # Pure tensor parallel, no expert parallel (STP) # 8k1k sa-bench concurrency sweep on GB300 # @@ -83,7 +83,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -125,7 +125,7 @@ backend: log-level: "info" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml index 00e576439a..fa43632fbc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 5P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP # Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. @@ -93,7 +93,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -128,7 +128,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml index ecab285097..691d0d72cf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP # Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml index d35f444694..44aa3934d8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 7P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP # Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e1..64bb3a6f7d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2131,7 +2131,7 @@ qwen3.5-fp8-b200-sglang-agentic: qwen3.5-fp4-b200-sglang: image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b200 precision: fp4 @@ -2152,7 +2152,7 @@ qwen3.5-fp4-b200-sglang: qwen3.5-fp4-b200-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b200 precision: fp4 @@ -2702,7 +2702,7 @@ qwen3.5-fp8-b300-sglang: qwen3.5-fp4-b300-sglang: image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b300 precision: fp4 @@ -2723,7 +2723,7 @@ qwen3.5-fp4-b300-sglang: qwen3.5-fp4-b300-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b300 precision: fp4 @@ -11403,7 +11403,7 @@ glm5-fp4-gb200-dynamo-sglang: qwen3.5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb300 precision: fp4 @@ -13411,7 +13411,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: qwen3.5-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b200 precision: fp4 @@ -13626,7 +13626,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: b200 precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1fd..bebf77ba0b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,16 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - qwen3.5-fp4-b200-sglang + - qwen3.5-fp4-b200-sglang-mtp + - qwen3.5-fp4-b200-trt + - qwen3.5-fp4-b200-trt-mtp + - qwen3.5-fp4-b300-sglang + - qwen3.5-fp4-b300-sglang-mtp + - qwen3.5-fp4-gb300-dynamo-sglang + description: + - "Switch all Qwen3.5 FP4 configs to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint" + - "Model pre-staged on all clusters as Qwen3.5-397B-A17B-NVFP4-V2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 58051e6405..a6a10c9093 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -40,7 +40,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4" + export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4-V2" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/GLM-5-FP8" diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 6810ee5d85..29c780f3d6 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -361,6 +361,7 @@ else Qwen3.5-397B-A17B Qwen3.5-397B-A17B-FP8 Qwen3.5-397B-A17B-NVFP4 + Qwen3.5-397B-A17B-NVFP4-V2 gpt-oss-120b ) diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1381bd5db7..4e96f95802 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -72,7 +72,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 + export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4-V2 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our From 15cd958a73970918ee1f9dd7a505269cb13eb460 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 14 Jul 2026 10:26:06 -0700 Subject: [PATCH 2/4] Update perf-changelog pr-link for #2205 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bebf77ba0b..fa7ce1c70c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4793,4 +4793,4 @@ description: - "Switch all Qwen3.5 FP4 configs to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint" - "Model pre-staged on all clusters as Qwen3.5-397B-A17B-NVFP4-V2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2205 From 25ef4a1526dcf3954e2a225fe473484566beae6b Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 14 Jul 2026 14:36:52 -0700 Subject: [PATCH 3/4] remove trt configs --- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml | 6 +++--- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 5 +---- runners/launch_b200-dgxc.sh | 2 +- runners/launch_gb300-nv.sh | 2 +- 8 files changed, 18 insertions(+), 21 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml index 9619a3fd59..71a0ffb9f3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode +# Qwen3.5-397B-A17B-NVFP4 Disaggregated 1P1D: TP4 Prefill + TP4 Decode # Pure tensor parallel, no expert parallel (STP) # 8k1k sa-bench concurrency sweep on GB300 # @@ -83,7 +83,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true @@ -125,7 +125,7 @@ backend: log-level: "info" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml index fa43632fbc..00e576439a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4 Disaggregated 5P1D wide-EP # Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. @@ -93,7 +93,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true @@ -128,7 +128,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml index 691d0d72cf..ecab285097 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP # Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml index 44aa3934d8..d35f444694 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4 Disaggregated 7P1D wide-EP # Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" model-path: "/model/" trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 64bb3a6f7d..3a5aac90f5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11403,7 +11403,7 @@ glm5-fp4-gb200-dynamo-sglang: qwen3.5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: gb300 precision: fp4 @@ -13411,7 +13411,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: qwen3.5-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: b200 precision: fp4 @@ -13626,7 +13626,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: b200 precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fa7ce1c70c..ec664727ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4785,12 +4785,9 @@ - config-keys: - qwen3.5-fp4-b200-sglang - qwen3.5-fp4-b200-sglang-mtp - - qwen3.5-fp4-b200-trt - - qwen3.5-fp4-b200-trt-mtp - qwen3.5-fp4-b300-sglang - qwen3.5-fp4-b300-sglang-mtp - - qwen3.5-fp4-gb300-dynamo-sglang description: - - "Switch all Qwen3.5 FP4 configs to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint" + - "Switch Qwen3.5 FP4 SGLang configs to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint" - "Model pre-staged on all clusters as Qwen3.5-397B-A17B-NVFP4-V2" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2205 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a6a10c9093..bdf24f7abf 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -40,7 +40,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4-V2" + export MODEL_PATH="/scratch/fsw/models/Qwen3.5-397B-A17B-NVFP4-V2" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/GLM-5-FP8" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 4e96f95802..1381bd5db7 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -72,7 +72,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4-V2 + export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our From f85d3de53e17ad4a143175e8cad6f03b64283bd4 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Wed, 29 Jul 2026 15:00:52 -0700 Subject: [PATCH 4/4] =?UTF-8?q?fix(runners):=20scope=20b200-dgxc=20qwen3.5?= =?UTF-8?q?=20fp4=20model=20path=20to=20the=20V2=20checkpoint=20/=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9A=E5=B0=86=20b200-dgxc=20=E7=9A=84=20?= =?UTF-8?q?qwen3.5=20fp4=20=E6=A8=A1=E5=9E=8B=E8=B7=AF=E5=BE=84=E9=99=90?= =?UTF-8?q?=E5=AE=9A=E5=88=B0=20V2=20=E6=A3=80=E6=9F=A5=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The qwen3.5 + fp4 MODEL_PATH branch in launch_b200-dgxc.sh keyed only on MODEL_PREFIX and PRECISION, but two checkpoints now share that pair: qwen3.5-fp4-b200-sglang / -sglang-mtp -> nvidia/Qwen3.5-397B-A17B-NVFP4-V2 qwen3.5-fp4-b200-trt / -trt-mtp -> nvidia/Qwen3.5-397B-A17B-NVFP4 Since this script later does `export MODEL="$MODEL_PATH"`, the shared branch made the two TRT configs serve V2 weights while still publishing results under the old checkpoint name whenever they landed on a b200-dgxc_* runner (launch_b200-cw.sh / launch_b200-nb.sh pin no path, so the same key served the old weights there -- i.e. runner-dependent). Introduced by 25ef4a15, which reverted the TRT keys to NVFP4 but left the launcher pointing at the V2 path. Branch on the checkpoint so each key resolves to its own weights. The old-checkpoint arm restores the exact /lustre path used on main, so TRT behaviour is unchanged and needs no re-sweep. 中文：launch_b200-dgxc.sh 中 qwen3.5 + fp4 的 MODEL_PATH 分支仅依据 MODEL_PREFIX 与 PRECISION 判断，而这两个字段现已对应两个不同的检查点，导致 TRT 配置在 b200-dgxc 运行器上会加载 V2 权重，却仍以旧检查点名称发布结果。 改为按检查点分支：V2 走 /scratch，旧检查点恢复为 main 上的 /lustre 路径， 因此 TRT 行为保持不变，无需重跑 sweep。 --- runners/launch_b200-dgxc.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index bdf24f7abf..61dc845985 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -39,9 +39,19 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" -elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then +# qwen3.5 fp4 spans two checkpoints, so this must branch on the checkpoint and +# not on MODEL_PREFIX+PRECISION alone: the sglang keys moved to NVFP4-V2 while +# qwen3.5-fp4-b200-trt / -trt-mtp still declare plain NVFP4. Both share +# model-prefix qwen3.5 + precision fp4 + runner b200, and further down this +# script does `export MODEL="$MODEL_PATH"`, so a single shared branch would +# serve V2 weights to the TRT configs while publishing them under the old +# checkpoint name. +elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" && $MODEL == *NVFP4-V2 ]]; then export MODEL_PATH="/scratch/fsw/models/Qwen3.5-397B-A17B-NVFP4-V2" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" +elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B-NVFP4" + export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/GLM-5-FP8" export SRT_SLURM_MODEL_PREFIX="glm5-fp8"