feat: add Qwen3.5 RTX PRO 6000 SGLang recipe / 新增 Qwen3.5 RTX PRO 6000 SGLang 基准配置

benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs.
+# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs
+# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is
+# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# copy. Either way, SERVE_MODEL is what the server is launched with.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    SERVE_MODEL="$MODEL_PATH"
+else
+    hf download "$MODEL"
+    SERVE_MODEL="$MODEL"
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+nvidia-smi
+
+export SGLANG_ENABLE_JIT_DEEPGEMM=false
+export PYTHONUNBUFFERED=1
+
+SERVER_LOG=/workspace/server.log
+
+# 96 GiB per GPU leaves far less headroom than the B300 recipe assumes. The
+# weights take ~56 GiB per rank and the prefill/decode CUDA graphs another
+# ~7 GiB, so the static fraction has to stay low enough that a prefill chunk's
+# activations still fit: at 0.8 the KV pool grew to 2.2M tokens (30x what
+# concurrency 64 needs) and the first 8k prefill OOM'd. 0.7 still leaves ~1M
+# KV tokens, and a 2-request prefill chunk keeps the activation peak bounded.
+MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.7}"
+CHUNKED_PREFILL_SIZE=$((ISL * 2))
+MAX_PREFILL_TOKENS=$((ISL * 2))
+MAX_RUNNING_REQUESTS=128
+CONTEXT_LENGTH=$((ISL + OSL + 20))
+
+# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls.
+if [[ $CONC -ge 16 ]]; then
+    SCHEDULER_RECV_INTERVAL=30
+else
+    SCHEDULER_RECV_INTERVAL=10
+fi
+
+if [[ "$EVAL_ONLY" == "true" ]]; then
+    setup_eval_context
+    CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN"
+fi
+
+echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL"
+
+start_gpu_monitor
+
+set -x
+PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \
+    --model-path "$SERVE_MODEL" \
+    --served-model-name "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --trust-remote-code \
+    --tensor-parallel-size "$TP" \
+    --data-parallel-size 1 \
+    --ep-size "$EP_SIZE" \
+    --reasoning-parser qwen3 \
+    --tool-call-parser qwen3_coder \
+    --quantization modelopt_fp4 \
+    --fp4-gemm-backend flashinfer_cutlass \
+    --moe-runner-backend flashinfer_cutlass \
+    --attention-backend flashinfer \
+    --kv-cache-dtype fp8_e4m3 \
+    --mamba-ssm-dtype bfloat16 \
+    --mamba-scheduler-strategy no_buffer \
+    --disable-custom-all-reduce \
+    --disable-radix-cache \
+    --mem-fraction-static "$MEM_FRAC_STATIC" \
+    --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \
+    --max-prefill-tokens "$MAX_PREFILL_TOKENS" \
+    --context-length "$CONTEXT_LENGTH" \
+    --cuda-graph-max-bs-decode "$CONC" \
+    --max-running-requests "$MAX_RUNNING_REQUESTS" \
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \
+    --stream-interval 20 > "$SERVER_LOG" 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready \
+    --port "$PORT" \
+    --server-log "$SERVER_LOG" \
+    --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --trust-remote-code
+
+if [[ "$RUN_EVAL" == "true" ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x

benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs, with the
+# built-in MTP draft head driven through SGLang's EAGLE speculative path.
+# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs
+# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is
+# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# copy. Either way, SERVE_MODEL is what the server is launched with.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    SERVE_MODEL="$MODEL_PATH"
+else
+    hf download "$MODEL"
+    SERVE_MODEL="$MODEL"
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+nvidia-smi
+
+export SGLANG_ENABLE_JIT_DEEPGEMM=false
+export PYTHONUNBUFFERED=1
+
+SERVER_LOG=/workspace/server.log
+
+# SGLang holds back total * (1 - mem-fraction-static) as slack and gives the
+# rest to the KV and Mamba pools, so the MTP draft head (Qwen3_5ForCausalLMMTP,
+# 4.05 GiB per rank on top of the 56.12 GiB target model) has to be paid for by
+# raising the fraction, not lowering it. 0.65 and 0.75 both left the pools
+# empty on this 96 GiB SKU once the draft's own Mamba state was accounted for.
+# 0.85 came up but left only 4.56 GiB free per rank, too thin for a 16k prefill
+# chunk, so 0.80 trades surplus KV (1.7M tokens, ~3x what concurrency 64 needs)
+# for activation headroom.
+MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.80}"
+CHUNKED_PREFILL_SIZE=$((ISL * 2))
+MAX_PREFILL_TOKENS=$((ISL * 2))
+# The client never opens more than CONC connections, so sizing the Mamba state
+# pool for 128 requests (as the B300 recipe does) just strands memory that the
+# draft head needs here.
+MAX_RUNNING_REQUESTS="$CONC"
+CONTEXT_LENGTH=$((ISL + OSL + 20))
+
+# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls.
+if [[ $CONC -ge 16 ]]; then
+    SCHEDULER_RECV_INTERVAL=30
+else
+    SCHEDULER_RECV_INTERVAL=10
+fi
+
+if [[ "$EVAL_ONLY" == "true" ]]; then
+    setup_eval_context
+    CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN"
+fi
+
+echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL"
+
+start_gpu_monitor
+
+set -x
+PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \
+    --model-path "$SERVE_MODEL" \
+    --served-model-name "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --trust-remote-code \
+    --tensor-parallel-size "$TP" \
+    --data-parallel-size 1 \
+    --ep-size "$EP_SIZE" \
+    --reasoning-parser qwen3 \
+    --tool-call-parser qwen3_coder \
+    --quantization modelopt_fp4 \
+    --fp4-gemm-backend flashinfer_cutlass \
+    --moe-runner-backend flashinfer_cutlass \
+    --attention-backend flashinfer \
+    --kv-cache-dtype fp8_e4m3 \
+    --mamba-ssm-dtype bfloat16 \
+    --mamba-scheduler-strategy no_buffer \
+    --disable-custom-all-reduce \
+    --disable-radix-cache \
+    --mem-fraction-static "$MEM_FRAC_STATIC" \
+    --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \
+    --max-prefill-tokens "$MAX_PREFILL_TOKENS" \
+    --context-length "$CONTEXT_LENGTH" \
+    --cuda-graph-max-bs-decode "$CONC" \
+    --max-running-requests "$MAX_RUNNING_REQUESTS" \
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \
+    --stream-interval 20 \
+    --speculative-algorithm EAGLE \
+    --speculative-num-steps 3 \
+    --speculative-eagle-topk 1 \
+    --speculative-num-draft-tokens 4 > "$SERVER_LOG" 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready \
+    --port "$PORT" \
+    --server-log "$SERVER_LOG" \
+    --server-pid "$SERVER_PID"
+
+# EAGLE-style spec decoding is trained against chat-formatted inputs, so the
+# benchmark must send chat prompts or the acceptance rate silently collapses.
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --use-chat-template \
+    --trust-remote-code
+
+if [[ "$RUN_EVAL" == "true" ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x

configs/nvidia-master.yaml

@@ -1343,6 +1343,42 @@ qwen3.5-fp4-b300-sglang:
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 }
       - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 }
 
+# Qwen3.5-397B-A17B NVFP4 single-node SGLang sweep using 4 of 8 RTX PRO
+# 6000 Blackwell GPUs. Both arms use ordinary NCCL collectives on PCIe.
+qwen3.5-fp4-rtx6000pro-sglang:
+  image: lmsysorg/sglang:v0.5.16-cu130
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: rtx6000pro-lat
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 4, conc-list: [1, 4, 16, 64] }
+      - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64] }
+
+# Same sweep with the built-in MTP draft head driven through SGLang's EAGLE
+# speculative path.
+qwen3.5-fp4-rtx6000pro-sglang-mtp:
+  image: lmsysorg/sglang:v0.5.16-cu130
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: rtx6000pro-lat
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp }
+      - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp }
+
 qwen3.5-fp4-b300-sglang-mtp:
   image: lmsysorg/sglang:v0.5.14-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4

configs/runners.yaml

@@ -164,6 +164,10 @@ labels:
     - gb300-nv_0
     - gb300-nv_1
     - gb300-nv_2
+  rtx6000pro:
+    - rtx6000pro-lat_00
+  rtx6000pro-lat:
+    - rtx6000pro-lat_00
   cluster:h100-cw:
     - h100-cw_00
     - h100-cw_01
@@ -251,6 +255,8 @@ labels:
     - gb300-nv_0
     - gb300-nv_1
     - gb300-nv_2
+  cluster:rtx6000pro-lat:
+    - rtx6000pro-lat_00
   cluster:mi300x-amds:
     - mi300x-amds_00
     - mi300x-amds_01
@@ -318,6 +324,9 @@ hardware:
   cluster:gb300-nv:
     available-cpu-dram-mib: 860_160
     gpus-per-node: 4
+  cluster:rtx6000pro-lat:
+    available-cpu-dram-mib: 1_500_000
+    gpus-per-node: 8
   cluster:mi300x-amds:
     available-cpu-dram-mib: 2_321_924
     gpus-per-node: 8

perf-changelog.yaml

@@ -5311,3 +5311,31 @@
     - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs."
     - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418
+
+- config-keys:
+    - qwen3.5-fp4-rtx6000pro-sglang
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Add Qwen3.5-397B-A17B NVFP4 single-node SGLang benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner"
+    - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)"
+    - "Pin SGLang v0.5.16-cu130, the first tag whose auto backend resolution knows that trtllm-gen MoE is SM100-only and routes modelopt_fp4 experts to FlashInfer CUTLASS on SM120"
+    - "Use FlashInfer CUTLASS for both the NVFP4 GEMMs and the routed experts and the FlashInfer attention backend on SM120, with FP8 KV cache, bf16 Mamba SSM state, the no_buffer Mamba scheduler strategy, radix cache disabled, and decode CUDA graph capture capped to each tested concurrency"
+    - "Use standard NCCL collectives on the PCIe topology (custom all-reduce disabled explicitly) and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM"
+    - "Deviate from the B300 SGLang recipe where 96 GiB per GPU forces it: mem-fraction-static 0.7 with a 2-request prefill chunk (0.8 with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first 8k prefill), and a single tokenizer worker (the B300 recipe's six each pin a ~0.7 GiB CUDA context on GPU 0)"
+    - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist"
+    - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312
+
+- config-keys:
+    - qwen3.5-fp4-rtx6000pro-sglang-mtp
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Add an MTP arm to the Qwen3.5-397B-A17B NVFP4 RTX PRO 6000 SGLang sweep, driving the checkpoint's built-in MTP draft head through SGLang's EAGLE path with 3 steps, top-k 1, and 4 draft tokens"
+    - "Sweep the same 8k/1k grid as the non-MTP config: concurrency 1, 4, 16, and 64 for both TP4 and TEP4"
+    - "Raise mem-fraction-static to 0.80 for this arm: the fraction bounds weights plus the KV and Mamba pools, so the 4.05 GiB per-rank draft head has to be paid for by raising it, not lowering it (0.65 and 0.75 both left the pools empty on this 96 GiB SKU, and 0.85 came up with only 4.56 GiB free per rank)"
+    - "Size max-running-requests to the tested concurrency instead of a fixed 128, since the client never opens more connections and the oversized Mamba state pool is what starved the draft head"
+    - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding"
+    - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312