From 47133a8bd251fb145322bd8d961d719a80ca6e99 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 23 Jul 2026 23:42:39 +0800
Subject: [PATCH 1/7] feat: add RTX PRO 6000 Latitude runner

---
 configs/runners.yaml             |   9 ++
 runners/launch_rtx6000pro-lat.sh | 137 +++++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100755 runners/launch_rtx6000pro-lat.sh

diff --git a/configs/runners.yaml b/configs/runners.yaml
index 851b821ba2..4c6b11069d 100644
--- a/configs/runners.yaml
+++ b/configs/runners.yaml
@@ -164,6 +164,10 @@ labels:
     - gb300-nv_0
     - gb300-nv_1
     - gb300-nv_2
+  rtx6000pro:
+    - rtx6000pro-lat_00
+  rtx6000pro-lat:
+    - rtx6000pro-lat_00
   cluster:h100-cw:
     - h100-cw_00
     - h100-cw_01
@@ -251,6 +255,8 @@ labels:
     - gb300-nv_0
     - gb300-nv_1
     - gb300-nv_2
+  cluster:rtx6000pro-lat:
+    - rtx6000pro-lat_00
   cluster:mi300x-amds:
     - mi300x-amds_00
     - mi300x-amds_01
@@ -315,6 +321,9 @@ hardware:
   cluster:gb200-nv:
     available-cpu-dram-mib: 860_160
     gpus-per-node: 4
+  cluster:rtx6000pro-lat:
+    available-cpu-dram-mib: 1_500_000
+    gpus-per-node: 8
   cluster:mi300x-amds:
     available-cpu-dram-mib: 2_321_924
     gpus-per-node: 8
diff --git a/runners/launch_rtx6000pro-lat.sh b/runners/launch_rtx6000pro-lat.sh
new file mode 100755
index 0000000000..f9d96642eb
--- /dev/null
+++ b/runners/launch_rtx6000pro-lat.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+# This runner executes directly on the single RTX PRO 6000 GPU node. Docker
+# therefore owns a separate image cache from the node's RKE2/containerd cache.
+HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/var/lib/inferencex/hf-hub-cache}"
+export HF_HUB_CACHE="${HF_HUB_CACHE:-/mnt/hf_hub_cache/}"
+PORT="${PORT:-8888}"
+
+# NCCL 2.28.9 segfaults while probing this node's bnxt_re RDMA devices.
+# Disable that RDMA path by default while preserving local CUDA P2P/SHM and
+# allowing an explicit caller override.
+export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}"
+
+: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set}"
+: "${IMAGE:?IMAGE must be set}"
+: "${EXP_NAME:?EXP_NAME must be set}"
+: "${PRECISION:?PRECISION must be set}"
+
+mkdir -p "$HF_HUB_CACHE_MOUNT"
+
+export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
+if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
+    echo "GPU_COUNT must be a positive integer, got: $GPU_COUNT" >&2
+    exit 1
+fi
+
+export CUDA_VISIBLE_DEVICES
+CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((GPU_COUNT - 1))")"
+
+# Some Slurm/enroot configs spell registry paths as nvcr.io#namespace/image.
+# Docker requires the normal slash form.
+DOCKER_IMAGE="${IMAGE//#//}"
+
+SPEC_SUFFIX=""
+if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then
+    SPEC_SUFFIX="_mtp"
+fi
+
+export SCENARIO_SUBDIR="${SCENARIO_SUBDIR:-fixed_seq_len/}"
+SCENARIO_SUBDIR="${SCENARIO_SUBDIR#/}"
+SCENARIO_SUBDIR="${SCENARIO_SUBDIR%/}/"
+BENCH_SCRIPT="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_rtx6000pro${SPEC_SUFFIX}.sh"
+
+if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then
+    echo "Benchmark script not found: $GITHUB_WORKSPACE/$BENCH_SCRIPT" >&2
+    exit 1
+fi
+
+server_name="bmk-server-${RUNNER_NAME:-rtx6000pro-lat}"
+server_name="${server_name//[^a-zA-Z0-9_.-]/-}"
+
+cleanup() {
+    docker rm -f "$server_name" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+# Clear a container left behind by a cancelled or interrupted workflow.
+cleanup
+
+docker run \
+    --rm \
+    --pull=missing \
+    --name="$server_name" \
+    --runtime=nvidia \
+    --gpus="$GPU_COUNT" \
+    --network=host \
+    --ipc=host \
+    --privileged \
+    --shm-size=32g \
+    --ulimit memlock=-1 \
+    --ulimit stack=67108864 \
+    --security-opt seccomp=unconfined \
+    --cap-add=SYS_PTRACE \
+    --volume "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \
+    --volume "$GITHUB_WORKSPACE:/workspace/" \
+    --workdir=/workspace/ \
+    --env HF_TOKEN \
+    --env HF_HUB_CACHE \
+    --env MODEL \
+    --env MODEL_PREFIX \
+    --env MODEL_PATH \
+    --env TP \
+    --env PP_SIZE \
+    --env DCP_SIZE \
+    --env PCP_SIZE \
+    --env EP_SIZE \
+    --env DP_SIZE \
+    --env DP_ATTENTION \
+    --env GPU_COUNT \
+    --env CONC \
+    --env MAX_MODEL_LEN \
+    --env ISL \
+    --env OSL \
+    --env FRAMEWORK \
+    --env PRECISION \
+    --env DISAGG \
+    --env SPEC_DECODING \
+    --env NUM_SPEC_TOKENS \
+    --env RUN_EVAL \
+    --env EVAL_ONLY \
+    --env EVAL_LIMIT \
+    --env EVAL_MAX_MODEL_LEN \
+    --env RUNNER_TYPE \
+    --env RUNNER_NAME \
+    --env RESULT_FILENAME \
+    --env RESULT_DIR \
+    --env RANDOM_RANGE_RATIO \
+    --env GPU_MEM_UTIL \
+    --env AIPERF_FAILED_REQUEST_THRESHOLD \
+    --env KV_OFFLOADING \
+    --env KV_OFFLOAD_BACKEND \
+    --env KV_OFFLOAD_BACKEND_METADATA \
+    --env ROUTER_METADATA \
+    --env KV_P2P_TRANSFER \
+    --env TOTAL_CPU_DRAM_GB \
+    --env DURATION \
+    --env SCENARIO_TYPE \
+    --env SCENARIO_SUBDIR \
+    --env IS_AGENTIC \
+    --env SWEBENCH_GEN_MODE \
+    --env SWEBENCH_USE_MODAL \
+    --env MODAL_TOKEN_ID \
+    --env MODAL_TOKEN_SECRET \
+    --env PROFILE \
+    --env SGLANG_TORCH_PROFILER_DIR \
+    --env VLLM_TORCH_PROFILER_DIR \
+    --env VLLM_RPC_TIMEOUT \
+    --env PYTHONDONTWRITEBYTECODE \
+    --env PYTHONPYCACHEPREFIX=/tmp/pycache/ \
+    --env PORT="$PORT" \
+    --env CUDA_DEVICE_ORDER=PCI_BUS_ID \
+    --env CUDA_VISIBLE_DEVICES \
+    --env NCCL_IB_DISABLE \
+    --entrypoint=/bin/bash \
+    "$DOCKER_IMAGE" \
+    "$BENCH_SCRIPT"

From b29acc2376f1c5db80d0bbe3776b0db979671478 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 23 Jul 2026 23:43:17 +0800
Subject: [PATCH 2/7] feat: add Qwen3.5 RTX PRO 6000 vLLM sweep

---
 .../fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh   | 113 ++++++++++++++++++
 configs/nvidia-master.yaml                    |  18 +++
 2 files changed, 131 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh

diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh
new file mode 100755
index 0000000000..fa47f89829
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs.
+# SM120 uses Marlin for the routed experts and Triton for attention. The node is
+# PCIe-only, so collectives use NCCL rather than NVLink-tuned custom all-reduce.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    EP_SIZE \
+    DP_ATTENTION \
+    CONC \
+    ISL \
+    OSL \
+    MAX_MODEL_LEN \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    SERVE_MODEL="$MODEL_PATH"
+else
+    hf download "$MODEL"
+    SERVE_MODEL="$MODEL"
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+nvidia-smi
+
+SERVER_LOG=/workspace/server.log
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
+
+export VLLM_ENGINE_READY_TIMEOUT_S=3600
+
+if [[ "$DP_ATTENTION" == "true" ]]; then
+    PARALLEL_ARGS=(
+        --tensor-parallel-size 1
+        --data-parallel-size "$TP"
+        --enable-expert-parallel
+    )
+elif [[ "$EP_SIZE" -gt 1 ]]; then
+    PARALLEL_ARGS=(
+        --tensor-parallel-size "$TP"
+        --enable-expert-parallel
+    )
+else
+    PARALLEL_ARGS=(--tensor-parallel-size "$TP")
+fi
+
+if [[ "$EVAL_ONLY" == "true" ]]; then
+    setup_eval_context
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
+fi
+
+start_gpu_monitor
+
+set -x
+vllm serve "$SERVE_MODEL" \
+    --served-model-name "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    "${PARALLEL_ARGS[@]}" \
+    --disable-custom-all-reduce \
+    --gpu-memory-utilization "$GPU_MEM_UTIL" \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --max-num-seqs "$CONC" \
+    --max-num-batched-tokens "$((ISL * 2))" \
+    --max-cudagraph-capture-size "$CONC" \
+    --kv-cache-dtype fp8 \
+    --language-model-only \
+    --reasoning-parser qwen3 \
+    --tool-call-parser qwen3_coder \
+    --enable-auto-tool-choice \
+    --attention-backend TRITON_ATTN \
+    --moe-backend marlin \
+    --stream-interval 20 \
+    --no-enable-prefix-caching \
+    --trust-remote-code > "$SERVER_LOG" 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready \
+    --port "$PORT" \
+    --server-log "$SERVER_LOG" \
+    --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --trust-remote-code
+
+if [[ "$RUN_EVAL" == "true" ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index a167f49270..6641cc22cc 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1343,6 +1343,24 @@ qwen3.5-fp4-b300-sglang:
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 }
       - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 }
 
+# Qwen3.5-397B-A17B NVFP4 single-node vLLM sweep using 4 of 8 RTX PRO
+# 6000 Blackwell GPUs. Both arms use ordinary NCCL collectives on PCIe.
+qwen3.5-fp4-rtx6000pro-vllm:
+  image: vllm/vllm-openai:v0.25.1@sha256:e4f88a835143cd22aee2397a26ec6bb80b3a4a6fe0c882bcbc63822904766089
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: rtx6000pro-lat
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 4, conc-list: [1, 4, 16, 64] }
+      - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64] }
+
 qwen3.5-fp4-b300-sglang-mtp:
   image: lmsysorg/sglang:v0.5.14-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4

From 905f89dd4ee149ab4c72cd07dc4ed68173c60bc5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 23 Jul 2026 23:44:59 +0800
Subject: [PATCH 3/7] docs: record Qwen3.5 RTX PRO 6000 sweep

---
 perf-changelog.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 335245a6c3..158a2b3903 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5060,3 +5060,16 @@
     - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)"
     - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301
+
+- config-keys:
+    - qwen3.5-fp4-rtx6000pro-vllm
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Add Qwen3.5-397B-A17B NVFP4 single-node vLLM benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner"
+    - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)"
+    - "Pin vLLM v0.25.1 by digest so the mixed-dtype all-reduce/RMSNorm correctness fix is present without a runtime patch"
+    - "Use Marlin NVFP4 MoE and Triton attention on SM120, with language-model-only serving, FP8 KV cache, prefix caching disabled, and CUDA graph capture capped to each tested concurrency"
+    - "Use standard NCCL collectives on the PCIe topology and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM"
+    - "Validated over SSH: exact 8192/1024 generation passed on both TP4 and TEP4, and the TEP4 concurrency-64 envelope captured all graph sizes and completed 64/64 simultaneous requests without OOM"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312

From cca657c3593f930e05019b50e7f3a957aa8159cd Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 27 Jul 2026 12:46:36 +0800
Subject: [PATCH 4/7] feat: switch Qwen3.5 RTX PRO 6000 recipe from vLLM to
 SGLang
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the vLLM benchmark script with an SGLang one and rename the
config key to qwen3.5-fp4-rtx6000pro-sglang. Pin
lmsysorg/sglang:v0.5.16-cu130, whose auto backend resolution knows that
trtllm-gen MoE is SM100-only and routes modelopt_fp4 routed experts to
FlashInfer CUTLASS on SM120.

96 GiB per GPU forces two deviations from the B300 SGLang recipe:
mem-fraction-static drops to 0.7 with a 2-request prefill chunk (0.8
with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first
8k prefill), and the six tokenizer workers are dropped because each pins
a ~0.7 GiB CUDA context on GPU 0.

The launcher now prefers a framework-tagged benchmark script, matching
the B200/B300 launchers, so vLLM and SGLang recipes for the same model
can coexist.

Validated on the node through the real launcher at all four corners of
the sweep with every request successful: TEP4 conc 64 640/640 at 6198
tok/s, TP4 conc 64 640/640 at 5999 tok/s, TEP4 conc 1 10/10 at 672
tok/s, TP4 conc 1 10/10 at 658 tok/s.

中文：将 Qwen3.5 RTX PRO 6000 基准配置从 vLLM 切换为 SGLang。配置键更名为
qwen3.5-fp4-rtx6000pro-sglang，镜像固定为 lmsysorg/sglang:v0.5.16-cu130
——该版本的自动后端选择已知 trtllm-gen MoE 仅支持 SM100，会在 SM120 上将
modelopt_fp4 路由专家切换到 FlashInfer CUTLASS。受限于每卡 96 GiB 显存，
相比 B300 配方做了两处调整：mem-fraction-static 降到 0.7 并将预填充分块
缩小到两个请求（0.8 配 32k 分块会把 KV 池撑到 220 万 token，首次 8k 预填充
即 OOM），并移除六个 tokenizer worker（每个都会在 GPU 0 上占用约 0.7 GiB
CUDA 上下文）。启动器改为优先选择带框架后缀的基准脚本，与 B200/B300 启动器
一致，使同一模型的 vLLM 与 SGLang 配方可以共存。已在节点上通过真实启动器
验证扫描的四个角落，全部请求成功。
---
 .../fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh   | 113 ----------------
 .../qwen3.5_fp4_rtx6000pro_sglang.sh          | 128 ++++++++++++++++++
 configs/nvidia-master.yaml                    |   8 +-
 perf-changelog.yaml                           |  14 +-
 runners/launch_rtx6000pro-lat.sh              |   9 +-
 5 files changed, 148 insertions(+), 124 deletions(-)
 delete mode 100755 benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh
 create mode 100755 benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh

diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh
deleted file mode 100755
index fa47f89829..0000000000
--- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs.
-# SM120 uses Marlin for the routed experts and Triton for attention. The node is
-# PCIe-only, so collectives use NCCL rather than NVLink-tuned custom all-reduce.
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars \
-    MODEL \
-    TP \
-    EP_SIZE \
-    DP_ATTENTION \
-    CONC \
-    ISL \
-    OSL \
-    MAX_MODEL_LEN \
-    RANDOM_RANGE_RATIO \
-    RESULT_FILENAME
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
-    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
-        hf download "$MODEL" --local-dir "$MODEL_PATH"
-    fi
-    SERVE_MODEL="$MODEL_PATH"
-else
-    hf download "$MODEL"
-    SERVE_MODEL="$MODEL"
-fi
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
-    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
-fi
-
-nvidia-smi
-
-SERVER_LOG=/workspace/server.log
-GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-
-if [[ "$DP_ATTENTION" == "true" ]]; then
-    PARALLEL_ARGS=(
-        --tensor-parallel-size 1
-        --data-parallel-size "$TP"
-        --enable-expert-parallel
-    )
-elif [[ "$EP_SIZE" -gt 1 ]]; then
-    PARALLEL_ARGS=(
-        --tensor-parallel-size "$TP"
-        --enable-expert-parallel
-    )
-else
-    PARALLEL_ARGS=(--tensor-parallel-size "$TP")
-fi
-
-if [[ "$EVAL_ONLY" == "true" ]]; then
-    setup_eval_context
-    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
-fi
-
-start_gpu_monitor
-
-set -x
-vllm serve "$SERVE_MODEL" \
-    --served-model-name "$MODEL" \
-    --host 0.0.0.0 \
-    --port "$PORT" \
-    "${PARALLEL_ARGS[@]}" \
-    --disable-custom-all-reduce \
-    --gpu-memory-utilization "$GPU_MEM_UTIL" \
-    --max-model-len "$MAX_MODEL_LEN" \
-    --max-num-seqs "$CONC" \
-    --max-num-batched-tokens "$((ISL * 2))" \
-    --max-cudagraph-capture-size "$CONC" \
-    --kv-cache-dtype fp8 \
-    --language-model-only \
-    --reasoning-parser qwen3 \
-    --tool-call-parser qwen3_coder \
-    --enable-auto-tool-choice \
-    --attention-backend TRITON_ATTN \
-    --moe-backend marlin \
-    --stream-interval 20 \
-    --no-enable-prefix-caching \
-    --trust-remote-code > "$SERVER_LOG" 2>&1 &
-
-SERVER_PID=$!
-
-wait_for_server_ready \
-    --port "$PORT" \
-    --server-log "$SERVER_LOG" \
-    --server-pid "$SERVER_PID"
-
-run_benchmark_serving \
-    --model "$MODEL" \
-    --port "$PORT" \
-    --backend vllm \
-    --input-len "$ISL" \
-    --output-len "$OSL" \
-    --random-range-ratio "$RANDOM_RANGE_RATIO" \
-    --num-prompts "$((CONC * 10))" \
-    --max-concurrency "$CONC" \
-    --result-filename "$RESULT_FILENAME" \
-    --result-dir /workspace/ \
-    --trust-remote-code
-
-if [[ "$RUN_EVAL" == "true" ]]; then
-    run_eval --framework lm-eval --port "$PORT"
-    append_lm_eval_summary
-fi
-
-stop_gpu_monitor
-set +x
diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh
new file mode 100755
index 0000000000..f3e92ad622
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs.
+# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs
+# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is
+# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# copy. Either way, SERVE_MODEL is what the server is launched with.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    SERVE_MODEL="$MODEL_PATH"
+else
+    hf download "$MODEL"
+    SERVE_MODEL="$MODEL"
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+nvidia-smi
+
+export SGLANG_ENABLE_JIT_DEEPGEMM=false
+export PYTHONUNBUFFERED=1
+
+SERVER_LOG=/workspace/server.log
+
+# 96 GiB per GPU leaves far less headroom than the B300 recipe assumes. The
+# weights take ~56 GiB per rank and the prefill/decode CUDA graphs another
+# ~7 GiB, so the static fraction has to stay low enough that a prefill chunk's
+# activations still fit: at 0.8 the KV pool grew to 2.2M tokens (30x what
+# concurrency 64 needs) and the first 8k prefill OOM'd. 0.7 still leaves ~1M
+# KV tokens, and a 2-request prefill chunk keeps the activation peak bounded.
+MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.7}"
+CHUNKED_PREFILL_SIZE=$((ISL * 2))
+MAX_PREFILL_TOKENS=$((ISL * 2))
+MAX_RUNNING_REQUESTS=128
+CONTEXT_LENGTH=$((ISL + OSL + 20))
+
+# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls.
+if [[ $CONC -ge 16 ]]; then
+    SCHEDULER_RECV_INTERVAL=30
+else
+    SCHEDULER_RECV_INTERVAL=10
+fi
+
+if [[ "$EVAL_ONLY" == "true" ]]; then
+    setup_eval_context
+    CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN"
+fi
+
+echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL"
+
+start_gpu_monitor
+
+set -x
+PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \
+    --model-path "$SERVE_MODEL" \
+    --served-model-name "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --trust-remote-code \
+    --tensor-parallel-size "$TP" \
+    --data-parallel-size 1 \
+    --ep-size "$EP_SIZE" \
+    --reasoning-parser qwen3 \
+    --tool-call-parser qwen3_coder \
+    --quantization modelopt_fp4 \
+    --fp4-gemm-backend flashinfer_cutlass \
+    --moe-runner-backend flashinfer_cutlass \
+    --attention-backend flashinfer \
+    --kv-cache-dtype fp8_e4m3 \
+    --mamba-ssm-dtype bfloat16 \
+    --mamba-scheduler-strategy no_buffer \
+    --disable-custom-all-reduce \
+    --disable-radix-cache \
+    --mem-fraction-static "$MEM_FRAC_STATIC" \
+    --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \
+    --max-prefill-tokens "$MAX_PREFILL_TOKENS" \
+    --context-length "$CONTEXT_LENGTH" \
+    --cuda-graph-max-bs-decode "$CONC" \
+    --max-running-requests "$MAX_RUNNING_REQUESTS" \
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \
+    --stream-interval 20 > "$SERVER_LOG" 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready \
+    --port "$PORT" \
+    --server-log "$SERVER_LOG" \
+    --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --trust-remote-code
+
+if [[ "$RUN_EVAL" == "true" ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 6641cc22cc..c281031332 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1343,15 +1343,15 @@ qwen3.5-fp4-b300-sglang:
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 }
       - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 }
 
-# Qwen3.5-397B-A17B NVFP4 single-node vLLM sweep using 4 of 8 RTX PRO
+# Qwen3.5-397B-A17B NVFP4 single-node SGLang sweep using 4 of 8 RTX PRO
 # 6000 Blackwell GPUs. Both arms use ordinary NCCL collectives on PCIe.
-qwen3.5-fp4-rtx6000pro-vllm:
-  image: vllm/vllm-openai:v0.25.1@sha256:e4f88a835143cd22aee2397a26ec6bb80b3a4a6fe0c882bcbc63822904766089
+qwen3.5-fp4-rtx6000pro-sglang:
+  image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
   model-prefix: qwen3.5
   runner: rtx6000pro-lat
   precision: fp4
-  framework: vllm
+  framework: sglang
   multinode: false
   scenarios:
     fixed-seq-len:
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 158a2b3903..ac304922d3 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5062,14 +5062,16 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301
 
 - config-keys:
-    - qwen3.5-fp4-rtx6000pro-vllm
+    - qwen3.5-fp4-rtx6000pro-sglang
   scenario-type:
     - fixed-seq-len
   description:
-    - "Add Qwen3.5-397B-A17B NVFP4 single-node vLLM benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner"
+    - "Add Qwen3.5-397B-A17B NVFP4 single-node SGLang benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner"
     - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)"
-    - "Pin vLLM v0.25.1 by digest so the mixed-dtype all-reduce/RMSNorm correctness fix is present without a runtime patch"
-    - "Use Marlin NVFP4 MoE and Triton attention on SM120, with language-model-only serving, FP8 KV cache, prefix caching disabled, and CUDA graph capture capped to each tested concurrency"
-    - "Use standard NCCL collectives on the PCIe topology and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM"
-    - "Validated over SSH: exact 8192/1024 generation passed on both TP4 and TEP4, and the TEP4 concurrency-64 envelope captured all graph sizes and completed 64/64 simultaneous requests without OOM"
+    - "Pin SGLang v0.5.16-cu130, the first tag whose auto backend resolution knows that trtllm-gen MoE is SM100-only and routes modelopt_fp4 experts to FlashInfer CUTLASS on SM120"
+    - "Use FlashInfer CUTLASS for both the NVFP4 GEMMs and the routed experts and the FlashInfer attention backend on SM120, with FP8 KV cache, bf16 Mamba SSM state, the no_buffer Mamba scheduler strategy, radix cache disabled, and decode CUDA graph capture capped to each tested concurrency"
+    - "Use standard NCCL collectives on the PCIe topology (custom all-reduce disabled explicitly) and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM"
+    - "Deviate from the B300 SGLang recipe where 96 GiB per GPU forces it: mem-fraction-static 0.7 with a 2-request prefill chunk (0.8 with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first 8k prefill), and a single tokenizer worker (the B300 recipe's six each pin a ~0.7 GiB CUDA context on GPU 0)"
+    - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist"
+    - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312
diff --git a/runners/launch_rtx6000pro-lat.sh b/runners/launch_rtx6000pro-lat.sh
index f9d96642eb..6ad56e7092 100755
--- a/runners/launch_rtx6000pro-lat.sh
+++ b/runners/launch_rtx6000pro-lat.sh
@@ -40,7 +40,14 @@ fi
 export SCENARIO_SUBDIR="${SCENARIO_SUBDIR:-fixed_seq_len/}"
 SCENARIO_SUBDIR="${SCENARIO_SUBDIR#/}"
 SCENARIO_SUBDIR="${SCENARIO_SUBDIR%/}/"
-BENCH_SCRIPT="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_rtx6000pro${SPEC_SUFFIX}.sh"
+# Prefer a framework-tagged script (e.g. qwen3.5_fp4_rtx6000pro_sglang.sh) so
+# models with multiple inference engines can coexist; fall back to the name
+# without an engine suffix for scripts that haven't been retagged.
+BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_rtx6000pro"
+BENCH_SCRIPT="${BENCH_BASE}_${FRAMEWORK:-}${SPEC_SUFFIX}.sh"
+if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then
+    BENCH_SCRIPT="${BENCH_BASE}${SPEC_SUFFIX}.sh"
+fi
 
 if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then
     echo "Benchmark script not found: $GITHUB_WORKSPACE/$BENCH_SCRIPT" >&2

From a28493b570278be7f12b3791f233082cba617da2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 27 Jul 2026 12:59:46 +0800
Subject: [PATCH 5/7] fix: restore Qwen3.5 RTX PRO 6000 SGLang changelog entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The merge with main resolved perf-changelog.yaml by taking main's copy
and re-appending this PR's entry, but only main's copy was staged, so
the merge commit dropped the entry. Without it the PR diff no longer
touches perf-changelog.yaml and run-sweep.yml's path filter never
matches.

中文：与 main 合并时对 perf-changelog.yaml 采用了「取 main 版本再追加本 PR
条目」的方式，但只暂存了 main 的版本，导致合并提交丢失了该条目。缺少该条目
后 PR 的改动不再涉及 perf-changelog.yaml，run-sweep.yml 的路径过滤器便不会
匹配。此提交恢复该条目。
---
 perf-changelog.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index c3602ff749..0ed64152f4 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5085,3 +5085,18 @@
   description:
     - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313
+
+- config-keys:
+    - qwen3.5-fp4-rtx6000pro-sglang
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Add Qwen3.5-397B-A17B NVFP4 single-node SGLang benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner"
+    - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)"
+    - "Pin SGLang v0.5.16-cu130, the first tag whose auto backend resolution knows that trtllm-gen MoE is SM100-only and routes modelopt_fp4 experts to FlashInfer CUTLASS on SM120"
+    - "Use FlashInfer CUTLASS for both the NVFP4 GEMMs and the routed experts and the FlashInfer attention backend on SM120, with FP8 KV cache, bf16 Mamba SSM state, the no_buffer Mamba scheduler strategy, radix cache disabled, and decode CUDA graph capture capped to each tested concurrency"
+    - "Use standard NCCL collectives on the PCIe topology (custom all-reduce disabled explicitly) and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM"
+    - "Deviate from the B300 SGLang recipe where 96 GiB per GPU forces it: mem-fraction-static 0.7 with a 2-request prefill chunk (0.8 with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first 8k prefill), and a single tokenizer worker (the B300 recipe's six each pin a ~0.7 GiB CUDA context on GPU 0)"
+    - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist"
+    - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312

From a071d99f55fd71c0280a7dbc3c514c566a2105cc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 27 Jul 2026 17:00:52 +0800
Subject: [PATCH 6/7] feat: add MTP arm to the Qwen3.5 RTX PRO 6000 SGLang
 recipe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Drive the checkpoint's built-in MTP draft head through SGLang's EAGLE
path (3 steps, top-k 1, 4 draft tokens) over the same 8k/1k grid as the
non-MTP config.

mem-fraction-static bounds weights plus the KV and Mamba pools, so the
4.05 GiB per-rank draft head has to be paid for by raising the fraction,
not lowering it: 0.65 and 0.75 both left the pools empty, and 0.85 came
up with only 4.56 GiB free per rank. 0.80 lands at 1.08M KV tokens with
9.6 GiB free. max-running-requests now tracks the tested concurrency
rather than a fixed 128, since the client never opens more connections
and the oversized Mamba state pool was what starved the draft head.

Validated on the node through the real launcher, every request
successful: TEP4 conc 64 640/640 at 6895 tok/s (+11% over the non-MTP
arm, mean TTFT 27.2s to 9.6s) and TEP4 conc 1 10/10 at 1233 tok/s (+83%,
mean TPOT 12.6ms to 6.5ms).

中文：为 Qwen3.5 RTX PRO 6000 SGLang 配方新增 MTP 分支，通过 SGLang 的 EAGLE
路径驱动检查点内置的 MTP 草稿头（3 步、top-k 1、4 个草稿 token），扫描网格与
非 MTP 配置相同。mem-fraction-static 同时约束权重与 KV／Mamba 池，因此每卡
4.05 GiB 的草稿头必须通过调高该比例来容纳，而非调低：0.65 与 0.75 都会让池为
空，0.85 虽能启动但每卡仅剩 4.56 GiB。0.80 可得到 108 万 KV token 且剩余
9.6 GiB。max-running-requests 改为跟随被测并发数而非固定 128，因为客户端不会
建立更多连接，过大的 Mamba 状态池正是草稿头显存不足的原因。已在节点上通过真实
启动器验证，全部请求成功。
---
 .../qwen3.5_fp4_rtx6000pro_sglang_mtp.sh      | 141 ++++++++++++++++++
 configs/nvidia-master.yaml                    |  18 +++
 perf-changelog.yaml                           |  13 ++
 3 files changed, 172 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh

diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh
new file mode 100755
index 0000000000..2692860015
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs, with the
+# built-in MTP draft head driven through SGLang's EAGLE speculative path.
+# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs
+# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is
+# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+# `hf download` creates the target dir if missing and is itself idempotent.
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# copy. Either way, SERVE_MODEL is what the server is launched with.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    SERVE_MODEL="$MODEL_PATH"
+else
+    hf download "$MODEL"
+    SERVE_MODEL="$MODEL"
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+nvidia-smi
+
+export SGLANG_ENABLE_JIT_DEEPGEMM=false
+export PYTHONUNBUFFERED=1
+
+SERVER_LOG=/workspace/server.log
+
+# SGLang holds back total * (1 - mem-fraction-static) as slack and gives the
+# rest to the KV and Mamba pools, so the MTP draft head (Qwen3_5ForCausalLMMTP,
+# 4.05 GiB per rank on top of the 56.12 GiB target model) has to be paid for by
+# raising the fraction, not lowering it. 0.65 and 0.75 both left the pools
+# empty on this 96 GiB SKU once the draft's own Mamba state was accounted for.
+# 0.85 came up but left only 4.56 GiB free per rank, too thin for a 16k prefill
+# chunk, so 0.80 trades surplus KV (1.7M tokens, ~3x what concurrency 64 needs)
+# for activation headroom.
+MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.80}"
+CHUNKED_PREFILL_SIZE=$((ISL * 2))
+MAX_PREFILL_TOKENS=$((ISL * 2))
+# The client never opens more than CONC connections, so sizing the Mamba state
+# pool for 128 requests (as the B300 recipe does) just strands memory that the
+# draft head needs here.
+MAX_RUNNING_REQUESTS="$CONC"
+CONTEXT_LENGTH=$((ISL + OSL + 20))
+
+# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls.
+if [[ $CONC -ge 16 ]]; then
+    SCHEDULER_RECV_INTERVAL=30
+else
+    SCHEDULER_RECV_INTERVAL=10
+fi
+
+if [[ "$EVAL_ONLY" == "true" ]]; then
+    setup_eval_context
+    CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN"
+fi
+
+echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL"
+
+start_gpu_monitor
+
+set -x
+PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \
+    --model-path "$SERVE_MODEL" \
+    --served-model-name "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --trust-remote-code \
+    --tensor-parallel-size "$TP" \
+    --data-parallel-size 1 \
+    --ep-size "$EP_SIZE" \
+    --reasoning-parser qwen3 \
+    --tool-call-parser qwen3_coder \
+    --quantization modelopt_fp4 \
+    --fp4-gemm-backend flashinfer_cutlass \
+    --moe-runner-backend flashinfer_cutlass \
+    --attention-backend flashinfer \
+    --kv-cache-dtype fp8_e4m3 \
+    --mamba-ssm-dtype bfloat16 \
+    --mamba-scheduler-strategy no_buffer \
+    --disable-custom-all-reduce \
+    --disable-radix-cache \
+    --mem-fraction-static "$MEM_FRAC_STATIC" \
+    --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \
+    --max-prefill-tokens "$MAX_PREFILL_TOKENS" \
+    --context-length "$CONTEXT_LENGTH" \
+    --cuda-graph-max-bs-decode "$CONC" \
+    --max-running-requests "$MAX_RUNNING_REQUESTS" \
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \
+    --stream-interval 20 \
+    --speculative-algorithm EAGLE \
+    --speculative-num-steps 3 \
+    --speculative-eagle-topk 1 \
+    --speculative-num-draft-tokens 4 > "$SERVER_LOG" 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready \
+    --port "$PORT" \
+    --server-log "$SERVER_LOG" \
+    --server-pid "$SERVER_PID"
+
+# EAGLE-style spec decoding is trained against chat-formatted inputs, so the
+# benchmark must send chat prompts or the acceptance rate silently collapses.
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --use-chat-template \
+    --trust-remote-code
+
+if [[ "$RUN_EVAL" == "true" ]]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 5bd56b5f37..cf4a58384c 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1361,6 +1361,24 @@ qwen3.5-fp4-rtx6000pro-sglang:
       - { tp: 4, conc-list: [1, 4, 16, 64] }
       - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64] }
 
+# Same sweep with the built-in MTP draft head driven through SGLang's EAGLE
+# speculative path.
+qwen3.5-fp4-rtx6000pro-sglang-mtp:
+  image: lmsysorg/sglang:v0.5.16-cu130
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: rtx6000pro-lat
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp }
+      - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp }
+
 qwen3.5-fp4-b300-sglang-mtp:
   image: lmsysorg/sglang:v0.5.14-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 0ed64152f4..b39f8f1282 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5100,3 +5100,16 @@
     - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist"
     - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312
+
+- config-keys:
+    - qwen3.5-fp4-rtx6000pro-sglang-mtp
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Add an MTP arm to the Qwen3.5-397B-A17B NVFP4 RTX PRO 6000 SGLang sweep, driving the checkpoint's built-in MTP draft head through SGLang's EAGLE path with 3 steps, top-k 1, and 4 draft tokens"
+    - "Sweep the same 8k/1k grid as the non-MTP config: concurrency 1, 4, 16, and 64 for both TP4 and TEP4"
+    - "Raise mem-fraction-static to 0.80 for this arm: the fraction bounds weights plus the KV and Mamba pools, so the 4.05 GiB per-rank draft head has to be paid for by raising it, not lowering it (0.65 and 0.75 both left the pools empty on this 96 GiB SKU, and 0.85 came up with only 4.56 GiB free per rank)"
+    - "Size max-running-requests to the tested concurrency instead of a fixed 128, since the client never opens more connections and the oversized Mamba state pool is what starved the draft head"
+    - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding"
+    - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312

From a45bd3228029d70581988e60a6a9f6e269c9cab3 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 30 Jul 2026 01:58:37 -0400
Subject: [PATCH 7/7] chore: refresh PR #2312 for sweep reuse [skip-sweep]