Fix for https://github.com/sgl-project/sglang/issues/22072 / 修复 SGLang MoRI 分离式推理在 MI325X 上的挂起问题

benchmarks/multi_node/amd_utils/env.sh

@@ -62,6 +62,20 @@ set +x
 
 export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES}
 
+# Inter-node NCCL over this bnxt_re RoCE fabric. Only matters for multi-node TP/EP
+# workers (e.g. decode TP16 over 2 nodes); inert for the single-node-worker tests
+# (1-4), which do no inter-node NCCL.
+#   NCCL_IB_GID_INDEX=3: use the routable RoCEv2 GID (fd93:16d3:59b6:012*). Without
+#     it NCCL falls back to RoCEv1 (GID idx 0, link-local) and cross-node comm init
+#     hangs.
+#   NCCL_IB_TC=104 / NCCL_IB_SL=3: put NCCL's own RDMA on the PFC-protected lossless
+#     class (DSCP 24 / priority 3). Without it, NCCL's connection handshake can be
+#     dropped on the lossy default queue for some node pairs, hanging ncclCommInitRank.
+# Override for a cluster with a different GID layout / PFC class.
+export NCCL_IB_GID_INDEX="${NCCL_IB_GID_INDEX:-3}"
+export NCCL_IB_TC="${NCCL_IB_TC:-104}"
+export NCCL_IB_SL="${NCCL_IB_SL:-3}"
+
 # =============================================================================
 # MoRI-specific environment
 # =============================================================================
@@ -76,7 +90,12 @@ export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}"
 # QoS/DSCP configuration
 # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname
 if [[ -n "$MORI_RDMA_TC" ]]; then
-    echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)"
+    # Derive matching SL from TC; bnxt_re rejects inconsistent DSCP/SL pairs and
+    # silently downgrades to the lossy queue (RETRY_EXC_ERR / stalled KV under load).
+    [[ -z "${MORI_RDMA_SL:-}" ]] && export MORI_RDMA_SL=$(( MORI_RDMA_TC >> 5 ))
+    export MORI_IO_TC="${MORI_IO_TC:-$MORI_RDMA_TC}"
+    export MORI_IO_SL="${MORI_IO_SL:-$MORI_RDMA_SL}"
+    echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC MORI_RDMA_SL=$MORI_RDMA_SL MORI_IO_TC=$MORI_IO_TC MORI_IO_SL=$MORI_IO_SL (set by runner or environment)"
 elif command -v nicctl &> /dev/null; then
     ND_PRIO=$(nicctl show qos  2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}')
     ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" '
@@ -253,7 +272,9 @@ else
     export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32
 
     export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192
-    export MORI_MAX_DISPATCH_TOKENS_DECODE=512
+    # 512 undersizes the decode MoRI MoE dispatch buffer for conc-32/EP16, where the
+    # cross-node all-to-all stalls under load.
+    export MORI_MAX_DISPATCH_TOKENS_DECODE=4096
 
     export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768
     export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703

benchmarks/multi_node/amd_utils/models.yaml

@@ -142,7 +142,7 @@ DeepSeek-R1-0528:
   dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
   ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
   prefill:
-    mem_fraction_static: 0.8
+    mem_fraction_static: 0.9
     disable_radix_cache: true
     dp:
       max_running_requests: 24

benchmarks/multi_node/amd_utils/server_sglang.sh

@@ -226,14 +226,24 @@ fi
 
 # When both DP and EP are enabled, override max-running-requests and dispatch tokens
 if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then
-    decode_max_running_requests=$BENCH_MAX_CONC_VALUE
     decode_dp_ranks=$DECODE_TP_SIZE
-    MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks))
-    # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10))
-    # Update derived variable
-    SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2))
-    export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD
-    echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD"
+    # max-running-requests is split across DP ranks (sglang's req_to_token_pool is
+    # per-rank = max_running_requests // dp_ranks). It must be >= dp_ranks, else the
+    # per-rank pool floors to 0 and get_batch_sizes_to_capture collapses capture_bs
+    # to [0] (AssertionError). This happens when bench concurrency < dp_ranks.
+    # Old heuristic set max_running=BENCH_MAX_CONC (=conc) and then shrank the MoRI
+    # dispatch tokens to max_running/dp_ranks (=2 at conc32/DP16). That gives each DP
+    # rank only ~2 request slots and a 2-token MoE all-to-all dispatch buffer, which
+    # starves the cross-node EP path under load (TTFT ~11s, mass timeouts). The patch
+    # reference instead keeps a generous per-rank pool (conc*TP) and the env.sh MoRI
+    # dispatch / MOE token sizes (4096 / 2703). Floor at dp_ranks to keep capture_bs>0.
+    decode_max_running_requests=$((BENCH_MAX_CONC_VALUE * decode_dp_ranks))
+    if (( decode_max_running_requests < decode_dp_ranks )); then
+        decode_max_running_requests=$decode_dp_ranks
+    fi
+    # Keep MORI_MAX_DISPATCH_TOKENS_DECODE / MOE_MAX_INPUT / INTER_KERNEL_SWITCH from
+    # env.sh (do NOT shrink them to max_running/dp_ranks).
+    echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE (env), MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE (env)"
 fi
 
 # Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS)