[NVIDIA] Add SGlang B200 FP4

.github/workflows/dsr1-tmpl.yml

@@ -143,8 +143,8 @@ jobs:
     uses: ./.github/workflows/benchmark-tmpl.yml
     secrets: inherit
     with:
-      runner: b200
-      image: 'lmsysorg/sglang:v0.5.0rc1-cu128-b200'
+      runner: b200-trt
+      image: 'lmsysorg/sglang:v0.5.3rc1-cu129-b200'
       model: 'nvidia/DeepSeek-R1-0528-FP4'
       framework: 'sglang'
       precision: 'fp4'
@@ -154,6 +154,7 @@ jobs:
       max-model-len: ${{ inputs.max-model-len }}
       random-range-ratio: ${{ inputs.random-range-ratio }}
       tp-list: '[8]'
+      conc-list: '[4, 8, 16, 32, 64, 128, 256]'  # Custom concurrency values for this job
 
   bmk-b200-trt-fp4:
     if: ${{ inputs.use_b200 }}

.github/workflows/runner-sweep-test.yml

@@ -35,6 +35,7 @@ on:
           - 'rocm/vllm-dev:open-mi300-08052025'
           - 'rocm/vllm-dev:open-mi355-08052025'
           - 'vllm/vllm-openai:v0.10.2'
+          - 'lmsysorg/sglang:v0.5.3rc1-cu129-b200'
 
       model:
         description: 'Model'

.github/workflows/runner-test.yml

@@ -60,7 +60,7 @@ on:
           - 'rocm/vllm-dev:open-mi300-08052025'
           - 'rocm/vllm-dev:open-mi355-08052025'
           - 'vllm/vllm-openai:v0.10.2'
-
+          - 'lmsysorg/sglang:v0.5.3rc1-cu129-b200'
       model:
         description: 'Model'
         required: true

benchmarks/dsr1_fp4_b200_slurm.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+echo "JOB $SLURM_JOB_ID running on NODE $SLURMD_NODENAME"
+
+huggingface-cli download $MODEL
+SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
+
+set -x
+PORT=$(( 8888 + $PORT_OFFSET ))
+
+
+# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls.
+
+SCHEDULER_RECV_INTERVAL=10
+
+if [[ $CONC -ge 16 ]]; then
+    SCHEDULER_RECV_INTERVAL=30
+fi
+
+
+
+echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL"
+
+
+PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \
+--tensor-parallel-size=$TP --data-parallel-size=1 \
+--cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \
+--chunked-prefill-size 16384 \
+--enable-ep-moe --quantization modelopt_fp4  --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \
+--enable-symm-mem  --disable-radix-cache --attention-backend trtllm_mla --enable-flashinfer-trtllm-moe --stream-interval 10 \
+> $SERVER_LOG 2>&1 &
+
+set +x
+IGNORE_PAT="Ignore import error when loading sglang.srt.models.glm4v_moe: No module named 'transformers.models.glm4v_moe'"
+
+while IFS= read -r line; do
+  printf '%s\n' "$line"
+
+  # Skip the known benign "Ignore import error ..." line
+  if [[ "$line" == *"$IGNORE_PAT"* ]]; then
+    continue
+  fi
+
+  # Keep your original "error" trap for everything else
+  if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]]; then
+    sleep 5
+    tail -n100 "$SERVER_LOG"
+    echo "JOB ${SLURM_JOB_ID:-NA} ran on NODE ${SLURMD_NODENAME:-unknown}"
+    exit 1
+  fi
+
+  # Break when server is ready
+  if [[ "$line" == *"The server is fired up and ready to roll"* ]]; then
+    break
+  fi
+# Start tail from the beginning so we don't miss early lines
+done < <(tail -n +1 -F "$SERVER_LOG")
+
+set -x
+git clone https://github.com/kimbochen/bench_serving.git
+
+# warmup for JIT kernels
+python3 bench_serving/benchmark_serving.py \
+--model $MODEL --backend vllm \
+--base-url http://0.0.0.0:$PORT \
+--dataset-name random \
+--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \
+--num-prompts $CONC --max-concurrency $CONC \
+--request-rate inf --ignore-eos
+
+python3 bench_serving/benchmark_serving.py \
+--model $MODEL --backend vllm \
+--base-url http://0.0.0.0:$PORT \
+--dataset-name random \
+--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \
+--num-prompts $(( $CONC * 10 )) --max-concurrency $CONC \
+--request-rate inf --ignore-eos \
+--save-result --percentile-metrics 'ttft,tpot,itl,e2el' \
+--result-dir /workspace/ \
+--result-filename $RESULT_FILENAME.json