diff --git a/.github/workflows/dsr1-tmpl.yml b/.github/workflows/dsr1-tmpl.yml index df05305d4b..375e53291e 100644 --- a/.github/workflows/dsr1-tmpl.yml +++ b/.github/workflows/dsr1-tmpl.yml @@ -143,8 +143,8 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml secrets: inherit with: - runner: b200 - image: 'lmsysorg/sglang:v0.5.0rc1-cu128-b200' + runner: b200-trt + image: 'lmsysorg/sglang:v0.5.3rc1-cu129-b200' model: 'nvidia/DeepSeek-R1-0528-FP4' framework: 'sglang' precision: 'fp4' @@ -154,6 +154,7 @@ jobs: max-model-len: ${{ inputs.max-model-len }} random-range-ratio: ${{ inputs.random-range-ratio }} tp-list: '[8]' + conc-list: '[4, 8, 16, 32, 64, 128, 256]' # Custom concurrency values for this job bmk-b200-trt-fp4: if: ${{ inputs.use_b200 }} diff --git a/.github/workflows/runner-sweep-test.yml b/.github/workflows/runner-sweep-test.yml index 6e6ca023de..1eab7f9fd9 100644 --- a/.github/workflows/runner-sweep-test.yml +++ b/.github/workflows/runner-sweep-test.yml @@ -35,6 +35,7 @@ on: - 'rocm/vllm-dev:open-mi300-08052025' - 'rocm/vllm-dev:open-mi355-08052025' - 'vllm/vllm-openai:v0.10.2' + - 'lmsysorg/sglang:v0.5.3rc1-cu129-b200' model: description: 'Model' diff --git a/.github/workflows/runner-test.yml b/.github/workflows/runner-test.yml index 9db2644c02..21afc5e778 100644 --- a/.github/workflows/runner-test.yml +++ b/.github/workflows/runner-test.yml @@ -60,7 +60,7 @@ on: - 'rocm/vllm-dev:open-mi300-08052025' - 'rocm/vllm-dev:open-mi355-08052025' - 'vllm/vllm-openai:v0.10.2' - + - 'lmsysorg/sglang:v0.5.3rc1-cu129-b200' model: description: 'Model' required: true diff --git a/benchmarks/dsr1_fp4_b200_slurm.sh b/benchmarks/dsr1_fp4_b200_slurm.sh new file mode 100644 index 0000000000..d3a6b22225 --- /dev/null +++ b/benchmarks/dsr1_fp4_b200_slurm.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +echo "JOB $SLURM_JOB_ID running on NODE $SLURMD_NODENAME" + +huggingface-cli download $MODEL +SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) + +set -x +PORT=$(( 8888 + $PORT_OFFSET )) + + +# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. + +SCHEDULER_RECV_INTERVAL=10 + +if [[ $CONC -ge 16 ]]; then + SCHEDULER_RECV_INTERVAL=30 +fi + + + +echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" + + +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ +--tensor-parallel-size=$TP --data-parallel-size=1 \ +--cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \ +--chunked-prefill-size 16384 \ +--enable-ep-moe --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ +--enable-symm-mem --disable-radix-cache --attention-backend trtllm_mla --enable-flashinfer-trtllm-moe --stream-interval 10 \ +> $SERVER_LOG 2>&1 & + +set +x +IGNORE_PAT="Ignore import error when loading sglang.srt.models.glm4v_moe: No module named 'transformers.models.glm4v_moe'" + +while IFS= read -r line; do + printf '%s\n' "$line" + + # Skip the known benign "Ignore import error ..." line + if [[ "$line" == *"$IGNORE_PAT"* ]]; then + continue + fi + + # Keep your original "error" trap for everything else + if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]]; then + sleep 5 + tail -n100 "$SERVER_LOG" + echo "JOB ${SLURM_JOB_ID:-NA} ran on NODE ${SLURMD_NODENAME:-unknown}" + exit 1 + fi + + # Break when server is ready + if [[ "$line" == *"The server is fired up and ready to roll"* ]]; then + break + fi +# Start tail from the beginning so we don't miss early lines +done < <(tail -n +1 -F "$SERVER_LOG") + +set -x +git clone https://github.com/kimbochen/bench_serving.git + +# warmup for JIT kernels +python3 bench_serving/benchmark_serving.py \ +--model $MODEL --backend vllm \ +--base-url http://0.0.0.0:$PORT \ +--dataset-name random \ +--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \ +--num-prompts $CONC --max-concurrency $CONC \ +--request-rate inf --ignore-eos + +python3 bench_serving/benchmark_serving.py \ +--model $MODEL --backend vllm \ +--base-url http://0.0.0.0:$PORT \ +--dataset-name random \ +--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \ +--num-prompts $(( $CONC * 10 )) --max-concurrency $CONC \ +--request-rate inf --ignore-eos \ +--save-result --percentile-metrics 'ttft,tpot,itl,e2el' \ +--result-dir /workspace/ \ +--result-filename $RESULT_FILENAME.json