diff --git a/.github/workflows/dsr1-tmpl.yml b/.github/workflows/dsr1-tmpl.yml index d202f36ae6..25ba17f78b 100644 --- a/.github/workflows/dsr1-tmpl.yml +++ b/.github/workflows/dsr1-tmpl.yml @@ -151,7 +151,7 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml secrets: inherit with: - runner: b200-trt + runner: b200 image: 'lmsysorg/sglang:v0.5.3rc1-cu129-b200' model: 'nvidia/DeepSeek-R1-0528-FP4' framework: 'sglang' @@ -162,7 +162,7 @@ jobs: max-model-len: ${{ inputs.max-model-len }} random-range-ratio: ${{ inputs.random-range-ratio }} tp-list: '[8]' - conc-list: '[4, 8, 16, 32, 64, 128, 256]' # Custom concurrency values for this job + conc-list: '[4, 8, 16, 32, 64, 128]' # Custom concurrency values for this job bmk-b200-trt-fp4: if: ${{ inputs.use_b200 }} diff --git a/benchmarks/dsr1_fp4_b200_docker.sh b/benchmarks/dsr1_fp4_b200_docker.sh index fe0ac38024..88d68f6b6d 100644 --- a/benchmarks/dsr1_fp4_b200_docker.sh +++ b/benchmarks/dsr1_fp4_b200_docker.sh @@ -15,3 +15,4 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0. --chunked-prefill-size 16384 \ --enable-ep-moe --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ --enable-symm-mem --disable-radix-cache --attention-backend trtllm_mla --enable-flashinfer-trtllm-moe --stream-interval 10 + diff --git a/runners/launch_b200-nvd.sh b/runners/launch_b200-nvd.sh index b9a3be44a2..041b7756d9 100644 --- a/runners/launch_b200-nvd.sh +++ b/runners/launch_b200-nvd.sh @@ -28,6 +28,25 @@ done < <(docker logs -f --tail=0 $server_name 2>&1) git clone https://github.com/kimbochen/bench_serving.git +# warmup for JIT kernels - only for DeepSeek-R1-0528-FP4 model +if [[ "$MODEL" == "nvidia/DeepSeek-R1-0528-FP4" ]]; then + echo "Running warmup for DeepSeek-R1-0528-FP4 model..." + WARMUP_PROMPTS=$(( $CONC * 10 )) + echo "Warmup prompts: $WARMUP_PROMPTS" + docker run --rm --network host --name warmup-client \ + -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \ + -e HF_TOKEN -e PYTHONPYCACHEPREFIX=/tmp/pycache/ \ + --entrypoint=/bin/bash \ + $(echo "$IMAGE" | sed 's/#/\//') \ + -lc "pip install -q datasets pandas && \ + python3 bench_serving/benchmark_serving.py \ + --model $MODEL --backend vllm --base-url http://localhost:$PORT \ + --dataset-name random \ + --random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \ + --num-prompts $WARMUP_PROMPTS --max-concurrency $CONC \ + --request-rate inf --ignore-eos" +fi + set -x docker run --rm --network host --name $client_name \ -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \