diff --git a/.github/workflows/runner-sweep-test.yml b/.github/workflows/runner-sweep-test.yml index 20578f24e0..5f8ae61e0f 100644 --- a/.github/workflows/runner-sweep-test.yml +++ b/.github/workflows/runner-sweep-test.yml @@ -23,31 +23,30 @@ on: type: choice options: - 'kedarpotdar147/vllm0.1:latest' - - 'kedarpotdar147/vllm:05' - - 'vllm/vllm-openai:v0.10.2' - - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_vllm_0.10.1_instinct_rc1' - - 'rocm/vllm-dev:open-mi300-08052025' - - 'rocm/vllm-dev:open-mi355-08052025' - 'lmsysorg/sglang:v0.4.9.post1-cu126' - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200' - - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post2' + - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_vllm_0.10.1_instinct_rc1' + - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' + - 'rocm/vllm-dev:open-mi300-08052025' + - 'rocm/vllm-dev:open-mi355-08052025' + - 'vllm/vllm-openai:v0.10.2' model: description: 'Model' required: true type: choice options: - - 'nvidia/Llama-3.3-70B-Instruct-FP8' - - 'nvidia/Llama-3.3-70B-Instruct-FP4' + - 'amd/DeepSeek-R1-0528-MXFP4-Preview' - 'amd/Llama-3.3-70B-Instruct-FP8-KV' - 'amd/Llama-3.3-70B-Instruct-MXFP4-Preview' - 'deepseek-ai/DeepSeek-R1-0528' + - 'nvidia/Llama-3.3-70B-Instruct-FP8' + - 'nvidia/Llama-3.3-70B-Instruct-FP4' - 'nvidia/DeepSeek-R1-0528-FP4' - 'nvidia/DeepSeek-R1-0528-FP4-v2' - - 'amd/DeepSeek-R1-0528-MXFP4-Preview' - 'openai/gpt-oss-120b' framework: diff --git a/.github/workflows/runner-test.yml b/.github/workflows/runner-test.yml index 58dc969622..c5271c6c40 100644 --- a/.github/workflows/runner-test.yml +++ b/.github/workflows/runner-test.yml @@ -24,6 +24,8 @@ on: - 'h200-nv_3' - 'b200-nv_0' - 'b200-nv_1' + - 'b200-nvd_0' + - 'b200-nvd_1' - 'b200-tg_0' - 'mi300x-amd_0' - 'mi300x-amd_1' @@ -46,31 +48,30 @@ on: type: choice options: - 'kedarpotdar147/vllm0.1:latest' - - 'kedarpotdar147/vllm:05' - - 'vllm/vllm-openai:v0.10.2' - - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_vllm_0.10.1_instinct_rc1' - - 'rocm/vllm-dev:open-mi300-08052025' - - 'rocm/vllm-dev:open-mi355-08052025' - 'lmsysorg/sglang:v0.4.9.post1-cu126' - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200' - - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post2' + - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_vllm_0.10.1_instinct_rc1' + - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' + - 'rocm/vllm-dev:open-mi300-08052025' + - 'rocm/vllm-dev:open-mi355-08052025' + - 'vllm/vllm-openai:v0.10.2' model: description: 'Model' required: true type: choice options: - - 'nvidia/Llama-3.3-70B-Instruct-FP8' - - 'nvidia/Llama-3.3-70B-Instruct-FP4' + - 'amd/DeepSeek-R1-0528-MXFP4-Preview' - 'amd/Llama-3.3-70B-Instruct-FP8-KV' - 'amd/Llama-3.3-70B-Instruct-MXFP4-Preview' - 'deepseek-ai/DeepSeek-R1-0528' + - 'nvidia/Llama-3.3-70B-Instruct-FP8' + - 'nvidia/Llama-3.3-70B-Instruct-FP4' - 'nvidia/DeepSeek-R1-0528-FP4' - 'nvidia/DeepSeek-R1-0528-FP4-v2' - - 'amd/DeepSeek-R1-0528-MXFP4-Preview' - 'openai/gpt-oss-120b' framework: diff --git a/runners/launch_b200-nvd.sh b/runners/launch_b200-nvd.sh new file mode 100644 index 0000000000..3b61edad0a --- /dev/null +++ b/runners/launch_b200-nvd.sh @@ -0,0 +1,54 @@ +#!/usr/bin/bash + +HF_HUB_CACHE_MOUNT="/raid/hf_hub_cache/" +PORT=8888 + +network_name="bmk-net" +server_name="bmk-server" +client_name="bmk-client" + +docker network create $network_name + +set -x +docker run --rm -d --network $network_name --name $server_name \ +--runtime nvidia --gpus all --ipc host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \ +-v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \ +-v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \ +-e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e PORT=$PORT \ +-e TORCH_CUDA_ARCH_LIST="10.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" \ +--entrypoint=/bin/bash \ +$IMAGE \ +benchmarks/"${EXP_NAME%%_*}_${PRECISION}_b200_docker.sh" + +set +x +while IFS= read -r line; do + printf '%s\n' "$line" + if [[ "$line" =~ Application\ startup\ complete ]]; then + break + fi +done < <(docker logs -f --tail=0 $server_name 2>&1) + +git clone https://github.com/kimbochen/bench_serving.git + +set -x +docker run --rm --network $network_name --name $client_name \ +-v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \ +-e HF_TOKEN -e PYTHONPYCACHEPREFIX=/tmp/pycache/ \ +--entrypoint=/bin/bash \ +$IMAGE \ +-lc "pip install -q datasets pandas && \ +python3 bench_serving/benchmark_serving.py \ +--model $MODEL --backend vllm --base-url http://$server_name:$PORT \ +--dataset-name random \ +--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \ +--num-prompts $(( $CONC * 10 )) \ +--max-concurrency $CONC \ +--request-rate inf --ignore-eos \ +--save-result --percentile-metrics 'ttft,tpot,itl,e2el' \ +--result-dir /workspace/ --result-filename $RESULT_FILENAME.json" + +while [ -n "$(docker ps -aq)" ]; do + docker stop $server_name + docker network rm $network_name + sleep 5 +done