diff --git a/benchmarks/llm-d/Dockerfile b/benchmarks/llm-d/Dockerfile index 8494027604..cfc04557c5 100644 --- a/benchmarks/llm-d/Dockerfile +++ b/benchmarks/llm-d/Dockerfile @@ -22,7 +22,7 @@ # --build-arg EPP_FROM_IMAGE="$EPP_FROM_IMAGE" \ # --build-arg ROUTING_SIDECAR_IMAGE="$ROUTING_SIDECAR_IMAGE" \ # --build-arg ENVOY_FROM_IMAGE="$ENVOY_FROM_IMAGE" ... -ARG VLLM_BASE=vllm/vllm-openai:v0.23.0 +ARG VLLM_BASE=vllm/vllm-openai:v0.26.0 ARG EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 ARG ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0 ARG ENVOY_FROM_IMAGE=envoyproxy/envoy:distroless-v1.33.2 diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml index f0cd876e4c..be307349ec 100644 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml +++ b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml @@ -18,7 +18,7 @@ # decode ep:1/dp-attn:false in the master key is what selects the # pure-TP path in server.sh. Decode knobs are latency-sized: max-num-seqs # 256, max-num-batched-tokens 256, max-cudagraph-capture-size 256, -# FULL_DECODE_ONLY graphs, stream-interval 50, gpu-memory-utilization 0.9. +# FULL_DECODE_ONLY graphs, gpu-memory-utilization 0.9. # 2. PREFILL is IDENTICAL to the mid-curve/high-tpt prefill (copied verbatim, # DEP8 with enable-ep-weight-filter + moe-backend deep_gemm_mega_moe). An # earlier srt-slurm-style low-latency prefill (weight-offload, no filter, @@ -51,7 +51,9 @@ plugins: - type: prefix-cache-scorer - type: queue-scorer - type: active-request-scorer - - type: max-score-picker + - type: weighted-random-picker + parameters: + threshold: "0.1" schedulingProfiles: - name: prefill @@ -63,12 +65,13 @@ schedulingProfiles: weight: 2 - pluginRef: active-request-scorer weight: 2 - - pluginRef: max-score-picker + - pluginRef: weighted-random-picker - name: decode plugins: - pluginRef: decode-filter - pluginRef: active-request-scorer - - pluginRef: max-score-picker + weight: 2 + - pluginRef: weighted-random-picker dataLayer: discovery: @@ -89,7 +92,10 @@ prefill: extra-args: >- --kv-cache-dtype fp8 --enforce-eager - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.95 + --max-model-len 9280 + --max-num-seqs 16 + --max-num-batched-tokens 32768 --enable-cumem-allocator --no-enable-prefix-caching --no-async-scheduling @@ -98,6 +104,7 @@ prefill: --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune --numa-bind env: NCCL_CUMEM_ENABLE: "1" @@ -123,14 +130,16 @@ prefill: NVSHMEM_REMOTE_TRANSPORT: "none" NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" TILELANG_CLEANUP_TEMP_FILES: "1" - TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" NVIDIA_GDRCOPY: "enabled" VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" # Harden distributed coordination against transient gloo/NCCL resets under @@ -156,7 +165,6 @@ decode: --no-enable-flashinfer-autotune --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --stream-interval 50 --tokenizer-mode deepseek_v4 --no-disable-hybrid-kv-cache-manager --enable-sleep-mode @@ -171,9 +179,13 @@ decode: # rc added so KV falls back to InfiniBand, not TCP (see prefill note). UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" UCX_CUDA_IPC_ENABLE_MNNVL: "y" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" TILELANG_CLEANUP_TEMP_FILES: "1" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" VLLM_SKIP_P2P_CHECK: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_RUST_FRONTEND: "1" NVIDIA_GDRCOPY: "enabled" TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" diff --git a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml index 2103ecf5c7..16032ca877 100644 --- a/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml +++ b/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml @@ -45,7 +45,9 @@ plugins: - type: prefix-cache-scorer - type: queue-scorer - type: active-request-scorer - - type: max-score-picker + - type: weighted-random-picker + parameters: + threshold: "0.1" schedulingProfiles: - name: prefill @@ -57,12 +59,13 @@ schedulingProfiles: weight: 2 - pluginRef: active-request-scorer weight: 2 - - pluginRef: max-score-picker + - pluginRef: weighted-random-picker - name: decode plugins: - pluginRef: decode-filter - pluginRef: active-request-scorer - - pluginRef: max-score-picker + weight: 2 + - pluginRef: weighted-random-picker dataLayer: discovery: @@ -87,7 +90,10 @@ prefill: extra-args: >- --kv-cache-dtype fp8 --enforce-eager - --gpu-memory-utilization 0.9 + --gpu-memory-utilization 0.95 + --max-model-len 9280 + --max-num-seqs 16 + --max-num-batched-tokens 32768 --enable-cumem-allocator --no-enable-prefix-caching --no-async-scheduling @@ -96,6 +102,7 @@ prefill: --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter --no-disable-hybrid-kv-cache-manager + --no-enable-flashinfer-autotune --numa-bind env: NCCL_CUMEM_ENABLE: "1" @@ -121,14 +128,16 @@ prefill: NVSHMEM_REMOTE_TRANSPORT: "none" NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" TILELANG_CLEANUP_TEMP_FILES: "1" - TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" NVIDIA_GDRCOPY: "enabled" VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" # Harden distributed coordination against transient gloo/NCCL connection @@ -146,12 +155,13 @@ decode: --max-num-seqs 512 --max-num-batched-tokens 512 --max-cudagraph-capture-size 512 - --gpu-memory-utilization 0.85 + --gpu-memory-utilization 0.9 + --max-model-len 9280 --enable-cumem-allocator --no-enable-prefix-caching + --no-enable-flashinfer-autotune --block-size 256 --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --stream-interval 50 --tokenizer-mode deepseek_v4 --moe-backend deep_gemm_mega_moe --enable-ep-weight-filter @@ -170,12 +180,14 @@ decode: NVSHMEM_REMOTE_TRANSPORT: "none" NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" TILELANG_CLEANUP_TEMP_FILES: "1" - TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" + NVSHMEM_DISABLE_CUDA_VMM: "0" VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" + VLLM_USE_RUST_FRONTEND: "1" NVIDIA_GDRCOPY: "enabled" # See prefill note: harden the decode's cross-node DP8/EP8 coordination # against transient gloo resets under the conc4096 warmup burst. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b850f7d6ec..19deb94e52 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5272,7 +5272,7 @@ dsv4-fp4-gb200-dynamo-vllm: # TODO: change image to official llmd image. # Build source: benchmarks/llm-d/Dockerfile. dsv4-fp4-gb200-llmd-vllm: - image: ghcr.io/ezrasilvera/llm-d-nokube-vllm:vllm0.23 + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb200 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b5aa7a3857..7b1cdd8e36 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5356,6 +5356,16 @@ - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + description: + - "Decode: gpu-memory-utilization 0.85->0.9, add max-model-len 9280, max-num-seqs 512->1024, disable NCCL symmetric memory, add no-enable-flashinfer-autotune, drop stream-interval (unsupported by rust frontend)" + - "Prefill: gpu-memory-utilization 0.9->0.95, add max-model-len 9280, add max-num-seqs 16 + max-num-batched-tokens 32768, enable rust frontend, enable randomize-dp-dummy-inputs" + - "EPP: switch from max-score-picker to weighted-random-picker (threshold=0.1) for better load distribution under high concurrency" + - "Low-latency recipe: same prefill optimizations, decode NCCL symmetric memory disabled + rust frontend enabled, drop stream-interval" + - "Bump llmd-vllm image vllm0.23 -> vllm0.26 (v0.26 strips data_parallel_hybrid_lb from rust args-json; required for VLLM_USE_RUST_FRONTEND=1 on multi-node DEP8)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2242 + - config-keys: - qwen3.5-fp4-b300-sglang-agentic-mtp description: @@ -5373,7 +5383,6 @@ - "Use the shared AIPerf watchdog to cap whole-trajectory runtime idle gaps at 300 seconds" - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 - - config-keys: - minimaxm3-fp4-b200-dynamo-vllm