diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml new file mode 100644 index 0000000000..5cb6c798b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep4-dep8-c1024 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1024' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml new file mode 100644 index 0000000000..e3c08dc477 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2048' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml new file mode 100644 index 0000000000..1c0abe2fa0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml @@ -0,0 +1,110 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '0' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + max-model-len: 9216 + max-num-seqs: 1024 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml new file mode 100644 index 0000000000..1ca87ad9b8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p4d-dep4-tp4-c512 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml new file mode 100644 index 0000000000..44f8ff7849 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c128 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml new file mode 100644 index 0000000000..c748107fec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c32 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.93 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '32' + req_rate: inf + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml new file mode 100644 index 0000000000..54e623d767 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-2p1d-dep8-dep8-c8192 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8192' + req_rate: inf + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9fc14fd4e8..a30af59641 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4977,6 +4977,114 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 16 dp-attn: true +kimik2.6-fp4-b200-dynamo-vllm: + image: vllm/vllm-openai:v0.25.1 + model: nvidia/Kimi-K2.6-NVFP4 + model-prefix: kimik2.6 + runner: b200-multinode + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260721" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [32] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [128] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [512] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [1024] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [2048] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [8192] + prefill: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + dsv4-fp4-b200-dynamo-vllm: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8f5831507e..4c22ed9198 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5339,3 +5339,11 @@ - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding" - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + description: + - "Add Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration." + - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries." + - "Image: vllm/vllm-openai:v0.25.1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index b304ad4c1f..be68ce073b 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -6,13 +6,11 @@ SLURM_ACCOUNT="benchmark" set -x -# MODEL_PATH: Override with pre-downloaded paths on the shared Lustre tree. +# MODEL_PATH: Override with pre-downloaded paths on cluster-accessible storage. # Bench scripts and srt-slurm yaml configs specify HuggingFace model IDs for -# portability, but we resolve to /lustre/fsw/models/* here to avoid repeated +# portability, but we resolve to pre-staged paths here to avoid repeated # downloading on every dgxc node. Runs for both single-node and multinode # launches. -# NOTE: per-node /raid/models/* would be faster but is only populated on a -# subset of dgxc nodes today, so we use Lustre for reliability. if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2" export SRT_SLURM_MODEL_PREFIX="dsr1" @@ -54,6 +52,9 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" +elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" + export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" @@ -127,6 +128,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 + mkdir -p recipes/vllm/kimi-k2.6 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -299,7 +306,14 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + + SRTCTL_PREFLIGHT_ARGS=() + # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. + if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) + fi + + SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output