diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml new file mode 100644 index 000000000..5cb6c798b --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep4-dep8-c1024 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1024' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml new file mode 100644 index 000000000..e3c08dc47 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2048' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml new file mode 100644 index 000000000..1c0abe2fa --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml @@ -0,0 +1,110 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '0' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + max-model-len: 9216 + max-num-seqs: 1024 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml new file mode 100644 index 000000000..1ca87ad9b --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p4d-dep4-tp4-c512 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml new file mode 100644 index 000000000..44f8ff784 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c128 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml new file mode 100644 index 000000000..c748107fe --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml @@ -0,0 +1,109 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c32 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.93 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '32' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml new file mode 100644 index 000000000..54e623d76 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -0,0 +1,112 @@ +name: kimi-k2.6-vllm-disagg-b200-2p1d-dep8-dep8-c8192 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8192' + req_rate: inf + use_chat_template: true