diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6b5319c667..0a666a6e90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -467,7 +467,7 @@ jobs: if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }} + name: eval_${{ env.RESULT_FILENAME }} path: | meta_env.json results*.json diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..7dc3d49793 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml @@ -0,0 +1,111 @@ +name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-agentic" + +# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates +# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. +# This keeps FlashInfer fused collectives node-local while using shipped vLLM. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval + # selects the paired real-verification recipe instead. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml new file mode 100644 index 0000000000..f562682522 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml @@ -0,0 +1,109 @@ +name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-eval-agentic" + +# Real-verification eval counterpart to the B200 DEP throughput recipe. +# It keeps TP8/DP2/EP16 and target settings unchanged; only DSpark acceptance +# changes from synthetic golden AL to target-verified block. +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Eval must verify every draft against real target logits. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..0119d79219 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,112 @@ +name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-agentic" + +# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates +# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. +# This keeps FlashInfer fused collectives node-local while using shipped vLLM. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval + # selects the paired real-verification recipe instead. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml new file mode 100644 index 0000000000..a3ae8342b9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml @@ -0,0 +1,110 @@ +name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-eval-agentic" + +# Real-verification eval counterpart to the B200 DEP offload throughput recipe. +# It keeps TP8/DP2/EP16 and the CPU KV connector unchanged; only DSpark +# acceptance changes from synthetic golden AL to target-verified block. +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Eval must verify every draft against real target logits. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ab43c7573a..c752b04f67 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8035,6 +8035,90 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false +# DSpark uses two node-local TP8 attention/dense groups and DP2 across nodes. +# Expert parallel spans all 16 ranks, so the 896 experts remain EP16 while +# FlashInfer fused collectives stay inside each B200 NVSwitch domain. +kimik3-fp4-b200-dynamo-vllm-agentic-dspark: + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.63 + search-space: + # Low-latency and high-interactivity points. + - spec-decoding: mtp + conc-list: [1, 2, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 16 + dp-attn: true + # Balanced medium-concurrency points. + - spec-decoding: mtp + conc-list: [8, 12, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 16 + dp-attn: true + # GPU-resident throughput points around the prior c16-c32 KV cliff. + - spec-decoding: mtp + conc-list: [20, 24, 28, 32] + prefill: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 16 + dp-attn: true + # CPU KV-offload crossover and capacity points. Keep the resident points + # above so the same concurrency can be compared with one variable changed. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "13c59a3" } + conc-list: [8, 12, 16, 20, 24, 28, 32, 48, 64] + prefill: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 16 + dp-attn: true + qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 23369f7273..0bce8d894f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5784,7 +5784,6 @@ - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 - - config-keys: - kimik3-fp4-b300-vllm-agentic-dspark description: @@ -5803,4 +5802,19 @@ - "Use Dynamo header-based session affinity and AIPerf agentx-v1.0.1 (b7b16cf851885567988a643282266bce74e34437)." - "Enable SGLang engine metrics and require the sglang: server-metric prefix so the published trace artifacts carry backend data." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2364 - + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on a two-node TP8/DP2/EP16 DSpark topology with tensor parallelism local to each B200" + - "Add a TP8/DP2/EP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Keep throughput on DSpark level 7 probabilistic drafting with committed golden synthetic AL 3.84; EVAL_ONLY selects committed rejection_sample_method=block recipes for real target verification" + - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" + - "Use node-local TP8 FlashInfer collectives and cross-node DP2/EP16 so the digest-pinned upstream serving engine runs without a compatibility layer or source rewrite" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 2cd2f2ee81..8c2b357b00 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -165,21 +165,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor + # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct + # multi-node vLLM. Pin the tested renderer so branch movement cannot change + # generated rank commands between sweep points. if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 - # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm - # serve` owns the OpenAI port itself, no Dynamo layer. The fork branch - # carries PR #278 plus the multi-node extension (vLLM-native - # --master-addr/--nnodes/--node-rank serve + headless non-leader - # ranks) so the 2-node TP8xPP2 topology can run. git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - fi + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -352,6 +347,15 @@ EOF echo "MODEL_PATH=$MODEL_PATH (exists=$(test -d "$MODEL_PATH" && echo yes || echo NO))" ls -ld "$MODEL_PATH" 2>&1 || ls /lustre/fsw/models/ 2>&1 | head -40 + # An eval row may point at a committed real-verification recipe while its + # throughput row keeps synthetic golden acceptance. Only configs that set + # EVAL_CONFIG_FILE opt into this selection; all other configs keep using + # CONFIG_FILE unchanged. + if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" + fi + if [[ -z "$CONFIG_FILE" ]]; then echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 echo "Config: MODEL_PREFIX=${MODEL_PREFIX} PRECISION=${PRECISION} FRAMEWORK=${FRAMEWORK}" >&2