[AgentX] Tune Kimi K3 DSpark on B200

.github/workflows/benchmark-multinode-tmpl.yml

@@ -467,7 +467,7 @@ jobs:
         if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }}
+          name: eval_${{ env.RESULT_FILENAME }}
           path: |
             meta_env.json
             results*.json

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml

@@ -0,0 +1,111 @@
+name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-agentic"
+
+# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates
+# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs.
+# This keeps FlashInfer fused collectives node-local while using shipped vLLM.
+# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    # TP8 collectives stay within each B200 NVSwitch domain.
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 2
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: auto
+      # Keep headroom for the first FlashInfer MXFP4 MoE workspace.
+      gpu-memory-utilization: 0.90
+      no-enable-flashinfer-autotune: true
+      max-model-len: 1048576
+      kv-cache-dtype: fp8
+      attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+      max-num-seqs: 8
+      max-num-batched-tokens: 8192
+      # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval
+      # selects the paired real-verification recipe instead.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}'
+
+sbatch_directives:
+  segment: "1"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  aiperf_server_metrics: true
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml

@@ -0,0 +1,109 @@
+name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-eval-agentic"
+
+# Real-verification eval counterpart to the B200 DEP throughput recipe.
+# It keeps TP8/DP2/EP16 and target settings unchanged; only DSpark acceptance
+# changes from synthetic golden AL to target-verified block.
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    # TP8 collectives stay within each B200 NVSwitch domain.
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 2
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: auto
+      # Keep headroom for the first FlashInfer MXFP4 MoE workspace.
+      gpu-memory-utilization: 0.90
+      no-enable-flashinfer-autotune: true
+      max-model-len: 1048576
+      kv-cache-dtype: fp8
+      attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+      max-num-seqs: 8
+      max-num-batched-tokens: 8192
+      # Eval must verify every draft against real target logits.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+
+sbatch_directives:
+  segment: "1"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  aiperf_server_metrics: true
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml

@@ -0,0 +1,112 @@
+name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-agentic"
+
+# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates
+# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs.
+# This keeps FlashInfer fused collectives node-local while using shipped vLLM.
+# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    # TP8 collectives stay within each B200 NVSwitch domain.
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 2
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: auto
+      # Keep headroom for the first FlashInfer MXFP4 MoE workspace.
+      gpu-memory-utilization: 0.90
+      no-enable-flashinfer-autotune: true
+      max-model-len: 1048576
+      kv-cache-dtype: fp8
+      attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+      max-num-seqs: 8
+      max-num-batched-tokens: 8192
+      # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval
+      # selects the paired real-verification recipe instead.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}'
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}'
+
+sbatch_directives:
+  segment: "1"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  aiperf_server_metrics: true
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"