diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..12208e12fe --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,186 @@ +schema: 2 + +base: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg + identity: + model: {} + frameworks: + dynamo: "1.7.0.dev20261030" + sglang: "0.0.0.dev1+ge3718813e" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + environment: + HF_HUB_CACHE: /hf_hub_cache + observability: + tachometer: + enabled: false + slurm: + time_limit: "4:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + FMHA_SM100_ALLOW_JIT: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + moe-runner-backend: flashinfer_trtllm_routed + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + cuda-graph-backend-prefill: breakable + router-mode: kv + router-session-affinity-ttl-secs: "14400" + watchdog-timeout: 3600 + enable-metrics: true + kv_events: true + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + +override_tp4_c1: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-c1 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + mem-fraction-static: 0.8 + +override_tp4_c4: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-c4 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + mem-fraction-static: 0.8 + +override_tp4_c8: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-c8 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + mem-fraction-static: 0.8 + +override_tp4_c16: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-c16 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + mem-fraction-static: 0.8 + +override_tp4_hicache_c32: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-hicache-c32 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + mem-fraction-static: 0.81 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_back + +override_tp4_hicache_c40: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-hicache-c40 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 80 + cuda-graph-max-bs-decode: 80 + mem-fraction-static: 0.81 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_back + +override_tp4_hicache_c48: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-hicache-c48 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 96 + cuda-graph-max-bs-decode: 96 + mem-fraction-static: 0.81 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_back + +override_tp4_hicache_c56: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-agg-tp4-hicache-c56 + roles: + agg: + args: + tensor-parallel-size: 4 + chunked-prefill-size: 32768 + max-running-requests: 112 + cuda-graph-max-bs-decode: 112 + mem-fraction-static: 0.81 + enable-hierarchical-cache: true + hicache-size: 180 + hicache-write-policy: write_back diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..69adf1c7a1 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,212 @@ +schema: 2 + +base: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-disagg + identity: + model: {} + frameworks: + dynamo: "1.7.0.dev20261030" + sglang: "0.0.0.dev1+ge3718813e" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb200 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + # Keeps tokenize, route and stream CPU work off the engine trays. + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + # The default frontend tokenizer disables its prefix cache for this model, + # so every turn would re-tokenize the full prompt. + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + FMHA_SM100_ALLOW_JIT: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # Some nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + # Dynamo 1.7 backend admission rejects turns with 529 once a DP rank is full; + # the SGLang scheduler queues them instead. + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + # flashinfer_trtllm_routed needs attention TP to match EP, which attention DP breaks. + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + chunked-prefill-size: 32768 + # Shortest uncached prefill first: short cached turns stop queueing + # behind long uncached prompts, which cuts the TTFT tail. + schedule-policy: shortest-prefill-first + # 64 per DP rank. + max-running-requests: 256 + mem-fraction-static: 0.85 + enable-hierarchical-cache: true + # 180 GB per rank: 720 GB per node, the matrix's 0.80 DRAM share, leaving + # headroom for the engine, Mooncake and the OS. + hicache-size: 180 + hicache-write-policy: write_back + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + # A session's turns stay on one prefill worker. + router-mode: kv + router-session-affinity-ttl-secs: "14400" + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: true + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + FMHA_SM100_ALLOW_JIT: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + DYN_SGL_GATEWAY_WORKERS: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + # The frontend picks the decode DP rank; least-loaded sends each request to + # the rank with the fewest in flight, where round-robin left the busiest rank + # at 1.45x the mean and every DP rank waits for it each step. + router-mode: least-loaded + load-balance-method: total_tokens + # 48 per DP rank. + max-running-requests: 384 + cuda-graph-max-bs-decode: 48 + mem-fraction-static: 0.90 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + watchdog-timeout: 3600 + enable-metrics: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + +override_2p_dep4_dep8: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-disagg-2p-dep4-dep8 + +override_3p_dep4_dep8: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-disagg-3p-dep4-dep8 + roles: + prefill: + nodes: 3 + workers: 3 + +override_3p_dep4_dep16: + name: minimaxm31flash-fp4-gb200-dynamo-sglang-agentx-disagg-3p-dep4-dep16 + roles: + prefill: + nodes: 3 + workers: 3 + decode: + nodes: 4 + gpus: 16 + args: + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + max-running-requests: 768 + mem-fraction-static: 0.85 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ca18213d0c..08cf5a9fa3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5332,6 +5332,113 @@ minimaxm31flash-fp4-gb200-sglang-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [72, 80], srt-recipe: agentic.yaml } +minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg: + image: lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.7.0.dev20261030" } + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/sglang/gb200-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c8 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_hicache_c32 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_hicache_c40 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [48] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_hicache_c48 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [56] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_hicache_c56 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + +minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg: + image: lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.7.0.dev20261030" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: minimaxm31flash/sglang/gb200-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [224, 256] + srt-recipe: disagg-variants.yaml:override_2p_dep4_dep8 + prefill: { num-worker: 2, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [256, 288] + srt-recipe: disagg-variants.yaml:override_3p_dep4_dep8 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [280, 320, 360] + srt-recipe: disagg-variants.yaml:override_3p_dep4_dep16 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + # Day-0 Kimi K3 AgentX coverage on GB200. The three aggregate srt-slurm # profiles mirror the serving strategies synthesized by the official recipe: # latency: multi-node TP16 (4 GB200 nodes / 16 GPUs) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 88ea87392c..877d72282b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11769,3 +11769,14 @@ - "6 DEP2 prefill workers with a DEP8 decode at conc 280-400 or a DEP16 decode at conc 300; MegaMoE, DSpark block 7 with fp8 draft KV, Dynamo KV router with fastokens for prefill and least-loaded routing across decode DP ranks." - "Prefill: shortest-prefill-first, chunk 8192, mem-fraction-static 0.85, HiCache 200 GB per rank write_back." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4686 + +- config-keys: + - minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg + - minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg + scenario-type: [agentic-coding] + description: + - "Add disaggregated SGLang behind Dynamo for MiniMax-M3.1-Flash AgentX on GB200 with lmsysorg/sglang:nightly-cu134-20261030-e371881 (Dynamo 1.7.0.dev20261030, Mooncake transfer, minimax-msa kernels)." + - "2 DEP4 prefill workers with a DEP8 decode at conc 224 and 256, 3 DEP4 prefill workers with a DEP8 decode at conc 256 and 288, or 3 DEP4 prefill workers with a DEP16 decode at conc 280-360; MegaMoE, DSpark block 7 with fp8 draft KV, Dynamo KV router with fastokens for prefill and least-loaded routing across decode DP ranks." + - "Prefill: shortest-prefill-first, chunk 32768 (8192 per DP rank), mem-fraction-static 0.85, HiCache 180 GB per rank write_back. Decode: mem-fraction-static 0.90 on DEP8, 0.85 on DEP16." + - "Agg: TP4 at conc 1-16, and TP4 with HiCache 180 GB per rank at conc 32-56, behind the same Dynamo frontend." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4695