Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
base:
schema: 2
name: minimaxm31flash-fp4-gb300-sglang-agentic-tp2
resources:
gpu_type: gb300
gpus_per_node: 4
frontend:
type: sglang
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine: sglang
# Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default.
health_check:
max_attempts: 1440
interval_seconds: 5
roles:
agg:
nodes: 1
workers: 1
gpus: 2
# Serve flags follow the cookbook's GB300 cells at
# https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1.
args:
served-model-name: MiniMaxAI/MiniMax-M3.1-Flash
trust-remote-code: true
reasoning-parser: minimax-m3
tool-call-parser: minimax-m3
# Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs
# switch to synthetic acceptance at the golden acceptance length.
speculative-algorithm: DSPARK
speculative-dspark-block-size: 7
speculative-use-block-verification: true
cuda-graph-backend-prefill: breakable
tokenizer-worker-num: 8
watchdog-timeout: 3600
enable-metrics: true
env:
PYTHONNOUSERSITE: '1'
PYTHONUNBUFFERED: '1'
SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
FMHA_SM100_ALLOW_JIT: '1'
benchmark:
env:
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

zip_override_tp2_hicache:
roles:
agg:
args:
tensor-parallel-size: 2
moe-runner-backend: flashinfer_trtllm_routed
chunked-prefill-size: 32768
# 2x CONC (AgentX subagent fan-out); the decode graph batch matches it.
max-running-requests: [40, 48, 56, 72]
cuda-graph-max-bs-decode: [40, 48, 56, 72]
mem-fraction-static: 0.87
enable-hierarchical-cache: true
hicache-size: 180
hicache-write-policy: write_back
benchmark:
env:
CONC: ['20', '24', '28', '36']
KV_OFFLOADING: dram
1 change: 1 addition & 0 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8957,6 +8957,7 @@ minimaxm31flash-fp4-gb300-sglang-agentic-dspark:
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64], srt-recipe: agentic.yaml }
# TP2 with L2 HiCache: 180 GB per rank at conc 32-44.
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 44], srt-recipe: agentic.yaml }
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 36], srt-recipe: agentic-tp2.yaml }
# High-Throughput with L2 HiCache: 175 GB per rank at conc 64 and 128.
- { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 128], srt-recipe: agentic.yaml }

Expand Down
8 changes: 8 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11244,6 +11244,14 @@
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX
scenario-type: [agentic-coding]

- config-keys:
- minimaxm31flash-fp4-gb300-sglang-agentic-dspark
description:
- "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 20, 24, 28 and 36 to the existing curve."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4525
scenario-type: [agentic-coding]
append-only: true

- config-keys:
- minimaxm31flash-fp4-b200-sglang-agentic-dspark
description:
Expand Down
Loading