diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx
index 7fa34007b5bb..4a468739ef20 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx
@@ -79,13 +79,8 @@ Agentic workloads, such as coding agents and other multi-turn, tool-using assist
import { AgentX } from "/src/snippets/_agentx.jsx";
import { agentx } from "/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx";
-import { b300Cells } from "/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx";
-The B300 HiCache configurations reserve 298 GB of host memory for the combined target/draft KV and Mamba pool, in addition to pinned PLE embeddings and process overhead. Provision sufficient free host RAM and container shared memory before starting them. GPU KV configurations also offload PLE embeddings to host memory.
-
-For the B300 NVIDIA configurations at concurrency 12 and above, `--mamba-track-interval 262144` reduces intermediate state checkpoints. The concurrency-80 configuration also uses `--strip-thinking-cache` and `--mamba-max-states-per-path 2`. These settings target replay traffic that does not resend model output; reassess prefix reuse for conversational clients that do.
-
-
+
### MI355X deployment measured with SemiAnalysis InferenceX
diff --git a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx
index fdb18ccba66e..5c947fd9e24a 100644
--- a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx
+++ b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx
@@ -1,51 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670, #4675, #4676, #4698), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "MiniMax-M3.1-Flash",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4698": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271829525",
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "status": "verified-pr"
+ "4726": {
+ "run": null,
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "status": "in-progress"
},
- "4676": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41054395691",
- "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
"status": "verified-pr"
},
- "4675": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41085412898",
- "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e",
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
"status": "verified-pr"
},
- "4670": {
- "run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
@@ -73,7 +73,72 @@ export const agentx = {
"id": "MiniMaxAI/MiniMax-M3.1-Flash"
}
],
- "envs": [],
+ "envs": [
+ {
+ "DYN_EVENT_PLANE": "zmq",
+ "DYN_REQUEST_PLANE": "tcp",
+ "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600",
+ "DYN_ROUTER_QUEUE_THRESHOLD": "None",
+ "DYN_ROUTER_TEMPERATURE": "0",
+ "DYN_TCP_CONNECT_TIMEOUT": "120",
+ "DYN_TOKENIZER": "fastokens",
+ "DYN_TOKENIZER_CACHE": "1",
+ "DYN_TOKENIZER_CACHE_BYTES": "8000000000",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
+ "DYN_EVENT_PLANE": "zmq",
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "FMHA_SM100_ALLOW_JIT": "1",
+ "NCCL_CUMEM_ENABLE": "1",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
+ },
+ {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
+ "DYN_EVENT_PLANE": "zmq",
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "FMHA_SM100_ALLOW_JIT": "1",
+ "MC_ENABLE_DEST_DEVICE_AFFINITY": "1",
+ "MC_FORCE_MNNVL": "1",
+ "MC_TCP_BIND_ADDRESS": "$NODE_IP",
+ "NCCL_CUMEM_ENABLE": "1",
+ "NCCL_MNNVL_ENABLE": "1",
+ "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
+ "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
+ "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
+ "WITH_NVIDIA_PEERMEM": "0"
+ },
+ {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
+ "DYN_EVENT_PLANE": "zmq",
+ "DYN_REQUEST_PLANE": "tcp",
+ "DYN_SGL_GATEWAY_WORKERS": "8",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "FMHA_SM100_ALLOW_JIT": "1",
+ "MC_ENABLE_DEST_DEVICE_AFFINITY": "1",
+ "MC_FORCE_MNNVL": "1",
+ "MC_TCP_BIND_ADDRESS": "$NODE_IP",
+ "NCCL_CUMEM_ENABLE": "1",
+ "NCCL_MNNVL_ENABLE": "1",
+ "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
+ "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
+ "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
+ "WITH_NVIDIA_PEERMEM": "0"
+ }
+ ],
"kvMooncake": {
"donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg",
"donorHardware": "gb200",
@@ -138,17 +203,17 @@ export const agentx = {
"files": {},
"cells": [
{
- "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#0",
"hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
+ "detail": "DSpark 7",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -165,29 +230,31 @@ export const agentx = {
}
},
"prov": {
- "sha": "3b96f3383ade71707f456573909b65a84b0b74d1",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml",
- "run": "40634232927/attempts/2",
- "pr": "4600",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-28"
+ "date": "2026-11-02"
},
"points": [
{
"concs": [1],
"item": 0,
- "override": "zip_override_tp4[0]",
+ "override": "override_tp4_c1",
"goldenAL": "4.04",
"kv": "none"
},
{
"concs": [4],
- "item": 0,
- "override": "zip_override_tp4[1]",
+ "item": 1,
+ "override": "override_tp4_c4",
"goldenAL": "4.04",
"kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
@@ -195,26 +262,27 @@ export const agentx = {
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"]
}
- }
+ },
+ null
]
}
},
{
"concs": [8],
- "item": 0,
- "override": "zip_override_tp4[2]",
+ "item": 2,
+ "override": "override_tp4_c8",
"goldenAL": "4.04",
"kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["16"],
@@ -222,26 +290,27 @@ export const agentx = {
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["16"],
"--max-running-requests": ["16"]
}
- }
+ },
+ null
]
}
},
{
"concs": [16],
- "item": 0,
- "override": "zip_override_tp4[3]",
+ "item": 3,
+ "override": "override_tp4_c16",
"goldenAL": "4.04",
"kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["32"],
@@ -249,67 +318,105 @@ export const agentx = {
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["32"],
"--max-running-requests": ["32"]
}
- }
+ },
+ null
]
}
},
{
- "concs": [32],
- "item": 0,
- "override": "zip_override_tp4[4]",
+ "concs": [24],
+ "item": 4,
+ "override": "override_tp4_hicache_spf_c24",
"goldenAL": "4.04",
- "kv": "none",
+ "kv": "hicache",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--chunked-prefill-size": ["8192"],
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["48"],
+ "--mem-fraction-static": ["0.87"],
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"]
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--chunked-prefill-size": ["8192"],
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["48"],
+ "--mem-fraction-static": ["0.87"],
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"]
}
- }
+ },
+ null
]
}
}
],
"routers": {
- "sglang": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
+ },
{
"kind": "worker",
"role": "agg",
"workers": 1,
"nodes_per_worker": 1,
- "cmd": "python3 -m sglang.launch_server",
- "env": {
- "FMHA_SM100_ALLOW_JIT": "1",
- "NCCL_CUMEM_ENABLE": "1",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
- },
+ "cmd": "python3 -m dynamo.sglang",
"args": [
["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
["--host", "0.0.0.0"],
["--port", "30000"],
+ ["--request-plane", "tcp"],
+ ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"],
["--chunked-prefill-size", "32768"],
["--cuda-graph-backend-prefill", "breakable"],
["--cuda-graph-max-bs-decode", "2"],
@@ -317,70 +424,66 @@ export const agentx = {
["--mem-fraction-static", "0.8"],
["--moe-runner-backend", "flashinfer_trtllm_routed"],
["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
["--speculative-algorithm", "DSPARK"],
["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
["--speculative-dspark-block-size", "7"],
["--speculative-use-block-verification"],
["--tensor-parallel-size", "4"],
- ["--tokenizer-worker-num", "8"],
["--tool-call-parser", "minimax-m3"],
["--trust-remote-code"],
["--watchdog-timeout", "3600"]
- ]
+ ],
+ "envRef": 1
}
]
},
- "dynamo": {
+ "sglang": {
"install": [],
"files": [],
"blocks": [
{
- "kind": "etcd",
- "count_nodes": 1,
- "cmd": "etcd",
- "env": {},
- "args": [
- ["--data-dir", "/tmp/etcd"],
- ["--listen-client-urls", "http://0.0.0.0:2379"],
- ["--advertise-client-urls", "http://$ETCD_IP:2379"]
- ]
- },
- {
- "kind": "dynamo-frontend",
- "count": 1,
- "cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
- "args": [
- ["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
- },
- {
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#2",
+ "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0",
"hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
"gpus": 4,
"configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
@@ -403,29 +506,241 @@ export const agentx = {
}
},
"prov": {
- "sha": "3b96f3383ade71707f456573909b65a84b0b74d1",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-low-latency.yaml",
- "run": "40634232927/attempts/2",
- "pr": "4600",
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
"status": "verified",
- "date": "2026-10-28"
+ "date": "2026-10-31"
},
"points": [
{
"concs": [1],
- "item": 2,
- "override": "override_tp4_c1",
+ "item": 0,
+ "override": "zip_override_tp4[0]",
"goldenAL": "4.04",
"kv": "none"
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
+ },
+ {
+ "concs": [4],
+ "item": 0,
+ "override": "zip_override_tp4[1]",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [8],
+ "item": 0,
+ "override": "zip_override_tp4[2]",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [16],
+ "item": 0,
+ "override": "zip_override_tp4[3]",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [32],
+ "item": 0,
+ "override": "zip_override_tp4[4]",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--tokenizer-worker-num": ["8"]
+ }
+ },
+ "ref": [0, "sglang", 0]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#2",
+ "hw": "gb300",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Agg TP4",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "imageStatus": "ok",
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "donorHardware": "gb300"
+ }
+ },
+ "prov": {
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-low-latency.yaml",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
+ "status": "verified",
+ "date": "2026-10-31"
+ },
+ "points": [
+ {
+ "concs": [1],
+ "item": 2,
+ "override": "override_tp4_c1",
+ "goldenAL": "4.04",
+ "kv": "none"
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
"args": {
"--disable-attn-tp-gather": [],
"--max-running-requests": ["8"],
@@ -437,8 +752,7 @@ export const agentx = {
"--chunked-prefill-size": null,
"--cuda-graph-backend-prefill": null,
"--cuda-graph-max-bs-decode": null,
- "--speculative-draft-kv-cache-dtype": null,
- "--tokenizer-worker-num": null
+ "--speculative-draft-kv-cache-dtype": null
},
"env": {
"ALLOW_JIT": "1",
@@ -499,7 +813,7 @@ export const agentx = {
"hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP2",
- "detail": "DSpark 7",
+ "detail": "DSpark 7 · --chunked-prefill-size 32768 · minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
"gpus": 2,
"configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
@@ -522,11 +836,11 @@ export const agentx = {
}
},
"prov": {
- "sha": "3b96f3383ade71707f456573909b65a84b0b74d1",
- "run": "40634232927/attempts/2",
- "pr": "4600",
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
"status": "verified",
- "date": "2026-10-28"
+ "date": "2026-10-31"
},
"points": [
{
@@ -723,7 +1037,7 @@ export const agentx = {
"--tensor-parallel-size": ["2"]
}
},
- "ref": [0, "sglang", 0]
+ "ref": [1, "sglang", 0]
}
]
},
@@ -773,12 +1087,12 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#1",
+ "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#10",
"hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4·EP4",
- "detail": "DSpark 7",
- "gpus": 4,
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --chunked-prefill-size 8192 · minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "gpus": 2,
"configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
@@ -800,36 +1114,34 @@ export const agentx = {
}
},
"prov": {
- "sha": "3b96f3383ade71707f456573909b65a84b0b74d1",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml",
- "run": "40634232927/attempts/2",
- "pr": "4600",
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
"status": "verified",
- "date": "2026-10-28"
+ "date": "2026-10-31"
},
"points": [
{
- "concs": [32],
- "item": 1,
- "override": "override_tp4_ep4_c32",
+ "concs": [20],
+ "item": 10,
+ "override": "zip_override_tp2_hicache_spf_interval32[0]",
"goldenAL": "4.04",
- "kv": "none"
+ "kv": "hicache"
},
{
- "concs": [64],
- "item": 6,
- "override": "override_tp4_ep4_hicache_c64",
+ "concs": [24],
+ "item": 11,
+ "override": "zip_override_tp2_hicache_spf_interval32[1]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["175"],
- "--hicache-write-policy": ["write_back"],
- "--mem-fraction-static": ["0.87"]
- }
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
+ }
}
],
"dynamo": [
@@ -837,29 +1149,25 @@ export const agentx = {
null,
{
"args": {
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["175"],
- "--hicache-write-policy": ["write_back"],
- "--mem-fraction-static": ["0.87"]
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
}
}
]
}
},
{
- "concs": [128],
- "item": 6,
- "override": "override_tp4_ep4_hicache_c128",
+ "concs": [28],
+ "item": 12,
+ "override": "zip_override_tp2_hicache_spf_interval32[2]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["175"],
- "--hicache-write-policy": ["write_back"],
- "--mem-fraction-static": ["0.87"]
+ "--cuda-graph-max-bs-decode": ["56"],
+ "--max-running-requests": ["56"]
}
}
],
@@ -868,10 +1176,89 @@ export const agentx = {
null,
{
"args": {
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["175"],
- "--hicache-write-policy": ["write_back"],
- "--mem-fraction-static": ["0.87"]
+ "--cuda-graph-max-bs-decode": ["56"],
+ "--max-running-requests": ["56"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [32],
+ "item": 13,
+ "override": "zip_override_tp2_hicache_spf_interval32[3]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [36],
+ "item": 14,
+ "override": "zip_override_tp2_hicache_spf_interval32[4]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [40],
+ "item": 15,
+ "override": "zip_override_tp2_hicache_spf_interval32[5]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
}
}
]
@@ -886,16 +1273,13 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--expert-parallel-size": ["4"],
- "--max-running-requests": ["128"],
- "--moe-runner-backend": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
+ "--chunked-prefill-size": ["8192"],
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"]
}
},
- "ref": [0, "sglang", 0]
+ "ref": [3, "sglang", 0]
}
]
},
@@ -972,20 +1356,112 @@ export const agentx = {
}
},
"prov": {
- "sha": "3b96f3383ade71707f456573909b65a84b0b74d1",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml",
- "run": "40634232927/attempts/2",
- "pr": "4600",
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
"status": "verified",
- "date": "2026-10-28"
+ "date": "2026-10-31"
},
"points": [
+ {
+ "concs": [24],
+ "item": 7,
+ "override": "zip_override_tp4_hicache_spf_interval32[0]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml"
+ },
+ {
+ "concs": [32],
+ "item": 8,
+ "override": "zip_override_tp4_hicache_spf_interval32[1]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml"
+ },
+ {
+ "concs": [40],
+ "item": 9,
+ "override": "zip_override_tp4_hicache_spf_interval32[2]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml"
+ },
{
"concs": [64],
"item": 3,
"override": "override_tp4_hicache_c64",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--chunked-prefill-size": ["32768"],
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"],
+ "--prefill-decode-interval": null,
+ "--schedule-policy": null,
+ "--stream-interval": null
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--chunked-prefill-size": ["32768"],
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"],
+ "--prefill-decode-interval": null,
+ "--schedule-policy": null,
+ "--stream-interval": null
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml"
}
],
"routers": {
@@ -996,12 +1472,12 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"],
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"],
"--tensor-parallel-size": ["4"]
}
},
- "ref": [2, "sglang", 0]
+ "ref": [4, "sglang", 0]
}
]
},
@@ -1051,13 +1527,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#0",
- "hw": "gb200",
+ "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#1",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
+ "label": "Agg TP4·EP4",
+ "detail": "DSpark 7",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -1073,38 +1549,40 @@ export const agentx = {
["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "donorHardware": "gb200"
+ "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
- "run": "40739138596/attempts/1",
- "pr": "4599",
+ "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml",
+ "run": "41054395691/attempts/2",
+ "pr": "4676",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-10-31"
},
"points": [
{
- "concs": [1],
- "item": 0,
- "override": "override_tp4_c1",
+ "concs": [32],
+ "item": 1,
+ "override": "override_tp4_ep4_c32",
"goldenAL": "4.04",
"kv": "none"
},
{
- "concs": [4],
- "item": 0,
- "override": "override_tp4_c4",
+ "concs": [64],
+ "item": 6,
+ "override": "override_tp4_ep4_hicache_c64",
"goldenAL": "4.04",
- "kv": "none",
+ "kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["8"]
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["175"],
+ "--hicache-write-policy": ["write_back"],
+ "--mem-fraction-static": ["0.87"]
}
}
],
@@ -1113,79 +1591,29 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["8"]
- }
- }
- ]
- }
- },
- {
- "concs": [8],
- "item": 0,
- "override": "override_tp4_c8",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["16"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["16"]
- }
- }
- ]
- }
- },
- {
- "concs": [16],
- "item": 0,
- "override": "override_tp4_c16",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["175"],
+ "--hicache-write-policy": ["write_back"],
+ "--mem-fraction-static": ["0.87"]
}
}
]
}
},
{
- "concs": [32],
- "item": 0,
- "override": "override_tp4_c32",
+ "concs": [128],
+ "item": 6,
+ "override": "override_tp4_ep4_hicache_c128",
"goldenAL": "4.04",
- "kv": "none",
+ "kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["175"],
+ "--hicache-write-policy": ["write_back"],
+ "--mem-fraction-static": ["0.87"]
}
}
],
@@ -1194,8 +1622,10 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["175"],
+ "--hicache-write-policy": ["write_back"],
+ "--mem-fraction-static": ["0.87"]
}
}
]
@@ -1208,38 +1638,18 @@ export const agentx = {
"files": [],
"blocks": [
{
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
- "cmd": "python3 -m sglang.launch_server",
- "env": {
- "FMHA_SM100_ALLOW_JIT": "1",
- "NCCL_CUMEM_ENABLE": "1",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--expert-parallel-size": ["4"],
+ "--max-running-requests": ["128"],
+ "--moe-runner-backend": null
+ },
+ "env": {
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
+ }
},
- "args": [
- ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--chunked-prefill-size", "32768"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-max-bs-decode", "2"],
- ["--max-running-requests", "2"],
- ["--mem-fraction-static", "0.8"],
- ["--moe-runner-backend", "flashinfer_trtllm_routed"],
- ["--reasoning-parser", "minimax-m3"],
- ["--speculative-algorithm", "DSPARK"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-dspark-block-size", "7"],
- ["--speculative-use-block-verification"],
- ["--tensor-parallel-size", "4"],
- ["--tokenizer-worker-num", "8"],
- ["--tool-call-parser", "minimax-m3"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "3600"]
- ]
+ "ref": [1, "sglang", 0]
}
]
},
@@ -1289,17 +1699,17 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#1",
- "hw": "gb200",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#5",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --chunked-prefill-size 8192 · minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -1311,57 +1721,28 @@ export const agentx = {
["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "donorHardware": "gb200"
+ "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic-low-latency.yaml",
- "run": "40739138596/attempts/1",
- "pr": "4599",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
{
- "concs": [1],
- "item": 1,
- "override": "override_tp4_c1",
+ "concs": [40],
+ "item": 5,
+ "override": "override_tp2_hicache_spf_c40",
"goldenAL": "4.04",
- "kv": "none"
+ "kv": "hicache"
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--disable-attn-tp-gather": [],
- "--max-running-requests": ["8"],
- "--monokernel-backend": ["msa"],
- "--page-size": ["128"],
- "--speculative-eagle-topk": ["1"],
- "--speculative-num-draft-tokens": ["8"],
- "--speculative-num-steps": ["1"],
- "--chunked-prefill-size": null,
- "--cuda-graph-backend-prefill": null,
- "--cuda-graph-max-bs-decode": null,
- "--speculative-draft-kv-cache-dtype": null,
- "--tokenizer-worker-num": null
- },
- "env": {
- "ALLOW_JIT": "1",
- "SGLANG_FORCE_COARSE_WAR_BARRIER": "1"
- }
- },
- "ref": [5, "sglang", 0]
- }
- ]
- },
"dynamo": {
"install": [],
"files": [],
@@ -1381,44 +1762,87 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--chunked-prefill-size": ["8192"],
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["80"],
+ "--mem-fraction-static": ["0.87"],
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"],
+ "--tensor-parallel-size": ["2"]
+ }
+ },
+ "ref": [0, "dynamo", 2]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#2",
- "hw": "gb200",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#6",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
- "detail": "DSpark 7 · measured with HiCache",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
- "imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --chunked-prefill-size 32768 · minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "imageStatus": "ok",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -1430,155 +1854,56 @@ export const agentx = {
["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "donorHardware": "gb200"
+ "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
- "run": "40739138596/attempts/1",
- "pr": "4599",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
- {
- "concs": [32],
- "item": 2,
- "override": "override_tp4_dram_c32",
- "goldenAL": "4.04",
- "kv": "hicache"
- },
{
"concs": [40],
- "item": 2,
- "override": "override_tp4_dram_c40",
+ "item": 6,
+ "override": "override_tp2_hicache_c40",
"goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
- }
- }
- ]
- }
+ "kv": "hicache"
},
{
- "concs": [48],
- "item": 2,
- "override": "override_tp4_dram_c48",
+ "concs": [44],
+ "item": 7,
+ "override": "override_tp2_hicache_c44",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"]
- }
- }
- ],
"dynamo": [
null,
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"]
- }
- }
- ]
- }
- },
- {
- "concs": [56],
- "item": 2,
- "override": "override_tp4_dram_c56",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
}
],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
- }
- }
- ]
- }
- },
- {
- "concs": [64],
- "item": 2,
- "override": "override_tp4_dram_c64",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
- }
+ },
+ null
]
}
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["180"],
- "--hicache-write-policy": ["write_back"],
- "--max-running-requests": ["64"],
- "--mem-fraction-static": ["0.81"]
- }
- },
- "ref": [5, "sglang", 0]
- }
- ]
- },
"dynamo": {
"install": [],
"files": [],
@@ -1598,45 +1923,87 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--chunked-prefill-size": ["32768"],
+ "--prefill-decode-interval": null,
+ "--schedule-policy": null,
+ "--stream-interval": null
+ }
+ },
+ "ref": [7, "dynamo", 2]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#3",
- "hw": "gb200",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4·EP4",
+ "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)",
"detail": "DSpark 7",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "gpus": 16,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "note",
+ "script": "Placement",
+ "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ }
+ ],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -1644,61 +2011,32 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "200"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
- "donorHardware": "gb200"
+ "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
- "run": "40739138596/attempts/1",
- "pr": "4599",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
{
- "concs": [72],
- "item": 3,
- "override": "override_tp4_ep4_dram_c72",
- "goldenAL": "4.04",
- "kv": "hicache"
- },
- {
- "concs": [80],
- "item": 3,
- "override": "override_tp4_ep4_dram_c80",
+ "concs": [224, 256],
+ "item": 1,
+ "override": "override_4p_dep2_dep8",
"goldenAL": "4.04",
"kv": "hicache"
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--cuda-graph-max-bs-prefill": ["16384"],
- "--expert-parallel-size": ["4"],
- "--max-running-requests": ["128"],
- "--mem-fraction-static": ["0.8"],
- "--moe-runner-backend": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
- }
- },
- "ref": [7, "sglang", 0]
- }
- ]
- },
- "dynamo": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
@@ -1717,45 +2055,192 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
+ "kind": "worker",
+ "role": "prefill",
+ "workers": 4,
+ "nodes_per_worker": 1,
+ "gpu_pinning": [
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"]
+ ],
+ "local_ranks": [0, 1, 0, 1],
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""],
+ "cmd": "python3 -m dynamo.sglang",
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "$PORT"],
+ ["--disaggregation-mode", "prefill"],
+ ["--disaggregation-bootstrap-port", "$BOOTSTRAP_PORT"],
+ ["--request-plane", "tcp"],
+ ["--kv-events-config", "$KV_EVENTS_CONFIG"],
+ ["--chunked-prefill-size", "16384"],
+ ["--data-parallel-size", "2"],
+ ["--disaggregation-transfer-backend", "mooncake"],
+ ["--enable-dp-attention"],
+ ["--enable-dp-lm-head"],
+ ["--enable-hierarchical-cache"],
+ ["--expert-parallel-size", "2"],
+ ["--hicache-size", "200"],
+ ["--hicache-write-policy", "write_back"],
+ ["--max-running-requests", "128"],
+ ["--mem-fraction-static", "0.85"],
+ ["--moe-a2a-backend", "megamoe"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--schedule-policy", "shortest-prefill-first"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "2"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ],
+ "envRef": 2
+ },
+ {
+ "kind": "worker",
+ "role": "decode",
+ "workers": 1,
+ "nodes_per_worker": 2,
"cmd": "python3 -m dynamo.sglang",
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--disaggregation-mode", "decode"],
+ ["--dist-init-addr", "$DECODE_HEAD_IP:20000"],
+ ["--nnodes", "2"],
+ ["--node-rank", "$NODE_RANK"],
+ ["--request-plane", "tcp"],
+ ["--cuda-graph-max-bs-decode", "48"],
+ ["--data-parallel-size", "8"],
+ ["--disaggregation-transfer-backend", "mooncake"],
+ ["--enable-dp-attention"],
+ ["--enable-dp-lm-head"],
+ ["--expert-parallel-size", "8"],
+ ["--load-balance-method", "total_tokens"],
+ ["--max-running-requests", "384"],
+ ["--mem-fraction-static", "0.85"],
+ ["--moe-a2a-backend", "megamoe"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "least-loaded"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "8"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ],
+ "envRef": 3
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
+ "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#0",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 20,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "note",
+ "script": "Placement",
+ "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ }
+ ],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -1763,237 +2248,165 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "200"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "40739166138/attempts/1",
- "pr": "4524",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
{
- "concs": [1],
+ "concs": [280, 320, 340, 360, 400],
"item": 0,
- "override": "override_tp4_c1",
+ "override": "override_6p_dep2_dep8",
"goldenAL": "4.04",
- "kv": "none"
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
+ },
+ {
+ "workers": 6,
+ "gpu_pinning": [
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"]
+ ],
+ "local_ranks": [0, 1, 0, 1, 0, 1],
+ "ref": [9, "dynamo", 2]
+ },
+ {
+ "ref": [9, "dynamo", 3]
+ }
+ ]
},
- {
- "concs": [4],
- "item": 0,
- "override": "override_tp4_c4",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["8"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["8"]
- }
- }
- ]
- }
- },
- {
- "concs": [8],
- "item": 0,
- "override": "override_tp4_c8",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["16"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["16"]
- }
- }
- ]
- }
- },
- {
- "concs": [16],
- "item": 0,
- "override": "override_tp4_c16",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
- }
- }
- ]
- }
- },
- {
- "concs": [32],
- "item": 0,
- "override": "override_tp4_c32",
- "goldenAL": "4.04",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
- }
- }
- ]
- }
- }
- ],
- "routers": {
"sglang": {
"install": [],
"files": [],
"blocks": [
{
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
+ "from": 2,
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
"cmd": "python3 -m sglang.launch_server",
- "env": {
- "FMHA_SM100_ALLOW_JIT": "1",
- "NCCL_CUMEM_ENABLE": "1",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
- },
- "args": [
- ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--chunked-prefill-size", "32768"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-max-bs-decode", "2"],
- ["--max-running-requests", "2"],
- ["--mem-fraction-static", "0.8"],
- ["--moe-runner-backend", "flashinfer_trtllm_routed"],
- ["--reasoning-parser", "minimax-m3"],
- ["--speculative-algorithm", "DSPARK"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-dspark-block-size", "7"],
- ["--speculative-use-block-verification"],
- ["--tensor-parallel-size", "4"],
- ["--tokenizer-worker-num", "8"],
- ["--tool-call-parser", "minimax-m3"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "3600"]
- ]
- }
- ]
- },
- "dynamo": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "kind": "etcd",
- "count_nodes": 1,
- "cmd": "etcd",
- "env": {},
- "args": [
- ["--data-dir", "/tmp/etcd"],
- ["--listen-client-urls", "http://0.0.0.0:2379"],
- ["--advertise-client-urls", "http://$ETCD_IP:2379"]
- ]
- },
- {
- "kind": "dynamo-frontend",
- "count": 1,
- "cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
- "args": [
- ["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#1",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#3",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
- "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 28,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "note",
+ "script": "Placement",
+ "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ }
+ ],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -2001,60 +2414,31 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "200"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-low-latency.yaml",
- "run": "40739166138/attempts/1",
- "pr": "4524",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
{
- "concs": [1],
- "item": 1,
- "override": "override_tp4_c1",
+ "concs": [300, 360, 400],
+ "item": 3,
+ "override": "override_6p_dep2_dep16",
"goldenAL": "4.04",
- "kv": "none"
+ "kv": "hicache"
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--disable-attn-tp-gather": [],
- "--max-running-requests": ["8"],
- "--monokernel-backend": ["msa"],
- "--page-size": ["128"],
- "--speculative-eagle-topk": ["1"],
- "--speculative-num-draft-tokens": ["8"],
- "--speculative-num-steps": ["1"],
- "--chunked-prefill-size": null,
- "--cuda-graph-backend-prefill": null,
- "--cuda-graph-max-bs-decode": null,
- "--speculative-draft-kv-cache-dtype": null,
- "--tokenizer-worker-num": null
- },
- "env": {
- "ALLOW_JIT": "1",
- "SGLANG_FORCE_COARSE_WAR_BARRIER": "1"
- }
- },
- "ref": [9, "sglang", 0]
- }
- ]
- },
"dynamo": {
"install": [],
"files": [],
@@ -2074,45 +2458,121 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "ref": [10, "dynamo", 2]
+ },
+ {
+ "nodes_per_worker": 4,
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--nnodes": ["4"],
+ "--data-parallel-size": ["16"],
+ "--expert-parallel-size": ["16"],
+ "--max-running-requests": ["768"],
+ "--tensor-parallel-size": ["16"]
+ }
+ },
+ "ref": [9, "dynamo", 3]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#3",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#2",
+ "hw": "gb300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP2",
- "detail": "DSpark 7 · --prefill-decode-interval unset",
- "gpus": 2,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "label": "Disagg 8P1D · prefill DEP2 · decode DEP8 (2 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 24,
+ "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "note",
+ "script": "Placement",
+ "reason": "Workers share nodes: 8 prefill workers on 4 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ }
+ ],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -2120,192 +2580,429 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "200"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml",
- "run": "40739166138/attempts/1",
- "pr": "4524",
+ "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
+ "run": "41271829525/attempts/1",
+ "pr": "4698",
"status": "verified",
- "date": "2026-10-29"
+ "date": "2026-11-02"
},
"points": [
{
- "concs": [32],
- "item": 3,
- "override": "zip_override_tp2_hicache[0]",
+ "concs": [440, 480],
+ "item": 2,
+ "override": "override_8p_dep2_dep8",
"goldenAL": "4.04",
"kv": "hicache"
+ }
+ ],
+ "routers": {
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
+ },
+ {
+ "workers": 8,
+ "gpu_pinning": [
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"]
+ ],
+ "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1],
+ "ref": [9, "dynamo", 2]
+ },
+ {
+ "ref": [9, "dynamo", 3]
+ }
+ ]
},
- {
- "concs": [36],
- "item": 3,
- "override": "zip_override_tp2_hicache[1]",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
}
}
- ],
- "dynamo": [
- null,
- null,
- {
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
- }
- }
- ]
- }
- },
- {
- "concs": [40],
- "item": 3,
- "override": "zip_override_tp2_hicache[2]",
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_7_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_6_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg#0",
+ "hw": "gb200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Agg TP4",
+ "detail": "DSpark 7",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "imageStatus": "ok",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "setup",
+ "command": "pip install torchao",
+ "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed."
+ }
+ ],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "donorHardware": "gb200"
+ }
+ },
+ "prov": {
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/agg-variants.yaml",
+ "run": "41271615328",
+ "pr": "4695",
+ "comment": "6576168162",
+ "status": "verified-pr"
+ },
+ "points": [
+ {
+ "concs": [1],
+ "item": 0,
+ "override": "override_tp4_c1",
"goldenAL": "4.04",
- "kv": "hicache",
+ "kv": "none"
+ },
+ {
+ "concs": [4],
+ "item": 1,
+ "override": "override_tp4_c4",
+ "goldenAL": "4.04",
+ "kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
}
}
],
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
+ }
+ },
+ null
+ ]
+ }
+ },
+ {
+ "concs": [8],
+ "item": 2,
+ "override": "override_tp4_c8",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
"dynamo": [
null,
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
}
}
+ ],
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
+ }
+ },
+ null
]
}
},
{
- "concs": [44],
+ "concs": [16],
"item": 3,
- "override": "zip_override_tp2_hicache_lpm[0]",
+ "override": "override_tp4_c16",
"goldenAL": "4.04",
- "kv": "hicache",
+ "kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"],
- "--schedule-policy": ["lpm"]
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"],
- "--schedule-policy": ["lpm"]
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
}
- }
+ },
+ null
]
}
},
{
- "concs": [46],
- "item": 3,
- "override": "zip_override_tp2_hicache_lpm[1]",
+ "concs": [32],
+ "item": 4,
+ "override": "override_tp4_hicache_c32",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["92"],
- "--max-running-requests": ["92"],
- "--schedule-policy": ["lpm"]
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["64"],
+ "--mem-fraction-static": ["0.81"]
}
}
],
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["64"],
+ "--mem-fraction-static": ["0.81"]
+ }
+ },
+ null
+ ]
+ }
+ },
+ {
+ "concs": [40],
+ "item": 5,
+ "override": "override_tp4_hicache_c40",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
"dynamo": [
null,
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["92"],
- "--max-running-requests": ["92"],
- "--schedule-policy": ["lpm"]
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["80"],
+ "--mem-fraction-static": ["0.81"]
}
}
+ ],
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["80"],
+ "--mem-fraction-static": ["0.81"]
+ }
+ },
+ null
]
}
},
{
"concs": [48],
- "item": 3,
- "override": "zip_override_tp2_hicache_lpm[2]",
+ "item": 6,
+ "override": "override_tp4_hicache_c48",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["96"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
"--max-running-requests": ["96"],
- "--schedule-policy": ["lpm"]
+ "--mem-fraction-static": ["0.81"]
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["96"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
"--max-running-requests": ["96"],
- "--schedule-policy": ["lpm"]
+ "--mem-fraction-static": ["0.81"]
}
- }
+ },
+ null
]
}
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
+ },
+ {
+ "concs": [56],
+ "item": 7,
+ "override": "override_tp4_hicache_c56",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "dynamo": [
+ null,
+ null,
+ {
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
+ "--cuda-graph-max-bs-decode": ["112"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["370"],
+ "--hicache-size": ["180"],
"--hicache-write-policy": ["write_back"],
- "--max-running-requests": ["64"],
- "--mem-fraction-static": ["0.87"],
- "--tensor-parallel-size": ["2"]
+ "--max-running-requests": ["112"],
+ "--mem-fraction-static": ["0.81"]
+ }
+ }
+ ],
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["112"],
+ "--mem-fraction-static": ["0.81"]
}
},
- "ref": [9, "sglang", 0]
- }
- ]
- },
+ null
+ ]
+ }
+ }
+ ],
+ "routers": {
"dynamo": {
"install": [],
"files": [],
@@ -2325,40 +3022,98 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
+ "kind": "worker",
+ "role": "agg",
+ "workers": 1,
+ "nodes_per_worker": 1,
"cmd": "python3 -m dynamo.sglang",
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--request-plane", "tcp"],
+ ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"],
+ ["--chunked-prefill-size", "32768"],
+ ["--cuda-graph-backend-prefill", "breakable"],
+ ["--cuda-graph-max-bs-decode", "2"],
+ ["--max-running-requests", "2"],
+ ["--mem-fraction-static", "0.8"],
+ ["--moe-runner-backend", "flashinfer_trtllm_routed"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "4"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ],
+ "envRef": 1
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#2",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#0",
+ "hw": "gb200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · measured with HiCache",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -2371,41 +3126,41 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "donorHardware": "gb200"
}
},
"prov": {
- "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "40739166138/attempts/1",
- "pr": "4524",
+ "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
+ "run": "40739138596/attempts/1",
+ "pr": "4599",
"status": "verified",
"date": "2026-10-29"
},
"points": [
{
- "concs": [32],
- "item": 2,
- "override": "override_tp4_dram_c32",
+ "concs": [1],
+ "item": 0,
+ "override": "override_tp4_c1",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "none"
},
{
- "concs": [64],
- "item": 2,
- "override": "override_tp4_dram_c64",
+ "concs": [4],
+ "item": 0,
+ "override": "override_tp4_c4",
"goldenAL": "4.04",
- "kv": "hicache",
+ "kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
}
}
],
@@ -2414,25 +3169,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--max-running-requests": ["8"]
}
}
]
}
},
{
- "concs": [80],
- "item": 2,
- "override": "override_tp4_dram_c80",
+ "concs": [8],
+ "item": 0,
+ "override": "override_tp4_c8",
"goldenAL": "4.04",
- "kv": "hicache",
+ "kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
}
}
],
@@ -2441,25 +3196,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["16"],
+ "--max-running-requests": ["16"]
}
}
]
}
},
{
- "concs": [96],
- "item": 2,
- "override": "override_tp4_dram_c96",
+ "concs": [16],
+ "item": 0,
+ "override": "override_tp4_c16",
"goldenAL": "4.04",
- "kv": "hicache",
+ "kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
}
}
],
@@ -2468,8 +3223,35 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [32],
+ "item": 0,
+ "override": "override_tp4_c32",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
}
}
]
@@ -2484,11 +3266,10 @@ export const agentx = {
{
"diff": {
"args": {
- "--hicache-size": ["290"],
- "--tensor-parallel-size": ["4"]
+ "--tokenizer-worker-num": ["8"]
}
},
- "ref": [11, "sglang", 0]
+ "ref": [13, "sglang", 0]
}
]
},
@@ -2538,13 +3319,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#4",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#1",
+ "hw": "gb200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP2",
- "detail": "DSpark 7 · --prefill-decode-interval 16",
- "gpus": 2,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "label": "Agg TP4",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -2557,42 +3338,160 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "donorHardware": "gb200"
}
},
"prov": {
- "sha": "b623efbb700ba428206bda393658559d3c7dc631",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml",
- "run": "40895671658/attempts/1",
- "pr": "4647",
+ "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic-low-latency.yaml",
+ "run": "40739138596/attempts/1",
+ "pr": "4599",
"status": "verified",
"date": "2026-10-29"
},
"points": [
{
- "concs": [40],
- "item": 4,
- "override": "zip_override_tp2_hicache_spf[0]",
- "goldenAL": "4.04",
- "kv": "hicache"
- },
- {
- "concs": [44],
- "item": 5,
- "override": "zip_override_tp2_hicache_spf[1]",
+ "concs": [1],
+ "item": 1,
+ "override": "override_tp4_c1",
"goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
+ "kv": "none"
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"]
- }
+ "--disable-attn-tp-gather": [],
+ "--max-running-requests": ["8"],
+ "--monokernel-backend": ["msa"],
+ "--page-size": ["128"],
+ "--speculative-eagle-topk": ["1"],
+ "--speculative-num-draft-tokens": ["8"],
+ "--speculative-num-steps": ["1"],
+ "--chunked-prefill-size": null,
+ "--cuda-graph-backend-prefill": null,
+ "--cuda-graph-max-bs-decode": null,
+ "--speculative-draft-kv-cache-dtype": null
+ },
+ "env": {
+ "ALLOW_JIT": "1",
+ "SGLANG_FORCE_COARSE_WAR_BARRIER": "1"
+ }
+ },
+ "ref": [13, "sglang", 0]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#2",
+ "hw": "gb200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Agg TP4",
+ "detail": "DSpark 7 · measured with HiCache",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "imageStatus": "ok",
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "donorHardware": "gb200"
+ }
+ },
+ "prov": {
+ "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
+ "run": "40739138596/attempts/1",
+ "pr": "4599",
+ "status": "verified",
+ "date": "2026-10-29"
+ },
+ "points": [
+ {
+ "concs": [32],
+ "item": 2,
+ "override": "override_tp4_dram_c32",
+ "goldenAL": "4.04",
+ "kv": "hicache"
+ },
+ {
+ "concs": [40],
+ "item": 2,
+ "override": "override_tp4_dram_c40",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
}
],
"dynamo": [
@@ -2600,8 +3499,8 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"]
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
}
}
]
@@ -2609,8 +3508,8 @@ export const agentx = {
},
{
"concs": [48],
- "item": 6,
- "override": "zip_override_tp2_hicache_spf[2]",
+ "item": 2,
+ "override": "override_tp4_dram_c48",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
@@ -2635,17 +3534,17 @@ export const agentx = {
}
},
{
- "concs": [52],
- "item": 7,
- "override": "zip_override_tp2_hicache_spf[3]",
+ "concs": [56],
+ "item": 2,
+ "override": "override_tp4_dram_c56",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["104"],
- "--max-running-requests": ["104"]
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
}
}
],
@@ -2654,25 +3553,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["104"],
- "--max-running-requests": ["104"]
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
}
}
]
}
},
{
- "concs": [56],
- "item": 8,
- "override": "zip_override_tp2_hicache_spf[4]",
+ "concs": [64],
+ "item": 2,
+ "override": "override_tp4_dram_c64",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
],
@@ -2681,8 +3580,8 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
]
@@ -2690,26 +3589,466 @@ export const agentx = {
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"],
- "--mem-fraction-static": ["0.92"],
- "--prefill-decode-interval": ["16"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--stream-interval": ["20"]
- }
- },
- "ref": [11, "sglang", 0]
- }
- ]
- },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["180"],
+ "--hicache-write-policy": ["write_back"],
+ "--max-running-requests": ["64"],
+ "--mem-fraction-static": ["0.81"]
+ }
+ },
+ "ref": [14, "sglang", 0]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#3",
+ "hw": "gb200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Agg TP4·EP4",
+ "detail": "DSpark 7",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "imageStatus": "ok",
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark",
+ "donorHardware": "gb200"
+ }
+ },
+ "prov": {
+ "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml",
+ "run": "40739138596/attempts/1",
+ "pr": "4599",
+ "status": "verified",
+ "date": "2026-10-29"
+ },
+ "points": [
+ {
+ "concs": [72],
+ "item": 3,
+ "override": "override_tp4_ep4_dram_c72",
+ "goldenAL": "4.04",
+ "kv": "hicache"
+ },
+ {
+ "concs": [80],
+ "item": 3,
+ "override": "override_tp4_ep4_dram_c80",
+ "goldenAL": "4.04",
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--cuda-graph-max-bs-prefill": ["16384"],
+ "--expert-parallel-size": ["4"],
+ "--max-running-requests": ["128"],
+ "--mem-fraction-static": ["0.8"],
+ "--moe-runner-backend": null
+ },
+ "env": {
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
+ }
+ },
+ "ref": [16, "sglang", 0]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#0",
+ "hw": "gb200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Disagg 2P1D · prefill DEP4 · decode DEP8 (2 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 16,
+ "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "imageStatus": "ok",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "setup",
+ "command": "pip install torchao",
+ "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed."
+ }
+ ],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb200"
+ }
+ },
+ "prov": {
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml",
+ "run": "41271615328",
+ "pr": "4695",
+ "comment": "6576168162",
+ "status": "verified-pr"
+ },
+ "points": [
+ {
+ "concs": [224, 256],
+ "item": 0,
+ "override": "override_2p_dep4_dep8",
+ "goldenAL": "4.04",
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
+ },
+ {
+ "kind": "worker",
+ "role": "prefill",
+ "workers": 2,
+ "nodes_per_worker": 1,
+ "cmd": "python3 -m dynamo.sglang",
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--disaggregation-mode", "prefill"],
+ ["--disaggregation-bootstrap-port", "8998"],
+ ["--request-plane", "tcp"],
+ ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"],
+ ["--chunked-prefill-size", "32768"],
+ ["--data-parallel-size", "4"],
+ ["--disaggregation-transfer-backend", "mooncake"],
+ ["--enable-dp-attention"],
+ ["--enable-dp-lm-head"],
+ ["--enable-hierarchical-cache"],
+ ["--expert-parallel-size", "4"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"],
+ ["--max-running-requests", "256"],
+ ["--mem-fraction-static", "0.85"],
+ ["--moe-a2a-backend", "megamoe"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--schedule-policy", "shortest-prefill-first"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "4"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ],
+ "envRef": 2
+ },
+ {
+ "kind": "worker",
+ "role": "decode",
+ "workers": 1,
+ "nodes_per_worker": 2,
+ "cmd": "python3 -m dynamo.sglang",
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--disaggregation-mode", "decode"],
+ ["--dist-init-addr", "$DECODE_HEAD_IP:20000"],
+ ["--nnodes", "2"],
+ ["--node-rank", "$NODE_RANK"],
+ ["--request-plane", "tcp"],
+ ["--cuda-graph-max-bs-decode", "48"],
+ ["--data-parallel-size", "8"],
+ ["--disaggregation-transfer-backend", "mooncake"],
+ ["--enable-dp-attention"],
+ ["--enable-dp-lm-head"],
+ ["--expert-parallel-size", "8"],
+ ["--load-balance-method", "total_tokens"],
+ ["--max-running-requests", "384"],
+ ["--mem-fraction-static", "0.9"],
+ ["--moe-a2a-backend", "megamoe"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--router-mode", "least-loaded"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "8"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ],
+ "envRef": 3
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#1",
+ "hw": "gb200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Disagg 3P1D · prefill DEP4 · decode DEP8 (2 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 20,
+ "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "imageStatus": "ok",
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "setup",
+ "command": "pip install torchao",
+ "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed."
+ }
+ ],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "180"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb200"
+ }
+ },
+ "prov": {
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml",
+ "run": "41271615328",
+ "pr": "4695",
+ "comment": "6576168162",
+ "status": "verified-pr"
+ },
+ "points": [
+ {
+ "concs": [256, 288],
+ "item": 1,
+ "override": "override_3p_dep4_dep8",
+ "goldenAL": "4.04",
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
"dynamo": {
"install": [],
"files": [],
@@ -2729,45 +4068,108 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "workers": 3,
+ "ref": [18, "dynamo", 2]
+ },
+ {
+ "ref": [18, "dynamo", 3]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#9",
- "hw": "b300",
+ "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#2",
+ "hw": "gb200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP2",
- "detail": "DSpark 7 · --prefill-decode-interval 32",
- "gpus": 2,
- "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "label": "Disagg 3P1D · prefill DEP4 · decode DEP16 (4 nodes)",
+ "detail": "DSpark 7",
+ "gpus": 28,
+ "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
+ "imageCommit": "e371881",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "setup",
+ "command": "pip install torchao",
+ "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed."
+ }
+ ],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -2775,47 +4177,31 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "370"],
+ ["--hicache-size", "180"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg",
+ "donorHardware": "gb200"
}
},
"prov": {
- "sha": "b623efbb700ba428206bda393658559d3c7dc631",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml",
- "run": "40895671658/attempts/1",
- "pr": "4647",
- "status": "verified",
- "date": "2026-10-29"
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml",
+ "run": "41271615328",
+ "pr": "4695",
+ "comment": "6576168162",
+ "status": "verified-pr"
},
"points": [
{
- "concs": [48],
- "item": 9,
- "override": "override_tp2_c48_spf_interval32",
+ "concs": [280, 320, 360],
+ "item": 2,
+ "override": "override_3p_dep4_dep16",
"goldenAL": "4.04",
"kv": "hicache"
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"],
- "--prefill-decode-interval": ["32"]
- }
- },
- "ref": [13, "sglang", 0]
- }
- ]
- },
"dynamo": {
"install": [],
"files": [],
@@ -2835,40 +4221,107 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "ref": [19, "dynamo", 2]
+ },
+ {
+ "nodes_per_worker": 4,
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--nnodes": ["4"],
+ "--data-parallel-size": ["16"],
+ "--expert-parallel-size": ["16"],
+ "--max-running-requests": ["768"],
+ "--mem-fraction-static": ["0.85"],
+ "--tensor-parallel-size": ["16"]
+ }
+ },
+ "ref": [18, "dynamo", 3]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#0",
- "hw": "b200",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#0",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
"detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -2881,20 +4334,19 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "210"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
- "run": "40739120064/attempts/1",
- "pr": "4598",
- "status": "verified",
- "date": "2026-10-28"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
@@ -3100,13 +4552,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#1",
- "hw": "b200",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#2",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
"detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -3119,25 +4571,24 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "210"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic-low-latency.yaml",
- "run": "40739120064/attempts/1",
- "pr": "4598",
- "status": "verified",
- "date": "2026-10-28"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-low-latency.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
"concs": [1],
- "item": 1,
+ "item": 2,
"override": "override_tp4_c1",
"goldenAL": "4.04",
"kv": "none"
@@ -3169,7 +4620,7 @@ export const agentx = {
"SGLANG_FORCE_COARSE_WAR_BARRIER": "1"
}
},
- "ref": [15, "sglang", 0]
+ "ref": [21, "sglang", 0]
}
]
},
@@ -3219,13 +4670,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#2",
- "hw": "b200",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#4",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · measured with HiCache",
+ "detail": "DSpark 7 · measured with HiCache · --mem-fraction-static 0.9",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -3238,33 +4689,144 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "210"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
- "run": "40739120064/attempts/1",
- "pr": "4598",
- "status": "verified",
- "date": "2026-10-28"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
+ {
+ "concs": [8],
+ "item": 7,
+ "override": "zip_override_tp4_dram_spf_interval32[3]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml"
+ },
+ {
+ "concs": [16],
+ "item": 8,
+ "override": "zip_override_tp4_dram_spf_interval32[4]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml"
+ },
+ {
+ "concs": [20],
+ "item": 9,
+ "override": "zip_override_tp4_dram_spf_interval32[0]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["40"],
+ "--max-running-requests": ["40"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["40"],
+ "--max-running-requests": ["40"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml"
+ },
+ {
+ "concs": [24],
+ "item": 4,
+ "override": "zip_override_tp4_dram_spf_interval32[0]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml"
+ },
{
"concs": [32],
- "item": 2,
- "override": "override_tp4_dram_c32",
+ "item": 5,
+ "override": "zip_override_tp4_dram_spf_interval32[1]",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml"
},
{
"concs": [40],
- "item": 2,
- "override": "override_tp4_dram_c40",
+ "item": 6,
+ "override": "zip_override_tp4_dram_spf_interval32[2]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
@@ -3286,20 +4848,21 @@ export const agentx = {
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml"
},
{
- "concs": [48],
- "item": 2,
- "override": "override_tp4_dram_c48",
+ "concs": [44],
+ "item": 10,
+ "override": "zip_override_tp4_dram_spf_interval32[1]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
}
],
@@ -3308,25 +4871,26 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml"
},
{
- "concs": [56],
- "item": 2,
- "override": "override_tp4_dram_c56",
+ "concs": [52],
+ "item": 11,
+ "override": "zip_override_tp4_dram_spf_interval32[2]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"]
}
}
],
@@ -3335,25 +4899,26 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["112"],
- "--max-running-requests": ["112"]
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"]
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml"
},
{
- "concs": [64],
- "item": 2,
- "override": "override_tp4_dram_c64",
+ "concs": [60],
+ "item": 12,
+ "override": "zip_override_tp4_dram_spf_interval32[3]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["120"],
+ "--max-running-requests": ["120"]
}
}
],
@@ -3362,12 +4927,13 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--max-running-requests": ["128"]
+ "--cuda-graph-max-bs-decode": ["120"],
+ "--max-running-requests": ["120"]
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml"
}
],
"routers": {
@@ -3378,15 +4944,19 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
+ "--chunked-prefill-size": ["8192"],
+ "--cuda-graph-max-bs-decode": ["16"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["210"],
+ "--hicache-size": ["290"],
"--hicache-write-policy": ["write_back"],
- "--max-running-requests": ["64"],
- "--mem-fraction-static": ["0.81"]
+ "--max-running-requests": ["16"],
+ "--mem-fraction-static": ["0.9"],
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"]
}
},
- "ref": [15, "sglang", 0]
+ "ref": [21, "sglang", 0]
}
]
},
@@ -3436,13 +5006,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#3",
- "hw": "b200",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#1",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4·EP4",
- "detail": "DSpark 7",
+ "label": "Agg TP4",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 8192",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -3455,57 +5025,45 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "210"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
- "run": "40739120064/attempts/1",
- "pr": "4598",
- "status": "verified",
- "date": "2026-10-28"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [72],
- "item": 3,
- "override": "override_tp4_ep4_dram_c72",
- "goldenAL": "4.04",
- "kv": "hicache"
- },
- {
- "concs": [80],
- "item": 3,
- "override": "override_tp4_ep4_dram_c80",
+ "concs": [16],
+ "item": 1,
+ "override": "override_tp4_c16_spf_interval32",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "none"
}
],
"routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--cuda-graph-max-bs-decode": ["128"],
- "--cuda-graph-max-bs-prefill": ["16384"],
- "--expert-parallel-size": ["4"],
- "--max-running-requests": ["128"],
- "--mem-fraction-static": ["0.8"],
- "--moe-runner-backend": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"],
+ "--enable-hierarchical-cache": null,
+ "--hicache-size": null,
+ "--hicache-write-policy": null
}
},
- "ref": [17, "sglang", 0]
+ "ref": [23, "sglang", 0]
}
]
},
@@ -3555,13 +5113,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#7",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#19",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP4",
+ "label": "Agg TP2",
"detail": "DSpark 7 · --prefill-decode-interval 32",
- "gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -3574,32 +5132,88 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml",
- "run": "41054395691",
- "pr": "4676",
- "status": "verified-pr"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
+ {
+ "concs": [20],
+ "item": 20,
+ "override": "zip_override_tp2_spf_interval32[0]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
+ },
{
"concs": [24],
- "item": 7,
- "override": "zip_override_tp4_hicache_spf_interval32[0]",
+ "item": 21,
+ "override": "zip_override_tp2_spf_interval32[1]",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--max-running-requests": ["48"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
+ },
+ {
+ "concs": [28],
+ "item": 22,
+ "override": "zip_override_tp2_spf_interval32[2]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["56"],
+ "--max-running-requests": ["56"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["56"],
+ "--max-running-requests": ["56"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
},
{
"concs": [32],
- "item": 8,
- "override": "zip_override_tp4_hicache_spf_interval32[1]",
+ "item": 23,
+ "override": "zip_override_tp2_spf_interval32[3]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
@@ -3621,12 +5235,41 @@ export const agentx = {
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
+ },
+ {
+ "concs": [36],
+ "item": 24,
+ "override": "zip_override_tp2_spf_interval32[4]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
},
{
"concs": [40],
- "item": 9,
- "override": "zip_override_tp4_hicache_spf_interval32[2]",
+ "item": 25,
+ "override": "zip_override_tp2_spf_interval32[5]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
@@ -3648,7 +5291,64 @@ export const agentx = {
}
}
]
- }
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
+ },
+ {
+ "concs": [44],
+ "item": 28,
+ "override": "override_tp2_c44_spf_interval32",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml"
+ },
+ {
+ "concs": [48],
+ "item": 19,
+ "override": "override_tp2_c48_spf_interval32",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml"
}
],
"routers": {
@@ -3659,15 +5359,14 @@ export const agentx = {
{
"diff": {
"args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"],
- "--prefill-decode-interval": ["32"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--stream-interval": ["20"]
+ "--cuda-graph-max-bs-decode": ["40"],
+ "--hicache-size": ["370"],
+ "--max-running-requests": ["40"],
+ "--mem-fraction-static": ["0.92"],
+ "--tensor-parallel-size": ["2"]
}
},
- "ref": [4, "sglang", 0]
+ "ref": [23, "sglang", 0]
}
]
},
@@ -3717,13 +5416,13 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#10",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#13",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP2",
- "detail": "DSpark 7 · --prefill-decode-interval 32",
+ "detail": "DSpark 7 · --prefill-decode-interval unset",
"gpus": 2,
- "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
"imageCommit": null,
@@ -3736,40 +5435,40 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml",
- "run": "41054395691",
- "pr": "4676",
- "status": "verified-pr"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [20],
- "item": 10,
- "override": "zip_override_tp2_hicache_spf_interval32[0]",
+ "concs": [32],
+ "item": 13,
+ "override": "zip_override_tp2_hicache[0]",
"goldenAL": "4.04",
"kv": "hicache"
},
{
- "concs": [24],
- "item": 11,
- "override": "zip_override_tp2_hicache_spf_interval32[1]",
+ "concs": [36],
+ "item": 13,
+ "override": "zip_override_tp2_hicache[1]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
}
}
],
@@ -3778,25 +5477,53 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
+ "--cuda-graph-max-bs-decode": ["72"],
+ "--max-running-requests": ["72"]
}
}
]
}
},
{
- "concs": [28],
- "item": 12,
- "override": "zip_override_tp2_hicache_spf_interval32[2]",
+ "concs": [40],
+ "item": 13,
+ "override": "zip_override_tp2_hicache[2]",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [44],
+ "item": 13,
+ "override": "zip_override_tp2_hicache_lpm[0]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["56"],
- "--max-running-requests": ["56"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"],
+ "--schedule-policy": ["lpm"]
}
}
],
@@ -3805,25 +5532,27 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["56"],
- "--max-running-requests": ["56"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"],
+ "--schedule-policy": ["lpm"]
}
}
]
}
},
{
- "concs": [36],
+ "concs": [46],
"item": 13,
- "override": "zip_override_tp2_hicache_spf_interval32[4]",
+ "override": "zip_override_tp2_hicache_lpm[1]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
+ "--cuda-graph-max-bs-decode": ["92"],
+ "--max-running-requests": ["92"],
+ "--schedule-policy": ["lpm"]
}
}
],
@@ -3832,25 +5561,27 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
+ "--cuda-graph-max-bs-decode": ["92"],
+ "--max-running-requests": ["92"],
+ "--schedule-policy": ["lpm"]
}
}
]
}
},
{
- "concs": [40],
- "item": 14,
- "override": "zip_override_tp2_hicache_spf_interval32[5]",
+ "concs": [48],
+ "item": 13,
+ "override": "zip_override_tp2_hicache_lpm[2]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"],
+ "--schedule-policy": ["lpm"]
}
}
],
@@ -3859,8 +5590,9 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"],
+ "--schedule-policy": ["lpm"]
}
}
]
@@ -3875,13 +5607,16 @@ export const agentx = {
{
"diff": {
"args": {
- "--chunked-prefill-size": ["8192"],
- "--prefill-decode-interval": ["32"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--stream-interval": ["20"]
+ "--chunked-prefill-size": ["32768"],
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"],
+ "--mem-fraction-static": ["0.87"],
+ "--prefill-decode-interval": null,
+ "--schedule-policy": null,
+ "--stream-interval": null
}
},
- "ref": [2, "sglang", 0]
+ "ref": [25, "sglang", 0]
}
]
},
@@ -3931,11 +5666,11 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#6",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#3",
"hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · --prefill-decode-interval 32",
+ "detail": "DSpark 7 · measured with HiCache · --mem-fraction-static 0.87",
"gpus": 4,
"configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
@@ -3954,63 +5689,36 @@ export const agentx = {
["--hicache-write-policy", "write_back"]
],
"donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml",
- "run": "41085412898",
- "pr": "4675",
- "status": "verified-pr"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [8],
- "item": 6,
- "override": "zip_override_tp4_dram_spf_interval32[3]",
+ "concs": [32],
+ "item": 3,
+ "override": "override_tp4_dram_c32",
"goldenAL": "4.04",
"kv": "hicache"
},
{
- "concs": [16],
- "item": 7,
- "override": "zip_override_tp4_dram_spf_interval32[4]",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
- }
- }
- ]
- }
- },
- {
- "concs": [24],
+ "concs": [64],
"item": 3,
- "override": "zip_override_tp4_dram_spf_interval32[0]",
+ "override": "override_tp4_dram_c64",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
],
@@ -4019,25 +5727,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
]
}
},
{
- "concs": [32],
- "item": 4,
- "override": "zip_override_tp4_dram_spf_interval32[1]",
+ "concs": [80],
+ "item": 3,
+ "override": "override_tp4_dram_c80",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
],
@@ -4046,25 +5754,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
]
}
},
{
- "concs": [40],
- "item": 5,
- "override": "zip_override_tp4_dram_spf_interval32[2]",
+ "concs": [96],
+ "item": 3,
+ "override": "override_tp4_dram_c96",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
],
@@ -4073,8 +5781,8 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
]
@@ -4089,15 +5797,11 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["16"],
"--hicache-size": ["290"],
- "--max-running-requests": ["16"],
- "--mem-fraction-static": ["0.90"],
- "--prefill-decode-interval": ["32"],
"--tensor-parallel-size": ["4"]
}
},
- "ref": [13, "sglang", 0]
+ "ref": [26, "sglang", 0]
}
]
},
@@ -4147,11 +5851,11 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#15",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#14",
"hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP2",
- "detail": "DSpark 7 · --prefill-decode-interval 20–32",
+ "detail": "DSpark 7 · --prefill-decode-interval 16",
"gpus": 2,
"configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
"image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
@@ -4170,117 +5874,36 @@ export const agentx = {
["--hicache-write-policy", "write_back"]
],
"donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
- "donorHardware": "b300"
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml",
- "run": "41085412898",
- "pr": "4675",
- "status": "verified-pr"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [20],
- "item": 15,
- "override": "zip_override_tp2_spf_interval32[0]",
+ "concs": [40],
+ "item": 14,
+ "override": "zip_override_tp2_hicache_spf[0]",
"goldenAL": "4.04",
"kv": "hicache"
},
{
- "concs": [24],
- "item": 16,
- "override": "zip_override_tp2_spf_interval32[1]",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"]
- }
- }
- ]
- }
- },
- {
- "concs": [28],
- "item": 17,
- "override": "zip_override_tp2_spf_interval32[2]",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["56"],
- "--max-running-requests": ["56"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["56"],
- "--max-running-requests": ["56"]
- }
- }
- ]
- }
- },
- {
- "concs": [32],
- "item": 18,
- "override": "zip_override_tp2_spf_interval32[3]",
- "goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["64"],
- "--max-running-requests": ["64"]
- }
- }
- ]
- }
- },
- {
- "concs": [36],
- "item": 19,
- "override": "zip_override_tp2_spf_interval32[4]",
+ "concs": [44],
+ "item": 15,
+ "override": "zip_override_tp2_hicache_spf[1]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
}
],
@@ -4289,25 +5912,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["72"],
- "--max-running-requests": ["72"]
+ "--cuda-graph-max-bs-decode": ["88"],
+ "--max-running-requests": ["88"]
}
}
]
}
},
{
- "concs": [40],
- "item": 20,
- "override": "zip_override_tp2_spf_interval32[5]",
+ "concs": [48],
+ "item": 16,
+ "override": "zip_override_tp2_hicache_spf[2]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
}
}
],
@@ -4316,26 +5939,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"]
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
}
}
]
}
},
- {
- "concs": [48],
- "item": 21,
- "override": "override_tp2_c48_spf_interval24",
+ {
+ "concs": [52],
+ "item": 17,
+ "override": "zip_override_tp2_hicache_spf[3]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"],
- "--prefill-decode-interval": ["24"]
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"]
}
}
],
@@ -4344,27 +5966,25 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["96"],
- "--max-running-requests": ["96"],
- "--prefill-decode-interval": ["24"]
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"]
}
}
]
}
},
{
- "concs": [50],
- "item": 22,
- "override": "override_tp2_c50_spf_interval20",
+ "concs": [56],
+ "item": 18,
+ "override": "zip_override_tp2_hicache_spf[4]",
"goldenAL": "4.04",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["100"],
- "--max-running-requests": ["100"],
- "--prefill-decode-interval": ["20"]
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
}
}
],
@@ -4373,9 +5993,8 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["100"],
- "--max-running-requests": ["100"],
- "--prefill-decode-interval": ["20"]
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
}
}
]
@@ -4390,12 +6009,12 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["40"],
- "--max-running-requests": ["40"],
- "--prefill-decode-interval": ["32"]
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"],
+ "--prefill-decode-interval": ["16"]
}
},
- "ref": [13, "sglang", 0]
+ "ref": [25, "sglang", 0]
}
]
},
@@ -4445,24 +6064,18 @@ export const agentx = {
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#26",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)",
- "detail": "DSpark 7 · shortest-prefill-first",
- "gpus": 20,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.92",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
- "setup": [
- {
- "kind": "note",
- "script": "Placement",
- "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
- }
- ],
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -4470,31 +6083,46 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "200"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [280, 320, 340, 360, 400],
- "item": 0,
- "override": "override_6p_dep2_dep8",
+ "concs": [48],
+ "item": 26,
+ "override": "override_tp2_c48_spf_interval24",
"goldenAL": "4.04",
"kv": "hicache"
}
],
"routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"],
+ "--prefill-decode-interval": ["24"]
+ }
+ },
+ "ref": [25, "sglang", 0]
+ }
+ ]
+ },
"dynamo": {
"install": [],
"files": [],
@@ -4515,245 +6143,44 @@ export const agentx = {
"count": 1,
"cmd": "python3 -m dynamo.frontend",
"env": {
- "DYN_EVENT_PLANE": "zmq",
"DYN_REQUEST_PLANE": "tcp",
- "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600",
- "DYN_ROUTER_QUEUE_THRESHOLD": "None",
- "DYN_ROUTER_TEMPERATURE": "0",
- "DYN_TCP_CONNECT_TIMEOUT": "120",
- "DYN_TOKENIZER": "fastokens",
- "DYN_TOKENIZER_CACHE": "1",
- "DYN_TOKENIZER_CACHE_BYTES": "8000000000",
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
},
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"],
- ["--router-session-affinity-ttl-secs", "14400"],
- ["--active-decode-blocks-threshold", "None"],
- ["--active-prefill-tokens-threshold", "None"],
- ["--active-prefill-tokens-threshold-frac", "None"]
- ]
- },
- {
- "kind": "worker",
- "role": "prefill",
- "workers": 6,
- "nodes_per_worker": 1,
- "gpu_pinning": [
- ["0,1"],
- ["2,3"],
- ["0,1"],
- ["2,3"],
- ["0,1"],
- ["2,3"]
- ],
- "local_ranks": [0, 1, 0, 1, 0, 1],
- "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""],
- "cmd": "python3 -m dynamo.sglang",
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
- "DYN_EVENT_PLANE": "zmq",
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "FMHA_SM100_ALLOW_JIT": "1",
- "MC_ENABLE_DEST_DEVICE_AFFINITY": "1",
- "MC_FORCE_MNNVL": "1",
- "MC_TCP_BIND_ADDRESS": "$NODE_IP",
- "NCCL_CUMEM_ENABLE": "1",
- "NCCL_MNNVL_ENABLE": "1",
- "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
- "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
- "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "WITH_NVIDIA_PEERMEM": "0"
- },
- "args": [
- ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--host", "0.0.0.0"],
- ["--port", "$PORT"],
- ["--disaggregation-mode", "prefill"],
- ["--disaggregation-bootstrap-port", "$BOOTSTRAP_PORT"],
- ["--request-plane", "tcp"],
- ["--kv-events-config", "$KV_EVENTS_CONFIG"],
- ["--chunked-prefill-size", "16384"],
- ["--data-parallel-size", "2"],
- ["--disaggregation-transfer-backend", "mooncake"],
- ["--enable-dp-attention"],
- ["--enable-dp-lm-head"],
- ["--enable-hierarchical-cache"],
- ["--expert-parallel-size", "2"],
- ["--hicache-size", "200"],
- ["--hicache-write-policy", "write_back"],
- ["--max-running-requests", "128"],
- ["--mem-fraction-static", "0.85"],
- ["--moe-a2a-backend", "megamoe"],
- ["--reasoning-parser", "minimax-m3"],
- ["--router-mode", "kv"],
- ["--router-session-affinity-ttl-secs", "14400"],
- ["--schedule-policy", "shortest-prefill-first"],
- ["--speculative-algorithm", "DSPARK"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-dspark-block-size", "7"],
- ["--speculative-use-block-verification"],
- ["--tensor-parallel-size", "2"],
- ["--tool-call-parser", "minimax-m3"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "3600"]
- ]
- },
- {
- "kind": "worker",
- "role": "decode",
- "workers": 1,
- "nodes_per_worker": 2,
- "cmd": "python3 -m dynamo.sglang",
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
- "DYN_EVENT_PLANE": "zmq",
- "DYN_REQUEST_PLANE": "tcp",
- "DYN_SGL_GATEWAY_WORKERS": "8",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "FMHA_SM100_ALLOW_JIT": "1",
- "MC_ENABLE_DEST_DEVICE_AFFINITY": "1",
- "MC_FORCE_MNNVL": "1",
- "MC_TCP_BIND_ADDRESS": "$NODE_IP",
- "NCCL_CUMEM_ENABLE": "1",
- "NCCL_MNNVL_ENABLE": "1",
- "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600",
- "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600",
- "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "WITH_NVIDIA_PEERMEM": "0"
- },
- "args": [
- ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--disaggregation-mode", "decode"],
- ["--dist-init-addr", "$DECODE_HEAD_IP:20000"],
- ["--nnodes", "2"],
- ["--node-rank", "$NODE_RANK"],
- ["--request-plane", "tcp"],
- ["--cuda-graph-max-bs-decode", "48"],
- ["--data-parallel-size", "8"],
- ["--disaggregation-transfer-backend", "mooncake"],
- ["--enable-dp-attention"],
- ["--enable-dp-lm-head"],
- ["--expert-parallel-size", "8"],
- ["--load-balance-method", "total_tokens"],
- ["--max-running-requests", "384"],
- ["--mem-fraction-static", "0.85"],
- ["--moe-a2a-backend", "megamoe"],
- ["--reasoning-parser", "minimax-m3"],
- ["--router-mode", "least-loaded"],
- ["--speculative-algorithm", "DSPARK"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-dspark-block-size", "7"],
- ["--speculative-use-block-verification"],
- ["--tensor-parallel-size", "8"],
- ["--tool-call-parser", "minimax-m3"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "3600"]
- ]
- }
- ]
- },
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "from": 2,
- "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--skip-server-warmup": [],
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
- },
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
- }
- }
- },
- {
- "from": 3,
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--skip-server-warmup": [],
- "--request-plane": null,
- "--router-mode": null
- },
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "DYN_SGL_GATEWAY_WORKERS": null
- }
- }
- },
- {
- "kind": "sglang-router",
- "cmd": "python3 -m sglang_router.launch_router",
- "env": {},
- "args": [
- ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
- ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"],
- ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"],
- ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
- ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
- ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
- ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
- ["--pd-disaggregation"],
- ["--host", "0.0.0.0"],
- ["--port", "8000"],
- ["--policy", "cache_aware"]
+ ["--router-mode", "kv"]
]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#30",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)",
- "detail": "DSpark 7 · shortest-prefill-first",
- "gpus": 28,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.93",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
- "setup": [
- {
- "kind": "note",
- "script": "Placement",
- "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
- }
- ],
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -4761,112 +6188,102 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "200"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [300, 360, 400],
- "item": 3,
- "override": "override_6p_dep2_dep16",
+ "concs": [48],
+ "item": 30,
+ "override": "override_tp2_c48_spf_interval24_mem093",
"goldenAL": "4.04",
"kv": "hicache"
}
],
"routers": {
- "dynamo": {
+ "sglang": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
- },
- {
- "ref": [23, "dynamo", 2]
- },
- {
- "nodes_per_worker": 4,
- "ref": [23, "dynamo", 3],
"diff": {
"args": {
- "--nnodes": ["4"],
- "--data-parallel-size": ["16"],
- "--expert-parallel-size": ["16"],
- "--tensor-parallel-size": ["16"],
- "--max-running-requests": ["768"]
+ "--mem-fraction-static": ["0.93"]
}
- }
+ },
+ "ref": [29, "sglang", 0]
}
]
},
- "sglang": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "sglang", 0]
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
},
{
- "from": 3,
- "cmd": "python3 -m sglang.launch_server",
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--skip-server-warmup": [],
- "--request-plane": null,
- "--router-mode": null
+ "--request-plane": ["tcp"]
},
"env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "DYN_SGL_GATEWAY_WORKERS": null
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
}
}
- },
- {
- "ref": [23, "sglang", 2]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#2",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#27",
+ "hw": "b300",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)",
- "detail": "DSpark 7 · shortest-prefill-first",
- "gpus": 16,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "label": "Agg TP2",
+ "detail": "DSpark 7 · --prefill-decode-interval 20",
+ "gpus": 2,
+ "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
- "setup": [
- {
- "kind": "note",
- "script": "Placement",
- "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
- }
- ],
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none", "hicache", "mooncake"],
@@ -4874,135 +6291,133 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "200"],
+ ["--hicache-size", "370"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark",
+ "donorHardware": "b300",
+ "donorRef": "main"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
+ "pr": "4726",
+ "status": "in-progress"
},
"points": [
{
- "concs": [224, 256],
- "item": 1,
- "override": "override_4p_dep2_dep8",
+ "concs": [50],
+ "item": 27,
+ "override": "override_tp2_c50_spf_interval20",
"goldenAL": "4.04",
- "kv": "hicache"
+ "kv": "hicache",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml"
+ },
+ {
+ "concs": [52],
+ "item": 29,
+ "override": "override_tp2_c52_spf_interval20_mem093",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"],
+ "--mem-fraction-static": ["0.93"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["104"],
+ "--max-running-requests": ["104"],
+ "--mem-fraction-static": ["0.93"]
+ }
+ }
+ ]
+ },
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml"
}
],
"routers": {
- "dynamo": {
+ "sglang": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
- },
- {
- "workers": 4,
- "gpu_pinning": [
- ["0,1"],
- ["2,3"],
- ["0,1"],
- ["2,3"]
- ],
- "local_ranks": [0, 1, 0, 1],
- "ref": [23, "dynamo", 2]
- },
- {
- "ref": [23, "dynamo", 3]
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["100"],
+ "--max-running-requests": ["100"],
+ "--prefill-decode-interval": ["20"]
+ }
+ },
+ "ref": [25, "sglang", 0]
}
]
},
- "sglang": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
{
- "from": 2,
- "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--skip-server-warmup": [],
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
- },
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
- }
- }
- },
- {
- "from": 3,
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--skip-server-warmup": [],
- "--request-plane": null,
- "--router-mode": null
- },
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "DYN_SGL_GATEWAY_WORKERS": null
- }
- }
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
},
{
- "kind": "sglang-router",
- "cmd": "python3 -m sglang_router.launch_router",
- "env": {},
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
"args": [
- ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
- ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"],
- ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
- ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
- ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
- ["--pd-disaggregation"],
- ["--host", "0.0.0.0"],
- ["--port", "8000"],
- ["--policy", "cache_aware"]
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#0",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#0",
+ "hw": "b200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
+ "imageCommit": null,
+ "submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -5011,20 +6426,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "210"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "donorHardware": "b200"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
+ "run": "40739120064/attempts/1",
+ "pr": "4598",
+ "status": "verified",
+ "date": "2026-10-28"
},
"points": [
{
@@ -5036,14 +6451,12 @@ export const agentx = {
},
{
"concs": [4],
- "item": 1,
+ "item": 0,
"override": "override_tp4_c4",
"goldenAL": "4.04",
"kv": "none",
"patch": {
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
@@ -5051,27 +6464,26 @@ export const agentx = {
}
}
],
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"]
}
- },
- null
+ }
]
}
},
{
"concs": [8],
- "item": 2,
+ "item": 0,
"override": "override_tp4_c8",
"goldenAL": "4.04",
"kv": "none",
"patch": {
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["16"],
@@ -5079,24 +6491,33 @@ export const agentx = {
}
}
],
- "sglang": [
+ "dynamo": [
+ null,
+ null,
{
"args": {
"--cuda-graph-max-bs-decode": ["16"],
"--max-running-requests": ["16"]
}
- },
- null
+ }
]
}
},
{
"concs": [16],
- "item": 3,
+ "item": 0,
"override": "override_tp4_c16",
"goldenAL": "4.04",
"kv": "none",
"patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["32"],
+ "--max-running-requests": ["32"]
+ }
+ }
+ ],
"dynamo": [
null,
null,
@@ -5106,125 +6527,254 @@ export const agentx = {
"--max-running-requests": ["32"]
}
}
- ],
+ ]
+ }
+ },
+ {
+ "concs": [32],
+ "item": 0,
+ "override": "override_tp4_c32",
+ "goldenAL": "4.04",
+ "kv": "none",
+ "patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"]
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--max-running-requests": ["64"]
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "worker",
+ "role": "agg",
+ "workers": 1,
+ "nodes_per_worker": 1,
+ "cmd": "python3 -m sglang.launch_server",
+ "env": {
+ "FMHA_SM100_ALLOW_JIT": "1",
+ "NCCL_CUMEM_ENABLE": "1",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
+ },
+ "args": [
+ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--chunked-prefill-size", "32768"],
+ ["--cuda-graph-backend-prefill", "breakable"],
+ ["--cuda-graph-max-bs-decode", "2"],
+ ["--max-running-requests", "2"],
+ ["--mem-fraction-static", "0.8"],
+ ["--moe-runner-backend", "flashinfer_trtllm_routed"],
+ ["--reasoning-parser", "minimax-m3"],
+ ["--speculative-algorithm", "DSPARK"],
+ ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
+ ["--speculative-dspark-block-size", "7"],
+ ["--speculative-use-block-verification"],
+ ["--tensor-parallel-size", "4"],
+ ["--tokenizer-worker-num", "8"],
+ ["--tool-call-parser", "minimax-m3"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "3600"]
+ ]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#1",
+ "hw": "b200",
+ "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
+ "label": "Agg TP4",
+ "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
+ "imageStatus": "ok",
+ "imageCommit": null,
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-size", "210"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "donorHardware": "b200"
+ }
+ },
+ "prov": {
+ "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic-low-latency.yaml",
+ "run": "40739120064/attempts/1",
+ "pr": "4598",
+ "status": "verified",
+ "date": "2026-10-28"
+ },
+ "points": [
+ {
+ "concs": [1],
+ "item": 1,
+ "override": "override_tp4_c1",
+ "goldenAL": "4.04",
+ "kv": "none"
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--disable-attn-tp-gather": [],
+ "--max-running-requests": ["8"],
+ "--monokernel-backend": ["msa"],
+ "--page-size": ["128"],
+ "--speculative-eagle-topk": ["1"],
+ "--speculative-num-draft-tokens": ["8"],
+ "--speculative-num-steps": ["1"],
+ "--chunked-prefill-size": null,
+ "--cuda-graph-backend-prefill": null,
+ "--cuda-graph-max-bs-decode": null,
+ "--speculative-draft-kv-cache-dtype": null,
+ "--tokenizer-worker-num": null
+ },
+ "env": {
+ "ALLOW_JIT": "1",
+ "SGLANG_FORCE_COARSE_WAR_BARRIER": "1"
}
},
- null
- ]
- }
- }
- ],
- "routers": {
+ "ref": [32, "sglang", 0]
+ }
+ ]
+ },
"dynamo": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
},
{
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
- "cmd": "python3 -m dynamo.sglang",
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
"env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000",
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000",
- "DYN_EVENT_PLANE": "zmq",
"DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "FMHA_SM100_ALLOW_JIT": "1",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
},
"args": [
- ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--request-plane", "tcp"],
- ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"],
- ["--chunked-prefill-size", "32768"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-max-bs-decode", "2"],
- ["--max-running-requests", "2"],
- ["--mem-fraction-static", "0.8"],
- ["--moe-runner-backend", "flashinfer_trtllm_routed"],
- ["--reasoning-parser", "minimax-m3"],
- ["--router-mode", "kv"],
- ["--router-session-affinity-ttl-secs", "14400"],
- ["--speculative-algorithm", "DSPARK"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-dspark-block-size", "7"],
- ["--speculative-use-block-verification"],
- ["--tensor-parallel-size", "4"],
- ["--tool-call-parser", "minimax-m3"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "3600"]
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
]
- }
- ]
- },
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
+ },
{
- "from": 2,
- "cmd": "python3 -m sglang.launch_server",
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
+ "--request-plane": ["tcp"]
},
"env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
}
}
- },
- {
- "kind": "sglang-router",
- "cmd": "python3 -m sglang_router.launch_router",
- "env": {},
- "args": [
- ["--host", "0.0.0.0"],
- ["--port", "8000"],
- ["--policy", "cache_aware"],
- ["--worker-urls", "http://127.0.0.1:30000"]
- ]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#1",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#2",
+ "hw": "b200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
"label": "Agg TP4",
- "detail": "DSpark 7 · shortest-prefill-first · --prefill-decode-interval 32",
+ "detail": "DSpark 7 · measured with HiCache",
"gpus": 4,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
+ "imageCommit": null,
+ "submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -5233,209 +6783,215 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "210"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "donorHardware": "b200"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
+ "run": "40739120064/attempts/1",
+ "pr": "4598",
+ "status": "verified",
+ "date": "2026-10-28"
},
"points": [
{
- "concs": [24],
- "item": 4,
- "override": "override_tp4_hicache_spf_c24",
+ "concs": [32],
+ "item": 2,
+ "override": "override_tp4_dram_c32",
"goldenAL": "4.04",
"kv": "hicache"
- }
- ],
- "routers": {
- "dynamo": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
- },
- {
- "ref": [26, "dynamo", 2],
- "diff": {
+ },
+ {
+ "concs": [40],
+ "item": 2,
+ "override": "override_tp4_dram_c40",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
"args": {
- "--chunked-prefill-size": ["8192"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--prefill-decode-interval": ["32"],
- "--stream-interval": ["20"],
- "--cuda-graph-max-bs-decode": ["48"],
- "--max-running-requests": ["48"],
- "--mem-fraction-static": ["0.87"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["180"],
- "--hicache-write-policy": ["write_back"]
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
}
}
- }
- ]
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["80"],
+ "--max-running-requests": ["80"]
+ }
+ }
+ ]
+ }
},
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "from": 2,
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
+ {
+ "concs": [48],
+ "item": 2,
+ "override": "override_tp4_dram_c48",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["96"],
+ "--max-running-requests": ["96"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [56],
+ "item": 2,
+ "override": "override_tp4_dram_c56",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["112"],
+ "--max-running-requests": ["112"]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "concs": [64],
+ "item": 2,
+ "override": "override_tp4_dram_c64",
+ "goldenAL": "4.04",
+ "kv": "hicache",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
"args": {
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
- },
- "env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--max-running-requests": ["128"]
}
}
- },
- {
- "ref": [26, "sglang", 1]
- }
- ]
- }
- }
- },
- {
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#2",
- "hw": "gb300",
- "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP2",
- "detail": "DSpark 7 · shortest-prefill-first · --prefill-decode-interval 32",
- "gpus": 2,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
- "imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
- "setup": [],
- "disclosures": ["synthetic-acceptance"],
- "kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
- "hicache": {
- "args": [
- ["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
- ["--hicache-write-policy", "write_back"]
- ],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "donorHardware": "gb300"
- }
- },
- "prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
- },
- "points": [
- {
- "concs": [40],
- "item": 5,
- "override": "override_tp2_hicache_spf_c40",
- "goldenAL": "4.04",
- "kv": "hicache"
+ ]
+ }
}
],
"routers": {
- "dynamo": {
+ "sglang": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
- },
- {
- "ref": [26, "dynamo", 2],
"diff": {
"args": {
- "--tensor-parallel-size": ["2"],
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"],
- "--mem-fraction-static": ["0.87"],
+ "--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["180"],
+ "--hicache-size": ["210"],
"--hicache-write-policy": ["write_back"],
- "--chunked-prefill-size": ["8192"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--prefill-decode-interval": ["32"],
- "--stream-interval": ["20"]
+ "--max-running-requests": ["64"],
+ "--mem-fraction-static": ["0.81"]
}
- }
+ },
+ "ref": [32, "sglang", 0]
}
]
},
- "sglang": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
{
- "from": 2,
- "cmd": "python3 -m sglang.launch_server",
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
+ "--request-plane": ["tcp"]
},
"env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
}
}
- },
- {
- "ref": [26, "sglang", 1]
}
]
}
}
},
{
- "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#3",
- "hw": "gb300",
+ "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#3",
+ "hw": "b200",
"ckpt": "MiniMaxAI/MiniMax-M3.1-Flash",
- "label": "Agg TP2",
+ "label": "Agg TP4·EP4",
"detail": "DSpark 7",
- "gpus": 2,
- "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a",
+ "gpus": 4,
+ "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87",
"imageStatus": "ok",
- "imageCommit": "e3718813",
- "submitted": "dynamo",
+ "imageCommit": null,
+ "submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -5444,111 +7000,100 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "180"],
+ ["--hicache-size", "210"],
["--hicache-write-policy", "write_back"]
],
- "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg",
- "donorHardware": "gb300"
+ "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark",
+ "donorHardware": "b200"
}
},
"prov": {
- "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "41271829525",
- "pr": "4698",
- "status": "verified-pr",
- "date": "2026-11-01"
+ "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml",
+ "run": "40739120064/attempts/1",
+ "pr": "4598",
+ "status": "verified",
+ "date": "2026-10-28"
},
"points": [
{
- "concs": [40],
- "item": 6,
- "override": "override_tp2_hicache_c40",
+ "concs": [72],
+ "item": 3,
+ "override": "override_tp4_ep4_dram_c72",
"goldenAL": "4.04",
"kv": "hicache"
},
{
- "concs": [44],
- "item": 7,
- "override": "override_tp2_hicache_c44",
+ "concs": [80],
+ "item": 3,
+ "override": "override_tp4_ep4_dram_c80",
"goldenAL": "4.04",
- "kv": "hicache",
- "patch": {
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"]
- }
- }
- ],
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["88"],
- "--max-running-requests": ["88"]
- }
- },
- null
- ]
- }
+ "kv": "hicache"
}
],
"routers": {
- "dynamo": {
+ "sglang": {
"install": [],
"files": [],
"blocks": [
{
- "ref": [23, "dynamo", 0]
- },
- {
- "ref": [23, "dynamo", 1]
- },
- {
- "ref": [26, "dynamo", 2],
"diff": {
"args": {
- "--tensor-parallel-size": ["2"],
- "--cuda-graph-max-bs-decode": ["80"],
- "--max-running-requests": ["80"],
- "--mem-fraction-static": ["0.87"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["180"],
- "--hicache-write-policy": ["write_back"]
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--cuda-graph-max-bs-prefill": ["16384"],
+ "--expert-parallel-size": ["4"],
+ "--max-running-requests": ["128"],
+ "--mem-fraction-static": ["0.8"],
+ "--moe-runner-backend": null
+ },
+ "env": {
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768"
}
- }
+ },
+ "ref": [34, "sglang", 0]
}
]
},
- "sglang": {
+ "dynamo": {
"install": [],
"files": [],
"blocks": [
{
- "from": 2,
- "cmd": "python3 -m sglang.launch_server",
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--request-plane": null,
- "--kv-events-config": null,
- "--router-mode": null,
- "--router-session-affinity-ttl-secs": null
+ "--request-plane": ["tcp"]
},
"env": {
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
- "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
- "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
- "DYN_EVENT_PLANE": null,
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
}
}
- },
- {
- "ref": [26, "sglang", 1]
}
]
}
diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx
index 7d466caaa4f0..69507c2a9748 100644
--- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx
+++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "Qwen3.5",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
+ },
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
+ },
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
@@ -184,10 +199,6 @@ export const agentx = {
"nginx.conf#033bddfb": {
"name": "nginx.conf",
"content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n server $FRONTEND_1_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}"
- },
- "router_policy_config.yaml#005cce35": {
- "name": "router_policy_config.yaml",
- "content": "worker_selection:\n decode: dynamo-two-tier-cost-fn\n instances:\n - name: dynamo-two-tier-cost-fn\n type: dynamo-two-tier-cost-fn\n"
}
},
"cells": [
@@ -213,7 +224,7 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -405,7 +416,7 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -525,7 +536,7 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -642,7 +653,7 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -757,8 +768,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1084,8 +1095,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1262,8 +1273,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1416,8 +1427,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1573,8 +1584,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1731,8 +1742,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -1890,8 +1901,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -2052,8 +2063,8 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -2195,25 +2206,27 @@ export const agentx = {
"id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#0",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Agg TP4",
- "detail": "MTP 6-1-7",
- "gpus": 4,
+ "label": "Agg TP2",
+ "detail": "MTP 3-1-4",
+ "gpus": 2,
"configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
- "imageStatus": "ok",
- "imageCommit": null,
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1",
+ "imageStatus": "pruned",
+ "imageCommit": "433429b1",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -2221,276 +2234,61 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "eb1e4107ac078fa736735066c90b2ef457babab0",
+ "run": "31042542308/attempts/1",
+ "pr": "2477",
+ "status": "verified",
+ "date": "2026-08-07"
},
"points": [
{
- "concs": [8, 9],
- "item": 1,
- "override": "override_tp4_mrr8_c8",
- "goldenAL": "4.8",
- "kv": "hicache"
+ "concs": [1],
+ "item": 0,
+ "goldenAL": "3.39",
+ "kv": "hicache",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml"
},
{
- "concs": [12, 18],
- "item": 0,
- "override": "override_tp4_mrr8",
- "goldenAL": "4.8",
+ "concs": [24],
+ "item": 1,
+ "goldenAL": "3.39",
"kv": "hicache",
- "patch": {
- "sglang": [
- {
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- },
- null
- ],
- "dynamo": [
- null,
- null,
- {
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- }
- ]
- }
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml"
},
{
- "concs": [13],
- "item": 5,
- "override": "override_tp4_mrr1",
- "goldenAL": "4.8",
+ "concs": [32],
+ "item": 2,
+ "goldenAL": "3.39",
"kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1"],
- "--cuda-graph-max-bs-decode": ["1"],
- "--max-running-requests": ["1"],
- "--pp-max-micro-batch-size": ["1"]
- },
- "env": {
- "SGLANG_EAGER_INPUT_NO_COPY": "1",
- "SGLANG_Q35MF_MKA_PUSH": "1",
- "SGLANG_Q35MF_MKA_Q1": "1",
- "SGLANG_Q35MF_MKA_SKV2": "1",
- "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192",
- "SGLANG_Q35MF_MKD_HEAD": "1",
- "SGLANG_Q35MF_MKD_HOP": "1",
- "SGLANG_Q35MF_MKD_HOP_EARLY": "1",
- "SGLANG_Q35MF_MKD_HOP_PUSH": "1",
- "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1",
- "SGLANG_Q35MF_MKH_N3B": "1",
- "SGLANG_Q35MF_MKH_N3C": "1",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1"
- }
- },
- null
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-bs-decode": ["1"],
- "--cuda-graph-max-bs-decode": ["1"],
- "--max-running-requests": ["1"],
- "--pp-max-micro-batch-size": ["1"]
- },
- "env": {
- "SGLANG_EAGER_INPUT_NO_COPY": "1",
- "SGLANG_Q35MF_MKA_PUSH": "1",
- "SGLANG_Q35MF_MKA_Q1": "1",
- "SGLANG_Q35MF_MKA_SKV2": "1",
- "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192",
- "SGLANG_Q35MF_MKD_HEAD": "1",
- "SGLANG_Q35MF_MKD_HOP": "1",
- "SGLANG_Q35MF_MKD_HOP_EARLY": "1",
- "SGLANG_Q35MF_MKD_HOP_PUSH": "1",
- "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1",
- "SGLANG_Q35MF_MKH_N3B": "1",
- "SGLANG_Q35MF_MKH_N3C": "1",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1"
- }
- }
- ]
- }
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml"
},
{
- "concs": [16],
+ "concs": [40],
"item": 3,
- "override": "override_tp4_mrr2_k64",
- "goldenAL": "4.8",
+ "goldenAL": "3.39",
"kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--pp-max-micro-batch-size": ["2"],
- "--prefill-decode-interval": ["64"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- },
- null
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--pp-max-micro-batch-size": ["2"],
- "--prefill-decode-interval": ["64"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- }
- ]
- }
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml"
},
{
- "concs": [20],
+ "concs": [48],
"item": 4,
- "override": "override_tp4_mrr2_k32",
- "goldenAL": "4.8",
+ "goldenAL": "3.39",
"kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--pp-max-micro-batch-size": ["2"],
- "--prefill-decode-interval": ["32"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- },
- null
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--pp-max-micro-batch-size": ["2"],
- "--prefill-decode-interval": ["32"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- }
- ]
- }
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml"
},
{
- "concs": [23],
- "item": 2,
- "override": "override_tp4_mrr3_k48",
- "goldenAL": "4.8",
+ "concs": [52],
+ "item": 5,
+ "goldenAL": "3.39",
"kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2", "3"],
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--pp-max-micro-batch-size": ["3"],
- "--prefill-decode-interval": ["48"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8",
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- },
- null
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2", "3"],
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--pp-max-micro-batch-size": ["3"],
- "--prefill-decode-interval": ["48"]
- },
- "env": {
- "SGLANG_GDN_CHUNK_H_BV": "16",
- "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3",
- "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8",
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8",
- "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1",
- "SGLANG_Q35MF_MKM_FUSED_M16": "1",
- "SGLANG_Q35MF_MKM_FUSED_M21": "1"
- }
- }
- ]
- }
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml"
+ },
+ {
+ "concs": [64],
+ "item": 6,
+ "goldenAL": "3.39",
+ "kv": "hicache",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml"
}
],
"routers": {
@@ -2505,131 +2303,50 @@ export const agentx = {
"nodes_per_worker": 1,
"cmd": "python3 -m sglang.launch_server",
"env": {
- "FLASHINFER_DISABLE_VERSION_CHECK": "1",
- "MC_FORCE_MNNVL": "1",
"NCCL_CUMEM_ENABLE": "1",
- "NCCL_MNNVL_ENABLE": "1",
- "NCCL_NVLS_ENABLE": "0",
- "NVSHMEM_REMOTE_TRANSPORT": "none",
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
- "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1",
- "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1",
+ "NCCL_NVLS_ENABLE": "1",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
- "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0",
- "SGLANG_ENABLE_JIT_DEEPGEMM": "true",
- "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "1",
- "SGLANG_ENABLE_SPEC_V2": "1",
- "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
- "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass",
- "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1",
- "SGLANG_HEALTH_CHECK_TIMEOUT": "1800",
- "SGLANG_MOE_NVFP4_DISPATCH": "1",
- "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1",
- "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
- "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1",
- "SGLANG_Q35MF_ARRIVE_D2H_RING": "1",
- "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1",
- "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1",
- "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1",
- "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1",
- "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1",
- "SGLANG_Q35MF_ATTN_FRONT": "1",
- "SGLANG_Q35MF_ATTN_GATE_QUANT": "1",
- "SGLANG_Q35MF_ATTN_SPLIT_KV": "1",
- "SGLANG_Q35MF_BK_DRAFT_META": "1",
- "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1",
- "SGLANG_Q35MF_BK_STASH_SCATTER": "1",
- "SGLANG_Q35MF_BK_VEC_COPY": "1",
- "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1",
- "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1",
- "SGLANG_Q35MF_DRAFT_FC_GEMV": "1",
- "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1",
- "SGLANG_Q35MF_DRAFT_MNNVL_AR": "1",
- "SGLANG_Q35MF_DRAFT_MOE_GEMV": "1",
- "SGLANG_Q35MF_DRAFT_MOE_ROUTER": "1",
- "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1",
- "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1",
- "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1",
- "SGLANG_Q35MF_EDGE_SHM_SYNC": "1",
- "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1",
- "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1",
- "SGLANG_Q35MF_GDN_CONV_TILE": "1",
- "SGLANG_Q35MF_GDN_LAZY_VALID": "1",
- "SGLANG_Q35MF_GDN_NORM_FP8": "1",
- "SGLANG_Q35MF_GDN_PDL": "1",
- "SGLANG_Q35MF_GDN_TILED_VERIFY": "1",
- "SGLANG_Q35MF_GDN_VIEWS": "1",
- "SGLANG_Q35MF_K4_LOCAL_VERIFY_SAMPLING": "1",
- "SGLANG_Q35MF_MKA_FUSED": "1",
- "SGLANG_Q35MF_MKD_EXTEND_FC": "1",
- "SGLANG_Q35MF_MKD_MOE1": "1",
- "SGLANG_Q35MF_MKD_TTABLE": "1",
- "SGLANG_Q35MF_MKD_XCHG": "1",
- "SGLANG_Q35MF_MKH_HOP": "1",
- "SGLANG_Q35MF_MKH_OPROJ": "1",
- "SGLANG_Q35MF_MKM_FUSED": "1",
- "SGLANG_Q35MF_ROUTE_FUSED": "1",
- "SGLANG_Q35MF_ROUTE_SHARED": "1",
- "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1",
- "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
- "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800"
+ "SGL_ENABLE_JIT_DEEPGEMM": "false",
+ "TORCH_CUDA_ARCH_LIST": "10.0"
},
"args": [
["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
["--host", "0.0.0.0"],
["--port", "30000"],
+ ["--allow-auto-truncate"],
["--attention-backend", "trtllm_mha"],
["--chunked-prefill-size", "16384"],
- ["--context-length", "262144"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8"],
- ["--cuda-graph-max-bs-decode", "8"],
- ["--cuda-graph-max-bs-prefill", "16384"],
+ ["--cuda-graph-max-bs", "80"],
["--data-parallel-size", "1"],
- ["--disable-attn-tp-gather"],
- ["--disable-shared-experts-fusion"],
["--enable-hierarchical-cache"],
- ["--enable-linear-replayssm-spec"],
["--enable-symm-mem"],
["--expert-parallel-size", "1"],
- ["--flashinfer-allreduce-fusion-backend", "cutedsl"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"],
["--kv-cache-dtype", "fp8_e4m3"],
- ["--linear-attn-decode-backend", "triton"],
- ["--load-balance-method", "round_robin"],
- ["--mamba-max-states-per-path", "3"],
- ["--mamba-radix-cache-strategy", "extra_buffer"],
+ ["--mamba-max-states-per-path", "1"],
+ ["--mamba-scheduler-strategy", "extra_buffer"],
["--mamba-ssm-dtype", "bfloat16"],
["--mamba-track-interval", "8192"],
- ["--max-mamba-cache-size", "1536"],
+ ["--max-mamba-cache-size", "360"],
["--max-prefill-tokens", "16384"],
- ["--max-running-requests", "8"],
- ["--mem-fraction-static", "0.85"],
- ["--moe-a2a-backend", "none"],
- ["--moe-dense-tp-size", "4"],
+ ["--max-running-requests", "72"],
+ ["--mem-fraction-static", "0.75"],
["--moe-runner-backend", "flashinfer_trtllm"],
["--page-size", "64"],
- ["--pipeline-parallel-size", "1"],
- ["--pp-max-micro-batch-size", "8"],
- ["--prefill-max-requests", "1"],
["--quantization", "modelopt_fp4"],
+ ["--scheduler-recv-interval", "10"],
["--speculative-algorithm", "NEXTN"],
["--speculative-eagle-topk", "1"],
- ["--speculative-moe-a2a-backend", "none"],
- ["--speculative-moe-runner-backend", "flashinfer_cutedsl"],
- ["--speculative-num-draft-tokens", "7"],
- ["--speculative-num-steps", "6"],
- ["--stream-interval", "20"],
- ["--tensor-parallel-size", "4"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "1000000"],
- ["--weight-loader-prefetch-checkpoints"],
- ["--weight-loader-prefetch-num-threads", "4"]
+ ["--speculative-num-draft-tokens", "4"],
+ ["--speculative-num-steps", "3"],
+ ["--stream-interval", "50"],
+ ["--tensor-parallel-size", "2"],
+ ["--tokenizer-worker-num", "6"],
+ ["--trust-remote-code"]
]
},
{
@@ -2639,14 +2356,14 @@ export const agentx = {
"args": [
["--host", "0.0.0.0"],
["--port", "8000"],
- ["--worker-urls", "http://127.0.0.1:30000"]
+ ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"]
]
}
]
},
"dynamo": {
"install": [],
- "files": [],
+ "files": ["nats.conf#48a4de48"],
"blocks": [
{
"kind": "etcd",
@@ -2659,13 +2376,21 @@ export const agentx = {
["--advertise-client-urls", "http://$ETCD_IP:2379"]
]
},
+ {
+ "kind": "nats",
+ "count_nodes": 1,
+ "cmd": "nats-server -c nats.conf",
+ "env": {},
+ "args": []
+ },
{
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
"env": {
"DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
},
"args": [
["--http-port", "8000"],
@@ -2681,7 +2406,8 @@ export const agentx = {
},
"env": {
"DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
}
}
}
@@ -2690,28 +2416,30 @@ export const agentx = {
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#6",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#2",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
"label": "Agg TP8 (2 nodes)",
"detail": "MTP 6-1-7",
"gpus": 8,
- "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
+ "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198",
"imageStatus": "ok",
- "imageCommit": null,
+ "imageCommit": "bb5e6198",
"submitted": "sglang",
"setup": [],
- "disclosures": ["synthetic-acceptance"],
+ "disclosures": [],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-size", "32"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
@@ -2719,19 +2447,17 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml",
+ "run": "34345050756/attempts/1",
+ "pr": "2812",
+ "status": "verified",
+ "date": "2026-09-09"
},
"points": [
{
- "concs": [11, 18],
- "item": 6,
- "override": "override_tp8_mrr1",
- "goldenAL": "4.8",
+ "concs": [7],
+ "item": 2,
"kv": "hicache"
}
],
@@ -2741,40 +2467,93 @@ export const agentx = {
"files": [],
"blocks": [
{
+ "kind": "worker",
+ "role": "agg",
+ "workers": 1,
"nodes_per_worker": 2,
- "diff": {
- "args": {
- "--dist-init-addr": ["$AGG_HEAD_IP:20000"],
- "--nnodes": ["2"],
- "--node-rank": ["$NODE_RANK"],
- "--cuda-graph-bs-decode": ["1"],
- "--cuda-graph-max-bs-decode": ["1"],
- "--max-running-requests": ["1"],
- "--moe-dense-tp-size": ["8"],
- "--pp-max-micro-batch-size": ["1"],
- "--tensor-parallel-size": ["8"]
- },
- "env": {
- "SGLANG_EAGER_INPUT_NO_COPY": "1",
- "SGLANG_Q35MF_EDGE_MULTIHOST_TCP": "1",
- "SGLANG_Q35MF_EDGE_SHM_SYNC_MULTIHOST": "1",
- "SGLANG_Q35MF_MKA_PUSH": "1",
- "SGLANG_Q35MF_MKA_Q1": "1",
- "SGLANG_Q35MF_MKA_SKV2": "1",
- "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192",
- "SGLANG_Q35MF_MKD_GEMV": "1",
- "SGLANG_Q35MF_MKD_HEAD": "1",
- "SGLANG_Q35MF_MKD_HOP": "1",
- "SGLANG_Q35MF_MKD_HOP_EARLY": "1",
- "SGLANG_Q35MF_MKG_BA_TC": "1",
- "SGLANG_Q35MF_MKG_FUSED": "1",
- "SGLANG_Q35MF_MKH_N3B": "1",
- "SGLANG_Q35MF_MKH_N3C": "1",
- "SGLANG_Q35MF_MKH_PF_GRAPH": "1",
- "SGLANG_Q35MF_MKM_GDN_FRONT": "1"
- }
+ "cmd": "python3 -m sglang.launch_server",
+ "env": {
+ "FLASHINFER_DISABLE_VERSION_CHECK": "1",
+ "MC_FORCE_MNNVL": "1",
+ "NCCL_CUMEM_ENABLE": "1",
+ "NCCL_MNNVL_ENABLE": "1",
+ "NCCL_NVLS_ENABLE": "0",
+ "NVSHMEM_REMOTE_TRANSPORT": "none",
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1",
+ "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1",
+ "SGLANG_ENABLE_FLASHINFER_GEMM": "true",
+ "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0",
+ "SGLANG_ENABLE_JIT_DEEPGEMM": "true",
+ "SGLANG_ENABLE_SPEC_V2": "1",
+ "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
+ "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass",
+ "SGLANG_HEALTH_CHECK_TIMEOUT": "1800",
+ "SGLANG_MOE_NVFP4_DISPATCH": "1",
+ "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1",
+ "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1",
+ "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
+ "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1",
+ "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1",
+ "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
+ "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800"
},
- "ref": [12, "sglang", 0]
+ "args": [
+ ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
+ ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
+ ["--host", "0.0.0.0"],
+ ["--port", "30000"],
+ ["--dist-init-addr", "$AGG_HEAD_IP:20000"],
+ ["--nnodes", "2"],
+ ["--node-rank", "$NODE_RANK"],
+ ["--attention-backend", "trtllm_mha"],
+ ["--chunked-prefill-size", "8192"],
+ ["--context-length", "262144"],
+ ["--cuda-graph-bs-decode", "1"],
+ ["--cuda-graph-max-bs-decode", "1"],
+ ["--data-parallel-size", "1"],
+ ["--disable-attn-tp-gather"],
+ ["--disable-prefill-cuda-graph"],
+ ["--disable-shared-experts-fusion"],
+ ["--enable-hierarchical-cache"],
+ ["--enable-linear-replayssm-spec"],
+ ["--expert-parallel-size", "1"],
+ ["--hicache-io-backend", "kernel"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-size", "32"],
+ ["--hicache-write-policy", "write_back"],
+ ["--kv-cache-dtype", "fp8_e4m3"],
+ ["--linear-attn-decode-backend", "triton"],
+ ["--load-balance-method", "round_robin"],
+ ["--mamba-max-states-per-path", "3"],
+ ["--mamba-radix-cache-strategy", "extra_buffer"],
+ ["--mamba-ssm-dtype", "bfloat16"],
+ ["--mamba-track-interval", "8192"],
+ ["--max-mamba-cache-size", "1536"],
+ ["--max-prefill-tokens", "8192"],
+ ["--max-running-requests", "1"],
+ ["--mem-fraction-static", "0.85"],
+ ["--moe-a2a-backend", "none"],
+ ["--moe-dense-tp-size", "8"],
+ ["--moe-runner-backend", "flashinfer_cutedsl"],
+ ["--page-size", "64"],
+ ["--pipeline-parallel-size", "1"],
+ ["--pp-max-micro-batch-size", "1"],
+ ["--prefill-max-requests", "1"],
+ ["--quantization", "modelopt_fp4"],
+ ["--speculative-algorithm", "NEXTN"],
+ ["--speculative-eagle-topk", "1"],
+ ["--speculative-moe-a2a-backend", "none"],
+ ["--speculative-moe-runner-backend", "flashinfer_cutedsl"],
+ ["--speculative-num-draft-tokens", "7"],
+ ["--speculative-num-steps", "6"],
+ ["--stream-interval", "20"],
+ ["--tensor-parallel-size", "8"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "1000000"],
+ ["--weight-loader-prefetch-checkpoints"],
+ ["--weight-loader-prefetch-num-threads", "4"]
+ ]
},
{
"kind": "sglang-router",
@@ -2790,7 +2569,7 @@ export const agentx = {
},
"dynamo": {
"install": [],
- "files": [],
+ "files": ["nats.conf#48a4de48"],
"blocks": [
{
"kind": "etcd",
@@ -2803,13 +2582,21 @@ export const agentx = {
["--advertise-client-urls", "http://$ETCD_IP:2379"]
]
},
+ {
+ "kind": "nats",
+ "count_nodes": 1,
+ "cmd": "nats-server -c nats.conf",
+ "env": {},
+ "args": []
+ },
{
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
"env": {
"DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
},
"args": [
["--http-port", "8000"],
@@ -2825,7 +2612,8 @@ export const agentx = {
},
"env": {
"DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
}
}
}
@@ -2834,34 +2622,30 @@ export const agentx = {
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#5",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#0",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Disagg 1P1D · prefill TP1 · decode TP2",
- "detail": "MTP 6-1-7",
- "gpus": 3,
+ "label": "Disagg 1P1D · prefill TP4 · decode TP4",
+ "detail": "no spec",
+ "gpus": 8,
"configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
- "imageStatus": "ok",
- "imageCommit": null,
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1",
+ "imageStatus": "pruned",
+ "imageCommit": "433429b1",
"submitted": "dynamo",
- "setup": [
- {
- "kind": "note",
- "script": "Placement",
- "reason": "Workers share nodes: 1 prefill worker on 1 node, 1 decode worker on 1 node. The prefill and decode workers share a node. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
- }
- ],
+ "setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -2869,26 +2653,38 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "eb1e4107ac078fa736735066c90b2ef457babab0",
+ "run": "31042542308/attempts/1",
+ "pr": "2477",
+ "status": "verified",
+ "date": "2026-08-07"
},
"points": [
{
- "concs": [20],
- "item": 5,
- "override": "override_1p1d_tp2_cap10",
- "goldenAL": "4.8",
- "kv": "hicache"
+ "concs": [8],
+ "item": 0,
+ "goldenAL": "3.39",
+ "kv": "hicache",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml"
},
{
- "concs": [26],
- "item": 8,
- "override": "override_1p1d_tp2_cap4",
- "goldenAL": "4.8",
+ "concs": [16],
+ "item": 1,
+ "goldenAL": "3.39",
+ "kv": "hicache",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml"
+ },
+ {
+ "concs": [32],
+ "item": 2,
+ "goldenAL": "3.39",
+ "kv": "hicache",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml"
+ },
+ {
+ "concs": [64],
+ "item": 3,
+ "goldenAL": "3.39",
"kv": "hicache",
"patch": {
"dynamo": [
@@ -2896,19 +2692,10 @@ export const agentx = {
null,
null,
null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["4"],
- "--pp-max-micro-batch-size": ["4"],
- "--cuda-graph-bs-decode": null
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["80"]
}
}
],
@@ -2916,28 +2703,18 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["4"],
- "--pp-max-micro-batch-size": ["4"],
- "--cuda-graph-bs-decode": null
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["80"]
}
},
null
]
- }
+ },
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml"
},
{
- "concs": [36],
- "item": 7,
- "override": "override_1p1d_tp2_cap5",
- "goldenAL": "4.8",
+ "concs": [96],
+ "item": 4,
+ "goldenAL": "3.39",
"kv": "hicache",
"patch": {
"dynamo": [
@@ -2945,19 +2722,10 @@ export const agentx = {
null,
null,
null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["5"],
- "--pp-max-micro-batch-size": ["8"],
- "--cuda-graph-bs-decode": null
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["128"]
}
}
],
@@ -2965,28 +2733,18 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["5"],
- "--pp-max-micro-batch-size": ["8"],
- "--cuda-graph-bs-decode": null
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["128"]
}
},
null
]
- }
+ },
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml"
},
{
- "concs": [38],
- "item": 6,
- "override": "override_1p1d_tp2_cap12",
- "goldenAL": "4.8",
+ "concs": [128],
+ "item": 5,
+ "goldenAL": "3.39",
"kv": "hicache",
"patch": {
"dynamo": [
@@ -2994,19 +2752,10 @@ export const agentx = {
null,
null,
null,
+ null,
{
"args": {
- "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["12"],
- "--pp-max-micro-batch-size": ["12"]
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["160"]
}
}
],
@@ -3014,28 +2763,19 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"],
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["12"],
- "--pp-max-micro-batch-size": ["12"]
- },
- "env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "--cuda-graph-max-bs": ["160"]
}
},
null
]
- }
+ },
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml"
}
],
"routers": {
"dynamo": {
- "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"],
- "files": ["nats.conf#48a4de48"],
+ "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"],
+ "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"],
"blocks": [
{
"kind": "etcd",
@@ -3056,24 +2796,27 @@ export const agentx = {
"args": []
},
{
- "kind": "dynamo-frontend",
- "count": 1,
- "cmd": "python3 -m dynamo.frontend",
+ "kind": "nginx",
+ "cmd": "nginx -c nginx.conf -g daemon off;",
+ "env": {},
+ "args": []
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
"env": {
"DYN_REQUEST_PLANE": "tcp",
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
"NATS_SERVER": "nats://$NATS_IP:4222"
},
"args": [
- ["--http-port", "8000"],
+ ["--http-port", "8180"],
["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "3600"],
["--active-decode-blocks-threshold", "None"],
["--active-prefill-tokens-threshold", "None"],
- ["--active-prefill-tokens-threshold-frac", "None"],
- ["--router-replica-sync"],
- ["--router-queue-threshold", "None"],
- ["--router-session-affinity-ttl-secs", "3600"],
- ["--router-temperature", "1.0"]
+ ["--active-prefill-tokens-threshold-frac", "None"]
]
},
{
@@ -3081,9 +2824,6 @@ export const agentx = {
"role": "prefill",
"workers": 1,
"nodes_per_worker": 1,
- "gpu_pinning": [
- ["0"]
- ],
"cmd": "python3 -m dynamo.sglang",
"env": {
"DYN_REQUEST_PLANE": "tcp",
@@ -3093,29 +2833,20 @@ export const agentx = {
"NATS_SERVER": "nats://$NATS_IP:4222",
"NCCL_CUMEM_ENABLE": "1",
"NCCL_MNNVL_ENABLE": "1",
- "NCCL_NVLS_ENABLE": "0",
- "NVSHMEM_REMOTE_TRANSPORT": "none",
- "PYTHONOPTIMIZE": "1",
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "NCCL_NVLS_ENABLE": "1",
"SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000",
"SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000",
"SGLANG_DISAGG_STAGING_BUFFER": "1",
+ "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024",
"SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
- "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
- "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
- "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1",
+ "SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass",
- "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1",
- "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192",
"SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
- "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
- "SGLANG_Q35MF_K4_MROPE_GPU": "1",
"SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
- "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800",
- "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800"
+ "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800"
},
"args": [
["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
@@ -3126,49 +2857,30 @@ export const agentx = {
["--disaggregation-bootstrap-port", "8998"],
["--request-plane", "tcp"],
["--attention-backend", "trtllm_mha"],
- ["--chunked-prefill-size", "8192"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-max-bs-prefill", "8192"],
+ ["--chunked-prefill-size", "65536"],
["--data-parallel-size", "1"],
["--disable-cuda-graph"],
- ["--disable-overlap-schedule"],
- ["--disable-shared-experts-fusion"],
- ["--dist-timeout", "1800"],
["--enable-hierarchical-cache"],
["--expert-parallel-size", "1"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"],
["--kv-cache-dtype", "fp8_e4m3"],
- ["--linear-attn-decode-backend", "triton"],
+ ["--linear-attn-decode-backend", "flashinfer"],
["--load-balance-method", "round_robin"],
["--mamba-max-states-per-path", "3"],
- ["--mamba-radix-cache-strategy", "extra_buffer"],
+ ["--mamba-scheduler-strategy", "extra_buffer"],
["--mamba-ssm-dtype", "bfloat16"],
- ["--mamba-track-interval", "1048576"],
- ["--max-mamba-cache-size", "128"],
- ["--max-prefill-tokens", "8192"],
- ["--max-running-requests", "16"],
- ["--mem-fraction-static", "0.94"],
- ["--moe-a2a-backend", "none"],
- ["--moe-dense-tp-size", "1"],
- ["--moe-runner-backend", "flashinfer_cutedsl"],
+ ["--mamba-track-interval", "2048"],
+ ["--max-mamba-cache-size", "1100"],
+ ["--mem-fraction-static", "0.8"],
+ ["--moe-runner-backend", "flashinfer_trtllm"],
["--page-size", "64"],
- ["--pipeline-parallel-size", "1"],
["--quantization", "modelopt_fp4"],
- ["--scheduler-recv-interval", "1"],
- ["--speculative-algorithm", "NEXTN"],
- ["--speculative-eagle-topk", "1"],
- ["--speculative-moe-a2a-backend", "none"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
- ["--speculative-num-draft-tokens", "7"],
- ["--speculative-num-steps", "6"],
- ["--tensor-parallel-size", "1"],
+ ["--tensor-parallel-size", "4"],
["--trust-remote-code"],
- ["--watchdog-timeout", "1000000"],
- ["--weight-loader-prefetch-checkpoints"],
- ["--weight-loader-prefetch-num-threads", "4"]
+ ["--watchdog-timeout", "1000000"]
]
},
{
@@ -3176,151 +2888,77 @@ export const agentx = {
"role": "decode",
"workers": 1,
"nodes_per_worker": 1,
- "gpu_pinning": [
- ["1,2"]
- ],
"cmd": "python3 -m dynamo.sglang",
"env": {
"DYN_REQUEST_PLANE": "tcp",
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
"FLASHINFER_DISABLE_VERSION_CHECK": "1",
- "FLASHINFER_ROUTING_FORCE_BLOCK_PER_TOKEN": "0",
"MC_FORCE_MNNVL": "1",
"NATS_SERVER": "nats://$NATS_IP:4222",
"NCCL_CUMEM_ENABLE": "1",
"NCCL_MNNVL_ENABLE": "1",
- "NCCL_NVLS_ENABLE": "0",
- "NVSHMEM_REMOTE_TRANSPORT": "none",
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "NCCL_NVLS_ENABLE": "1",
"SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1",
+ "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": "1000",
+ "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
"SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000",
"SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000",
"SGLANG_DISAGG_STAGING_BUFFER": "1",
+ "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024",
"SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
"SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0",
"SGLANG_ENABLE_JIT_DEEPGEMM": "true",
- "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "0",
"SGLANG_ENABLE_SPEC_V2": "1",
- "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
- "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
"SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass",
- "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1",
+ "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": "1",
"SGLANG_HEALTH_CHECK_TIMEOUT": "1800",
+ "SGLANG_HEALTH_STARTING_OK": "1",
"SGLANG_MOE_NVFP4_DISPATCH": "1",
"SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
"SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1",
- "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
- "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1",
- "SGLANG_Q35MF_ARRIVE_D2H_RING": "1",
- "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1",
- "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1",
- "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1",
- "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1",
- "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1",
- "SGLANG_Q35MF_ATTN_FRONT": "1",
- "SGLANG_Q35MF_ATTN_GATE_QUANT": "1",
- "SGLANG_Q35MF_BK_DRAFT_META": "1",
- "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1",
- "SGLANG_Q35MF_BK_STASH_SCATTER": "1",
- "SGLANG_Q35MF_BK_VEC_COPY": "1",
- "SGLANG_Q35MF_CUTEDSL_LL_MAX_M": "96",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": "1",
- "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1",
- "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1",
- "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1",
- "SGLANG_Q35MF_DRAFT_VOCAB_EXCHANGE_MAX_ROWS": "12",
- "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1",
- "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1",
- "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1",
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "10",
- "SGLANG_Q35MF_EDGE_SHM_SYNC": "1",
- "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1",
- "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1",
- "SGLANG_Q35MF_FP8_BMM_TACTICS": "/sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/fp8_bmm_tactics_gb300_tp2.json",
- "SGLANG_Q35MF_GDN_CONV_TILE": "1",
- "SGLANG_Q35MF_GDN_DIRECT_SOLVE": "0",
- "SGLANG_Q35MF_GDN_EMPTY_HISTORY": "0",
- "SGLANG_Q35MF_GDN_LAZY_VALID": "1",
- "SGLANG_Q35MF_GDN_NORM_FP8": "1",
- "SGLANG_Q35MF_GDN_PDL": "1",
- "SGLANG_Q35MF_GDN_TILED_VERIFY": "1",
- "SGLANG_Q35MF_GDN_VERIFY_BV64": "1",
- "SGLANG_Q35MF_GDN_VIEWS": "1",
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "84",
- "SGLANG_Q35MF_K4_SIDE_PRIO_MAX_TOKENS": "16",
- "SGLANG_Q35MF_KDENSE_DRAFT_ARGMAX_XCHG": "1",
- "SGLANG_Q35MF_KDENSE_DRAFT_FC_XCHG": "1",
- "SGLANG_Q35MF_KDENSE_DRAFT_META_BRANCH": "1",
- "SGLANG_Q35MF_KGDN_VERIFY_REG168": "1",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "84",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MIN_TOKENS": "17",
- "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1",
- "SGLANG_Q35MF_PDDEC_FENCE": "1",
- "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1",
- "SGLANG_Q35MF_PDDEC_SHARDED_VERIFY_SAMPLING": "1",
- "SGLANG_Q35MF_ROUTE2_EARLY_TRIGGER": "1",
- "SGLANG_Q35MF_ROUTE2_SMALL_TABLES": "1",
- "SGLANG_Q35MF_ROUTE2_SMALL_TABLES_MIN_TOKENS": "1",
- "SGLANG_Q35MF_ROUTE_FUSED": "1",
- "SGLANG_Q35MF_ROUTE_PARQ": "1",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "84",
- "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1",
+ "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1",
"SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
- "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800",
- "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800"
+ "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800"
},
"args": [
["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
["--host", "0.0.0.0"],
- ["--port", "30032"],
+ ["--port", "30000"],
["--disaggregation-mode", "decode"],
- ["--disaggregation-bootstrap-port", "8999"],
+ ["--disaggregation-bootstrap-port", "8998"],
["--request-plane", "tcp"],
["--attention-backend", "trtllm_mha"],
["--chunked-prefill-size", "4096"],
- ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"],
- ["--cuda-graph-max-bs-decode", "10"],
+ ["--cuda-graph-max-bs", "64"],
["--data-parallel-size", "1"],
- ["--disable-attn-tp-gather"],
["--disable-radix-cache"],
["--disable-shared-experts-fusion"],
- ["--dist-timeout", "1800"],
- ["--enable-linear-replayssm-spec"],
+ ["--ep-dispatch-algorithm", "static"],
+ ["--eplb-algorithm", "deepseek"],
["--expert-parallel-size", "1"],
- ["--flashinfer-allreduce-fusion-backend", "cutedsl"],
["--kv-cache-dtype", "fp8_e4m3"],
- ["--linear-attn-decode-backend", "triton"],
- ["--load-balance-method", "round_robin"],
- ["--mamba-radix-cache-strategy", "no_buffer"],
+ ["--linear-attn-decode-backend", "flashinfer"],
+ ["--mamba-scheduler-strategy", "no_buffer"],
["--mamba-ssm-dtype", "bfloat16"],
["--mamba-track-interval", "128"],
- ["--max-mamba-cache-size", "936"],
- ["--max-running-requests", "10"],
- ["--mem-fraction-static", "0.85"],
- ["--moe-a2a-backend", "none"],
- ["--moe-dense-tp-size", "2"],
+ ["--max-mamba-cache-size", "1024"],
+ ["--max-running-requests", "512"],
+ ["--mem-fraction-static", "0.75"],
["--moe-runner-backend", "flashinfer_trtllm"],
["--page-size", "64"],
- ["--pipeline-parallel-size", "1"],
- ["--pp-max-micro-batch-size", "10"],
+ ["--prefill-round-robin-balance"],
["--quantization", "modelopt_fp4"],
["--speculative-algorithm", "NEXTN"],
["--speculative-eagle-topk", "1"],
- ["--speculative-moe-a2a-backend", "none"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
- ["--speculative-num-draft-tokens", "7"],
- ["--speculative-num-steps", "6"],
- ["--stream-interval", "30"],
- ["--tensor-parallel-size", "2"],
+ ["--speculative-num-draft-tokens", "4"],
+ ["--speculative-num-steps", "3"],
+ ["--tensor-parallel-size", "4"],
["--trust-remote-code"],
- ["--watchdog-timeout", "1000000"],
- ["--weight-loader-prefetch-checkpoints"],
- ["--weight-loader-prefetch-num-threads", "4"]
+ ["--watchdog-timeout", "1000000"]
]
}
]
@@ -3330,7 +2968,7 @@ export const agentx = {
"files": [],
"blocks": [
{
- "from": 3,
+ "from": 4,
"cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
@@ -3345,7 +2983,7 @@ export const agentx = {
}
},
{
- "from": 4,
+ "from": 5,
"cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
@@ -3364,7 +3002,7 @@ export const agentx = {
"cmd": "python3 -m sglang_router.launch_router",
"env": {},
"args": [
- ["--decode", "http://$DECODE_0_HEAD_IP:30032"],
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
["--pd-disaggregation"],
["--host", "0.0.0.0"],
@@ -3377,61 +3015,94 @@ export const agentx = {
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#9",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#0",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Disagg 1P2D · prefill TP1 · decode TP2",
- "detail": "MTP 6-1-7",
- "gpus": 5,
- "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
+ "label": "Agg TP2",
+ "detail": "MTP 6-1-7 · --max-running-requests 2",
+ "gpus": 2,
+ "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198",
"imageStatus": "ok",
- "imageCommit": null,
- "submitted": "dynamo",
- "setup": [
- {
- "kind": "note",
- "script": "Placement",
- "reason": "Workers share nodes: 1 prefill worker on 1 node, 2 decode workers on 1 node. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
- }
- ],
- "disclosures": ["synthetic-acceptance"],
+ "imageCommit": "bb5e6198",
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": [],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
- "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
+ "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
"donorHardware": "gb300"
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml",
+ "run": "34345050756/attempts/1",
+ "pr": "2812",
+ "status": "verified",
+ "date": "2026-09-09"
},
"points": [
{
- "concs": [48],
- "item": 9,
- "override": "override_1p2d_tp2_cap3",
- "goldenAL": "4.8",
+ "concs": [44],
+ "item": 0,
"kv": "hicache"
}
],
"routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "nodes_per_worker": 1,
+ "diff": {
+ "args": {
+ "--cuda-graph-bs-decode": ["1", "2"],
+ "--cuda-graph-max-bs-decode": ["2"],
+ "--hicache-size": ["128"],
+ "--max-running-requests": ["2"],
+ "--moe-dense-tp-size": ["2"],
+ "--moe-runner-backend": ["flashinfer_trtllm"],
+ "--pp-max-micro-batch-size": ["2"],
+ "--prefill-decode-interval": ["0"],
+ "--prefill-max-requests": ["2"],
+ "--speculative-moe-runner-backend": ["flashinfer_trtllm"],
+ "--tensor-parallel-size": ["2"],
+ "--dist-init-addr": null,
+ "--nnodes": null,
+ "--node-rank": null,
+ "--disable-prefill-cuda-graph": null
+ }
+ },
+ "ref": [13, "sglang", 0]
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
+ }
+ ]
+ },
"dynamo": {
- "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"],
- "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"],
+ "install": [],
+ "files": ["nats.conf#48a4de48"],
"blocks": [
{
"kind": "etcd",
@@ -3451,12 +3122,6 @@ export const agentx = {
"env": {},
"args": []
},
- {
- "kind": "nginx",
- "cmd": "nginx -c nginx.conf -g daemon off;",
- "env": {},
- "args": []
- },
{
"kind": "dynamo-frontend",
"count": 1,
@@ -3467,125 +3132,183 @@ export const agentx = {
"NATS_SERVER": "nats://$NATS_IP:4222"
},
"args": [
- ["--http-port", "8180"],
- ["--router-mode", "kv"],
- ["--active-decode-blocks-threshold", "None"],
- ["--active-prefill-tokens-threshold", "None"],
- ["--active-prefill-tokens-threshold-frac", "None"],
- ["--router-replica-sync"],
- ["--router-queue-threshold", "None"],
- ["--router-temperature", "0.0"],
- ["--router-decode-active-request-weight", "8192"]
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
]
},
{
- "drop": ["gpu_pinning"],
- "ref": [14, "dynamo", 3]
- },
- {
- "workers": 2,
- "gpu_pinning": [
- ["0,1"],
- ["2,3"]
- ],
- "local_ranks": [0, 1],
- "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
"diff": {
"args": {
- "--port": ["$PORT"],
- "--disaggregation-bootstrap-port": ["$BOOTSTRAP_PORT"],
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--pp-max-micro-batch-size": ["3"],
- "--cuda-graph-bs-decode": null
+ "--request-plane": ["tcp"]
},
"env": {
- "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70",
- "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70",
- "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70",
- "SGLANG_Q35MF_D2R_PVG_SHARDED": null,
- "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
}
- },
- "ref": [14, "dynamo", 4]
+ }
}
]
+ }
+ }
+ },
+ {
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#1",
+ "hw": "gb300",
+ "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
+ "label": "Agg TP2",
+ "detail": "MTP 6-1-7 · --max-running-requests 1",
+ "gpus": 2,
+ "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198",
+ "imageStatus": "ok",
+ "imageCommit": "bb5e6198",
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": [],
+ "kv": {
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-io-backend", "kernel"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg",
+ "donorHardware": "gb300"
+ }
+ },
+ "prov": {
+ "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml",
+ "run": "34345050756/attempts/1",
+ "pr": "2812",
+ "status": "verified",
+ "date": "2026-09-09"
+ },
+ "points": [
+ {
+ "concs": [44],
+ "item": 1,
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
"sglang": {
"install": [],
"files": [],
"blocks": [
{
- "from": 4,
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--skip-server-warmup": [],
- "--request-plane": null
- },
- "env": {
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "NATS_SERVER": null
- }
- }
- },
- {
- "from": 5,
- "cmd": "python3 -m sglang.launch_server",
"diff": {
"args": {
- "--skip-server-warmup": [],
- "--request-plane": null
- },
- "env": {
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "NATS_SERVER": null
+ "--cuda-graph-bs-decode": ["1"],
+ "--cuda-graph-max-bs-decode": ["1"],
+ "--hicache-size": ["32"],
+ "--max-running-requests": ["1"],
+ "--pp-max-micro-batch-size": ["1"],
+ "--prefill-max-requests": ["1"]
}
- }
+ },
+ "ref": [15, "sglang", 0]
},
{
"kind": "sglang-router",
"cmd": "python3 -m sglang_router.launch_router",
"env": {},
"args": [
- ["--decode", "http://$DECODE_1_HEAD_IP:30032"],
- ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
- ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
- ["--pd-disaggregation"],
["--host", "0.0.0.0"],
["--port", "8000"],
- ["--policy", "cache_aware"]
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": ["nats.conf#48a4de48"],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "nats",
+ "count_nodes": 1,
+ "cmd": "nats-server -c nats.conf",
+ "env": {},
+ "args": []
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
+ }
+ }
}
]
}
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#4",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#6",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Disagg 2P1D · prefill PP4 · decode DEP16 (4 nodes)",
- "detail": "MTP 6-1-7",
- "gpus": 24,
+ "label": "Disagg 1P1D · prefill TP2 · decode TP2",
+ "detail": "MTP 3-1-4",
+ "gpus": 4,
"configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
- "imageStatus": "ok",
- "imageCommit": null,
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1",
+ "imageStatus": "pruned",
+ "imageCommit": "433429b1",
"submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -3593,26 +3316,25 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "eb1e4107ac078fa736735066c90b2ef457babab0",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml",
+ "run": "31042542308/attempts/1",
+ "pr": "2477",
+ "status": "verified",
+ "date": "2026-08-07"
},
"points": [
{
- "concs": [488],
- "item": 4,
- "override": "override_2p1d_dep16_cap6",
- "goldenAL": "4.8",
+ "concs": [72],
+ "item": 6,
+ "goldenAL": "3.39",
"kv": "hicache"
}
],
"routers": {
"dynamo": {
- "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"],
- "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"],
+ "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"],
+ "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"],
"blocks": [
{
"kind": "etcd",
@@ -3634,164 +3356,53 @@ export const agentx = {
},
{
"kind": "nginx",
- "cmd": "nginx -c nginx.conf -g daemon off;",
- "env": {},
- "args": []
- },
- {
- "kind": "dynamo-frontend",
- "count": 2,
- "cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "NATS_SERVER": "nats://$NATS_IP:4222"
- },
- "args": [
- ["--http-port", "8180"],
- ["--router-mode", "kv"],
- ["--active-decode-blocks-threshold", "None"],
- ["--active-prefill-tokens-threshold", "None"],
- ["--active-prefill-tokens-threshold-frac", "None"],
- ["--router-replica-sync"],
- ["--router-queue-threshold", "None"],
- ["--router-session-affinity-ttl-secs", "3600"],
- ["--router-session-affinity-mode", "soft"],
- ["--router-temperature", "1.0"],
- ["--router-policy-config", "router_policy_config.yaml"]
- ]
- },
- {
- "workers": 2,
- "diff": {
- "args": {
- "--chunked-prefill-size": ["32768"],
- "--hicache-mem-layout": ["page_first_direct"],
- "--hicache-ratio": ["0.9"],
- "--linear-attn-prefill-backend": ["flashinfer"],
- "--max-mamba-cache-size": ["1536"],
- "--max-prefill-tokens": ["32768"],
- "--max-running-requests": ["128"],
- "--mem-fraction-static": ["0.85"],
- "--pipeline-parallel-size": ["4"],
- "--pp-async-batch-depth": ["1"],
- "--cuda-graph-backend-prefill": null,
- "--cuda-graph-max-bs-prefill": null
- },
- "env": {
- "SGLANG_PP_LAYER_PARTITION": "16,16,16,12",
- "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": "1",
- "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": "1",
- "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": "1",
- "SGLANG_Q35MF_GDN_PREFILL_SPLIT_CONV": "1",
- "SGLANG_Q35MF_GDN_PREFILL_TILES": "1",
- "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": "1",
- "SGLANG_Q35MF_K4_MROPE_GPU": null
- }
- },
- "ref": [15, "dynamo", 4]
- },
- {
- "kind": "worker",
- "role": "decode",
- "workers": 1,
- "nodes_per_worker": 4,
- "cmd": "python3 -m dynamo.sglang",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "FLASHINFER_DISABLE_VERSION_CHECK": "1",
- "MC_FORCE_MNNVL": "1",
- "NATS_SERVER": "nats://$NATS_IP:4222",
- "NCCL_CUMEM_ENABLE": "1",
- "NCCL_MNNVL_ENABLE": "1",
- "NCCL_NVLS_ENABLE": "0",
- "NVSHMEM_REMOTE_TRANSPORT": "none",
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
- "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1",
- "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1",
- "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000",
- "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000",
- "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000",
- "SGLANG_DISAGG_STAGING_BUFFER": "1",
- "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192",
- "SGLANG_ENABLE_FLASHINFER_GEMM": "true",
- "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0",
- "SGLANG_ENABLE_JIT_DEEPGEMM": "true",
- "SGLANG_ENABLE_SPEC_V2": "1",
- "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
- "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
- "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass",
- "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1",
- "SGLANG_HEALTH_CHECK_TIMEOUT": "1800",
- "SGLANG_MOE_NVFP4_DISPATCH": "1",
- "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True",
- "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1",
- "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": "1",
- "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": "2",
- "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1",
- "SGLANG_Q35MF_PDDEC_FENCE": "1",
- "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1",
- "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
- "SGLANG_USE_SYMM_MEM_DP_SYNC": "true",
- "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800",
- "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800"
+ "cmd": "nginx -c nginx.conf -g daemon off;",
+ "env": {},
+ "args": []
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
},
"args": [
- ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
- ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--disaggregation-mode", "decode"],
- ["--disaggregation-bootstrap-port", "8998"],
- ["--dist-init-addr", "$DECODE_HEAD_IP:20000"],
- ["--nnodes", "4"],
- ["--node-rank", "$NODE_RANK"],
- ["--request-plane", "tcp"],
- ["--attention-backend", "trtllm_mha"],
- ["--chunked-prefill-size", "4096"],
- ["--cuda-graph-max-bs-decode", "6"],
- ["--data-parallel-size", "16"],
- ["--disable-radix-cache"],
- ["--disable-shared-experts-fusion"],
- ["--dist-timeout", "1800"],
- ["--enable-dp-attention"],
- ["--enable-linear-replayssm-spec"],
- ["--ep-dispatch-algorithm", "static"],
- ["--eplb-algorithm", "deepseek"],
- ["--expert-parallel-size", "16"],
- ["--gc-threshold", "50000", "50", "1000"],
- ["--kv-cache-dtype", "fp8_e4m3"],
- ["--linear-attn-decode-backend", "triton"],
- ["--linear-replayssm-cache-len", "32"],
- ["--load-balance-method", "round_robin"],
- ["--mamba-radix-cache-strategy", "no_buffer"],
- ["--mamba-ssm-dtype", "bfloat16"],
- ["--mamba-track-interval", "128"],
- ["--max-mamba-cache-size", "1872"],
- ["--max-running-requests", "96"],
- ["--max-total-tokens", "13000000"],
- ["--mem-fraction-static", "0.85"],
- ["--moe-a2a-backend", "flashinfer"],
- ["--moe-dense-tp-size", "1"],
- ["--moe-runner-backend", "flashinfer_cutedsl"],
- ["--page-size", "64"],
- ["--pipeline-parallel-size", "1"],
- ["--quantization", "modelopt_fp4"],
- ["--speculative-algorithm", "NEXTN"],
- ["--speculative-eagle-topk", "1"],
- ["--speculative-moe-a2a-backend", "flashinfer"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm_routed"],
- ["--speculative-num-draft-tokens", "7"],
- ["--speculative-num-steps", "6"],
- ["--stream-interval", "30"],
- ["--tensor-parallel-size", "16"],
- ["--trust-remote-code"],
- ["--watchdog-timeout", "1000000"],
- ["--weight-loader-prefetch-checkpoints"],
- ["--weight-loader-prefetch-num-threads", "4"]
+ ["--http-port", "8180"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "3600"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
]
+ },
+ {
+ "diff": {
+ "args": {
+ "--mamba-max-states-per-path": ["1"],
+ "--max-mamba-cache-size": ["320"],
+ "--speculative-algorithm": ["NEXTN"],
+ "--speculative-eagle-topk": ["1"],
+ "--speculative-num-draft-tokens": ["4"],
+ "--speculative-num-steps": ["3"],
+ "--tensor-parallel-size": ["2"]
+ }
+ },
+ "ref": [14, "dynamo", 4]
+ },
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs": ["128"],
+ "--mamba-max-states-per-path": ["-1"],
+ "--max-mamba-cache-size": ["80"],
+ "--max-running-requests": ["80"],
+ "--tensor-parallel-size": ["2"]
+ }
+ },
+ "ref": [14, "dynamo", 5]
}
]
},
@@ -3835,7 +3446,6 @@ export const agentx = {
"env": {},
"args": [
["--decode", "http://$DECODE_0_HEAD_IP:30000"],
- ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
["--pd-disaggregation"],
["--host", "0.0.0.0"],
@@ -3848,28 +3458,30 @@ export const agentx = {
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#1",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#1",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Disagg 3P1D · prefill PP4 · decode DEP16 (4 nodes)",
- "detail": "MTP 6-1-7",
+ "label": "Disagg 3P4D · prefill PP4 · decode DEP4",
+ "detail": "MTP 4-1-5",
"gpus": 28,
- "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
+ "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198",
"imageStatus": "ok",
- "imageCommit": null,
+ "imageCommit": "bb5e6198",
"submitted": "dynamo",
"setup": [],
- "disclosures": ["synthetic-acceptance"],
+ "disclosures": [],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -3877,120 +3489,24 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml",
+ "run": "34345050756/attempts/1",
+ "pr": "2812",
+ "status": "verified",
+ "date": "2026-09-09"
},
"points": [
{
- "concs": [530],
- "item": 3,
- "override": "override_3p1d_dep16_cap9",
- "goldenAL": "4.8",
- "kv": "hicache"
- },
- {
- "concs": [660],
- "item": 2,
- "override": "override_3p1d_dep16_cap12_combine_split",
- "goldenAL": "4.8",
- "kv": "hicache",
- "patch": {
- "dynamo": [
- null,
- null,
- null,
- null,
- {
- "env": {
- "SGLANG_Q35MF_PPO": null
- }
- },
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["192"]
- },
- "env": {
- "SGLANG_Q35MF_K75P_LMHEAD": null
- }
- }
- ],
- "sglang": [
- {
- "env": {
- "SGLANG_Q35MF_PPO": null
- }
- },
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["192"]
- },
- "env": {
- "SGLANG_Q35MF_K75P_LMHEAD": null
- }
- },
- null
- ]
- }
- },
- {
- "concs": [760, 840],
+ "concs": [565],
"item": 1,
- "override": "override_3p1d_dep16_cap12",
- "goldenAL": "4.8",
- "kv": "hicache",
- "patch": {
- "dynamo": [
- null,
- null,
- null,
- null,
- {
- "env": {
- "SGLANG_Q35MF_PPO": null
- }
- },
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["192"]
- },
- "env": {
- "SGLANG_Q35MF_K75P_LMHEAD": null,
- "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null
- }
- }
- ],
- "sglang": [
- {
- "env": {
- "SGLANG_Q35MF_PPO": null
- }
- },
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["192"]
- },
- "env": {
- "SGLANG_Q35MF_K75P_LMHEAD": null,
- "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null
- }
- },
- null
- ]
- }
+ "kv": "hicache"
}
],
"routers": {
"dynamo": {
- "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"],
- "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"],
+ "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"],
+ "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"],
"blocks": [
{
"kind": "etcd",
@@ -4028,37 +3544,107 @@ export const agentx = {
"args": [
["--http-port", "8180"],
["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "3600"],
["--active-decode-blocks-threshold", "None"],
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"],
["--router-replica-sync"],
["--router-queue-threshold", "None"],
- ["--router-session-affinity-ttl-secs", "3600"],
- ["--router-session-affinity-mode", "soft"],
- ["--router-temperature", "1.0"],
- ["--router-policy-config", "router_policy_config.yaml"]
+ ["--router-temperature", "1.0"]
]
},
{
"workers": 3,
"diff": {
+ "args": {
+ "--chunked-prefill-size": ["32768"],
+ "--disable-overlap-schedule": [],
+ "--disable-shared-experts-fusion": [],
+ "--linear-attn-decode-backend": ["triton"],
+ "--mamba-max-states-per-path": ["3"],
+ "--mamba-radix-cache-strategy": ["extra_buffer"],
+ "--mamba-track-interval": ["1048576"],
+ "--max-mamba-cache-size": ["1536"],
+ "--max-prefill-tokens": ["32768"],
+ "--max-running-requests": ["128"],
+ "--mem-fraction-static": ["0.85"],
+ "--moe-a2a-backend": ["none"],
+ "--moe-dense-tp-size": ["1"],
+ "--moe-runner-backend": ["flashinfer_cutedsl"],
+ "--pipeline-parallel-size": ["4"],
+ "--pp-async-batch-depth": ["1"],
+ "--scheduler-recv-interval": ["1"],
+ "--speculative-moe-a2a-backend": ["none"],
+ "--speculative-moe-runner-backend": ["flashinfer_trtllm"],
+ "--speculative-num-draft-tokens": ["5"],
+ "--speculative-num-steps": ["4"],
+ "--tensor-parallel-size": ["1"],
+ "--weight-loader-prefetch-checkpoints": [],
+ "--weight-loader-prefetch-num-threads": ["4"],
+ "--mamba-scheduler-strategy": null
+ },
"env": {
- "SGLANG_Q35MF_PPO": "1"
+ "NCCL_NVLS_ENABLE": "0",
+ "NVSHMEM_REMOTE_TRANSPORT": "none",
+ "PYTHONOPTIMIZE": "1",
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
+ "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1",
+ "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192",
+ "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
+ "SGLANG_PP_LAYER_PARTITION": "16,16,16,12",
+ "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null,
+ "SGLANG_ENABLE_SPEC_V2": null
}
},
- "ref": [16, "dynamo", 4]
+ "ref": [17, "dynamo", 4]
},
{
+ "workers": 4,
"diff": {
"args": {
- "--cuda-graph-max-bs-decode": ["9"],
- "--max-running-requests": ["144"]
+ "--cuda-graph-max-bs": ["80"],
+ "--data-parallel-size": ["4"],
+ "--disaggregation-decode-extra-slots": ["2"],
+ "--enable-dp-attention": [],
+ "--enable-dp-lm-head": [],
+ "--enable-linear-replayssm-spec": [],
+ "--expert-parallel-size": ["4"],
+ "--linear-attn-decode-backend": ["triton"],
+ "--linear-replayssm-cache-len": ["32"],
+ "--load-balance-method": ["round_robin"],
+ "--max-mamba-cache-size": ["468"],
+ "--max-running-requests": ["320"],
+ "--mem-fraction-static": ["0.7"],
+ "--moe-a2a-backend": ["flashinfer"],
+ "--moe-dense-tp-size": ["1"],
+ "--moe-runner-backend": ["flashinfer_cutedsl"],
+ "--pipeline-parallel-size": ["1"],
+ "--speculative-draft-model-quantization": ["nvfp4_online"],
+ "--speculative-moe-a2a-backend": ["flashinfer"],
+ "--speculative-moe-runner-backend": ["flashinfer_trtllm_routed"],
+ "--speculative-num-draft-tokens": ["5"],
+ "--speculative-num-steps": ["4"],
+ "--stream-interval": ["30"],
+ "--weight-loader-prefetch-checkpoints": [],
+ "--weight-loader-prefetch-num-threads": ["4"],
+ "--prefill-round-robin-balance": null
},
"env": {
- "SGLANG_Q35MF_K75P_LMHEAD": "1"
+ "NCCL_NVLS_ENABLE": "0",
+ "NVSHMEM_REMOTE_TRANSPORT": "none",
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
+ "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4",
+ "SGLANG_USE_SYMM_MEM_DP_SYNC": "true",
+ "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": null,
+ "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": null,
+ "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null,
+ "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": null,
+ "SGLANG_HEALTH_STARTING_OK": null
}
},
- "ref": [16, "dynamo", 5]
+ "ref": [14, "dynamo", 5]
}
]
},
@@ -4101,6 +3687,9 @@ export const agentx = {
"cmd": "python3 -m sglang_router.launch_router",
"env": {},
"args": [
+ ["--decode", "http://$DECODE_3_HEAD_IP:30000"],
+ ["--decode", "http://$DECODE_2_HEAD_IP:30000"],
+ ["--decode", "http://$DECODE_1_HEAD_IP:30000"],
["--decode", "http://$DECODE_0_HEAD_IP:30000"],
["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
@@ -4116,28 +3705,30 @@ export const agentx = {
}
},
{
- "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#0",
+ "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#0",
"hw": "gb300",
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
- "label": "Disagg 3P1D · prefill PP4 · decode DEP8 (2 nodes)",
- "detail": "MTP 6-1-7",
+ "label": "Disagg 3P2D · prefill PP4 · decode DEP4",
+ "detail": "MTP 5-1-6",
"gpus": 20,
- "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
- "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116",
+ "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198",
"imageStatus": "ok",
- "imageCommit": null,
+ "imageCommit": "bb5e6198",
"submitted": "dynamo",
"setup": [],
- "disclosures": ["synthetic-acceptance"],
+ "disclosures": [],
"kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
+ "available": ["none", "hicache"],
+ "unavailable": {
+ "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
+ },
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-ratio", "12"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "0.9"],
["--hicache-write-policy", "write_back"]
],
"donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg",
@@ -4145,26 +3736,24 @@ export const agentx = {
}
},
"prov": {
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml",
- "run": "39355392627",
- "pr": "4333",
- "comment": "6334247978",
- "status": "verified-pr"
+ "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc",
+ "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml",
+ "run": "34345050756/attempts/1",
+ "pr": "2812",
+ "status": "verified",
+ "date": "2026-09-09"
},
"points": [
{
"concs": [704],
"item": 0,
- "override": "override_3p1d_dep8_cap32",
- "goldenAL": "4.8",
"kv": "hicache"
}
],
"routers": {
"dynamo": {
- "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"],
- "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"],
+ "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"],
+ "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"],
"blocks": [
{
"kind": "etcd",
@@ -4202,56 +3791,33 @@ export const agentx = {
"args": [
["--http-port", "8180"],
["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "3600"],
["--active-decode-blocks-threshold", "None"],
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"],
["--router-replica-sync"],
["--router-queue-threshold", "None"],
- ["--router-session-affinity-ttl-secs", "3600"],
- ["--router-session-affinity-mode", "soft"],
- ["--router-temperature", "1.0"],
- ["--router-policy-config", "router_policy_config.yaml"]
+ ["--router-temperature", "1.0"]
]
},
{
- "workers": 3,
"diff": {
"args": {
- "--linear-attn-prefill-backend": null
- },
- "env": {
- "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": null,
- "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": null,
- "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": null,
- "SGLANG_Q35MF_GDN_PREFILL_TILES": null,
- "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": null
+ "--speculative-num-draft-tokens": ["6"],
+ "--speculative-num-steps": ["5"]
}
},
- "ref": [16, "dynamo", 4]
+ "ref": [18, "dynamo", 4]
},
{
- "nodes_per_worker": 2,
+ "workers": 2,
"diff": {
"args": {
- "--nnodes": ["2"],
- "--cuda-graph-max-bs-decode": ["32"],
- "--data-parallel-size": ["8"],
- "--disaggregation-decode-extra-slots": ["64"],
- "--enable-dp-lm-head": [],
- "--expert-parallel-size": ["8"],
- "--max-mamba-cache-size": ["936"],
- "--max-running-requests": ["256"],
- "--max-total-tokens": ["12000000"],
- "--tensor-parallel-size": ["8"]
- },
- "env": {
- "SGLANG_Q35MF_PDDEC_SPLIT_HOIST": "1",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4",
- "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": null,
- "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null
+ "--speculative-num-draft-tokens": ["6"],
+ "--speculative-num-steps": ["5"]
}
},
- "ref": [16, "dynamo", 5]
+ "ref": [18, "dynamo", 5]
}
]
},
@@ -4294,6 +3860,7 @@ export const agentx = {
"cmd": "python3 -m sglang_router.launch_router",
"env": {},
"args": [
+ ["--decode", "http://$DECODE_1_HEAD_IP:30000"],
["--decode", "http://$DECODE_0_HEAD_IP:30000"],
["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"],
@@ -4666,7 +4233,7 @@ export const agentx = {
"--enable-symm-mem": null
}
},
- "ref": [19, "dynamo", 3]
+ "ref": [20, "dynamo", 3]
}
]
},
@@ -4834,7 +4401,7 @@ export const agentx = {
"--tokenizer-worker-num": ["1"]
}
},
- "ref": [20, "dynamo", 3]
+ "ref": [21, "dynamo", 3]
}
]
},
@@ -5691,7 +5258,7 @@ export const agentx = {
"--tp": ["4"]
}
},
- "ref": [22, "sglang", 0]
+ "ref": [23, "sglang", 0]
}
]
},
@@ -6297,7 +5864,7 @@ export const agentx = {
"--page-size": ["64"]
}
},
- "ref": [22, "sglang", 0]
+ "ref": [23, "sglang", 0]
}
]
},
@@ -6562,7 +6129,7 @@ export const agentx = {
"--tensor-parallel-size": ["2"]
}
},
- "ref": [24, "dynamo", 2]
+ "ref": [25, "dynamo", 2]
},
{
"gpu_pinning": [
@@ -6577,7 +6144,7 @@ export const agentx = {
"--tensor-parallel-size": ["2"]
}
},
- "ref": [24, "dynamo", 3]
+ "ref": [25, "dynamo", 3]
}
]
},
@@ -6721,7 +6288,7 @@ export const agentx = {
"--expert-parallel-size": ["4"]
}
},
- "ref": [24, "dynamo", 2]
+ "ref": [25, "dynamo", 2]
},
{
"diff": {
@@ -6729,7 +6296,7 @@ export const agentx = {
"--cuda-graph-max-bs-decode": ["32"]
}
},
- "ref": [24, "dynamo", 3]
+ "ref": [25, "dynamo", 3]
}
]
},
@@ -7603,7 +7170,7 @@ export const agentx = {
"--tp": ["4"]
}
},
- "ref": [28, "sglang", 0]
+ "ref": [29, "sglang", 0]
}
]
},
@@ -8116,7 +7683,7 @@ export const agentx = {
"--page-size": ["64"]
}
},
- "ref": [28, "sglang", 0]
+ "ref": [29, "sglang", 0]
}
]
},
@@ -8269,7 +7836,7 @@ export const agentx = {
"--disable-prefill-cuda-graph": null
}
},
- "ref": [30, "sglang", 0]
+ "ref": [31, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -8390,7 +7957,7 @@ export const agentx = {
"--speculative-num-steps": ["5"]
}
},
- "ref": [32, "sglang", 0]
+ "ref": [33, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -8538,7 +8105,7 @@ export const agentx = {
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2"
}
},
- "ref": [32, "sglang", 0]
+ "ref": [33, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -9556,7 +9123,7 @@ export const agentx = {
"--tensor-parallel-size": ["8"]
}
},
- "ref": [35, "sglang", 0]
+ "ref": [36, "sglang", 0]
}
]
},
@@ -10218,7 +9785,7 @@ export const agentx = {
"--page-size": ["64"]
}
},
- "ref": [36, "sglang", 0]
+ "ref": [37, "sglang", 0]
}
]
},
@@ -11225,7 +10792,7 @@ export const agentx = {
"--tp": ["4"]
}
},
- "ref": [39, "sglang", 0]
+ "ref": [40, "sglang", 0]
}
]
},
@@ -11465,7 +11032,7 @@ export const agentx = {
"--page-size": ["64"]
}
},
- "ref": [40, "sglang", 0]
+ "ref": [41, "sglang", 0]
}
]
},
@@ -11745,7 +11312,7 @@ export const agentx = {
"--speculative-num-steps": ["5"]
}
},
- "ref": [42, "sglang", 0]
+ "ref": [43, "sglang", 0]
}
]
},
@@ -11875,7 +11442,7 @@ export const agentx = {
"--tokenizer-path": ["Qwen/Qwen3.5-397B-A17B-FP8"]
}
},
- "ref": [42, "sglang", 0]
+ "ref": [43, "sglang", 0]
}
]
},
@@ -12009,7 +11576,7 @@ export const agentx = {
"--page-size": null
}
},
- "ref": [44, "sglang", 0]
+ "ref": [45, "sglang", 0]
}
]
},
@@ -12142,7 +11709,7 @@ export const agentx = {
"--tensor-parallel-size": ["4"]
}
},
- "ref": [42, "sglang", 0]
+ "ref": [43, "sglang", 0]
}
]
},
@@ -12256,7 +11823,7 @@ export const agentx = {
"--speculative-num-steps": ["6"]
}
},
- "ref": [43, "sglang", 0]
+ "ref": [44, "sglang", 0]
}
]
},
@@ -12363,7 +11930,7 @@ export const agentx = {
"--speculative-num-steps": ["4"]
}
},
- "ref": [43, "sglang", 0]
+ "ref": [44, "sglang", 0]
}
]
},
@@ -12655,7 +12222,7 @@ export const agentx = {
"--page-size": ["64"]
}
},
- "ref": [49, "sglang", 0]
+ "ref": [50, "sglang", 0]
}
]
},
diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx
deleted file mode 100644
index afb8ee4e320c..000000000000
--- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx
+++ /dev/null
@@ -1,431 +0,0 @@
-export const b300Cells = [
- {
- "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp4",
- "hw": "b300",
- "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4",
- "label": "Agg TP4·EP4",
- "detail": "MTP 3-1-4",
- "gpus": 4,
- "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
- "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
- "imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
- "disclosures": [],
- "kv": {
- "available": ["none"],
- "unavailable": {
- "hicache": "Not qualified for this deployment.",
- "mooncake": "Not qualified for this deployment."
- }
- },
- "prov": {
- "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "41117066222/attempts/1",
- "pr": "4683",
- "status": "verified-pr",
- "date": "2026-10-30"
- },
- "points": [
- {
- "concs": [1],
- "override": "override_tp4_c1",
- "kv": "none"
- },
- {
- "concs": [2],
- "override": "override_tp4_c2",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--max-running-requests": ["4"]
- },
- "env": {}
- }
- ]
- }
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
- "cmd": "python3 -m sglang.launch_server",
- "env": {
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0",
- "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1"
- },
- "args": [
- ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--trust-remote-code"],
- ["--quantization", "modelopt_mixed"],
- ["--context-length", "262144"],
- ["--tensor-parallel-size", "4"],
- ["--pipeline-parallel-size", "1"],
- ["--data-parallel-size", "1"],
- ["--expert-parallel-size", "4"],
- ["--attention-backend", "trtllm_mha"],
- ["--linear-attn-prefill-backend", "flashinfer"],
- ["--linear-attn-decode-backend", "flashinfer"],
- ["--linear-attn-verify-backend", "triton"],
- ["--mamba-ssm-dtype", "bfloat16"],
- ["--mem-fraction-static", "0.88"],
- ["--max-running-requests", "2"],
- ["--page-size", "64"],
- ["--chunked-prefill-size", "16384"],
- ["--max-prefill-tokens", "16384"],
- ["--moe-runner-backend", "flashinfer_trtllm"],
- ["--moe-a2a-backend", "none"],
- ["--disable-shared-experts-fusion"],
- ["--speculative-algorithm", "NEXTN"],
- ["--speculative-num-steps", "3"],
- ["--speculative-eagle-topk", "1"],
- ["--speculative-num-draft-tokens", "4"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
- ["--enable-linear-replayssm-spec"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-backend-decode", "full"],
- ["--cuda-graph-max-bs-decode", "96"],
- ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
- ["--kv-cache-dtype", "bfloat16"],
- ["--speculative-draft-kv-cache-dtype", "bfloat16"],
- ["--ple-offload-embedding"],
- ["--ple-offload-backend", "pinned"],
- ["--reasoning-parser", "auto"],
- ["--tokenizer-worker-num", "6"],
- ["--scheduler-recv-interval", "1"],
- ["--stream-interval", "20"],
- ["--watchdog-timeout", "1800"],
- ["--flashinfer-allreduce-fusion-backend", "trtllm"]
- ]
- }
- ]
- }
- }
- },
- {
- "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp1",
- "hw": "b300",
- "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4",
- "label": "Agg TP1 · GPU KV",
- "detail": "MTP 3-1-4",
- "gpus": 1,
- "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
- "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
- "imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
- "disclosures": [],
- "kv": {
- "available": ["none"],
- "unavailable": {
- "hicache": "Not qualified for this deployment.",
- "mooncake": "Not qualified for this deployment."
- }
- },
- "prov": {
- "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "41117066222/attempts/1",
- "pr": "4683",
- "status": "verified-pr",
- "date": "2026-10-30"
- },
- "points": [
- {
- "concs": [1],
- "override": "override_c1",
- "kv": "none"
- },
- {
- "concs": [2],
- "override": "override_c2",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {},
- "env": {}
- }
- ]
- }
- },
- {
- "concs": [12],
- "override": "override_c12",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
- "--mamba-track-interval": ["262144"],
- "--kv-cache-dtype": ["fp8_e4m3"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
- },
- "env": {
- "SGLANG_Q35MF_MKH_N3B": null,
- "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
- "SGLANG_Q35MF_MKH_N3C": null,
- "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
- "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
- "SGLANG_Q35MF_BK_VEC_COPY": null,
- "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null
- }
- }
- ]
- }
- },
- {
- "concs": [24],
- "override": "override_c24",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
- "--mamba-track-interval": ["262144"],
- "--max-mamba-cache-size": ["960"],
- "--kv-cache-dtype": ["fp8_e4m3"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
- },
- "env": {
- "SGLANG_Q35MF_MKH_N3B": null,
- "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
- "SGLANG_Q35MF_MKH_N3C": null,
- "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
- "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
- "SGLANG_Q35MF_BK_VEC_COPY": null,
- "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null
- }
- }
- ]
- }
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
- "cmd": "python3 -m sglang.launch_server",
- "env": {
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0",
- "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1",
- "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": "1",
- "SGLANG_Q35MF_MKH_N3B": "1",
- "SGLANG_Q35MF_MKH_N3C": "1",
- "SGLANG_Q35MF_BK_VEC_COPY": "1",
- "SGLANG_Q35MF_EDGE_SHM_SYNC": "1",
- "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1",
- "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": "1"
- },
- "args": [
- ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--trust-remote-code"],
- ["--quantization", "modelopt_mixed"],
- ["--context-length", "262144"],
- ["--tensor-parallel-size", "1"],
- ["--pipeline-parallel-size", "1"],
- ["--data-parallel-size", "1"],
- ["--expert-parallel-size", "1"],
- ["--attention-backend", "trtllm_mha"],
- ["--linear-attn-prefill-backend", "flashinfer"],
- ["--linear-attn-decode-backend", "flashinfer"],
- ["--linear-attn-verify-backend", "triton"],
- ["--mamba-ssm-dtype", "bfloat16"],
- ["--mem-fraction-static", "0.88"],
- ["--max-running-requests", "96"],
- ["--page-size", "64"],
- ["--chunked-prefill-size", "16384"],
- ["--max-prefill-tokens", "16384"],
- ["--moe-runner-backend", "flashinfer_trtllm"],
- ["--moe-a2a-backend", "none"],
- ["--disable-shared-experts-fusion"],
- ["--speculative-algorithm", "NEXTN"],
- ["--speculative-num-steps", "3"],
- ["--speculative-eagle-topk", "1"],
- ["--speculative-num-draft-tokens", "4"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
- ["--enable-linear-replayssm-spec"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-backend-decode", "full"],
- ["--cuda-graph-max-bs-decode", "96"],
- ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
- ["--kv-cache-dtype", "bfloat16"],
- ["--speculative-draft-kv-cache-dtype", "bfloat16"],
- ["--ple-offload-embedding"],
- ["--ple-offload-backend", "pinned"],
- ["--reasoning-parser", "auto"],
- ["--tokenizer-worker-num", "6"],
- ["--scheduler-recv-interval", "1"],
- ["--stream-interval", "20"],
- ["--watchdog-timeout", "1800"]
- ]
- }
- ]
- }
- }
- },
- {
- "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp1-hicache",
- "hw": "b300",
- "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4",
- "label": "Agg TP1 · HiCache",
- "detail": "MTP 3-1-4",
- "gpus": 1,
- "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
- "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
- "imageStatus": "ok",
- "imageCommit": null,
- "submitted": "sglang",
- "setup": [],
- "disclosures": [],
- "kv": {
- "available": ["hicache"],
- "unavailable": {
- "none": "Not qualified for this deployment.",
- "mooncake": "Not qualified for this deployment."
- }
- },
- "prov": {
- "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "41117066222/attempts/1",
- "pr": "4683",
- "status": "verified-pr",
- "date": "2026-10-30"
- },
- "points": [
- {
- "concs": [48],
- "override": "override_c48",
- "kv": "hicache"
- },
- {
- "concs": [80],
- "override": "override_c80",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"],
- "--max-running-requests": ["128"],
- "--mamba-max-states-per-path": ["2"],
- "--max-mamba-cache-size": ["640"],
- "--cuda-graph-max-bs-decode": ["128"],
- "--strip-thinking-cache": []
- },
- "env": {}
- }
- ]
- }
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "kind": "worker",
- "role": "agg",
- "workers": 1,
- "nodes_per_worker": 1,
- "cmd": "python3 -m sglang.launch_server",
- "env": {
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0",
- "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1"
- },
- "args": [
- ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--host", "0.0.0.0"],
- ["--port", "30000"],
- ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
- ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
- ["--trust-remote-code"],
- ["--quantization", "modelopt_mixed"],
- ["--context-length", "262144"],
- ["--tensor-parallel-size", "1"],
- ["--pipeline-parallel-size", "1"],
- ["--data-parallel-size", "1"],
- ["--expert-parallel-size", "1"],
- ["--attention-backend", "trtllm_mha"],
- ["--linear-attn-prefill-backend", "flashinfer"],
- ["--linear-attn-decode-backend", "flashinfer"],
- ["--linear-attn-verify-backend", "triton"],
- ["--mamba-ssm-dtype", "bfloat16"],
- ["--mamba-radix-cache-strategy", "extra_buffer_lazy"],
- ["--mamba-track-interval", "262144"],
- ["--max-mamba-cache-size", "720"],
- ["--mem-fraction-static", "0.88"],
- ["--max-running-requests", "96"],
- ["--page-size", "64"],
- ["--chunked-prefill-size", "16384"],
- ["--max-prefill-tokens", "16384"],
- ["--moe-runner-backend", "flashinfer_trtllm"],
- ["--moe-a2a-backend", "none"],
- ["--disable-shared-experts-fusion"],
- ["--speculative-algorithm", "NEXTN"],
- ["--speculative-num-steps", "3"],
- ["--speculative-eagle-topk", "1"],
- ["--speculative-num-draft-tokens", "4"],
- ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
- ["--enable-linear-replayssm-spec"],
- ["--cuda-graph-backend-prefill", "breakable"],
- ["--cuda-graph-backend-decode", "full"],
- ["--cuda-graph-max-bs-decode", "96"],
- ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"],
- ["--kv-cache-dtype", "fp8_e4m3"],
- ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"],
- ["--enable-hierarchical-cache"],
- ["--hicache-size", "298"],
- ["--hicache-io-backend", "kernel"],
- ["--hicache-mem-layout", "page_first"],
- ["--hicache-write-policy", "write_back"],
- ["--ple-offload-embedding"],
- ["--ple-offload-backend", "pinned"],
- ["--reasoning-parser", "auto"],
- ["--tokenizer-worker-num", "1"],
- ["--scheduler-recv-interval", "1"],
- ["--stream-interval", "20"],
- ["--watchdog-timeout", "1800"]
- ]
- }
- ]
- }
- }
- }
-];
diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx
index 5ab50e132a15..3ef7ca99320e 100644
--- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx
+++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "Qwen3.8-Flash-Next",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
+ },
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
+ },
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
@@ -64,16 +79,7 @@ export const agentx = {
"id": "nvidia/Qwen3.8-Flash-Next-NVFP4"
}
],
- "envs": [
- {
- "NCCL_CUMEM_ENABLE": "1",
- "NCCL_NVLS_ENABLE": "1",
- "SGLANG_ENABLE_FLASHINFER_GEMM": "true",
- "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
- "SGL_ENABLE_JIT_DEEPGEMM": "false",
- "TORCH_CUDA_ARCH_LIST": "10.0"
- }
- ],
+ "envs": [],
"kvMooncake": {
"donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg",
"donorHardware": "gb200",
@@ -353,53 +359,80 @@ export const agentx = {
}
},
{
- "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0",
+ "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#1",
"hw": "b300",
- "ckpt": "RadixArk/Qwen3.8-Flash-Next-NVFP4",
+ "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4",
"label": "Agg TP1",
"detail": "MTP 3-1-4",
"gpus": 1,
"configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
- "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
"imageStatus": "ok",
- "imageCommit": null,
+ "imageCommit": "586355a8",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
- "available": ["none", "mooncake"],
- "unavailable": {
- "hicache": "no verified HiCache settings for this model yet"
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-io-backend", "kernel"],
+ ["--hicache-mem-layout", "page_first"],
+ ["--hicache-size", "298"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
+ "donorHardware": "b300"
}
},
"prov": {
- "sha": "3c0d91ceb2334554aef03a8ddfec7e0377a3f274",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "38635046800/attempts/1",
- "pr": "4069",
- "status": "verified",
- "date": "2026-10-21"
+ "run": "41117066222",
+ "pr": "4683",
+ "comment": "6536482181",
+ "status": "verified-pr"
},
"points": [
{
"concs": [1],
- "item": 0,
+ "item": 1,
"override": "override_c1",
"goldenAL": "2.32",
"kv": "none"
},
{
- "concs": [4],
- "item": 0,
- "override": "override_c4",
+ "concs": [2],
+ "item": 1,
+ "override": "override_c2",
+ "goldenAL": "2.32",
+ "kv": "none"
+ },
+ {
+ "concs": [12],
+ "item": 1,
+ "override": "override_c12",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["8"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-track-interval": ["262144"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
],
@@ -408,25 +441,48 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["8"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-track-interval": ["262144"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
]
}
},
{
- "concs": [8],
- "item": 0,
- "override": "override_c8",
+ "concs": [24],
+ "item": 1,
+ "override": "override_c24",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["16"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["960"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
],
@@ -435,25 +491,56 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["8"],
- "--max-running-requests": ["16"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["960"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
]
}
},
{
- "concs": [12],
- "item": 0,
- "override": "override_c12",
+ "concs": [48],
+ "item": 2,
+ "override": "override_c48",
"goldenAL": "2.32",
- "kv": "none",
+ "kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["24"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-io-backend": ["kernel"],
+ "--hicache-mem-layout": ["page_first"],
+ "--hicache-size": ["298"],
+ "--hicache-write-policy": ["write_back"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-radix-cache-strategy": ["extra_buffer_lazy"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["720"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
+ "--tokenizer-worker-num": ["1"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
],
@@ -462,25 +549,67 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["24"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-io-backend": ["kernel"],
+ "--hicache-mem-layout": ["page_first"],
+ "--hicache-size": ["298"],
+ "--hicache-write-policy": ["write_back"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-radix-cache-strategy": ["extra_buffer_lazy"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["720"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
+ "--tokenizer-worker-num": ["1"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
]
}
},
{
- "concs": [16],
- "item": 0,
- "override": "override_c16",
+ "concs": [80],
+ "item": 2,
+ "override": "override_c80",
"goldenAL": "2.32",
- "kv": "none",
+ "kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["32"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"],
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-io-backend": ["kernel"],
+ "--hicache-mem-layout": ["page_first"],
+ "--hicache-size": ["298"],
+ "--hicache-write-policy": ["write_back"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-max-states-per-path": ["2"],
+ "--mamba-radix-cache-strategy": ["extra_buffer_lazy"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["640"],
+ "--max-running-requests": ["128"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
+ "--strip-thinking-cache": [],
+ "--tokenizer-worker-num": ["1"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
],
@@ -489,8 +618,31 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["16"],
- "--max-running-requests": ["32"]
+ "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"],
+ "--cuda-graph-max-bs-decode": ["128"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-io-backend": ["kernel"],
+ "--hicache-mem-layout": ["page_first"],
+ "--hicache-size": ["298"],
+ "--hicache-write-policy": ["write_back"],
+ "--kv-cache-dtype": ["fp8_e4m3"],
+ "--mamba-max-states-per-path": ["2"],
+ "--mamba-radix-cache-strategy": ["extra_buffer_lazy"],
+ "--mamba-track-interval": ["262144"],
+ "--max-mamba-cache-size": ["640"],
+ "--max-running-requests": ["128"],
+ "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
+ "--strip-thinking-cache": [],
+ "--tokenizer-worker-num": ["1"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
}
}
]
@@ -508,31 +660,210 @@ export const agentx = {
"workers": 1,
"nodes_per_worker": 1,
"cmd": "python3 -m sglang.launch_server",
+ "env": {
+ "NCCL_CUMEM_ENABLE": "1",
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": "1",
+ "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0",
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": "1",
+ "SGLANG_Q35MF_BK_VEC_COPY": "1",
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1",
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": "1",
+ "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1",
+ "SGLANG_Q35MF_MKH_N3B": "1",
+ "SGLANG_Q35MF_MKH_N3C": "1",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800"
+ },
"args": [
- ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
- ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
+ ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
+ ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"],
["--host", "0.0.0.0"],
["--port", "30000"],
- ["--cuda-graph-max-bs-decode", "1"],
+ ["--attention-backend", "trtllm_mha"],
+ ["--chunked-prefill-size", "16384"],
+ ["--context-length", "262144"],
+ ["--cuda-graph-backend-decode", "full"],
+ ["--cuda-graph-backend-prefill", "breakable"],
+ ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"],
+ ["--cuda-graph-max-bs-decode", "96"],
["--data-parallel-size", "1"],
+ ["--disable-shared-experts-fusion"],
+ ["--enable-linear-replayssm-spec"],
["--expert-parallel-size", "1"],
+ ["--kv-cache-dtype", "bfloat16"],
["--linear-attn-decode-backend", "flashinfer"],
["--linear-attn-prefill-backend", "flashinfer"],
+ ["--linear-attn-verify-backend", "triton"],
["--mamba-ssm-dtype", "bfloat16"],
- ["--max-running-requests", "2"],
- ["--mem-fraction-static", "0.8"],
+ ["--max-prefill-tokens", "16384"],
+ ["--max-running-requests", "96"],
+ ["--mem-fraction-static", "0.88"],
+ ["--moe-a2a-backend", "none"],
+ ["--moe-runner-backend", "flashinfer_trtllm"],
+ ["--page-size", "64"],
+ ["--pipeline-parallel-size", "1"],
+ ["--ple-offload-backend", "pinned"],
+ ["--ple-offload-embedding"],
+ ["--quantization", "modelopt_mixed"],
+ ["--random-seed", "35"],
["--reasoning-parser", "auto"],
- ["--scheduler-recv-interval", "10"],
+ ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
+ ["--scheduler-recv-interval", "1"],
["--speculative-algorithm", "NEXTN"],
+ ["--speculative-draft-kv-cache-dtype", "bfloat16"],
+ ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"],
["--speculative-eagle-topk", "1"],
+ ["--speculative-moe-runner-backend", "flashinfer_trtllm"],
["--speculative-num-draft-tokens", "4"],
["--speculative-num-steps", "3"],
- ["--stream-interval", "50"],
+ ["--stream-interval", "20"],
["--tensor-parallel-size", "1"],
- ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
- ["--trust-remote-code"]
- ],
- "envRef": 0
+ ["--tokenizer-worker-num", "6"],
+ ["--trust-remote-code"],
+ ["--watchdog-timeout", "1800"]
+ ]
+ }
+ ]
+ },
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ },
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"]
+ ]
+ },
+ {
+ "from": 0,
+ "cmd": "python3 -m dynamo.sglang",
+ "diff": {
+ "args": {
+ "--request-plane": ["tcp"]
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": "tcp",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0",
+ "hw": "b300",
+ "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4",
+ "label": "Agg TP4·EP4",
+ "detail": "MTP 3-1-4",
+ "gpus": 4,
+ "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
+ "imageStatus": "ok",
+ "imageCommit": "586355a8",
+ "submitted": "sglang",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-io-backend", "kernel"],
+ ["--hicache-mem-layout", "page_first"],
+ ["--hicache-size", "298"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
+ "donorHardware": "b300"
+ }
+ },
+ "prov": {
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
+ "run": "41117066222",
+ "pr": "4683",
+ "comment": "6536482181",
+ "status": "verified-pr"
+ },
+ "points": [
+ {
+ "concs": [1],
+ "item": 0,
+ "override": "override_tp4_c1",
+ "goldenAL": "2.32",
+ "kv": "none"
+ },
+ {
+ "concs": [2],
+ "item": 0,
+ "override": "override_tp4_c2",
+ "goldenAL": "2.32",
+ "kv": "none",
+ "patch": {
+ "sglang": [
+ {
+ "args": {
+ "--max-running-requests": ["4"]
+ }
+ }
+ ],
+ "dynamo": [
+ null,
+ null,
+ {
+ "args": {
+ "--max-running-requests": ["4"]
+ }
+ }
+ ]
+ }
+ }
+ ],
+ "routers": {
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "diff": {
+ "args": {
+ "--expert-parallel-size": ["4"],
+ "--flashinfer-allreduce-fusion-backend": ["trtllm"],
+ "--max-running-requests": ["2"],
+ "--tensor-parallel-size": ["4"]
+ },
+ "env": {
+ "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null,
+ "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null,
+ "SGLANG_Q35MF_BK_VEC_COPY": null,
+ "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null,
+ "SGLANG_Q35MF_EDGE_SHM_SYNC": null,
+ "SGLANG_Q35MF_MKH_N3B": null,
+ "SGLANG_Q35MF_MKH_N3C": null
+ }
+ },
+ "ref": [1, "sglang", 0]
}
]
},
@@ -715,6 +1046,14 @@ export const agentx = {
"workers": 1,
"nodes_per_worker": 1,
"cmd": "python3 -m sglang.launch_server",
+ "env": {
+ "NCCL_CUMEM_ENABLE": "1",
+ "NCCL_NVLS_ENABLE": "1",
+ "SGLANG_ENABLE_FLASHINFER_GEMM": "true",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
+ "SGL_ENABLE_JIT_DEEPGEMM": "false",
+ "TORCH_CUDA_ARCH_LIST": "10.0"
+ },
"args": [
["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
@@ -739,8 +1078,7 @@ export const agentx = {
["--tokenizer-worker-num", "6"],
["--tp", "1"],
["--trust-remote-code"]
- ],
- "envRef": 0
+ ]
}
]
},
@@ -1103,7 +1441,7 @@ export const agentx = {
"--tensor-parallel-size": ["4"]
}
},
- "ref": [3, "sglang", 0]
+ "ref": [4, "sglang", 0]
}
]
},
diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx
index d4dbe71f56b3..cb5db60af19d 100644
--- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx
+++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "DeepSeek-V4.1",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
+ },
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
+ },
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
@@ -67,6 +82,19 @@ export const agentx = {
}
],
"envs": [
+ {
+ "DYN_REQUEST_PLANE": "tcp",
+ "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600",
+ "DYN_ROUTER_QUEUE_THRESHOLD": "None",
+ "DYN_ROUTER_TEMPERATURE": "0",
+ "DYN_TCP_CHANNEL_BUFFER": "128",
+ "DYN_TCP_REQUEST_TIMEOUT": "60",
+ "DYN_TOKENIZER": "fastokens",
+ "DYN_TOKENIZER_CACHE": "1",
+ "DYN_TOKENIZER_CACHE_BYTES": "8000000000",
+ "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
+ "NATS_SERVER": "nats://$NATS_IP:4222"
+ },
{
"DYN_EVENT_PLANE": "zmq",
"DYN_REQUEST_PLANE": "tcp",
@@ -174,18 +202,20 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
"sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "40989481011/attempts/1",
"pr": "4595",
- "status": "in-progress"
+ "status": "verified",
+ "date": "2026-10-30"
},
"points": [
{
@@ -359,19 +389,6 @@ export const agentx = {
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600",
- "DYN_ROUTER_QUEUE_THRESHOLD": "None",
- "DYN_ROUTER_TEMPERATURE": "0",
- "DYN_TCP_CHANNEL_BUFFER": "128",
- "DYN_TCP_REQUEST_TIMEOUT": "60",
- "DYN_TOKENIZER": "fastokens",
- "DYN_TOKENIZER_CACHE": "1",
- "DYN_TOKENIZER_CACHE_BYTES": "8000000000",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
- "NATS_SERVER": "nats://$NATS_IP:4222"
- },
"args": [
["--http-port", "8000"],
["--router-mode", "kv"],
@@ -379,7 +396,8 @@ export const agentx = {
["--active-decode-blocks-threshold", "None"],
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"]
- ]
+ ],
+ "envRef": 0
},
{
"kind": "worker",
@@ -480,9 +498,9 @@ export const agentx = {
"detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark",
"gpus": 4,
"configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579",
"imageStatus": "ok",
- "imageCommit": "c10e0e17",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -494,18 +512,20 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml",
+ "run": "41261902147",
"pr": "4603",
- "status": "in-progress"
+ "comment": "6576133363",
+ "status": "verified-pr"
},
"points": [
{
@@ -715,16 +735,169 @@ export const agentx = {
}
},
{
- "id": "dsv41flash-fp4-gb300-sglang-agentic-dspark#0",
+ "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg#3",
"hw": "gb300",
"ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
"label": "Agg TP2·EP2",
- "detail": "DSpark 5",
+ "detail": "DSpark 5 · dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg",
"gpus": 2,
- "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark",
+ "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg",
"image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47",
"imageStatus": "ok",
"imageCommit": "c10e0e17",
+ "submitted": "dynamo",
+ "setup": [],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-io-backend", "direct"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "1.2"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
+ }
+ },
+ "prov": {
+ "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "41081085818/attempts/1",
+ "pr": "4679",
+ "status": "verified",
+ "date": "2026-10-30"
+ },
+ "points": [
+ {
+ "concs": [16],
+ "item": 3,
+ "override": "override_tp2_c16",
+ "goldenAL": "3.51",
+ "kv": "none"
+ }
+ ],
+ "routers": {
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "nats",
+ "count_nodes": 1,
+ "cmd": "nats-server -js -sd /tmp/nats",
+ "env": {},
+ "args": []
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
+ },
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-backend-prefill": ["disabled"],
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--expert-parallel-size": ["2"],
+ "--max-running-requests": ["32"],
+ "--mem-fraction-static": ["0.8"],
+ "--prefill-decode-interval": ["96"],
+ "--swa-prefix-tails": ["1024"],
+ "--tensor-parallel-size": ["2"],
+ "--context-length": null,
+ "--enable-nvidia-megakernel": null,
+ "--kv-cache-dtype": null,
+ "--moe-a2a-backend": null,
+ "--schedule-policy": null,
+ "--startup-weight-load-mode": null,
+ "--tokenizer-backend": null
+ },
+ "env": {
+ "NCCL_CUMEM_ENABLE": "1",
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_NUMA_BIND_V2": "0",
+ "NCCL_MNNVL_ENABLE": null,
+ "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null,
+ "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null,
+ "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null,
+ "SGLANG_SPEC_TP_SYNC": null
+ }
+ },
+ "ref": [0, "dynamo", 3]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--request-plane": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null,
+ "NATS_SERVER": null
+ }
+ }
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "dsv41flash-fp4-gb300-sglang-agentic-dspark#0",
+ "hw": "gb300",
+ "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
+ "label": "Agg TP2·EP2",
+ "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark",
+ "gpus": 2,
+ "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579",
+ "imageStatus": "ok",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -736,18 +909,20 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml",
+ "run": "41261902147",
"pr": "4603",
- "status": "in-progress"
+ "comment": "6576133363",
+ "status": "verified-pr"
},
"points": [
{
@@ -801,34 +976,18 @@ export const agentx = {
{
"diff": {
"args": {
- "--cuda-graph-backend-prefill": ["disabled"],
- "--cuda-graph-max-bs-decode": ["64"],
- "--expert-parallel-size": ["2"],
- "--max-running-requests": ["32"],
- "--mem-fraction-static": ["0.8"],
- "--prefill-decode-interval": ["96"],
- "--swa-prefix-tails": ["1024"],
- "--tensor-parallel-size": ["2"],
- "--tokenizer-worker-num": ["4"],
- "--context-length": null,
- "--enable-nvidia-megakernel": null,
- "--kv-cache-dtype": null,
- "--moe-a2a-backend": null,
- "--schedule-policy": null,
- "--startup-weight-load-mode": null
+ "--tokenizer-backend": ["fastokens"],
+ "--tokenizer-worker-num": ["4"]
},
"env": {
- "NCCL_CUMEM_ENABLE": "1",
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
- "SGLANG_NUMA_BIND_V2": "0",
- "NCCL_MNNVL_ENABLE": null,
- "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null,
- "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null,
- "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null,
- "SGLANG_SPEC_TP_SYNC": null
+ "SGLANG_DEFAULT_THINKING": "1",
+ "SGLANG_DSPARK_FOLDED_SAMPLING": "2",
+ "SGLANG_DSPARK_OPT_MARKOV_W2_BF16": "True",
+ "SGLANG_DSV41_REASONING_EFFORT": "high",
+ "SGLANG_TIMEOUT_KEEP_ALIVE": "900"
}
},
- "ref": [1, "sglang", 0]
+ "ref": [2, "sglang", 0]
}
]
},
@@ -885,9 +1044,9 @@ export const agentx = {
"detail": "DSpark 5",
"gpus": 2,
"configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark",
- "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579",
"imageStatus": "ok",
- "imageCommit": "c10e0e17",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -899,18 +1058,20 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml",
+ "run": "41261902147",
"pr": "4603",
- "status": "in-progress"
+ "comment": "6576133363",
+ "status": "verified-pr"
},
"points": [
{
@@ -1012,7 +1173,7 @@ export const agentx = {
"SGLANG_RAGGED_VERIFY_MODE": "static"
}
},
- "ref": [2, "sglang", 0]
+ "ref": [3, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -1082,9 +1243,9 @@ export const agentx = {
"id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#0",
"hw": "gb300",
"ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
- "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)",
+ "label": "Disagg 3P1D · prefill DEP2 · decode DEP16 (4 nodes)",
"detail": "DSpark 5",
- "gpus": 26,
+ "gpus": 22,
"configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
"image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
"imageStatus": "ok",
@@ -1094,7 +1255,7 @@ export const agentx = {
{
"kind": "note",
"script": "Placement",
- "reason": "Workers share nodes: 5 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ "reason": "Workers share nodes: 3 prefill workers on 2 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
}
],
"disclosures": ["synthetic-acceptance"],
@@ -1106,24 +1267,26 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
+ "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml",
- "pr": "4595",
- "status": "in-progress"
+ "run": "41081085818/attempts/1",
+ "pr": "4679",
+ "status": "verified",
+ "date": "2026-10-30"
},
"points": [
{
- "concs": [2884],
+ "concs": [1228],
"item": 0,
- "override": "override_5p1d_dep2_dep16_c2884",
+ "override": "override_3p1d_dep2_dep16_c1228",
"goldenAL": "3.51",
"kv": "hicache"
}
@@ -1156,21 +1319,19 @@ export const agentx = {
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"]
],
- "envRef": 0
+ "envRef": 1
},
{
"kind": "worker",
"role": "prefill",
- "workers": 5,
+ "workers": 3,
"nodes_per_worker": 1,
"gpu_pinning": [
- ["0,1"],
- ["2,3"],
["0,1"],
["2,3"],
["0,1"]
],
- "local_ranks": [0, 1, 0, 1, 0],
+ "local_ranks": [0, 1, 0],
"preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1); 0 for a worker alone on its node.", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""],
"cmd": "python3 -m dynamo.sglang",
"env": {
@@ -1285,9 +1446,9 @@ export const agentx = {
["--nnodes", "4"],
["--node-rank", "$NODE_RANK"],
["--request-plane", "tcp"],
- ["--cuda-graph-max-bs-decode", "40"],
+ ["--cuda-graph-max-bs-decode", "24"],
["--data-parallel-size", "16"],
- ["--disaggregation-decode-extra-slots", "80"],
+ ["--disaggregation-decode-extra-slots", "36"],
["--disaggregation-transfer-backend", "mooncake"],
["--dsv4-attn-backend", "flashmla_mega"],
["--enable-decoder-swa-bounded-replay"],
@@ -1301,7 +1462,7 @@ export const agentx = {
["--expert-distribution-recorder-buffer-size", "1000"],
["--expert-parallel-size", "16"],
["--load-balance-method", "total_tokens"],
- ["--max-running-requests", "640"],
+ ["--max-running-requests", "384"],
["--mem-fraction-static", "0.75"],
["--model-loader-extra-config", "{\"enable_multithread_load\":true}"],
["--moe-a2a-backend", "megamoe"],
@@ -1322,6 +1483,177 @@ export const agentx = {
}
]
},
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 2,
+ "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1); 0 for a worker alone on its node.", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"],
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--router-mode": null,
+ "--router-session-affinity-ttl-secs": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--skip-server-warmup": [],
+ "--request-plane": null,
+ "--router-mode": null
+ },
+ "env": {
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null,
+ "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null,
+ "DYN_EVENT_PLANE": null,
+ "DYN_REQUEST_PLANE": null,
+ "DYN_SGL_GATEWAY_WORKERS": null,
+ "ETCD_ENDPOINTS": null
+ }
+ }
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--decode", "http://$DECODE_0_HEAD_IP:30000"],
+ ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"],
+ ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"],
+ ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"],
+ ["--pd-disaggregation"],
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"]
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#1",
+ "hw": "gb300",
+ "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
+ "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)",
+ "detail": "DSpark 5",
+ "gpus": 26,
+ "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90",
+ "imageStatus": "ok",
+ "imageCommit": "586355a8",
+ "submitted": "dynamo",
+ "setup": [
+ {
+ "kind": "note",
+ "script": "Placement",
+ "reason": "Workers share nodes: 5 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity."
+ }
+ ],
+ "disclosures": ["synthetic-acceptance"],
+ "kv": {
+ "available": ["none", "hicache", "mooncake"],
+ "unavailable": {},
+ "hicache": {
+ "args": [
+ ["--enable-hierarchical-cache"],
+ ["--hicache-io-backend", "direct"],
+ ["--hicache-mem-layout", "page_first_direct"],
+ ["--hicache-ratio", "1.2"],
+ ["--hicache-write-policy", "write_back"]
+ ],
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
+ }
+ },
+ "prov": {
+ "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml",
+ "run": "40989481011/attempts/1",
+ "pr": "4595",
+ "status": "verified",
+ "date": "2026-10-30"
+ },
+ "points": [
+ {
+ "concs": [2884],
+ "item": 1,
+ "override": "override_5p1d_dep2_dep16_c2884",
+ "goldenAL": "3.51",
+ "kv": "hicache"
+ }
+ ],
+ "routers": {
+ "dynamo": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "kind": "etcd",
+ "count_nodes": 1,
+ "cmd": "etcd",
+ "env": {},
+ "args": [
+ ["--data-dir", "/tmp/etcd"],
+ ["--listen-client-urls", "http://0.0.0.0:2379"],
+ ["--advertise-client-urls", "http://$ETCD_IP:2379"]
+ ]
+ },
+ {
+ "kind": "dynamo-frontend",
+ "count": 1,
+ "cmd": "python3 -m dynamo.frontend",
+ "args": [
+ ["--http-port", "8000"],
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "14400"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 1
+ },
+ {
+ "workers": 5,
+ "gpu_pinning": [
+ ["0,1"],
+ ["2,3"],
+ ["0,1"],
+ ["2,3"],
+ ["0,1"]
+ ],
+ "local_ranks": [0, 1, 0, 1, 0],
+ "ref": [5, "dynamo", 2]
+ },
+ {
+ "diff": {
+ "args": {
+ "--cuda-graph-max-bs-decode": ["40"],
+ "--disaggregation-decode-extra-slots": ["80"],
+ "--max-running-requests": ["640"]
+ }
+ },
+ "ref": [5, "dynamo", 3]
+ }
+ ]
+ },
"sglang": {
"install": [],
"files": [],
@@ -1390,7 +1722,7 @@ export const agentx = {
}
},
{
- "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#1",
+ "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2",
"hw": "gb300",
"ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
"label": "Disagg 8P1D · prefill DEP2 · decode DEP16 (4 nodes)",
@@ -1417,23 +1749,25 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
"sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml",
+ "run": "40989481011/attempts/1",
"pr": "4595",
- "status": "in-progress"
+ "status": "verified",
+ "date": "2026-10-30"
},
"points": [
{
"concs": [4096],
- "item": 1,
+ "item": 2,
"override": "override_8p1d_dep2_dep16_c4096",
"goldenAL": "3.51",
"kv": "hicache"
@@ -1467,7 +1801,7 @@ export const agentx = {
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"]
],
- "envRef": 0
+ "envRef": 1
},
{
"workers": 8,
@@ -1483,7 +1817,7 @@ export const agentx = {
],
"local_ranks": [0, 1, 0, 1, 0, 1, 0, 1],
"preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""],
- "ref": [4, "dynamo", 2]
+ "ref": [5, "dynamo", 2]
},
{
"diff": {
@@ -1493,7 +1827,7 @@ export const agentx = {
"--max-running-requests": ["1024"]
}
},
- "ref": [4, "dynamo", 3]
+ "ref": [5, "dynamo", 3]
}
]
},
@@ -1568,7 +1902,7 @@ export const agentx = {
}
},
{
- "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2",
+ "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#3",
"hw": "gb300",
"ckpt": "deepseek-ai/DeepSeek-V4.1-Flash",
"label": "Disagg 9P1D · prefill DEP2 · decode DEP16 (4 nodes)",
@@ -1595,23 +1929,25 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
- ["--hicache-ratio", "2"],
+ ["--hicache-ratio", "1.2"],
["--hicache-write-policy", "write_back"]
],
- "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark",
- "donorHardware": "b200"
+ "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache",
+ "donorHardware": "gb300"
}
},
"prov": {
"sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml",
+ "run": "40989481011/attempts/1",
"pr": "4595",
- "status": "in-progress"
+ "status": "verified",
+ "date": "2026-10-30"
},
"points": [
{
"concs": [7480],
- "item": 2,
+ "item": 3,
"override": "override_9p1d_dep2_dep16_b160_c7480",
"goldenAL": "3.51",
"kv": "hicache"
@@ -1645,7 +1981,7 @@ export const agentx = {
["--active-prefill-tokens-threshold", "None"],
["--active-prefill-tokens-threshold-frac", "None"]
],
- "envRef": 0
+ "envRef": 1
},
{
"workers": 9,
@@ -1661,7 +1997,7 @@ export const agentx = {
["0,1"]
],
"local_ranks": [0, 1, 0, 1, 0, 1, 0, 1, 0],
- "ref": [4, "dynamo", 2]
+ "ref": [5, "dynamo", 2]
},
{
"diff": {
@@ -1671,7 +2007,7 @@ export const agentx = {
"--max-running-requests": ["2560"]
}
},
- "ref": [4, "dynamo", 3]
+ "ref": [5, "dynamo", 3]
}
]
},
@@ -2056,7 +2392,7 @@ export const agentx = {
"SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null
}
},
- "ref": [7, "sglang", 0]
+ "ref": [9, "sglang", 0]
}
]
},
@@ -2292,7 +2628,7 @@ export const agentx = {
"SGLANG_RAGGED_VERIFY_MODE": "static"
}
},
- "ref": [8, "sglang", 0]
+ "ref": [10, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -2650,7 +2986,7 @@ export const agentx = {
"--schedule-policy": null
}
},
- "ref": [10, "sglang", 0]
+ "ref": [12, "sglang", 0]
}
]
},
@@ -2833,7 +3169,7 @@ export const agentx = {
"--dsv4-attn-backend": null
}
},
- "ref": [11, "sglang", 0]
+ "ref": [13, "sglang", 0]
}
]
},
@@ -2998,7 +3334,7 @@ export const agentx = {
"SGLANG_RAGGED_VERIFY_MODE": "static"
}
},
- "ref": [10, "sglang", 0]
+ "ref": [12, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -3403,7 +3739,7 @@ export const agentx = {
"SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null
}
},
- "ref": [14, "sglang", 0]
+ "ref": [16, "sglang", 0]
}
]
},
@@ -3663,7 +3999,7 @@ export const agentx = {
"SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0"
}
},
- "ref": [15, "sglang", 0]
+ "ref": [17, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -4104,7 +4440,7 @@ export const agentx = {
"--tensor-parallel-size": ["8"]
}
},
- "ref": [17, "sglang", 0]
+ "ref": [19, "sglang", 0]
}
]
},
@@ -4366,7 +4702,7 @@ export const agentx = {
["--trust-remote-code"],
["--watchdog-timeout", "3600"]
],
- "envRef": 1
+ "envRef": 2
}
]
},
@@ -4554,7 +4890,7 @@ export const agentx = {
"--swa-prefix-tails": ["64"]
}
},
- "ref": [19, "sglang", 0]
+ "ref": [21, "sglang", 0]
},
{
"kind": "sglang-router",
@@ -4571,7 +4907,7 @@ export const agentx = {
["--disable-retries"],
["--worker-urls", "http://127.0.0.1:30000"]
],
- "envRef": 1
+ "envRef": 2
}
]
},
diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx
index 9877864d5891..9c6a9d43321e 100644
--- a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx
+++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "Muse Spark Open",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
+ },
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
+ },
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
@@ -146,9 +161,9 @@ export const agentx = {
"detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash",
"gpus": 4,
"configKey": "musespark-fp4-gb300-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc",
"imageStatus": "ok",
- "imageCommit": "c1c287a4",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -158,7 +173,7 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "160"],
+ ["--hicache-size", "150"],
["--hicache-write-policy", "write_back"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
@@ -166,12 +181,12 @@ export const agentx = {
}
},
"prov": {
- "sha": "316905a060115a9a393f6a464d3646b830a3c74e",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml",
- "run": "39794121900/attempts/1",
- "pr": "4404",
- "status": "verified",
- "date": "2026-10-23"
+ "run": "41327192200",
+ "pr": "4724",
+ "comment": "6576178636",
+ "status": "verified-pr"
},
"points": [
{
@@ -182,19 +197,16 @@ export const agentx = {
"kv": "none"
},
{
- "concs": [5],
+ "concs": [4],
"item": 0,
- "override": "override_tp4_c5",
+ "override": "override_tp4_c4",
"goldenAL": "4.05",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--speculative-draft-attention-backend": ["fa4"]
+ "--schedule-policy": ["shortest-prefill-first"]
}
}
],
@@ -203,10 +215,7 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--speculative-draft-attention-backend": ["fa4"]
+ "--schedule-policy": ["shortest-prefill-first"]
}
}
]
@@ -222,9 +231,12 @@ export const agentx = {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--prefill-decode-interval": ["64"]
+ "--cuda-graph-max-bs-decode": ["2"],
+ "--max-running-requests": ["2"],
+ "--prefill-decode-interval": ["128"],
+ "--prefill-decode-interval-min-bs": ["1"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--speculative-draft-attention-backend": ["fa4"]
}
}
],
@@ -233,9 +245,12 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--prefill-decode-interval": ["64"]
+ "--cuda-graph-max-bs-decode": ["2"],
+ "--max-running-requests": ["2"],
+ "--prefill-decode-interval": ["128"],
+ "--prefill-decode-interval-min-bs": ["1"],
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--speculative-draft-attention-backend": ["fa4"]
}
}
]
@@ -255,6 +270,7 @@ export const agentx = {
"--incremental-streaming-output": [],
"--max-running-requests": ["3"],
"--prefill-decode-interval": ["64"],
+ "--schedule-policy": ["shortest-prefill-first"],
"--speculative-draft-attention-backend": ["fa4"],
"--tokenizer-worker-num": ["4"]
}
@@ -269,6 +285,7 @@ export const agentx = {
"--incremental-streaming-output": [],
"--max-running-requests": ["3"],
"--prefill-decode-interval": ["64"],
+ "--schedule-policy": ["shortest-prefill-first"],
"--speculative-draft-attention-backend": ["fa4"],
"--tokenizer-worker-num": ["4"]
}
@@ -330,6 +347,7 @@ export const agentx = {
"--max-running-requests": ["48"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["96"],
+ "--prefill-decode-interval-min-bs": ["2"],
"--schedule-policy": ["shortest-prefill-first"],
"--swa-full-tokens-ratio": ["0.1"]
},
@@ -352,57 +370,7 @@ export const agentx = {
"--max-running-requests": ["48"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["96"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048"
- }
- }
- ]
- }
- },
- {
- "concs": [28],
- "item": 1,
- "override": "override_tp4_c28",
- "goldenAL": "4.05",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-backend-prefill": ["breakable"],
- "--cuda-graph-max-bs-decode": ["48"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
- "--hicache-write-policy": ["write_back"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["48"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["32"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048"
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-backend-prefill": ["breakable"],
- "--cuda-graph-max-bs-decode": ["48"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
- "--hicache-write-policy": ["write_back"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["48"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["32"],
+ "--prefill-decode-interval-min-bs": ["2"],
"--schedule-policy": ["shortest-prefill-first"],
"--swa-full-tokens-ratio": ["0.1"]
},
@@ -426,12 +394,13 @@ export const agentx = {
"--cuda-graph-backend-prefill": ["breakable"],
"--cuda-graph-max-bs-decode": ["48"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["8192"],
"--max-running-requests": ["48"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval-min-bs": ["3"],
"--schedule-policy": ["shortest-prefill-first"],
"--swa-full-tokens-ratio": ["0.1"]
},
@@ -448,12 +417,13 @@ export const agentx = {
"--cuda-graph-backend-prefill": ["breakable"],
"--cuda-graph-max-bs-decode": ["48"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["8192"],
"--max-running-requests": ["48"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval-min-bs": ["3"],
"--schedule-policy": ["shortest-prefill-first"],
"--swa-full-tokens-ratio": ["0.1"]
},
@@ -465,9 +435,9 @@ export const agentx = {
}
},
{
- "concs": [34],
+ "concs": [40],
"item": 1,
- "override": "override_tp4_c34",
+ "override": "override_tp4_c40",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
@@ -476,16 +446,15 @@ export const agentx = {
"args": {
"--chunked-prefill-size": ["16384"],
"--cuda-graph-backend-prefill": ["breakable"],
- "--cuda-graph-max-bs-decode": ["96"],
+ "--cuda-graph-max-bs-decode": ["18"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["96"],
+ "--max-running-requests": ["18"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
+ "--schedule-policy": ["shortest-prefill-first"]
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -500,16 +469,15 @@ export const agentx = {
"args": {
"--chunked-prefill-size": ["16384"],
"--cuda-graph-backend-prefill": ["breakable"],
- "--cuda-graph-max-bs-decode": ["96"],
+ "--cuda-graph-max-bs-decode": ["18"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["160"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["96"],
+ "--max-running-requests": ["18"],
"--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
+ "--schedule-policy": ["shortest-prefill-first"]
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -529,20 +497,20 @@ export const agentx = {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["24"],
+ "--chunked-prefill-size": ["16384"],
+ "--cuda-graph-backend-prefill": ["breakable"],
+ "--cuda-graph-max-bs-decode": ["20"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["149"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
- "--incremental-streaming-output": [],
- "--max-running-requests": ["24"],
- "--mem-fraction-static": ["0.9"],
- "--speculative-draft-attention-backend": ["trtllm_mha"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
- "--swa-full-tokens-ratio": ["0.1"],
- "--tokenizer-worker-num": ["4"],
- "--chunked-prefill-size": null
+ "--max-prefill-tokens": ["16384"],
+ "--max-running-requests": ["20"],
+ "--mem-fraction-static": ["0.95"],
+ "--prefill-decode-interval": ["48"],
+ "--schedule-policy": ["shortest-prefill-first"]
},
"env": {
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
}
}
@@ -552,20 +520,20 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["24"],
+ "--chunked-prefill-size": ["16384"],
+ "--cuda-graph-backend-prefill": ["breakable"],
+ "--cuda-graph-max-bs-decode": ["20"],
"--enable-hierarchical-cache": [],
- "--hicache-size": ["149"],
+ "--hicache-size": ["150"],
"--hicache-write-policy": ["write_back"],
- "--incremental-streaming-output": [],
- "--max-running-requests": ["24"],
- "--mem-fraction-static": ["0.9"],
- "--speculative-draft-attention-backend": ["trtllm_mha"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
- "--swa-full-tokens-ratio": ["0.1"],
- "--tokenizer-worker-num": ["4"],
- "--chunked-prefill-size": null
+ "--max-prefill-tokens": ["16384"],
+ "--max-running-requests": ["20"],
+ "--mem-fraction-static": ["0.95"],
+ "--prefill-decode-interval": ["48"],
+ "--schedule-policy": ["shortest-prefill-first"]
},
"env": {
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
}
}
@@ -723,7 +691,7 @@ export const agentx = {
"hw": "gb300",
"ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4",
"label": "Agg TP8·EP8 (2 nodes)",
- "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
+ "detail": "DFlash 8",
"gpus": 8,
"configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
"image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a",
@@ -738,7 +706,7 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "160"],
+ ["--hicache-size", "150"],
["--hicache-write-policy", "write_back"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
@@ -963,17 +931,17 @@ export const agentx = {
}
},
{
- "id": "musespark-fp4-gb300-sglang-agentic-dflash-multinode#0",
+ "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#0",
"hw": "gb300",
"ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4",
- "label": "Agg TP8·EP8 (2 nodes)",
- "detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash-multinode",
- "gpus": 8,
- "configKey": "musespark-fp4-gb300-sglang-agentic-dflash-multinode",
- "image": "lmsysorg/sglang:nightly-dev-20261019-3c560089@sha256:c1f4d9fc6fcf50c2269adbc757ce7b1778883af04155edb888fa9eea3b6648c0",
+ "label": "Agg TP4·EP4",
+ "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
+ "gpus": 4,
+ "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
+ "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a",
"imageStatus": "ok",
- "imageCommit": "3c560089",
- "submitted": "sglang",
+ "imageCommit": null,
+ "submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
@@ -982,7 +950,7 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "160"],
+ ["--hicache-size", "150"],
["--hicache-write-policy", "write_back"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
@@ -990,315 +958,105 @@ export const agentx = {
}
},
"prov": {
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-tp8-direct-variants.yaml",
- "pr": "4454",
- "status": "in-progress"
+ "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
+ "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml",
+ "run": "39427144732/attempts/4",
+ "pr": "4344",
+ "status": "verified",
+ "date": "2026-10-22"
},
"points": [
{
- "concs": [5],
- "item": 0,
- "override": "override_tp8_c5",
+ "concs": [8],
+ "item": 1,
+ "override": "override_tp4_c8",
"goldenAL": "4.05",
"kv": "none"
},
{
- "concs": [10],
- "item": 1,
- "override": "override_tp8_c10",
+ "concs": [16],
+ "item": 0,
+ "override": "override_tp4_c16",
"goldenAL": "4.05",
"kv": "none",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
+ null,
{
"args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["2"],
+ "--cuda-graph-max-bs-decode": ["12"],
+ "--max-running-requests": ["12"],
+ "--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["128"],
- "--page-size": null
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--swa-full-tokens-ratio": ["0.1"]
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["2"],
+ "--cuda-graph-max-bs-decode": ["12"],
+ "--max-running-requests": ["12"],
+ "--mem-fraction-static": ["0.95"],
"--prefill-decode-interval": ["128"],
- "--page-size": null
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--swa-full-tokens-ratio": ["0.1"]
}
- }
+ },
+ null
]
}
},
{
- "concs": [18],
- "item": 2,
- "override": "override_tp8_c18",
+ "concs": [20],
+ "item": 8,
+ "override": "override_tp4_c20",
"goldenAL": "4.05",
- "kv": "none",
+ "kv": "hicache",
"patch": {
- "sglang": [
+ "dynamo": [
+ null,
+ null,
+ null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--prefill-decode-interval": ["32"]
+ "--cuda-graph-max-bs-decode": ["10"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["120"],
+ "--hicache-write-policy": ["write_through"],
+ "--max-running-requests": ["10"],
+ "--mem-fraction-static": ["0.95"],
+ "--prefill-decode-interval": ["48"],
+ "--schedule-policy": ["shortest-prefill-first"]
}
}
],
- "dynamo": [
- null,
- null,
+ "sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"],
- "--prefill-decode-interval": ["32"]
+ "--cuda-graph-max-bs-decode": ["10"],
+ "--enable-hierarchical-cache": [],
+ "--hicache-size": ["120"],
+ "--hicache-write-policy": ["write_through"],
+ "--max-running-requests": ["10"],
+ "--mem-fraction-static": ["0.95"],
+ "--prefill-decode-interval": ["48"],
+ "--schedule-policy": ["shortest-prefill-first"]
}
- }
+ },
+ null
]
}
},
{
"concs": [32],
- "item": 3,
- "override": "override_tp8_c32",
+ "item": 7,
+ "override": "override_tp4_c32",
"goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["6"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["6"],
- "--prefill-decode-interval": ["24"],
- "--schedule-policy": ["shortest-prefill-first"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["6"],
- "--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["6"],
- "--prefill-decode-interval": ["24"],
- "--schedule-policy": ["shortest-prefill-first"]
- }
- }
- ]
- }
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--speculative-draft-attention-backend": ["fa4"],
- "--swa-full-tokens-ratio": ["0.1"],
- "--tokenizer-backend": ["fastokens"],
- "--tool-call-parser": ["muse"],
- "--model-loader-extra-config": null,
- "--preferred-sampling-params": null,
- "--prefill-decode-interval": null
- },
- "env": {
- "SGLANG_PARALLEL_PROMPT_ENCODE": "0"
- }
- },
- "ref": [1, "sglang", 0]
- }
- ]
- },
- "dynamo": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "kind": "etcd",
- "count_nodes": 1,
- "cmd": "etcd",
- "env": {},
- "args": [
- ["--data-dir", "/tmp/etcd"],
- ["--listen-client-urls", "http://0.0.0.0:2379"],
- ["--advertise-client-urls", "http://$ETCD_IP:2379"]
- ]
- },
- {
- "kind": "dynamo-frontend",
- "count": 1,
- "cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
- "args": [
- ["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
- },
- {
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
- "diff": {
- "args": {
- "--request-plane": ["tcp"]
- },
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- }
- }
- }
- ]
- }
- }
- },
- {
- "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#0",
- "hw": "gb300",
- "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4",
- "label": "Agg TP4·EP4",
- "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
- "gpus": 4,
- "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a",
- "imageStatus": "ok",
- "imageCommit": null,
- "submitted": "dynamo",
- "setup": [],
- "disclosures": ["synthetic-acceptance"],
- "kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
- "hicache": {
- "args": [
- ["--enable-hierarchical-cache"],
- ["--hicache-size", "160"],
- ["--hicache-write-policy", "write_back"]
- ],
- "donor": "musespark-fp4-gb300-sglang-agentic-dflash",
- "donorHardware": "gb300"
- }
- },
- "prov": {
- "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
- "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml",
- "run": "39427144732/attempts/4",
- "pr": "4344",
- "status": "verified",
- "date": "2026-10-22"
- },
- "points": [
- {
- "concs": [8],
- "item": 1,
- "override": "override_tp4_c8",
- "goldenAL": "4.05",
- "kv": "none"
- },
- {
- "concs": [16],
- "item": 0,
- "override": "override_tp4_c16",
- "goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "dynamo": [
- null,
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["12"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["128"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
- }
- }
- ],
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["12"],
- "--max-running-requests": ["12"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["128"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.1"]
- }
- },
- null
- ]
- }
- },
- {
- "concs": [20],
- "item": 8,
- "override": "override_tp4_c20",
- "goldenAL": "4.05",
- "kv": "hicache",
- "patch": {
- "dynamo": [
- null,
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["10"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["120"],
- "--hicache-write-policy": ["write_through"],
- "--max-running-requests": ["10"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"]
- }
- }
- ],
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["10"],
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["120"],
- "--hicache-write-policy": ["write_through"],
- "--max-running-requests": ["10"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"]
- }
- },
- null
- ]
- }
- },
- {
- "concs": [32],
- "item": 7,
- "override": "override_tp4_c32",
- "goldenAL": "4.05",
- "kv": "hicache",
+ "kv": "hicache",
"patch": {
"dynamo": [
null,
@@ -1375,241 +1133,23 @@ export const agentx = {
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["24"],
"--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["32"],
- "--schedule-policy": ["shortest-prefill-first"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
- }
- },
- null
- ]
- }
- }
- ],
- "routers": {
- "dynamo": {
- "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""],
- "files": ["nats.conf#13771a85"],
- "blocks": [
- {
- "kind": "etcd",
- "count_nodes": 1,
- "cmd": "etcd",
- "env": {},
- "args": [
- ["--data-dir", "/tmp/etcd"],
- ["--listen-client-urls", "http://0.0.0.0:2379"],
- ["--advertise-client-urls", "http://$ETCD_IP:2379"]
- ]
- },
- {
- "kind": "nats",
- "count_nodes": 1,
- "cmd": "nats-server -c nats.conf",
- "env": {},
- "args": []
- },
- {
- "kind": "dynamo-frontend",
- "count": 1,
- "cmd": "python3 -m dynamo.frontend",
- "args": [
- ["--http-port", "8000"],
- ["--router-mode", "kv"],
- ["--router-session-affinity-ttl-secs", "3600"],
- ["--active-decode-blocks-threshold", "None"],
- ["--active-prefill-tokens-threshold", "None"],
- ["--active-prefill-tokens-threshold-frac", "None"]
- ],
- "envRef": 0
- },
- {
- "nodes_per_worker": 1,
- "diff": {
- "args": {
- "--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["8"],
- "--expert-parallel-size": ["4"],
- "--max-running-requests": ["8"],
- "--tensor-parallel-size": ["4"],
- "--dist-init-addr": null,
- "--nnodes": null,
- "--node-rank": null,
- "--page-size": null
- },
- "env": {
- "NCCL_MNNVL_ENABLE": null,
- "PYTORCH_CUDA_ALLOC_CONF": null
- }
- },
- "ref": [1, "dynamo", 3]
- }
- ]
- },
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "from": 3,
- "cmd": "python3 -m sglang.launch_server",
- "diff": {
- "args": {
- "--request-plane": null,
- "--kv-events-config": null,
- "--dyn-reasoning-parser": null,
- "--dyn-tool-call-parser": null
- },
- "env": {
- "DYN_REQUEST_PLANE": null,
- "ETCD_ENDPOINTS": null,
- "NATS_SERVER": null
- }
- }
- },
- {
- "kind": "sglang-router",
- "cmd": "python3 -m sglang_router.launch_router",
- "env": {},
- "args": [
- ["--host", "0.0.0.0"],
- ["--port", "8000"],
- ["--policy", "cache_aware"],
- ["--worker-urls", "http://127.0.0.1:30000"]
- ]
- }
- ]
- }
- }
- },
- {
- "id": "musespark-fp4-gb300-sglang-agentic-dflash-nightly1019#0",
- "hw": "gb300",
- "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4",
- "label": "Agg TP4·EP4",
- "detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash-nightly1019",
- "gpus": 4,
- "configKey": "musespark-fp4-gb300-sglang-agentic-dflash-nightly1019",
- "image": "lmsysorg/sglang:nightly-dev-20261019-3c560089@sha256:c1f4d9fc6fcf50c2269adbc757ce7b1778883af04155edb888fa9eea3b6648c0",
- "imageStatus": "ok",
- "imageCommit": "3c560089",
- "submitted": "sglang",
- "setup": [],
- "disclosures": ["synthetic-acceptance"],
- "kv": {
- "available": ["none", "hicache", "mooncake"],
- "unavailable": {},
- "hicache": {
- "args": [
- ["--enable-hierarchical-cache"],
- ["--hicache-size", "160"],
- ["--hicache-write-policy", "write_back"]
- ],
- "donor": "musespark-fp4-gb300-sglang-agentic-dflash",
- "donorHardware": "gb300"
- }
- },
- "prov": {
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic-nightly1019.yaml",
- "pr": "4454",
- "status": "in-progress"
- },
- "points": [
- {
- "concs": [28],
- "item": 0,
- "override": "override_tp4_c28",
- "goldenAL": "4.05",
- "kv": "hicache"
- },
- {
- "concs": [52],
- "item": 1,
- "override": "override_tp4_c52",
- "goldenAL": "4.05",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--flashinfer-autotune-skip-ops": ["flashinfer::trtllm_fp4_block_scale_moe"],
- "--hicache-write-policy": ["write_back"],
- "--max-running-requests": ["32"],
- "--swa-full-tokens-ratio": ["0.15"],
- "--chunked-prefill-size": null,
- "--max-prefill-tokens": null,
- "--prefill-decode-interval": null,
- "--schedule-policy": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--flashinfer-autotune-skip-ops": ["flashinfer::trtllm_fp4_block_scale_moe"],
- "--hicache-write-policy": ["write_back"],
- "--max-running-requests": ["32"],
- "--swa-full-tokens-ratio": ["0.15"],
- "--chunked-prefill-size": null,
- "--max-prefill-tokens": null,
- "--prefill-decode-interval": null,
- "--schedule-policy": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
- }
- }
- ]
- }
- }
- ],
- "routers": {
- "sglang": {
- "install": [],
- "files": [],
- "blocks": [
- {
- "diff": {
- "args": {
- "--enable-hierarchical-cache": [],
- "--hicache-size": ["149"],
- "--hicache-write-policy": ["write_through"],
- "--max-prefill-tokens": ["8192"],
- "--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--speculative-draft-attention-backend": ["trtllm_mha"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
- "--tokenizer-backend": ["fastokens"],
- "--tool-call-parser": ["muse"],
- "--model-loader-extra-config": null,
- "--preferred-sampling-params": null,
- "--speculative-draft-model-revision": null
+ "--prefill-decode-interval": ["32"],
+ "--schedule-policy": ["shortest-prefill-first"]
},
"env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048",
- "SGLANG_PARALLEL_PROMPT_ENCODE": "0"
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
+ "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
}
},
- "ref": [3, "sglang", 0]
- }
- ]
- },
+ null
+ ]
+ }
+ }
+ ],
+ "routers": {
"dynamo": {
- "install": [],
- "files": [],
+ "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""],
+ "files": ["nats.conf#13771a85"],
"blocks": [
{
"kind": "etcd",
@@ -1622,31 +1162,81 @@ export const agentx = {
["--advertise-client-urls", "http://$ETCD_IP:2379"]
]
},
+ {
+ "kind": "nats",
+ "count_nodes": 1,
+ "cmd": "nats-server -c nats.conf",
+ "env": {},
+ "args": []
+ },
{
"kind": "dynamo-frontend",
"count": 1,
"cmd": "python3 -m dynamo.frontend",
- "env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
- },
"args": [
["--http-port", "8000"],
- ["--router-mode", "kv"]
- ]
+ ["--router-mode", "kv"],
+ ["--router-session-affinity-ttl-secs", "3600"],
+ ["--active-decode-blocks-threshold", "None"],
+ ["--active-prefill-tokens-threshold", "None"],
+ ["--active-prefill-tokens-threshold-frac", "None"]
+ ],
+ "envRef": 0
},
{
- "from": 0,
- "cmd": "python3 -m dynamo.sglang",
+ "nodes_per_worker": 1,
"diff": {
"args": {
- "--request-plane": ["tcp"]
+ "--chunked-prefill-size": ["8192"],
+ "--cuda-graph-max-bs-decode": ["8"],
+ "--expert-parallel-size": ["4"],
+ "--max-running-requests": ["8"],
+ "--tensor-parallel-size": ["4"],
+ "--dist-init-addr": null,
+ "--nnodes": null,
+ "--node-rank": null,
+ "--page-size": null
},
"env": {
- "DYN_REQUEST_PLANE": "tcp",
- "ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
+ "NCCL_MNNVL_ENABLE": null,
+ "PYTORCH_CUDA_ALLOC_CONF": null
+ }
+ },
+ "ref": [1, "dynamo", 3]
+ }
+ ]
+ },
+ "sglang": {
+ "install": [],
+ "files": [],
+ "blocks": [
+ {
+ "from": 3,
+ "cmd": "python3 -m sglang.launch_server",
+ "diff": {
+ "args": {
+ "--request-plane": null,
+ "--kv-events-config": null,
+ "--dyn-reasoning-parser": null,
+ "--dyn-tool-call-parser": null
+ },
+ "env": {
+ "DYN_REQUEST_PLANE": null,
+ "ETCD_ENDPOINTS": null,
+ "NATS_SERVER": null
}
}
+ },
+ {
+ "kind": "sglang-router",
+ "cmd": "python3 -m sglang_router.launch_router",
+ "env": {},
+ "args": [
+ ["--host", "0.0.0.0"],
+ ["--port", "8000"],
+ ["--policy", "cache_aware"],
+ ["--worker-urls", "http://127.0.0.1:30000"]
+ ]
}
]
}
@@ -1675,15 +1265,16 @@ export const agentx = {
["--hicache-size", "120"]
],
"donor": "musespark-fp4-gb200-sglang-agentic-dflash",
- "donorHardware": "gb200",
- "donorRef": "main"
+ "donorHardware": "gb200"
}
},
"prov": {
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml",
+ "run": "41096285400",
"pr": "4670",
- "status": "in-progress"
+ "comment": "6576162113",
+ "status": "verified-pr"
},
"points": [
{
@@ -2080,7 +1671,6 @@ export const agentx = {
"--mem-fraction-static": ["0.93"],
"--prefill-decode-interval": ["64"],
"--schedule-policy": ["shortest-prefill-first"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
"--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.1"],
"--cuda-graph-backend-prefill": null,
@@ -2106,7 +1696,6 @@ export const agentx = {
"--mem-fraction-static": ["0.93"],
"--prefill-decode-interval": ["64"],
"--schedule-policy": ["shortest-prefill-first"],
- "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"],
"--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.1"],
"--cuda-graph-backend-prefill": null,
@@ -2224,9 +1813,9 @@ export const agentx = {
"detail": "DFlash 8",
"gpus": 8,
"configKey": "musespark-fp4-b300-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc",
"imageStatus": "ok",
- "imageCommit": "c1c287a4",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -2236,19 +1825,19 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "170"]
+ ["--hicache-size", "250"]
],
"donor": "musespark-fp4-b300-sglang-agentic-dflash",
"donorHardware": "b300"
}
},
"prov": {
- "sha": "63c4ad272bdbad2150417e9215d0be3409f1e8cb",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "40285747600/attempts/4",
- "pr": "4417",
- "status": "verified",
- "date": "2026-10-28"
+ "run": "41328422543",
+ "pr": "4725",
+ "comment": "6576185043",
+ "status": "verified-pr"
},
"points": [
{
@@ -2268,43 +1857,8 @@ export const agentx = {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--stream-interval": ["20"],
- "--speculative-draft-attention-backend": null
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--stream-interval": ["20"],
- "--speculative-draft-attention-backend": null
- }
- }
- ]
- }
- },
- {
- "concs": [8],
- "item": 0,
- "override": "override_tp8_c8",
- "goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--speculative-draft-attention-backend": null
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"]
}
}
],
@@ -2313,19 +1867,17 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["2"],
- "--max-running-requests": ["2"],
- "--prefill-decode-interval": ["64"],
- "--speculative-draft-attention-backend": null
+ "--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"]
}
}
]
}
},
{
- "concs": [10],
+ "concs": [7],
"item": 0,
- "override": "override_tp8_c10",
+ "override": "override_tp8_c7",
"goldenAL": "4.05",
"kv": "none",
"patch": {
@@ -2336,6 +1888,7 @@ export const agentx = {
"--max-running-requests": ["2"],
"--prefill-decode-interval": ["64"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"],
"--speculative-draft-attention-backend": null
}
}
@@ -2349,6 +1902,7 @@ export const agentx = {
"--max-running-requests": ["2"],
"--prefill-decode-interval": ["64"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"],
"--speculative-draft-attention-backend": null
}
}
@@ -2356,9 +1910,9 @@ export const agentx = {
}
},
{
- "concs": [14],
+ "concs": [16],
"item": 0,
- "override": "override_tp8_c14",
+ "override": "override_tp8_c16",
"goldenAL": "4.05",
"kv": "none",
"patch": {
@@ -2366,9 +1920,11 @@ export const agentx = {
{
"args": {
"--cuda-graph-max-bs-decode": ["3"],
+ "--incremental-streaming-output": [],
"--max-running-requests": ["3"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["88"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"],
"--speculative-draft-attention-backend": null
}
}
@@ -2379,42 +1935,11 @@ export const agentx = {
{
"args": {
"--cuda-graph-max-bs-decode": ["3"],
+ "--incremental-streaming-output": [],
"--max-running-requests": ["3"],
- "--prefill-decode-interval": ["64"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--speculative-draft-attention-backend": null
- }
- }
- ]
- }
- },
- {
- "concs": [16],
- "item": 0,
- "override": "override_tp8_c16",
- "goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["4"],
- "--prefill-decode-interval": ["64"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--speculative-draft-attention-backend": null
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--max-running-requests": ["4"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["88"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["40"],
"--speculative-draft-attention-backend": null
}
}
@@ -2433,7 +1958,7 @@ export const agentx = {
"args": {
"--cuda-graph-max-bs-decode": ["6"],
"--max-running-requests": ["6"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["48"],
"--schedule-policy": ["shortest-prefill-first"],
"--stream-interval": ["20"],
"--speculative-draft-attention-backend": null
@@ -2447,7 +1972,7 @@ export const agentx = {
"args": {
"--cuda-graph-max-bs-decode": ["6"],
"--max-running-requests": ["6"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["48"],
"--schedule-policy": ["shortest-prefill-first"],
"--stream-interval": ["20"],
"--speculative-draft-attention-backend": null
@@ -2468,7 +1993,7 @@ export const agentx = {
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["80"],
"--schedule-policy": ["shortest-prefill-first"],
"--stream-interval": ["20"],
"--speculative-draft-attention-backend": null
@@ -2482,7 +2007,7 @@ export const agentx = {
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["80"],
"--schedule-policy": ["shortest-prefill-first"],
"--stream-interval": ["20"],
"--speculative-draft-attention-backend": null
@@ -2601,9 +2126,9 @@ export const agentx = {
"detail": "DFlash 8",
"gpus": 4,
"configKey": "musespark-fp4-b300-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc",
"imageStatus": "ok",
- "imageCommit": "c1c287a4",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -2613,19 +2138,19 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
- ["--hicache-size", "170"]
+ ["--hicache-size", "250"]
],
"donor": "musespark-fp4-b300-sglang-agentic-dflash",
"donorHardware": "b300"
}
},
"prov": {
- "sha": "63c4ad272bdbad2150417e9215d0be3409f1e8cb",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml",
- "run": "40285747600/attempts/4",
- "pr": "4417",
- "status": "verified",
- "date": "2026-10-28"
+ "run": "41328422543",
+ "pr": "4725",
+ "comment": "6576185043",
+ "status": "verified-pr"
},
"points": [
{
@@ -2650,8 +2175,8 @@ export const agentx = {
"--hicache-size": ["170"],
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["56"],
- "--prefill-decode-interval": ["128"],
- "--stream-interval": ["20"]
+ "--prefill-decode-interval": ["104"],
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
@@ -2668,8 +2193,8 @@ export const agentx = {
"--hicache-size": ["170"],
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["56"],
- "--prefill-decode-interval": ["128"],
- "--stream-interval": ["20"]
+ "--prefill-decode-interval": ["104"],
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
@@ -2690,54 +2215,11 @@ export const agentx = {
"args": {
"--chunked-prefill-size": ["16384"],
"--cuda-graph-max-bs-decode": ["64"],
- "--hicache-size": ["170"],
- "--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["64"],
- "--prefill-decode-interval": ["128"],
- "--stream-interval": ["20"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--chunked-prefill-size": ["16384"],
- "--cuda-graph-max-bs-decode": ["64"],
- "--hicache-size": ["170"],
+ "--hicache-size": ["250"],
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["64"],
- "--prefill-decode-interval": ["128"],
- "--stream-interval": ["20"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
- }
- }
- ]
- }
- },
- {
- "concs": [40],
- "item": 1,
- "override": "override_tp4_c40",
- "goldenAL": "4.05",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--chunked-prefill-size": ["16384"],
- "--cuda-graph-max-bs-decode": ["80"],
- "--hicache-size": ["170"],
- "--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["80"],
- "--stream-interval": ["20"],
- "--swa-full-tokens-ratio": ["0.2"]
+ "--swa-full-tokens-ratio": ["0.2"],
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -2751,12 +2233,12 @@ export const agentx = {
{
"args": {
"--chunked-prefill-size": ["16384"],
- "--cuda-graph-max-bs-decode": ["80"],
- "--hicache-size": ["170"],
+ "--cuda-graph-max-bs-decode": ["64"],
+ "--hicache-size": ["250"],
"--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["80"],
- "--stream-interval": ["20"],
- "--swa-full-tokens-ratio": ["0.2"]
+ "--max-running-requests": ["64"],
+ "--swa-full-tokens-ratio": ["0.2"],
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -2767,9 +2249,9 @@ export const agentx = {
}
},
{
- "concs": [44],
+ "concs": [38],
"item": 1,
- "override": "override_tp4_c44",
+ "override": "override_tp4_c38",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
@@ -2777,16 +2259,16 @@ export const agentx = {
{
"args": {
"--chunked-prefill-size": ["16384"],
- "--cuda-graph-max-bs-decode": ["24"],
- "--hicache-size": ["170"],
- "--hicache-write-policy": ["write_back"],
+ "--cuda-graph-max-bs-decode": ["76"],
+ "--hicache-size": ["250"],
"--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["24"],
- "--stream-interval": ["20"],
- "--prefill-decode-interval": null,
- "--schedule-policy": null
+ "--max-running-requests": ["16"],
+ "--prefill-decode-interval": ["56"],
+ "--swa-full-tokens-ratio": ["0.2"],
+ "--scheduler-recv-interval": null
},
"env": {
+ "SGLANG_DFLASH_EAGER_DRAFT_SAMPLER": "0",
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
}
@@ -2797,17 +2279,17 @@ export const agentx = {
null,
{
"args": {
- "--chunked-prefill-size": ["16384"],
- "--cuda-graph-max-bs-decode": ["24"],
- "--hicache-size": ["170"],
- "--hicache-write-policy": ["write_back"],
- "--max-prefill-tokens": ["16384"],
- "--max-running-requests": ["24"],
- "--stream-interval": ["20"],
- "--prefill-decode-interval": null,
- "--schedule-policy": null
+ "--chunked-prefill-size": ["16384"],
+ "--cuda-graph-max-bs-decode": ["76"],
+ "--hicache-size": ["250"],
+ "--max-prefill-tokens": ["16384"],
+ "--max-running-requests": ["16"],
+ "--prefill-decode-interval": ["56"],
+ "--swa-full-tokens-ratio": ["0.2"],
+ "--scheduler-recv-interval": null
},
"env": {
+ "SGLANG_DFLASH_EAGER_DRAFT_SAMPLER": "0",
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4"
}
@@ -2831,9 +2313,9 @@ export const agentx = {
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["26"],
- "--stream-interval": ["20"],
"--prefill-decode-interval": null,
- "--schedule-policy": null
+ "--schedule-policy": null,
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -2852,9 +2334,9 @@ export const agentx = {
"--hicache-write-policy": ["write_back"],
"--max-prefill-tokens": ["16384"],
"--max-running-requests": ["26"],
- "--stream-interval": ["20"],
"--prefill-decode-interval": null,
- "--schedule-policy": null
+ "--schedule-policy": null,
+ "--scheduler-recv-interval": null
},
"env": {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
@@ -2875,6 +2357,7 @@ export const agentx = {
"args": {
"--chunked-prefill-size": ["8192"],
"--cuda-graph-max-bs-decode": ["12"],
+ "--disable-prefill-cuda-graph": [],
"--enable-hierarchical-cache": [],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
@@ -2882,11 +2365,14 @@ export const agentx = {
"--hicache-size": ["120"],
"--max-prefill-tokens": ["8192"],
"--max-running-requests": ["12"],
- "--mem-fraction-static": ["0.95"],
+ "--mem-fraction-static": ["0.92"],
"--prefill-decode-interval": ["96"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--scheduler-recv-interval": ["20"],
+ "--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.1"],
"--tensor-parallel-size": ["4"],
+ "--cuda-graph-backend-prefill": null,
"--speculative-draft-attention-backend": null
},
"env": {
@@ -2894,7 +2380,7 @@ export const agentx = {
"PYTORCH_CUDA_ALLOC_CONF": null
}
},
- "ref": [6, "sglang", 0]
+ "ref": [4, "sglang", 0]
}
]
},
@@ -2951,9 +2437,9 @@ export const agentx = {
"detail": "DFlash 8",
"gpus": 8,
"configKey": "musespark-fp4-b200-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc",
"imageStatus": "ok",
- "imageCommit": "c1c287a4",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -2971,12 +2457,12 @@ export const agentx = {
}
},
"prov": {
- "sha": "11358f4843ec3e086014d3bba1a102e400944a59",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml",
- "run": "39756010759/attempts/1",
- "pr": "4400",
- "status": "verified",
- "date": "2026-10-23"
+ "run": "41297696082",
+ "pr": "4707",
+ "comment": "6576173162",
+ "status": "verified-pr"
},
"points": [
{
@@ -2986,13 +2472,6 @@ export const agentx = {
"goldenAL": "4.05",
"kv": "none"
},
- {
- "concs": [2],
- "item": 0,
- "override": "override_tp8_c2",
- "goldenAL": "4.05",
- "kv": "none"
- },
{
"concs": [4],
"item": 0,
@@ -3003,59 +2482,12 @@ export const agentx = {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--enable-layernorm-sp": [],
- "--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["4"],
- "--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
- "--swa-full-tokens-ratio": ["0.4"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192",
- "PYTORCH_CUDA_ALLOC_CONF": null,
- "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": null
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["4"],
- "--enable-layernorm-sp": [],
- "--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["4"],
- "--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
- "--swa-full-tokens-ratio": ["0.4"]
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192",
- "PYTORCH_CUDA_ALLOC_CONF": null,
- "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": null
- }
- }
- ]
- }
- },
- {
- "concs": [6],
- "item": 0,
- "override": "override_tp8_c6",
- "goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["6"],
+ "--cuda-graph-max-bs-decode": ["2"],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["6"],
+ "--max-running-requests": ["2"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["128"],
"--swa-full-tokens-ratio": ["0.4"]
},
"env": {
@@ -3070,12 +2502,12 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["6"],
+ "--cuda-graph-max-bs-decode": ["2"],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["6"],
+ "--max-running-requests": ["2"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["128"],
"--swa-full-tokens-ratio": ["0.4"]
},
"env": {
@@ -3087,33 +2519,6 @@ export const agentx = {
]
}
},
- {
- "concs": [8],
- "item": 0,
- "override": "override_tp8_c8",
- "goldenAL": "4.05",
- "kv": "none",
- "patch": {
- "sglang": [
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"]
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--cuda-graph-max-bs-decode": ["3"],
- "--max-running-requests": ["3"]
- }
- }
- ]
- }
- },
{
"concs": [10],
"item": 0,
@@ -3124,12 +2529,12 @@ export const agentx = {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["10"],
+ "--cuda-graph-max-bs-decode": ["3"],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["10"],
+ "--max-running-requests": ["3"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["128"],
"--swa-full-tokens-ratio": ["0.4"]
},
"env": {
@@ -3144,12 +2549,12 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["10"],
+ "--cuda-graph-max-bs-decode": ["3"],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
- "--max-running-requests": ["10"],
+ "--max-running-requests": ["3"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["128"],
"--swa-full-tokens-ratio": ["0.4"]
},
"env": {
@@ -3162,9 +2567,9 @@ export const agentx = {
}
},
{
- "concs": [56],
+ "concs": [64],
"item": 1,
- "override": "override_tp8_c56",
+ "override": "override_tp8_c64",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
@@ -3180,8 +2585,9 @@ export const agentx = {
"--max-prefill-tokens": ["32768"],
"--max-running-requests": ["56"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
+ "--prefill-decode-interval": ["48"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.25"],
"--speculative-draft-attention-backend": null
},
@@ -3206,67 +2612,9 @@ export const agentx = {
"--max-prefill-tokens": ["32768"],
"--max-running-requests": ["56"],
"--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["64"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.25"],
- "--speculative-draft-attention-backend": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4",
- "PYTORCH_CUDA_ALLOC_CONF": null
- }
- }
- ]
- }
- },
- {
- "concs": [64],
- "item": 1,
- "override": "override_tp8_c64",
- "goldenAL": "4.05",
- "kv": "hicache",
- "patch": {
- "sglang": [
- {
- "args": {
- "--chunked-prefill-size": ["32768"],
- "--cuda-graph-max-bs-decode": ["64"],
- "--enable-hierarchical-cache": [],
- "--enable-layernorm-sp": [],
- "--enable-megamoe-for-layernorm-sp": [],
- "--hicache-size": ["160"],
- "--max-prefill-tokens": ["32768"],
- "--max-running-requests": ["64"],
- "--mem-fraction-static": ["0.9"],
- "--prefill-decode-interval": ["48"],
- "--schedule-policy": ["shortest-prefill-first"],
- "--swa-full-tokens-ratio": ["0.25"],
- "--speculative-draft-attention-backend": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192",
- "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4",
- "PYTORCH_CUDA_ALLOC_CONF": null
- }
- }
- ],
- "dynamo": [
- null,
- null,
- {
- "args": {
- "--chunked-prefill-size": ["32768"],
- "--cuda-graph-max-bs-decode": ["64"],
- "--enable-hierarchical-cache": [],
- "--enable-layernorm-sp": [],
- "--enable-megamoe-for-layernorm-sp": [],
- "--hicache-size": ["160"],
- "--max-prefill-tokens": ["32768"],
- "--max-running-requests": ["64"],
- "--mem-fraction-static": ["0.9"],
"--prefill-decode-interval": ["48"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.25"],
"--speculative-draft-attention-backend": null
},
@@ -3324,6 +2672,7 @@ export const agentx = {
["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"],
["--speculative-num-draft-tokens", "8"],
["--speculative-use-block-verification"],
+ ["--stream-interval", "40"],
["--swa-full-tokens-ratio", "0.3"],
["--tensor-parallel-size", "8"],
["--tokenizer-worker-num", "4"],
@@ -3387,9 +2736,9 @@ export const agentx = {
"detail": "DFlash 8",
"gpus": 4,
"configKey": "musespark-fp4-b200-sglang-agentic-dflash",
- "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4",
+ "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc",
"imageStatus": "ok",
- "imageCommit": "c1c287a4",
+ "imageCommit": "302b78cc",
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -3407,39 +2756,35 @@ export const agentx = {
}
},
"prov": {
- "sha": "11358f4843ec3e086014d3bba1a102e400944a59",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml",
- "run": "39756010759/attempts/1",
- "pr": "4400",
- "status": "verified",
- "date": "2026-10-23"
+ "run": "41297696082",
+ "pr": "4707",
+ "comment": "6576173162",
+ "status": "verified-pr"
},
"points": [
{
- "concs": [8],
+ "concs": [7],
"item": 2,
- "override": "override_tp4_c8",
+ "override": "override_tp4_c7",
"goldenAL": "4.05",
"kv": "hicache"
},
{
- "concs": [10],
+ "concs": [16],
"item": 2,
- "override": "override_tp4_c10",
+ "override": "override_tp4_c16",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["10"],
- "--max-running-requests": ["10"],
- "--prefill-decode-interval": ["64"],
- "--enable-layernorm-sp": null,
- "--enable-megamoe-for-layernorm-sp": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": null
+ "--cuda-graph-max-bs-decode": ["6"],
+ "--incremental-streaming-output": [],
+ "--max-running-requests": ["6"],
+ "--prefill-decode-interval": ["44"]
}
}
],
@@ -3448,32 +2793,33 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["10"],
- "--max-running-requests": ["10"],
- "--prefill-decode-interval": ["64"],
- "--enable-layernorm-sp": null,
- "--enable-megamoe-for-layernorm-sp": null
- },
- "env": {
- "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": null
+ "--cuda-graph-max-bs-decode": ["6"],
+ "--incremental-streaming-output": [],
+ "--max-running-requests": ["6"],
+ "--prefill-decode-interval": ["44"]
}
}
]
}
},
{
- "concs": [16],
+ "concs": [24],
"item": 2,
- "override": "override_tp4_c16",
+ "override": "override_tp4_c24",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"],
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--hicache-size": ["180"],
+ "--max-running-requests": ["48"],
+ "--mem-fraction-static": ["0.98"],
"--prefill-decode-interval": ["48"]
+ },
+ "env": {
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
}
}
],
@@ -3482,32 +2828,41 @@ export const agentx = {
null,
{
"args": {
- "--cuda-graph-max-bs-decode": ["32"],
- "--max-running-requests": ["32"],
+ "--cuda-graph-max-bs-decode": ["48"],
+ "--hicache-size": ["180"],
+ "--max-running-requests": ["48"],
+ "--mem-fraction-static": ["0.98"],
"--prefill-decode-interval": ["48"]
+ },
+ "env": {
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
}
}
]
}
},
{
- "concs": [24],
+ "concs": [40],
"item": 2,
- "override": "override_tp4_c24",
+ "override": "override_tp4_c40",
"goldenAL": "4.05",
"kv": "hicache",
"patch": {
"sglang": [
{
"args": {
+ "--chunked-prefill-size": ["16384"],
"--cuda-graph-max-bs-decode": ["48"],
- "--hicache-size": ["180"],
+ "--disable-prefill-cuda-graph": [],
+ "--hicache-size": ["160"],
+ "--max-prefill-tokens": ["16384"],
"--max-running-requests": ["48"],
- "--mem-fraction-static": ["0.98"],
- "--prefill-decode-interval": ["48"]
+ "--mem-fraction-static": ["0.93"],
+ "--cuda-graph-backend-prefill": null
},
"env": {
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
}
}
],
@@ -3516,14 +2871,18 @@ export const agentx = {
null,
{
"args": {
+ "--chunked-prefill-size": ["16384"],
"--cuda-graph-max-bs-decode": ["48"],
- "--hicache-size": ["180"],
+ "--disable-prefill-cuda-graph": [],
+ "--hicache-size": ["160"],
+ "--max-prefill-tokens": ["16384"],
"--max-running-requests": ["48"],
- "--mem-fraction-static": ["0.98"],
- "--prefill-decode-interval": ["48"]
+ "--mem-fraction-static": ["0.93"],
+ "--cuda-graph-backend-prefill": null
},
"env": {
- "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
+ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096"
}
}
]
@@ -3539,7 +2898,7 @@ export const agentx = {
"diff": {
"args": {
"--chunked-prefill-size": ["8192"],
- "--cuda-graph-max-bs-decode": ["8"],
+ "--cuda-graph-max-bs-decode": ["2"],
"--enable-hierarchical-cache": [],
"--enable-layernorm-sp": [],
"--enable-megamoe-for-layernorm-sp": [],
@@ -3547,10 +2906,11 @@ export const agentx = {
"--hicache-size": ["120"],
"--hicache-write-policy": ["write_through"],
"--max-prefill-tokens": ["8192"],
- "--max-running-requests": ["8"],
+ "--max-running-requests": ["2"],
"--mem-fraction-static": ["0.95"],
- "--prefill-decode-interval": ["96"],
+ "--prefill-decode-interval": ["64"],
"--schedule-policy": ["shortest-prefill-first"],
+ "--stream-interval": ["20"],
"--swa-full-tokens-ratio": ["0.1"],
"--tensor-parallel-size": ["4"],
"--speculative-draft-attention-backend": null
@@ -3560,7 +2920,7 @@ export const agentx = {
"PYTORCH_CUDA_ALLOC_CONF": null
}
},
- "ref": [8, "sglang", 0]
+ "ref": [6, "sglang", 0]
}
]
},
diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx
index 31d0cfc258b4..e3e02fe358bc 100644
--- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx
+++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "GLM-5.2",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
+ },
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
+ },
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}
diff --git a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx
index 1f6551be7b7d..72904b8ea8a1 100644
--- a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx
+++ b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx
@@ -1,36 +1,51 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
-// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed.
+// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed.
export const agentx = {
"model": "GLM-5.3",
- "generatedOn": "2026-10-30",
+ "generatedOn": "2026-11-02",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
- "main": "ea8b34394a603396d681e3c3d466d81cc885779b",
+ "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553",
"prs": {
- "4670": {
+ "4726": {
"run": null,
- "sha": "5692839d90345486f1f90a1ea48b1d15ef755169",
+ "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab",
"status": "in-progress"
},
- "4603": {
- "run": null,
- "sha": "9bd047ac75b8275723fd864987639e29d03efc91",
- "status": "in-progress"
+ "4725": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543",
+ "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99",
+ "status": "verified-pr"
},
- "4595": {
- "run": null,
- "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e",
- "status": "in-progress"
+ "4724": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200",
+ "sha": "98ea294c289546dca624e1f07d9d3d2106715741",
+ "status": "verified-pr"
},
- "4454": {
- "run": null,
- "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b",
- "status": "in-progress"
+ "4707": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082",
+ "sha": "c965ca2eb81db733ad280ea693025389ac11014b",
+ "status": "verified-pr"
+ },
+ "4695": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328",
+ "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e",
+ "status": "verified-pr"
+ },
+ "4683": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222",
+ "sha": "d06564bc533cc54584ef6a35837336a2b4b84421",
+ "status": "verified-pr"
},
- "4333": {
- "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
- "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
+ "4670": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400",
+ "sha": "c9eb455f32e7f026d7424de723756de2e719d868",
+ "status": "verified-pr"
+ },
+ "4603": {
+ "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147",
+ "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3",
"status": "verified-pr"
}
}