From b721c54db337df58f942930f96976109a14181ee Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" Date: Mon, 2 Nov 2026 06:40:45 -0800 Subject: [PATCH 1/2] [Doc] cookbook: Revert hand-added Qwen3.8-Flash-Next B300 NVFP4 AgentX module (#47560) This reverts commit 26452a6cfa871def34341378d8c8877b7dbb6cfc. #47560 added a hand-written data module (qwen3.8-flash-next-b300.jsx) next to the generated Qwen3.8-Flash-Next AgentX data and merged it into the page, for configurations from the still-open InferenceX #4683. The Agentic Long-Context sections are generated from SemiAnalysis submissions by the cookbook sync; the next commit overlays InferenceX #4683 through it, at the run #47560 cited. Co-Authored-By: Claude Opus 5.5 --- .../Qwen/Qwen3.8-Flash-Next.mdx | 7 +- .../agentx/Qwen/qwen3.8-flash-next-b300.jsx | 431 ------------------ 2 files changed, 1 insertion(+), 437 deletions(-) delete mode 100644 docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx index 7fa34007b5bb..4a468739ef20 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next.mdx @@ -79,13 +79,8 @@ Agentic workloads, such as coding agents and other multi-turn, tool-using assist import { AgentX } from "/src/snippets/_agentx.jsx"; import { agentx } from "/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx"; -import { b300Cells } from "/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx"; -The B300 HiCache configurations reserve 298 GB of host memory for the combined target/draft KV and Mamba pool, in addition to pinned PLE embeddings and process overhead. Provision sufficient free host RAM and container shared memory before starting them. GPU KV configurations also offload PLE embeddings to host memory. - -For the B300 NVIDIA configurations at concurrency 12 and above, `--mamba-track-interval 262144` reduces intermediate state checkpoints. The concurrency-80 configuration also uses `--strip-thinking-cache` and `--mamba-max-states-per-path 2`. These settings target replay traffic that does not resend model output; reassess prefix reuse for conversational clients that do. - - + ### MI355X deployment measured with SemiAnalysis InferenceX diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx deleted file mode 100644 index afb8ee4e320c..000000000000 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next-b300.jsx +++ /dev/null @@ -1,431 +0,0 @@ -export const b300Cells = [ - { - "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp4", - "hw": "b300", - "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", - "label": "Agg TP4·EP4", - "detail": "MTP 3-1-4", - "gpus": 4, - "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": [], - "kv": { - "available": ["none"], - "unavailable": { - "hicache": "Not qualified for this deployment.", - "mooncake": "Not qualified for this deployment." - } - }, - "prov": { - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41117066222/attempts/1", - "pr": "4683", - "status": "verified-pr", - "date": "2026-10-30" - }, - "points": [ - { - "concs": [1], - "override": "override_tp4_c1", - "kv": "none" - }, - { - "concs": [2], - "override": "override_tp4_c2", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["4"] - }, - "env": {} - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1" - }, - "args": [ - ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--trust-remote-code"], - ["--quantization", "modelopt_mixed"], - ["--context-length", "262144"], - ["--tensor-parallel-size", "4"], - ["--pipeline-parallel-size", "1"], - ["--data-parallel-size", "1"], - ["--expert-parallel-size", "4"], - ["--attention-backend", "trtllm_mha"], - ["--linear-attn-prefill-backend", "flashinfer"], - ["--linear-attn-decode-backend", "flashinfer"], - ["--linear-attn-verify-backend", "triton"], - ["--mamba-ssm-dtype", "bfloat16"], - ["--mem-fraction-static", "0.88"], - ["--max-running-requests", "2"], - ["--page-size", "64"], - ["--chunked-prefill-size", "16384"], - ["--max-prefill-tokens", "16384"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--moe-a2a-backend", "none"], - ["--disable-shared-experts-fusion"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-num-steps", "3"], - ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "4"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--enable-linear-replayssm-spec"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-backend-decode", "full"], - ["--cuda-graph-max-bs-decode", "96"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - ["--kv-cache-dtype", "bfloat16"], - ["--speculative-draft-kv-cache-dtype", "bfloat16"], - ["--ple-offload-embedding"], - ["--ple-offload-backend", "pinned"], - ["--reasoning-parser", "auto"], - ["--tokenizer-worker-num", "6"], - ["--scheduler-recv-interval", "1"], - ["--stream-interval", "20"], - ["--watchdog-timeout", "1800"], - ["--flashinfer-allreduce-fusion-backend", "trtllm"] - ] - } - ] - } - } - }, - { - "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp1", - "hw": "b300", - "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", - "label": "Agg TP1 · GPU KV", - "detail": "MTP 3-1-4", - "gpus": 1, - "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": [], - "kv": { - "available": ["none"], - "unavailable": { - "hicache": "Not qualified for this deployment.", - "mooncake": "Not qualified for this deployment." - } - }, - "prov": { - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41117066222/attempts/1", - "pr": "4683", - "status": "verified-pr", - "date": "2026-10-30" - }, - "points": [ - { - "concs": [1], - "override": "override_c1", - "kv": "none" - }, - { - "concs": [2], - "override": "override_c2", - "kv": "none", - "patch": { - "sglang": [ - { - "args": {}, - "env": {} - } - ] - } - }, - { - "concs": [12], - "override": "override_c12", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--mamba-track-interval": ["262144"], - "--kv-cache-dtype": ["fp8_e4m3"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] - }, - "env": { - "SGLANG_Q35MF_MKH_N3B": null, - "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, - "SGLANG_Q35MF_MKH_N3C": null, - "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, - "SGLANG_Q35MF_EDGE_SHM_SYNC": null, - "SGLANG_Q35MF_BK_VEC_COPY": null, - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null - } - } - ] - } - }, - { - "concs": [24], - "override": "override_c24", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--mamba-track-interval": ["262144"], - "--max-mamba-cache-size": ["960"], - "--kv-cache-dtype": ["fp8_e4m3"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] - }, - "env": { - "SGLANG_Q35MF_MKH_N3B": null, - "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, - "SGLANG_Q35MF_MKH_N3C": null, - "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, - "SGLANG_Q35MF_EDGE_SHM_SYNC": null, - "SGLANG_Q35MF_BK_VEC_COPY": null, - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", - "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": "1", - "SGLANG_Q35MF_MKH_N3B": "1", - "SGLANG_Q35MF_MKH_N3C": "1", - "SGLANG_Q35MF_BK_VEC_COPY": "1", - "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", - "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": "1" - }, - "args": [ - ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--trust-remote-code"], - ["--quantization", "modelopt_mixed"], - ["--context-length", "262144"], - ["--tensor-parallel-size", "1"], - ["--pipeline-parallel-size", "1"], - ["--data-parallel-size", "1"], - ["--expert-parallel-size", "1"], - ["--attention-backend", "trtllm_mha"], - ["--linear-attn-prefill-backend", "flashinfer"], - ["--linear-attn-decode-backend", "flashinfer"], - ["--linear-attn-verify-backend", "triton"], - ["--mamba-ssm-dtype", "bfloat16"], - ["--mem-fraction-static", "0.88"], - ["--max-running-requests", "96"], - ["--page-size", "64"], - ["--chunked-prefill-size", "16384"], - ["--max-prefill-tokens", "16384"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--moe-a2a-backend", "none"], - ["--disable-shared-experts-fusion"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-num-steps", "3"], - ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "4"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--enable-linear-replayssm-spec"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-backend-decode", "full"], - ["--cuda-graph-max-bs-decode", "96"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - ["--kv-cache-dtype", "bfloat16"], - ["--speculative-draft-kv-cache-dtype", "bfloat16"], - ["--ple-offload-embedding"], - ["--ple-offload-backend", "pinned"], - ["--reasoning-parser", "auto"], - ["--tokenizer-worker-num", "6"], - ["--scheduler-recv-interval", "1"], - ["--stream-interval", "20"], - ["--watchdog-timeout", "1800"] - ] - } - ] - } - } - }, - { - "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#4683-tp1-hicache", - "hw": "b300", - "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", - "label": "Agg TP1 · HiCache", - "detail": "MTP 3-1-4", - "gpus": 1, - "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": [], - "kv": { - "available": ["hicache"], - "unavailable": { - "none": "Not qualified for this deployment.", - "mooncake": "Not qualified for this deployment." - } - }, - "prov": { - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41117066222/attempts/1", - "pr": "4683", - "status": "verified-pr", - "date": "2026-10-30" - }, - "points": [ - { - "concs": [48], - "override": "override_c48", - "kv": "hicache" - }, - { - "concs": [80], - "override": "override_c80", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"], - "--max-running-requests": ["128"], - "--mamba-max-states-per-path": ["2"], - "--max-mamba-cache-size": ["640"], - "--cuda-graph-max-bs-decode": ["128"], - "--strip-thinking-cache": [] - }, - "env": {} - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1" - }, - "args": [ - ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--trust-remote-code"], - ["--quantization", "modelopt_mixed"], - ["--context-length", "262144"], - ["--tensor-parallel-size", "1"], - ["--pipeline-parallel-size", "1"], - ["--data-parallel-size", "1"], - ["--expert-parallel-size", "1"], - ["--attention-backend", "trtllm_mha"], - ["--linear-attn-prefill-backend", "flashinfer"], - ["--linear-attn-decode-backend", "flashinfer"], - ["--linear-attn-verify-backend", "triton"], - ["--mamba-ssm-dtype", "bfloat16"], - ["--mamba-radix-cache-strategy", "extra_buffer_lazy"], - ["--mamba-track-interval", "262144"], - ["--max-mamba-cache-size", "720"], - ["--mem-fraction-static", "0.88"], - ["--max-running-requests", "96"], - ["--page-size", "64"], - ["--chunked-prefill-size", "16384"], - ["--max-prefill-tokens", "16384"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--moe-a2a-backend", "none"], - ["--disable-shared-experts-fusion"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-num-steps", "3"], - ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "4"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--enable-linear-replayssm-spec"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-backend-decode", "full"], - ["--cuda-graph-max-bs-decode", "96"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--enable-hierarchical-cache"], - ["--hicache-size", "298"], - ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-write-policy", "write_back"], - ["--ple-offload-embedding"], - ["--ple-offload-backend", "pinned"], - ["--reasoning-parser", "auto"], - ["--tokenizer-worker-num", "1"], - ["--scheduler-recv-interval", "1"], - ["--stream-interval", "20"], - ["--watchdog-timeout", "1800"] - ] - } - ] - } - } - } -]; From 3a3825dd7278d83fab6fa3fb059be6a15a487ae9 Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" Date: Mon, 2 Nov 2026 06:40:52 -0800 Subject: [PATCH 2/2] [Doc] cookbook: Refresh SemiAnalysis AgentX launches to InferenceX main c5e8b9b3 Mirror the SemiAnalysis InferenceX AgentX SGLang / Dynamo+SGLang lanes at InferenceX main c5e8b9b3 (MiniMax-M3.1-Flash GB300 SGLang and Dynamo+SGLang from InferenceX #4676 / #4698, DeepSeek-V4.1-Flash GB300 Dynamo+SGLang from #4595, now from their dashboard runs), with InferenceX PRs #4726 (MiniMax-M3.1-Flash B300), #4695 (MiniMax-M3.1-Flash GB200 Dynamo+SGLang), #4725 #4724 #4707 #4670 (Muse Spark Open B300 / GB300 / B200 / GB200), #4683 (Qwen3.8-Flash-Next B300 NVFP4, /use run 41117066222) and #4603 (DeepSeek-V4.1-Flash GB300) overlaid. This replaces the MiniMax data that #47623, #47699 and #47791 edited by hand. Co-Authored-By: Claude Opus 5.5 --- .../agentx/MiniMaxAI/minimax-m3.1.jsx | 6021 +++++++++++------ docs/src/snippets/agentx/Qwen/qwen3.5.jsx | 1989 +++--- .../agentx/Qwen/qwen3.8-flash-next.jsx | 512 +- .../agentx/deepseek-ai/deepseek-v4_1.jsx | 602 +- .../agentx/meta-models/muse-spark-open.jsx | 1448 ++-- docs/src/snippets/agentx/zai-org/glm-5.2.jsx | 55 +- docs/src/snippets/agentx/zai-org/glm-5.3.jsx | 55 +- 7 files changed, 5929 insertions(+), 4753 deletions(-) diff --git a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx index fdb18ccba66e..5c947fd9e24a 100644 --- a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx +++ b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx @@ -1,51 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670, #4675, #4676, #4698), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "MiniMax-M3.1-Flash", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4698": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271829525", - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "status": "verified-pr" + "4726": { + "run": null, + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "status": "in-progress" }, - "4676": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41054395691", - "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", "status": "verified-pr" }, - "4675": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41085412898", - "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e", + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", "status": "verified-pr" }, - "4670": { - "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } @@ -73,7 +73,72 @@ export const agentx = { "id": "MiniMaxAI/MiniMax-M3.1-Flash" } ], - "envs": [], + "envs": [ + { + "DYN_EVENT_PLANE": "zmq", + "DYN_REQUEST_PLANE": "tcp", + "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", + "DYN_ROUTER_QUEUE_THRESHOLD": "None", + "DYN_ROUTER_TEMPERATURE": "0", + "DYN_TCP_CONNECT_TIMEOUT": "120", + "DYN_TOKENIZER": "fastokens", + "DYN_TOKENIZER_CACHE": "1", + "DYN_TOKENIZER_CACHE_BYTES": "8000000000", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", + "DYN_EVENT_PLANE": "zmq", + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FMHA_SM100_ALLOW_JIT": "1", + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + }, + { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", + "DYN_EVENT_PLANE": "zmq", + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FMHA_SM100_ALLOW_JIT": "1", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_FORCE_MNNVL": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "WITH_NVIDIA_PEERMEM": "0" + }, + { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", + "DYN_EVENT_PLANE": "zmq", + "DYN_REQUEST_PLANE": "tcp", + "DYN_SGL_GATEWAY_WORKERS": "8", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FMHA_SM100_ALLOW_JIT": "1", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_FORCE_MNNVL": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "WITH_NVIDIA_PEERMEM": "0" + } + ], "kvMooncake": { "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", "donorHardware": "gb200", @@ -138,17 +203,17 @@ export const agentx = { "files": {}, "cells": [ { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#0", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", + "detail": "DSpark 7", "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", + "imageCommit": "e371881", + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -165,29 +230,31 @@ export const agentx = { } }, "prov": { - "sha": "3b96f3383ade71707f456573909b65a84b0b74d1", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "40634232927/attempts/2", - "pr": "4600", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-28" + "date": "2026-11-02" }, "points": [ { "concs": [1], "item": 0, - "override": "zip_override_tp4[0]", + "override": "override_tp4_c1", "goldenAL": "4.04", "kv": "none" }, { "concs": [4], - "item": 0, - "override": "zip_override_tp4[1]", + "item": 1, + "override": "override_tp4_c4", "goldenAL": "4.04", "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["8"], @@ -195,26 +262,27 @@ export const agentx = { } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["8"] } - } + }, + null ] } }, { "concs": [8], - "item": 0, - "override": "zip_override_tp4[2]", + "item": 2, + "override": "override_tp4_c8", "goldenAL": "4.04", "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["16"], @@ -222,26 +290,27 @@ export const agentx = { } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["16"] } - } + }, + null ] } }, { "concs": [16], - "item": 0, - "override": "zip_override_tp4[3]", + "item": 3, + "override": "override_tp4_c16", "goldenAL": "4.04", "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["32"], @@ -249,67 +318,105 @@ export const agentx = { } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["32"] } - } + }, + null ] } }, { - "concs": [32], - "item": 0, - "override": "zip_override_tp4[4]", + "concs": [24], + "item": 4, + "override": "override_tp4_hicache_spf_c24", "goldenAL": "4.04", - "kv": "none", + "kv": "hicache", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["48"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.87"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"] } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["48"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.87"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"] } - } + }, + null ] } } ], "routers": { - "sglang": { + "dynamo": { "install": [], "files": [], "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, { "kind": "worker", "role": "agg", "workers": 1, "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "FMHA_SM100_ALLOW_JIT": "1", - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" - }, + "cmd": "python3 -m dynamo.sglang", "args": [ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], ["--host", "0.0.0.0"], ["--port", "30000"], + ["--request-plane", "tcp"], + ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], ["--chunked-prefill-size", "32768"], ["--cuda-graph-backend-prefill", "breakable"], ["--cuda-graph-max-bs-decode", "2"], @@ -317,70 +424,66 @@ export const agentx = { ["--mem-fraction-static", "0.8"], ["--moe-runner-backend", "flashinfer_trtllm_routed"], ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], ["--speculative-algorithm", "DSPARK"], ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], ["--speculative-dspark-block-size", "7"], ["--speculative-use-block-verification"], ["--tensor-parallel-size", "4"], - ["--tokenizer-worker-num", "8"], ["--tool-call-parser", "minimax-m3"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"] - ] + ], + "envRef": 1 } ] }, - "dynamo": { + "sglang": { "install": [], "files": [], "blocks": [ { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "from": 2, + "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--request-plane": ["tcp"] + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#2", + "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", "gpus": 4, "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -403,29 +506,241 @@ export const agentx = { } }, "prov": { - "sha": "3b96f3383ade71707f456573909b65a84b0b74d1", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-low-latency.yaml", - "run": "40634232927/attempts/2", - "pr": "4600", + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41054395691/attempts/2", + "pr": "4676", "status": "verified", - "date": "2026-10-28" + "date": "2026-10-31" }, "points": [ { "concs": [1], - "item": 2, - "override": "override_tp4_c1", + "item": 0, + "override": "zip_override_tp4[0]", "goldenAL": "4.04", "kv": "none" - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { + }, + { + "concs": [4], + "item": 0, + "override": "zip_override_tp4[1]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "zip_override_tp4[2]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "zip_override_tp4[3]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "zip_override_tp4[4]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--tokenizer-worker-num": ["8"] + } + }, + "ref": [0, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#2", + "hw": "gb300", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-low-latency.yaml", + "run": "41054395691/attempts/2", + "pr": "4676", + "status": "verified", + "date": "2026-10-31" + }, + "points": [ + { + "concs": [1], + "item": 2, + "override": "override_tp4_c1", + "goldenAL": "4.04", + "kv": "none" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { "args": { "--disable-attn-tp-gather": [], "--max-running-requests": ["8"], @@ -437,8 +752,7 @@ export const agentx = { "--chunked-prefill-size": null, "--cuda-graph-backend-prefill": null, "--cuda-graph-max-bs-decode": null, - "--speculative-draft-kv-cache-dtype": null, - "--tokenizer-worker-num": null + "--speculative-draft-kv-cache-dtype": null }, "env": { "ALLOW_JIT": "1", @@ -499,7 +813,7 @@ export const agentx = { "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7", + "detail": "DSpark 7 · --chunked-prefill-size 32768 · minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "gpus": 2, "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -522,11 +836,11 @@ export const agentx = { } }, "prov": { - "sha": "3b96f3383ade71707f456573909b65a84b0b74d1", - "run": "40634232927/attempts/2", - "pr": "4600", + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "run": "41054395691/attempts/2", + "pr": "4676", "status": "verified", - "date": "2026-10-28" + "date": "2026-10-31" }, "points": [ { @@ -723,7 +1037,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [0, "sglang", 0] + "ref": [1, "sglang", 0] } ] }, @@ -773,12 +1087,12 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#1", + "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#10", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4·EP4", - "detail": "DSpark 7", - "gpus": 4, + "label": "Agg TP2", + "detail": "DSpark 7 · --chunked-prefill-size 8192 · minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "gpus": 2, "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", @@ -800,36 +1114,34 @@ export const agentx = { } }, "prov": { - "sha": "3b96f3383ade71707f456573909b65a84b0b74d1", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "40634232927/attempts/2", - "pr": "4600", + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml", + "run": "41054395691/attempts/2", + "pr": "4676", "status": "verified", - "date": "2026-10-28" + "date": "2026-10-31" }, "points": [ { - "concs": [32], - "item": 1, - "override": "override_tp4_ep4_c32", + "concs": [20], + "item": 10, + "override": "zip_override_tp2_hicache_spf_interval32[0]", "goldenAL": "4.04", - "kv": "none" + "kv": "hicache" }, { - "concs": [64], - "item": 6, - "override": "override_tp4_ep4_hicache_c64", + "concs": [24], + "item": 11, + "override": "zip_override_tp2_hicache_spf_interval32[1]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.87"] - } + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } } ], "dynamo": [ @@ -837,29 +1149,25 @@ export const agentx = { null, { "args": { - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.87"] + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] } } ] } }, { - "concs": [128], - "item": 6, - "override": "override_tp4_ep4_hicache_c128", + "concs": [28], + "item": 12, + "override": "zip_override_tp2_hicache_spf_interval32[2]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.87"] + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["56"] } } ], @@ -868,10 +1176,89 @@ export const agentx = { null, { "args": { - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.87"] + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["56"] + } + } + ] + } + }, + { + "concs": [32], + "item": 13, + "override": "zip_override_tp2_hicache_spf_interval32[3]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + }, + { + "concs": [36], + "item": 14, + "override": "zip_override_tp2_hicache_spf_interval32[4]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] + } + } + ] + } + }, + { + "concs": [40], + "item": 15, + "override": "zip_override_tp2_hicache_spf_interval32[5]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] } } ] @@ -886,16 +1273,13 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--expert-parallel-size": ["4"], - "--max-running-requests": ["128"], - "--moe-runner-backend": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" + "--chunked-prefill-size": ["8192"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"] } }, - "ref": [0, "sglang", 0] + "ref": [3, "sglang", 0] } ] }, @@ -972,20 +1356,112 @@ export const agentx = { } }, "prov": { - "sha": "3b96f3383ade71707f456573909b65a84b0b74d1", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "40634232927/attempts/2", - "pr": "4600", + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "run": "41054395691/attempts/2", + "pr": "4676", "status": "verified", - "date": "2026-10-28" + "date": "2026-10-31" }, "points": [ + { + "concs": [24], + "item": 7, + "override": "zip_override_tp4_hicache_spf_interval32[0]", + "goldenAL": "4.04", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml" + }, + { + "concs": [32], + "item": 8, + "override": "zip_override_tp4_hicache_spf_interval32[1]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml" + }, + { + "concs": [40], + "item": 9, + "override": "zip_override_tp4_hicache_spf_interval32[2]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml" + }, { "concs": [64], "item": 3, "override": "override_tp4_hicache_c64", "goldenAL": "4.04", - "kv": "hicache" + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["32768"], + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"], + "--prefill-decode-interval": null, + "--schedule-policy": null, + "--stream-interval": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["32768"], + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"], + "--prefill-decode-interval": null, + "--schedule-policy": null, + "--stream-interval": null + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml" } ], "routers": { @@ -996,12 +1472,12 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"], + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"], "--tensor-parallel-size": ["4"] } }, - "ref": [2, "sglang", 0] + "ref": [4, "sglang", 0] } ] }, @@ -1051,13 +1527,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#0", - "hw": "gb200", + "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#1", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", + "label": "Agg TP4·EP4", + "detail": "DSpark 7", "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -1073,38 +1549,40 @@ export const agentx = { ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "donorHardware": "gb200" + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" } }, "prov": { - "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", - "run": "40739138596/attempts/1", - "pr": "4599", + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41054395691/attempts/2", + "pr": "4676", "status": "verified", - "date": "2026-10-29" + "date": "2026-10-31" }, "points": [ { - "concs": [1], - "item": 0, - "override": "override_tp4_c1", + "concs": [32], + "item": 1, + "override": "override_tp4_ep4_c32", "goldenAL": "4.04", "kv": "none" }, { - "concs": [4], - "item": 0, - "override": "override_tp4_c4", + "concs": [64], + "item": 6, + "override": "override_tp4_ep4_hicache_c64", "goldenAL": "4.04", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] + "--enable-hierarchical-cache": [], + "--hicache-size": ["175"], + "--hicache-write-policy": ["write_back"], + "--mem-fraction-static": ["0.87"] } } ], @@ -1113,79 +1591,29 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ] - } - }, - { - "concs": [8], - "item": 0, - "override": "override_tp4_c8", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ] - } - }, - { - "concs": [16], - "item": 0, - "override": "override_tp4_c16", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] + "--enable-hierarchical-cache": [], + "--hicache-size": ["175"], + "--hicache-write-policy": ["write_back"], + "--mem-fraction-static": ["0.87"] } } ] } }, { - "concs": [32], - "item": 0, - "override": "override_tp4_c32", + "concs": [128], + "item": 6, + "override": "override_tp4_ep4_hicache_c128", "goldenAL": "4.04", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--enable-hierarchical-cache": [], + "--hicache-size": ["175"], + "--hicache-write-policy": ["write_back"], + "--mem-fraction-static": ["0.87"] } } ], @@ -1194,8 +1622,10 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--enable-hierarchical-cache": [], + "--hicache-size": ["175"], + "--hicache-write-policy": ["write_back"], + "--mem-fraction-static": ["0.87"] } } ] @@ -1208,38 +1638,18 @@ export const agentx = { "files": [], "blocks": [ { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "FMHA_SM100_ALLOW_JIT": "1", - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["128"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["128"], + "--moe-runner-backend": null + }, + "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" + } }, - "args": [ - ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--chunked-prefill-size", "32768"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-max-bs-decode", "2"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.8"], - ["--moe-runner-backend", "flashinfer_trtllm_routed"], - ["--reasoning-parser", "minimax-m3"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--speculative-dspark-block-size", "7"], - ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "4"], - ["--tokenizer-worker-num", "8"], - ["--tool-call-parser", "minimax-m3"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ] + "ref": [1, "sglang", 0] } ] }, @@ -1289,17 +1699,17 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#1", - "hw": "gb200", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#5", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "label": "Agg TP2", + "detail": "DSpark 7 · --chunked-prefill-size 8192 · minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", + "imageCommit": "e371881", + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -1311,57 +1721,28 @@ export const agentx = { ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "donorHardware": "gb200" + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" } }, "prov": { - "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic-low-latency.yaml", - "run": "40739138596/attempts/1", - "pr": "4599", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ { - "concs": [1], - "item": 1, - "override": "override_tp4_c1", + "concs": [40], + "item": 5, + "override": "override_tp2_hicache_spf_c40", "goldenAL": "4.04", - "kv": "none" + "kv": "hicache" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--disable-attn-tp-gather": [], - "--max-running-requests": ["8"], - "--monokernel-backend": ["msa"], - "--page-size": ["128"], - "--speculative-eagle-topk": ["1"], - "--speculative-num-draft-tokens": ["8"], - "--speculative-num-steps": ["1"], - "--chunked-prefill-size": null, - "--cuda-graph-backend-prefill": null, - "--cuda-graph-max-bs-decode": null, - "--speculative-draft-kv-cache-dtype": null, - "--tokenizer-worker-num": null - }, - "env": { - "ALLOW_JIT": "1", - "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" - } - }, - "ref": [5, "sglang", 0] - } - ] - }, "dynamo": { "install": [], "files": [], @@ -1381,44 +1762,87 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": ["tcp"] + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["80"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["80"], + "--mem-fraction-static": ["0.87"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#2", - "hw": "gb200", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#6", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · measured with HiCache", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", + "label": "Agg TP2", + "detail": "DSpark 7 · --chunked-prefill-size 32768 · minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -1430,155 +1854,56 @@ export const agentx = { ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "donorHardware": "gb200" + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" } }, "prov": { - "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", - "run": "40739138596/attempts/1", - "pr": "4599", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ - { - "concs": [32], - "item": 2, - "override": "override_tp4_dram_c32", - "goldenAL": "4.04", - "kv": "hicache" - }, { "concs": [40], - "item": 2, - "override": "override_tp4_dram_c40", + "item": 6, + "override": "override_tp2_hicache_c40", "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] - } - } - ] - } + "kv": "hicache" }, { - "concs": [48], - "item": 2, - "override": "override_tp4_dram_c48", + "concs": [44], + "item": 7, + "override": "override_tp2_hicache_c44", "goldenAL": "4.04", "kv": "hicache", "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"] - } - } - ], "dynamo": [ null, null, { "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"] - } - } - ] - } - }, - { - "concs": [56], - "item": 2, - "override": "override_tp4_dram_c56", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } } ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] - } - } - ] - } - }, - { - "concs": [64], - "item": 2, - "override": "override_tp4_dram_c64", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } - } + }, + null ] } } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.81"] - } - }, - "ref": [5, "sglang", 0] - } - ] - }, "dynamo": { "install": [], "files": [], @@ -1598,45 +1923,87 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": ["tcp"] + "--chunked-prefill-size": ["32768"], + "--prefill-decode-interval": null, + "--schedule-policy": null, + "--stream-interval": null + } + }, + "ref": [7, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#3", - "hw": "gb200", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4·EP4", + "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)", "detail": "DSpark 7", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "gpus": 16, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "note", + "script": "Placement", + "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + } + ], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -1644,61 +2011,32 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "200"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", - "donorHardware": "gb200" + "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { - "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", - "run": "40739138596/attempts/1", - "pr": "4599", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ { - "concs": [72], - "item": 3, - "override": "override_tp4_ep4_dram_c72", - "goldenAL": "4.04", - "kv": "hicache" - }, - { - "concs": [80], - "item": 3, - "override": "override_tp4_ep4_dram_c80", + "concs": [224, 256], + "item": 1, + "override": "override_4p_dep2_dep8", "goldenAL": "4.04", "kv": "hicache" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--cuda-graph-max-bs-prefill": ["16384"], - "--expert-parallel-size": ["4"], - "--max-running-requests": ["128"], - "--mem-fraction-static": ["0.8"], - "--moe-runner-backend": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" - } - }, - "ref": [7, "sglang", 0] - } - ] - }, - "dynamo": { + "dynamo": { "install": [], "files": [], "blocks": [ @@ -1717,45 +2055,192 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, + "kind": "worker", + "role": "prefill", + "workers": 4, + "nodes_per_worker": 1, + "gpu_pinning": [ + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"] + ], + "local_ranks": [0, 1, 0, 1], + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "$PORT"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "$BOOTSTRAP_PORT"], + ["--request-plane", "tcp"], + ["--kv-events-config", "$KV_EVENTS_CONFIG"], + ["--chunked-prefill-size", "16384"], + ["--data-parallel-size", "2"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-dp-attention"], + ["--enable-dp-lm-head"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "2"], + ["--hicache-size", "200"], + ["--hicache-write-policy", "write_back"], + ["--max-running-requests", "128"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "megamoe"], + ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--schedule-policy", "shortest-prefill-first"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "2"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 2 + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 2, "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--request-plane", "tcp"], + ["--cuda-graph-max-bs-decode", "48"], + ["--data-parallel-size", "8"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-dp-attention"], + ["--enable-dp-lm-head"], + ["--expert-parallel-size", "8"], + ["--load-balance-method", "total_tokens"], + ["--max-running-requests", "384"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "megamoe"], + ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "least-loaded"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "8"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 3 + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--request-plane": ["tcp"] + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#0", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)", + "detail": "DSpark 7", + "gpus": 20, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "note", + "script": "Placement", + "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + } + ], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -1763,237 +2248,165 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "200"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { - "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", - "run": "40739166138/attempts/1", - "pr": "4524", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ { - "concs": [1], + "concs": [280, 320, 340, 360, 400], "item": 0, - "override": "override_tp4_c1", + "override": "override_6p_dep2_dep8", "goldenAL": "4.04", - "kv": "none" + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "workers": 6, + "gpu_pinning": [ + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"] + ], + "local_ranks": [0, 1, 0, 1, 0, 1], + "ref": [9, "dynamo", 2] + }, + { + "ref": [9, "dynamo", 3] + } + ] }, - { - "concs": [4], - "item": 0, - "override": "override_tp4_c4", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ] - } - }, - { - "concs": [8], - "item": 0, - "override": "override_tp4_c8", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ] - } - }, - { - "concs": [16], - "item": 0, - "override": "override_tp4_c16", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ] - } - }, - { - "concs": [32], - "item": 0, - "override": "override_tp4_c32", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] - } - } - ] - } - } - ], - "routers": { "sglang": { "install": [], "files": [], "blocks": [ { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, + "from": 2, + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], "cmd": "python3 -m sglang.launch_server", - "env": { - "FMHA_SM100_ALLOW_JIT": "1", - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" - }, - "args": [ - ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--chunked-prefill-size", "32768"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-max-bs-decode", "2"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.8"], - ["--moe-runner-backend", "flashinfer_trtllm_routed"], - ["--reasoning-parser", "minimax-m3"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--speculative-dspark-block-size", "7"], - ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "4"], - ["--tokenizer-worker-num", "8"], - ["--tool-call-parser", "minimax-m3"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "from": 3, + "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--request-plane": ["tcp"] + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#1", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#3", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "detail": "DSpark 7", + "gpus": 28, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "note", + "script": "Placement", + "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + } + ], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -2001,60 +2414,31 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "200"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { - "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-low-latency.yaml", - "run": "40739166138/attempts/1", - "pr": "4524", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ { - "concs": [1], - "item": 1, - "override": "override_tp4_c1", + "concs": [300, 360, 400], + "item": 3, + "override": "override_6p_dep2_dep16", "goldenAL": "4.04", - "kv": "none" + "kv": "hicache" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--disable-attn-tp-gather": [], - "--max-running-requests": ["8"], - "--monokernel-backend": ["msa"], - "--page-size": ["128"], - "--speculative-eagle-topk": ["1"], - "--speculative-num-draft-tokens": ["8"], - "--speculative-num-steps": ["1"], - "--chunked-prefill-size": null, - "--cuda-graph-backend-prefill": null, - "--cuda-graph-max-bs-decode": null, - "--speculative-draft-kv-cache-dtype": null, - "--tokenizer-worker-num": null - }, - "env": { - "ALLOW_JIT": "1", - "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" - } - }, - "ref": [9, "sglang", 0] - } - ] - }, "dynamo": { "install": [], "files": [], @@ -2074,45 +2458,121 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "ref": [10, "dynamo", 2] + }, + { + "nodes_per_worker": 4, "diff": { "args": { - "--request-plane": ["tcp"] + "--nnodes": ["4"], + "--data-parallel-size": ["16"], + "--expert-parallel-size": ["16"], + "--max-running-requests": ["768"], + "--tensor-parallel-size": ["16"] + } + }, + "ref": [9, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#3", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#2", + "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval unset", - "gpus": 2, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "label": "Disagg 8P1D · prefill DEP2 · decode DEP8 (2 nodes)", + "detail": "DSpark 7", + "gpus": 24, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "note", + "script": "Placement", + "reason": "Workers share nodes: 8 prefill workers on 4 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + } + ], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -2120,192 +2580,429 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "200"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { - "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml", - "run": "40739166138/attempts/1", - "pr": "4524", + "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "41271829525/attempts/1", + "pr": "4698", "status": "verified", - "date": "2026-10-29" + "date": "2026-11-02" }, "points": [ { - "concs": [32], - "item": 3, - "override": "zip_override_tp2_hicache[0]", + "concs": [440, 480], + "item": 2, + "override": "override_8p_dep2_dep8", "goldenAL": "4.04", "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "workers": 8, + "gpu_pinning": [ + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"] + ], + "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1], + "ref": [9, "dynamo", 2] + }, + { + "ref": [9, "dynamo", 3] + } + ] }, - { - "concs": [36], - "item": 3, - "override": "zip_override_tp2_hicache[1]", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "cmd": "python3 -m sglang.launch_server", + "diff": { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } - ], - "dynamo": [ - null, - null, - { + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] - } - } - ] - } - }, - { - "concs": [40], - "item": 3, - "override": "zip_override_tp2_hicache[2]", + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_7_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_6_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg#0", + "hw": "gb200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/agg-variants.yaml", + "run": "41271615328", + "pr": "4695", + "comment": "6576168162", + "status": "verified-pr" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "override_tp4_c1", "goldenAL": "4.04", - "kv": "hicache", + "kv": "none" + }, + { + "concs": [4], + "item": 1, + "override": "override_tp4_c4", + "goldenAL": "4.04", + "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] } } ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + }, + null + ] + } + }, + { + "concs": [8], + "item": 2, + "override": "override_tp4_c8", + "goldenAL": "4.04", + "kv": "none", + "patch": { "dynamo": [ null, null, { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] } } + ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + }, + null ] } }, { - "concs": [44], + "concs": [16], "item": 3, - "override": "zip_override_tp2_hicache_lpm[0]", + "override": "override_tp4_c16", "goldenAL": "4.04", - "kv": "hicache", + "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"], - "--schedule-policy": ["lpm"] + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"], - "--schedule-policy": ["lpm"] + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] } - } + }, + null ] } }, { - "concs": [46], - "item": 3, - "override": "zip_override_tp2_hicache_lpm[1]", + "concs": [32], + "item": 4, + "override": "override_tp4_hicache_c32", "goldenAL": "4.04", "kv": "hicache", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["92"], - "--max-running-requests": ["92"], - "--schedule-policy": ["lpm"] + "--cuda-graph-max-bs-decode": ["64"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.81"] } } ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.81"] + } + }, + null + ] + } + }, + { + "concs": [40], + "item": 5, + "override": "override_tp4_hicache_c40", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { "dynamo": [ null, null, { "args": { - "--cuda-graph-max-bs-decode": ["92"], - "--max-running-requests": ["92"], - "--schedule-policy": ["lpm"] + "--cuda-graph-max-bs-decode": ["80"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["80"], + "--mem-fraction-static": ["0.81"] } } + ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["80"], + "--mem-fraction-static": ["0.81"] + } + }, + null ] } }, { "concs": [48], - "item": 3, - "override": "zip_override_tp2_hicache_lpm[2]", + "item": 6, + "override": "override_tp4_hicache_c48", "goldenAL": "4.04", "kv": "hicache", "patch": { - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["96"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], "--max-running-requests": ["96"], - "--schedule-policy": ["lpm"] + "--mem-fraction-static": ["0.81"] } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["96"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], "--max-running-requests": ["96"], - "--schedule-policy": ["lpm"] + "--mem-fraction-static": ["0.81"] } - } + }, + null ] } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { + }, + { + "concs": [56], + "item": 7, + "override": "override_tp4_hicache_c56", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + { "args": { - "--cuda-graph-max-bs-decode": ["64"], + "--cuda-graph-max-bs-decode": ["112"], "--enable-hierarchical-cache": [], - "--hicache-size": ["370"], + "--hicache-size": ["180"], "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.87"], - "--tensor-parallel-size": ["2"] + "--max-running-requests": ["112"], + "--mem-fraction-static": ["0.81"] + } + } + ], + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["112"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["112"], + "--mem-fraction-static": ["0.81"] } }, - "ref": [9, "sglang", 0] - } - ] - }, + null + ] + } + } + ], + "routers": { "dynamo": { "install": [], "files": [], @@ -2325,40 +3022,98 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--request-plane", "tcp"], + ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], + ["--chunked-prefill-size", "32768"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-max-bs-decode", "2"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm_routed"], + ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 1 + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--request-plane": ["tcp"] + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#2", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#0", + "hw": "gb200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with HiCache", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", "gpus": 4, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -2371,41 +3126,41 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "donorHardware": "gb200" } }, "prov": { - "sha": "51bb400fde2167c9a265ec9d7e19f61a61d7b7ee", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", - "run": "40739166138/attempts/1", - "pr": "4524", + "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "40739138596/attempts/1", + "pr": "4599", "status": "verified", "date": "2026-10-29" }, "points": [ { - "concs": [32], - "item": 2, - "override": "override_tp4_dram_c32", + "concs": [1], + "item": 0, + "override": "override_tp4_c1", "goldenAL": "4.04", - "kv": "hicache" + "kv": "none" }, { - "concs": [64], - "item": 2, - "override": "override_tp4_dram_c64", + "concs": [4], + "item": 0, + "override": "override_tp4_c4", "goldenAL": "4.04", - "kv": "hicache", + "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] } } ], @@ -2414,25 +3169,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] } } ] } }, { - "concs": [80], - "item": 2, - "override": "override_tp4_dram_c80", + "concs": [8], + "item": 0, + "override": "override_tp4_c8", "goldenAL": "4.04", - "kv": "hicache", + "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] } } ], @@ -2441,25 +3196,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] } } ] } }, { - "concs": [96], - "item": 2, - "override": "override_tp4_dram_c96", + "concs": [16], + "item": 0, + "override": "override_tp4_c16", "goldenAL": "4.04", - "kv": "hicache", + "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] } } ], @@ -2468,8 +3223,35 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "override_tp4_c32", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] } } ] @@ -2484,11 +3266,10 @@ export const agentx = { { "diff": { "args": { - "--hicache-size": ["290"], - "--tensor-parallel-size": ["4"] + "--tokenizer-worker-num": ["8"] } }, - "ref": [11, "sglang", 0] + "ref": [13, "sglang", 0] } ] }, @@ -2538,13 +3319,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#4", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#1", + "hw": "gb200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 16", - "gpus": 2, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -2557,42 +3338,160 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "donorHardware": "gb200" } }, "prov": { - "sha": "b623efbb700ba428206bda393658559d3c7dc631", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml", - "run": "40895671658/attempts/1", - "pr": "4647", + "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic-low-latency.yaml", + "run": "40739138596/attempts/1", + "pr": "4599", "status": "verified", "date": "2026-10-29" }, "points": [ { - "concs": [40], - "item": 4, - "override": "zip_override_tp2_hicache_spf[0]", - "goldenAL": "4.04", - "kv": "hicache" - }, - { - "concs": [44], - "item": 5, - "override": "zip_override_tp2_hicache_spf[1]", + "concs": [1], + "item": 1, + "override": "override_tp4_c1", "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { + "kv": "none" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"] - } + "--disable-attn-tp-gather": [], + "--max-running-requests": ["8"], + "--monokernel-backend": ["msa"], + "--page-size": ["128"], + "--speculative-eagle-topk": ["1"], + "--speculative-num-draft-tokens": ["8"], + "--speculative-num-steps": ["1"], + "--chunked-prefill-size": null, + "--cuda-graph-backend-prefill": null, + "--cuda-graph-max-bs-decode": null, + "--speculative-draft-kv-cache-dtype": null + }, + "env": { + "ALLOW_JIT": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" + } + }, + "ref": [13, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#2", + "hw": "gb200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with HiCache", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "40739138596/attempts/1", + "pr": "4599", + "status": "verified", + "date": "2026-10-29" + }, + "points": [ + { + "concs": [32], + "item": 2, + "override": "override_tp4_dram_c32", + "goldenAL": "4.04", + "kv": "hicache" + }, + { + "concs": [40], + "item": 2, + "override": "override_tp4_dram_c40", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } } ], "dynamo": [ @@ -2600,8 +3499,8 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"] + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] } } ] @@ -2609,8 +3508,8 @@ export const agentx = { }, { "concs": [48], - "item": 6, - "override": "zip_override_tp2_hicache_spf[2]", + "item": 2, + "override": "override_tp4_dram_c48", "goldenAL": "4.04", "kv": "hicache", "patch": { @@ -2635,17 +3534,17 @@ export const agentx = { } }, { - "concs": [52], - "item": 7, - "override": "zip_override_tp2_hicache_spf[3]", + "concs": [56], + "item": 2, + "override": "override_tp4_dram_c56", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["104"], - "--max-running-requests": ["104"] + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] } } ], @@ -2654,25 +3553,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["104"], - "--max-running-requests": ["104"] + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] } } ] } }, { - "concs": [56], - "item": 8, - "override": "zip_override_tp2_hicache_spf[4]", + "concs": [64], + "item": 2, + "override": "override_tp4_dram_c64", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ], @@ -2681,8 +3580,8 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ] @@ -2690,26 +3589,466 @@ export const agentx = { } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"], - "--mem-fraction-static": ["0.92"], - "--prefill-decode-interval": ["16"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"] - } - }, - "ref": [11, "sglang", 0] - } - ] - }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.81"] + } + }, + "ref": [14, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark#3", + "hw": "gb200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4·EP4", + "detail": "DSpark 7", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb200-sglang-agentic-dspark", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "c350a66f91b3a8b631b789fc1a1569514b336b7d", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "40739138596/attempts/1", + "pr": "4599", + "status": "verified", + "date": "2026-10-29" + }, + "points": [ + { + "concs": [72], + "item": 3, + "override": "override_tp4_ep4_dram_c72", + "goldenAL": "4.04", + "kv": "hicache" + }, + { + "concs": [80], + "item": 3, + "override": "override_tp4_ep4_dram_c80", + "goldenAL": "4.04", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["128"], + "--cuda-graph-max-bs-prefill": ["16384"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.8"], + "--moe-runner-backend": null + }, + "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" + } + }, + "ref": [16, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#0", + "hw": "gb200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Disagg 2P1D · prefill DEP4 · decode DEP8 (2 nodes)", + "detail": "DSpark 7", + "gpus": 16, + "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml", + "run": "41271615328", + "pr": "4695", + "comment": "6576168162", + "status": "verified-pr" + }, + "points": [ + { + "concs": [224, 256], + "item": 0, + "override": "override_2p_dep4_dep8", + "goldenAL": "4.04", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "kind": "worker", + "role": "prefill", + "workers": 2, + "nodes_per_worker": 1, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], + ["--chunked-prefill-size", "32768"], + ["--data-parallel-size", "4"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-dp-attention"], + ["--enable-dp-lm-head"], + ["--enable-hierarchical-cache"], + ["--expert-parallel-size", "4"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"], + ["--max-running-requests", "256"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "megamoe"], + ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--schedule-policy", "shortest-prefill-first"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 2 + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 2, + "cmd": "python3 -m dynamo.sglang", + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--request-plane", "tcp"], + ["--cuda-graph-max-bs-decode", "48"], + ["--data-parallel-size", "8"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--enable-dp-attention"], + ["--enable-dp-lm-head"], + ["--expert-parallel-size", "8"], + ["--load-balance-method", "total_tokens"], + ["--max-running-requests", "384"], + ["--mem-fraction-static", "0.9"], + ["--moe-a2a-backend", "megamoe"], + ["--reasoning-parser", "minimax-m3"], + ["--router-mode", "least-loaded"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "8"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ], + "envRef": 3 + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#1", + "hw": "gb200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Disagg 3P1D · prefill DEP4 · decode DEP8 (2 nodes)", + "detail": "DSpark 7", + "gpus": 20, + "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml", + "run": "41271615328", + "pr": "4695", + "comment": "6576168162", + "status": "verified-pr" + }, + "points": [ + { + "concs": [256, 288], + "item": 1, + "override": "override_3p_dep4_dep8", + "goldenAL": "4.04", + "kv": "hicache" + } + ], + "routers": { "dynamo": { "install": [], "files": [], @@ -2729,45 +4068,108 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "workers": 3, + "ref": [18, "dynamo", 2] + }, + { + "ref": [18, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--request-plane": ["tcp"] + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#9", - "hw": "b300", + "id": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg#2", + "hw": "gb200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 32", - "gpus": 2, - "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "label": "Disagg 3P1D · prefill DEP4 · decode DEP16 (4 nodes)", + "detail": "DSpark 7", + "gpus": 28, + "configKey": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [ + { + "kind": "setup", + "command": "pip install torchao", + "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." + } + ], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -2775,47 +4177,31 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "370"], + ["--hicache-size", "180"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donor": "minimaxm31flash-fp4-gb200-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb200" } }, "prov": { - "sha": "b623efbb700ba428206bda393658559d3c7dc631", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml", - "run": "40895671658/attempts/1", - "pr": "4647", - "status": "verified", - "date": "2026-10-29" + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4/agentx/disagg-variants.yaml", + "run": "41271615328", + "pr": "4695", + "comment": "6576168162", + "status": "verified-pr" }, "points": [ { - "concs": [48], - "item": 9, - "override": "override_tp2_c48_spf_interval32", + "concs": [280, 320, 360], + "item": 2, + "override": "override_3p_dep4_dep16", "goldenAL": "4.04", "kv": "hicache" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"], - "--prefill-decode-interval": ["32"] - } - }, - "ref": [13, "sglang", 0] - } - ] - }, "dynamo": { "install": [], "files": [], @@ -2835,40 +4221,107 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "ref": [19, "dynamo", 2] + }, + { + "nodes_per_worker": 4, "diff": { "args": { - "--request-plane": ["tcp"] + "--nnodes": ["4"], + "--data-parallel-size": ["16"], + "--expert-parallel-size": ["16"], + "--max-running-requests": ["768"], + "--mem-fraction-static": ["0.85"], + "--tensor-parallel-size": ["16"] + } + }, + "ref": [18, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] } ] } } }, { - "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#0", - "hw": "b200", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#0", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", "gpus": 4, - "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -2881,20 +4334,19 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "210"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "40739120064/attempts/1", - "pr": "4598", - "status": "verified", - "date": "2026-10-28" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { @@ -3100,13 +4552,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#1", - "hw": "b200", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#2", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", "gpus": 4, - "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -3119,25 +4571,24 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "210"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic-low-latency.yaml", - "run": "40739120064/attempts/1", - "pr": "4598", - "status": "verified", - "date": "2026-10-28" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-low-latency.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { "concs": [1], - "item": 1, + "item": 2, "override": "override_tp4_c1", "goldenAL": "4.04", "kv": "none" @@ -3169,7 +4620,7 @@ export const agentx = { "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" } }, - "ref": [15, "sglang", 0] + "ref": [21, "sglang", 0] } ] }, @@ -3219,13 +4670,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#2", - "hw": "b200", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#4", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with HiCache", + "detail": "DSpark 7 · measured with HiCache · --mem-fraction-static 0.9", "gpus": 4, - "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -3238,33 +4689,144 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "210"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "40739120064/attempts/1", - "pr": "4598", - "status": "verified", - "date": "2026-10-28" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "pr": "4726", + "status": "in-progress" }, "points": [ + { + "concs": [8], + "item": 7, + "override": "zip_override_tp4_dram_spf_interval32[3]", + "goldenAL": "4.04", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml" + }, + { + "concs": [16], + "item": 8, + "override": "zip_override_tp4_dram_spf_interval32[4]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml" + }, + { + "concs": [20], + "item": 9, + "override": "zip_override_tp4_dram_spf_interval32[0]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["40"], + "--max-running-requests": ["40"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["40"], + "--max-running-requests": ["40"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml" + }, + { + "concs": [24], + "item": 4, + "override": "zip_override_tp4_dram_spf_interval32[0]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml" + }, { "concs": [32], - "item": 2, - "override": "override_tp4_dram_c32", + "item": 5, + "override": "zip_override_tp4_dram_spf_interval32[1]", "goldenAL": "4.04", - "kv": "hicache" + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml" }, { "concs": [40], - "item": 2, - "override": "override_tp4_dram_c40", + "item": 6, + "override": "zip_override_tp4_dram_spf_interval32[2]", "goldenAL": "4.04", "kv": "hicache", "patch": { @@ -3286,20 +4848,21 @@ export const agentx = { } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml" }, { - "concs": [48], - "item": 2, - "override": "override_tp4_dram_c48", + "concs": [44], + "item": 10, + "override": "zip_override_tp4_dram_spf_interval32[1]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } } ], @@ -3308,25 +4871,26 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml" }, { - "concs": [56], - "item": 2, - "override": "override_tp4_dram_c56", + "concs": [52], + "item": 11, + "override": "zip_override_tp4_dram_spf_interval32[2]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"] } } ], @@ -3335,25 +4899,26 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["112"], - "--max-running-requests": ["112"] + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"] } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml" }, { - "concs": [64], - "item": 2, - "override": "override_tp4_dram_c64", + "concs": [60], + "item": 12, + "override": "zip_override_tp4_dram_spf_interval32[3]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["120"], + "--max-running-requests": ["120"] } } ], @@ -3362,12 +4927,13 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--max-running-requests": ["128"] + "--cuda-graph-max-bs-decode": ["120"], + "--max-running-requests": ["120"] } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml" } ], "routers": { @@ -3378,15 +4944,19 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["64"], + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["16"], "--enable-hierarchical-cache": [], - "--hicache-size": ["210"], + "--hicache-size": ["290"], "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.81"] + "--max-running-requests": ["16"], + "--mem-fraction-static": ["0.9"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"] } }, - "ref": [15, "sglang", 0] + "ref": [21, "sglang", 0] } ] }, @@ -3436,13 +5006,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#3", - "hw": "b200", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#1", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4·EP4", - "detail": "DSpark 7", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 8192", "gpus": 4, - "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -3455,57 +5025,45 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "210"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "40739120064/attempts/1", - "pr": "4598", - "status": "verified", - "date": "2026-10-28" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [72], - "item": 3, - "override": "override_tp4_ep4_dram_c72", - "goldenAL": "4.04", - "kv": "hicache" - }, - { - "concs": [80], - "item": 3, - "override": "override_tp4_ep4_dram_c80", + "concs": [16], + "item": 1, + "override": "override_tp4_c16_spf_interval32", "goldenAL": "4.04", - "kv": "hicache" + "kv": "none" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--cuda-graph-max-bs-decode": ["128"], - "--cuda-graph-max-bs-prefill": ["16384"], - "--expert-parallel-size": ["4"], - "--max-running-requests": ["128"], - "--mem-fraction-static": ["0.8"], - "--moe-runner-backend": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"], + "--enable-hierarchical-cache": null, + "--hicache-size": null, + "--hicache-write-policy": null } }, - "ref": [17, "sglang", 0] + "ref": [23, "sglang", 0] } ] }, @@ -3555,13 +5113,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#7", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#19", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", + "label": "Agg TP2", "detail": "DSpark 7 · --prefill-decode-interval 32", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -3574,32 +5132,88 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml", - "run": "41054395691", - "pr": "4676", - "status": "verified-pr" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "pr": "4726", + "status": "in-progress" }, "points": [ + { + "concs": [20], + "item": 20, + "override": "zip_override_tp2_spf_interval32[0]", + "goldenAL": "4.04", + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + }, { "concs": [24], - "item": 7, - "override": "zip_override_tp4_hicache_spf_interval32[0]", + "item": 21, + "override": "zip_override_tp2_spf_interval32[1]", "goldenAL": "4.04", - "kv": "hicache" + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + }, + { + "concs": [28], + "item": 22, + "override": "zip_override_tp2_spf_interval32[2]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["56"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["56"], + "--max-running-requests": ["56"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" }, { "concs": [32], - "item": 8, - "override": "zip_override_tp4_hicache_spf_interval32[1]", + "item": 23, + "override": "zip_override_tp2_spf_interval32[3]", "goldenAL": "4.04", "kv": "hicache", "patch": { @@ -3621,12 +5235,41 @@ export const agentx = { } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + }, + { + "concs": [36], + "item": 24, + "override": "zip_override_tp2_spf_interval32[4]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" }, { "concs": [40], - "item": 9, - "override": "zip_override_tp4_hicache_spf_interval32[2]", + "item": 25, + "override": "zip_override_tp2_spf_interval32[5]", "goldenAL": "4.04", "kv": "hicache", "patch": { @@ -3648,7 +5291,64 @@ export const agentx = { } } ] - } + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + }, + { + "concs": [44], + "item": 28, + "override": "override_tp2_c44_spf_interval32", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml" + }, + { + "concs": [48], + "item": 19, + "override": "override_tp2_c48_spf_interval32", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml" } ], "routers": { @@ -3659,15 +5359,14 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"] + "--cuda-graph-max-bs-decode": ["40"], + "--hicache-size": ["370"], + "--max-running-requests": ["40"], + "--mem-fraction-static": ["0.92"], + "--tensor-parallel-size": ["2"] } }, - "ref": [4, "sglang", 0] + "ref": [23, "sglang", 0] } ] }, @@ -3717,13 +5416,13 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#10", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#13", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 32", + "detail": "DSpark 7 · --prefill-decode-interval unset", "gpus": 2, - "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", "imageCommit": null, @@ -3736,40 +5435,40 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-spf.yaml", - "run": "41054395691", - "pr": "4676", - "status": "verified-pr" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [20], - "item": 10, - "override": "zip_override_tp2_hicache_spf_interval32[0]", + "concs": [32], + "item": 13, + "override": "zip_override_tp2_hicache[0]", "goldenAL": "4.04", "kv": "hicache" }, { - "concs": [24], - "item": 11, - "override": "zip_override_tp2_hicache_spf_interval32[1]", + "concs": [36], + "item": 13, + "override": "zip_override_tp2_hicache[1]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] } } ], @@ -3778,25 +5477,53 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] + "--cuda-graph-max-bs-decode": ["72"], + "--max-running-requests": ["72"] } } ] } }, { - "concs": [28], - "item": 12, - "override": "zip_override_tp2_hicache_spf_interval32[2]", + "concs": [40], + "item": 13, + "override": "zip_override_tp2_hicache[2]", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ] + } + }, + { + "concs": [44], + "item": 13, + "override": "zip_override_tp2_hicache_lpm[0]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["56"], - "--max-running-requests": ["56"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"], + "--schedule-policy": ["lpm"] } } ], @@ -3805,25 +5532,27 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["56"], - "--max-running-requests": ["56"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"], + "--schedule-policy": ["lpm"] } } ] } }, { - "concs": [36], + "concs": [46], "item": 13, - "override": "zip_override_tp2_hicache_spf_interval32[4]", + "override": "zip_override_tp2_hicache_lpm[1]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] + "--cuda-graph-max-bs-decode": ["92"], + "--max-running-requests": ["92"], + "--schedule-policy": ["lpm"] } } ], @@ -3832,25 +5561,27 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] + "--cuda-graph-max-bs-decode": ["92"], + "--max-running-requests": ["92"], + "--schedule-policy": ["lpm"] } } ] } }, { - "concs": [40], - "item": 14, - "override": "zip_override_tp2_hicache_spf_interval32[5]", + "concs": [48], + "item": 13, + "override": "zip_override_tp2_hicache_lpm[2]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"], + "--schedule-policy": ["lpm"] } } ], @@ -3859,8 +5590,9 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"], + "--schedule-policy": ["lpm"] } } ] @@ -3875,13 +5607,16 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["8192"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"] + "--chunked-prefill-size": ["32768"], + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.87"], + "--prefill-decode-interval": null, + "--schedule-policy": null, + "--stream-interval": null } }, - "ref": [2, "sglang", 0] + "ref": [25, "sglang", 0] } ] }, @@ -3931,11 +5666,11 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#6", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#3", "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · --prefill-decode-interval 32", + "detail": "DSpark 7 · measured with HiCache · --mem-fraction-static 0.87", "gpus": 4, "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -3954,63 +5689,36 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf.yaml", - "run": "41085412898", - "pr": "4675", - "status": "verified-pr" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [8], - "item": 6, - "override": "zip_override_tp4_dram_spf_interval32[3]", + "concs": [32], + "item": 3, + "override": "override_tp4_dram_c32", "goldenAL": "4.04", "kv": "hicache" }, { - "concs": [16], - "item": 7, - "override": "zip_override_tp4_dram_spf_interval32[4]", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ] - } - }, - { - "concs": [24], + "concs": [64], "item": 3, - "override": "zip_override_tp4_dram_spf_interval32[0]", + "override": "override_tp4_dram_c64", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ], @@ -4019,25 +5727,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ] } }, { - "concs": [32], - "item": 4, - "override": "zip_override_tp4_dram_spf_interval32[1]", + "concs": [80], + "item": 3, + "override": "override_tp4_dram_c80", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ], @@ -4046,25 +5754,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ] } }, { - "concs": [40], - "item": 5, - "override": "zip_override_tp4_dram_spf_interval32[2]", + "concs": [96], + "item": 3, + "override": "override_tp4_dram_c96", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ], @@ -4073,8 +5781,8 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } ] @@ -4089,15 +5797,11 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["16"], "--hicache-size": ["290"], - "--max-running-requests": ["16"], - "--mem-fraction-static": ["0.90"], - "--prefill-decode-interval": ["32"], "--tensor-parallel-size": ["4"] } }, - "ref": [13, "sglang", 0] + "ref": [26, "sglang", 0] } ] }, @@ -4147,11 +5851,11 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#15", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#14", "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 20–32", + "detail": "DSpark 7 · --prefill-decode-interval 16", "gpus": 2, "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -4170,117 +5874,36 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300" + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "35dc1a5b48eab58438e3c6c85faf084f7ce12d6e", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml", - "run": "41085412898", - "pr": "4675", - "status": "verified-pr" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [20], - "item": 15, - "override": "zip_override_tp2_spf_interval32[0]", + "concs": [40], + "item": 14, + "override": "zip_override_tp2_hicache_spf[0]", "goldenAL": "4.04", "kv": "hicache" }, { - "concs": [24], - "item": 16, - "override": "zip_override_tp2_spf_interval32[1]", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"] - } - } - ] - } - }, - { - "concs": [28], - "item": 17, - "override": "zip_override_tp2_spf_interval32[2]", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["56"], - "--max-running-requests": ["56"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["56"], - "--max-running-requests": ["56"] - } - } - ] - } - }, - { - "concs": [32], - "item": 18, - "override": "zip_override_tp2_spf_interval32[3]", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--max-running-requests": ["64"] - } - } - ] - } - }, - { - "concs": [36], - "item": 19, - "override": "zip_override_tp2_spf_interval32[4]", + "concs": [44], + "item": 15, + "override": "zip_override_tp2_hicache_spf[1]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } } ], @@ -4289,25 +5912,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["72"], - "--max-running-requests": ["72"] + "--cuda-graph-max-bs-decode": ["88"], + "--max-running-requests": ["88"] } } ] } }, { - "concs": [40], - "item": 20, - "override": "zip_override_tp2_spf_interval32[5]", + "concs": [48], + "item": 16, + "override": "zip_override_tp2_hicache_spf[2]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] } } ], @@ -4316,26 +5939,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"] + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] } } ] } }, - { - "concs": [48], - "item": 21, - "override": "override_tp2_c48_spf_interval24", + { + "concs": [52], + "item": 17, + "override": "zip_override_tp2_hicache_spf[3]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"], - "--prefill-decode-interval": ["24"] + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"] } } ], @@ -4344,27 +5966,25 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["96"], - "--max-running-requests": ["96"], - "--prefill-decode-interval": ["24"] + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"] } } ] } }, { - "concs": [50], - "item": 22, - "override": "override_tp2_c50_spf_interval20", + "concs": [56], + "item": 18, + "override": "zip_override_tp2_hicache_spf[4]", "goldenAL": "4.04", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["100"], - "--max-running-requests": ["100"], - "--prefill-decode-interval": ["20"] + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] } } ], @@ -4373,9 +5993,8 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["100"], - "--max-running-requests": ["100"], - "--prefill-decode-interval": ["20"] + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] } } ] @@ -4390,12 +6009,12 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["40"], - "--max-running-requests": ["40"], - "--prefill-decode-interval": ["32"] + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"], + "--prefill-decode-interval": ["16"] } }, - "ref": [13, "sglang", 0] + "ref": [25, "sglang", 0] } ] }, @@ -4445,24 +6064,18 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#26", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)", - "detail": "DSpark 7 · shortest-prefill-first", - "gpus": 20, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "label": "Agg TP2", + "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.92", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "Placement", - "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." - } - ], + "imageCommit": null, + "submitted": "sglang", + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -4470,31 +6083,46 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "200"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [280, 320, 340, 360, 400], - "item": 0, - "override": "override_6p_dep2_dep8", + "concs": [48], + "item": 26, + "override": "override_tp2_c48_spf_interval24", "goldenAL": "4.04", "kv": "hicache" } ], "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"], + "--prefill-decode-interval": ["24"] + } + }, + "ref": [25, "sglang", 0] + } + ] + }, "dynamo": { "install": [], "files": [], @@ -4515,245 +6143,44 @@ export const agentx = { "count": 1, "cmd": "python3 -m dynamo.frontend", "env": { - "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", - "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", - "DYN_ROUTER_QUEUE_THRESHOLD": "None", - "DYN_ROUTER_TEMPERATURE": "0", - "DYN_TCP_CONNECT_TIMEOUT": "120", - "DYN_TOKENIZER": "fastokens", - "DYN_TOKENIZER_CACHE": "1", - "DYN_TOKENIZER_CACHE_BYTES": "8000000000", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "14400"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"] - ] - }, - { - "kind": "worker", - "role": "prefill", - "workers": 6, - "nodes_per_worker": 1, - "gpu_pinning": [ - ["0,1"], - ["2,3"], - ["0,1"], - ["2,3"], - ["0,1"], - ["2,3"] - ], - "local_ranks": [0, 1, 0, 1, 0, 1], - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], - "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", - "DYN_EVENT_PLANE": "zmq", - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "FMHA_SM100_ALLOW_JIT": "1", - "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", - "MC_FORCE_MNNVL": "1", - "MC_TCP_BIND_ADDRESS": "$NODE_IP", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "WITH_NVIDIA_PEERMEM": "0" - }, - "args": [ - ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "$PORT"], - ["--disaggregation-mode", "prefill"], - ["--disaggregation-bootstrap-port", "$BOOTSTRAP_PORT"], - ["--request-plane", "tcp"], - ["--kv-events-config", "$KV_EVENTS_CONFIG"], - ["--chunked-prefill-size", "16384"], - ["--data-parallel-size", "2"], - ["--disaggregation-transfer-backend", "mooncake"], - ["--enable-dp-attention"], - ["--enable-dp-lm-head"], - ["--enable-hierarchical-cache"], - ["--expert-parallel-size", "2"], - ["--hicache-size", "200"], - ["--hicache-write-policy", "write_back"], - ["--max-running-requests", "128"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "megamoe"], - ["--reasoning-parser", "minimax-m3"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "14400"], - ["--schedule-policy", "shortest-prefill-first"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--speculative-dspark-block-size", "7"], - ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "2"], - ["--tool-call-parser", "minimax-m3"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ] - }, - { - "kind": "worker", - "role": "decode", - "workers": 1, - "nodes_per_worker": 2, - "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", - "DYN_EVENT_PLANE": "zmq", - "DYN_REQUEST_PLANE": "tcp", - "DYN_SGL_GATEWAY_WORKERS": "8", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "FMHA_SM100_ALLOW_JIT": "1", - "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", - "MC_FORCE_MNNVL": "1", - "MC_TCP_BIND_ADDRESS": "$NODE_IP", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "WITH_NVIDIA_PEERMEM": "0" - }, - "args": [ - ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--disaggregation-mode", "decode"], - ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], - ["--nnodes", "2"], - ["--node-rank", "$NODE_RANK"], - ["--request-plane", "tcp"], - ["--cuda-graph-max-bs-decode", "48"], - ["--data-parallel-size", "8"], - ["--disaggregation-transfer-backend", "mooncake"], - ["--enable-dp-attention"], - ["--enable-dp-lm-head"], - ["--expert-parallel-size", "8"], - ["--load-balance-method", "total_tokens"], - ["--max-running-requests", "384"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "megamoe"], - ["--reasoning-parser", "minimax-m3"], - ["--router-mode", "least-loaded"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--speculative-dspark-block-size", "7"], - ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "8"], - ["--tool-call-parser", "minimax-m3"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ] - } - ] - }, - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "from": 2, - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--skip-server-warmup": [], - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null - } - } - }, - { - "from": 3, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--skip-server-warmup": [], - "--request-plane": null, - "--router-mode": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "DYN_SGL_GATEWAY_WORKERS": null - } - } - }, - { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, - "args": [ - ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], - ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], - ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], - ["--pd-disaggregation"], - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--policy", "cache_aware"] + ["--router-mode", "kv"] ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#30", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)", - "detail": "DSpark 7 · shortest-prefill-first", - "gpus": 28, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "label": "Agg TP2", + "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.93", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "Placement", - "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." - } - ], + "imageCommit": null, + "submitted": "sglang", + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -4761,112 +6188,102 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "200"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [300, 360, 400], - "item": 3, - "override": "override_6p_dep2_dep16", + "concs": [48], + "item": 30, + "override": "override_tp2_c48_spf_interval24_mem093", "goldenAL": "4.04", "kv": "hicache" } ], "routers": { - "dynamo": { + "sglang": { "install": [], "files": [], "blocks": [ { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] - }, - { - "ref": [23, "dynamo", 2] - }, - { - "nodes_per_worker": 4, - "ref": [23, "dynamo", 3], "diff": { "args": { - "--nnodes": ["4"], - "--data-parallel-size": ["16"], - "--expert-parallel-size": ["16"], - "--tensor-parallel-size": ["16"], - "--max-running-requests": ["768"] + "--mem-fraction-static": ["0.93"] } - } + }, + "ref": [29, "sglang", 0] } ] }, - "sglang": { + "dynamo": { "install": [], "files": [], "blocks": [ { - "ref": [23, "sglang", 0] + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] }, { - "from": 3, - "cmd": "python3 -m sglang.launch_server", + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--skip-server-warmup": [], - "--request-plane": null, - "--router-mode": null + "--request-plane": ["tcp"] }, "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "DYN_SGL_GATEWAY_WORKERS": null + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - }, - { - "ref": [23, "sglang", 2] } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#2", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#27", + "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)", - "detail": "DSpark 7 · shortest-prefill-first", - "gpus": 16, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "label": "Agg TP2", + "detail": "DSpark 7 · --prefill-decode-interval 20", + "gpus": 2, + "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "Placement", - "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." - } - ], + "imageCommit": null, + "submitted": "sglang", + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { "available": ["none", "hicache", "mooncake"], @@ -4874,135 +6291,133 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "200"], + ["--hicache-size", "370"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300", + "donorRef": "main" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "pr": "4726", + "status": "in-progress" }, "points": [ { - "concs": [224, 256], - "item": 1, - "override": "override_4p_dep2_dep8", + "concs": [50], + "item": 27, + "override": "override_tp2_c50_spf_interval20", "goldenAL": "4.04", - "kv": "hicache" + "kv": "hicache", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + }, + { + "concs": [52], + "item": 29, + "override": "override_tp2_c52_spf_interval20_mem093", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"], + "--mem-fraction-static": ["0.93"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["104"], + "--max-running-requests": ["104"], + "--mem-fraction-static": ["0.93"] + } + } + ] + }, + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml" } ], "routers": { - "dynamo": { + "sglang": { "install": [], "files": [], "blocks": [ { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] - }, - { - "workers": 4, - "gpu_pinning": [ - ["0,1"], - ["2,3"], - ["0,1"], - ["2,3"] - ], - "local_ranks": [0, 1, 0, 1], - "ref": [23, "dynamo", 2] - }, - { - "ref": [23, "dynamo", 3] + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["100"], + "--max-running-requests": ["100"], + "--prefill-decode-interval": ["20"] + } + }, + "ref": [25, "sglang", 0] } ] }, - "sglang": { + "dynamo": { "install": [], "files": [], "blocks": [ { - "from": 2, - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--skip-server-warmup": [], - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null - } - } - }, - { - "from": 3, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--skip-server-warmup": [], - "--request-plane": null, - "--router-mode": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "DYN_SGL_GATEWAY_WORKERS": null - } - } + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] }, { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, "args": [ - ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], - ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], - ["--pd-disaggregation"], - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--policy", "cache_aware"] + ["--http-port", "8000"], + ["--router-mode", "kv"] ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#0", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#0", + "hw": "b200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", + "imageCommit": null, + "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -5011,20 +6426,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "210"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", + "run": "40739120064/attempts/1", + "pr": "4598", + "status": "verified", + "date": "2026-10-28" }, "points": [ { @@ -5036,14 +6451,12 @@ export const agentx = { }, { "concs": [4], - "item": 1, + "item": 0, "override": "override_tp4_c4", "goldenAL": "4.04", "kv": "none", "patch": { - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["8"], @@ -5051,27 +6464,26 @@ export const agentx = { } } ], - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["8"] } - }, - null + } ] } }, { "concs": [8], - "item": 2, + "item": 0, "override": "override_tp4_c8", "goldenAL": "4.04", "kv": "none", "patch": { - "dynamo": [ - null, - null, + "sglang": [ { "args": { "--cuda-graph-max-bs-decode": ["16"], @@ -5079,24 +6491,33 @@ export const agentx = { } } ], - "sglang": [ + "dynamo": [ + null, + null, { "args": { "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["16"] } - }, - null + } ] } }, { "concs": [16], - "item": 3, + "item": 0, "override": "override_tp4_c16", "goldenAL": "4.04", "kv": "none", "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], "dynamo": [ null, null, @@ -5106,125 +6527,254 @@ export const agentx = { "--max-running-requests": ["32"] } } - ], + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "override_tp4_c32", + "goldenAL": "4.04", + "kv": "none", + "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "FMHA_SM100_ALLOW_JIT": "1", + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + }, + "args": [ + ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--chunked-prefill-size", "32768"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-max-bs-decode", "2"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm_routed"], + ["--reasoning-parser", "minimax-m3"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], + ["--speculative-dspark-block-size", "7"], + ["--speculative-use-block-verification"], + ["--tensor-parallel-size", "4"], + ["--tokenizer-worker-num", "8"], + ["--tool-call-parser", "minimax-m3"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#1", + "hw": "b200", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size unset", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "210"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic-low-latency.yaml", + "run": "40739120064/attempts/1", + "pr": "4598", + "status": "verified", + "date": "2026-10-28" + }, + "points": [ + { + "concs": [1], + "item": 1, + "override": "override_tp4_c1", + "goldenAL": "4.04", + "kv": "none" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--disable-attn-tp-gather": [], + "--max-running-requests": ["8"], + "--monokernel-backend": ["msa"], + "--page-size": ["128"], + "--speculative-eagle-topk": ["1"], + "--speculative-num-draft-tokens": ["8"], + "--speculative-num-steps": ["1"], + "--chunked-prefill-size": null, + "--cuda-graph-backend-prefill": null, + "--cuda-graph-max-bs-decode": null, + "--speculative-draft-kv-cache-dtype": null, + "--tokenizer-worker-num": null + }, + "env": { + "ALLOW_JIT": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" } }, - null - ] - } - } - ], - "routers": { + "ref": [32, "sglang", 0] + } + ] + }, "dynamo": { "install": [], "files": [], "blocks": [ { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] }, { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m dynamo.sglang", + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": "10000", - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": "10000", - "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "FMHA_SM100_ALLOW_JIT": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" }, "args": [ - ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--served-model-name", "MiniMaxAI/MiniMax-M3.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--request-plane", "tcp"], - ["--kv-events-config", "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], - ["--chunked-prefill-size", "32768"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-max-bs-decode", "2"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.8"], - ["--moe-runner-backend", "flashinfer_trtllm_routed"], - ["--reasoning-parser", "minimax-m3"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "14400"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], - ["--speculative-dspark-block-size", "7"], - ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "4"], - ["--tool-call-parser", "minimax-m3"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] + ["--http-port", "8000"], + ["--router-mode", "kv"] ] - } - ] - }, - "sglang": { - "install": [], - "files": [], - "blocks": [ + }, { - "from": 2, - "cmd": "python3 -m sglang.launch_server", + "from": 0, + "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null + "--request-plane": ["tcp"] }, "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - }, - { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, - "args": [ - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--policy", "cache_aware"], - ["--worker-urls", "http://127.0.0.1:30000"] - ] } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#1", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#2", + "hw": "b200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · shortest-prefill-first · --prefill-decode-interval 32", + "detail": "DSpark 7 · measured with HiCache", "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", + "imageCommit": null, + "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -5233,209 +6783,215 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "210"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", + "run": "40739120064/attempts/1", + "pr": "4598", + "status": "verified", + "date": "2026-10-28" }, "points": [ { - "concs": [24], - "item": 4, - "override": "override_tp4_hicache_spf_c24", + "concs": [32], + "item": 2, + "override": "override_tp4_dram_c32", "goldenAL": "4.04", "kv": "hicache" - } - ], - "routers": { - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] - }, - { - "ref": [26, "dynamo", 2], - "diff": { + }, + { + "concs": [40], + "item": 2, + "override": "override_tp4_dram_c40", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { "args": { - "--chunked-prefill-size": ["8192"], - "--schedule-policy": ["shortest-prefill-first"], - "--prefill-decode-interval": ["32"], - "--stream-interval": ["20"], - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.87"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"] + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] } } - } - ] + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["80"], + "--max-running-requests": ["80"] + } + } + ] + } }, - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "from": 2, - "cmd": "python3 -m sglang.launch_server", - "diff": { + { + "concs": [48], + "item": 2, + "override": "override_tp4_dram_c48", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ] + } + }, + { + "concs": [56], + "item": 2, + "override": "override_tp4_dram_c56", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["112"], + "--max-running-requests": ["112"] + } + } + ] + } + }, + { + "concs": [64], + "item": 2, + "override": "override_tp4_dram_c64", + "goldenAL": "4.04", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { "args": { - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] } } - }, - { - "ref": [26, "sglang", 1] - } - ] - } - } - }, - { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#2", - "hw": "gb300", - "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP2", - "detail": "DSpark 7 · shortest-prefill-first · --prefill-decode-interval 32", - "gpus": 2, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", - "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" - }, - "points": [ - { - "concs": [40], - "item": 5, - "override": "override_tp2_hicache_spf_c40", - "goldenAL": "4.04", - "kv": "hicache" + ] + } } ], "routers": { - "dynamo": { + "sglang": { "install": [], "files": [], "blocks": [ { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] - }, - { - "ref": [26, "dynamo", 2], "diff": { "args": { - "--tensor-parallel-size": ["2"], - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"], - "--mem-fraction-static": ["0.87"], + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], + "--hicache-size": ["210"], "--hicache-write-policy": ["write_back"], - "--chunked-prefill-size": ["8192"], - "--schedule-policy": ["shortest-prefill-first"], - "--prefill-decode-interval": ["32"], - "--stream-interval": ["20"] + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.81"] } - } + }, + "ref": [32, "sglang", 0] } ] }, - "sglang": { + "dynamo": { "install": [], "files": [], "blocks": [ { - "from": 2, - "cmd": "python3 -m sglang.launch_server", + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null + "--request-plane": ["tcp"] }, "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - }, - { - "ref": [26, "sglang", 1] } ] } } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#3", - "hw": "gb300", + "id": "minimaxm31flash-fp4-b200-sglang-agentic-dspark#3", + "hw": "b200", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP2", + "label": "Agg TP4·EP4", "detail": "DSpark 7", - "gpus": 2, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", "imageStatus": "ok", - "imageCommit": "e3718813", - "submitted": "dynamo", + "imageCommit": null, + "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -5444,111 +7000,100 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], + ["--hicache-size", "210"], ["--hicache-write-policy", "write_back"] ], - "donor": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "donorHardware": "gb300" + "donor": "minimaxm31flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525", - "pr": "4698", - "status": "verified-pr", - "date": "2026-11-01" + "sha": "bb3cf1be81594ddbfadb183ed5159177697cb946", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b200-fp4-mtp/agentic.yaml", + "run": "40739120064/attempts/1", + "pr": "4598", + "status": "verified", + "date": "2026-10-28" }, "points": [ { - "concs": [40], - "item": 6, - "override": "override_tp2_hicache_c40", + "concs": [72], + "item": 3, + "override": "override_tp4_ep4_dram_c72", "goldenAL": "4.04", "kv": "hicache" }, { - "concs": [44], - "item": 7, - "override": "override_tp2_hicache_c44", + "concs": [80], + "item": 3, + "override": "override_tp4_ep4_dram_c80", "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["88"], - "--max-running-requests": ["88"] - } - }, - null - ] - } + "kv": "hicache" } ], "routers": { - "dynamo": { + "sglang": { "install": [], "files": [], "blocks": [ { - "ref": [23, "dynamo", 0] - }, - { - "ref": [23, "dynamo", 1] - }, - { - "ref": [26, "dynamo", 2], "diff": { "args": { - "--tensor-parallel-size": ["2"], - "--cuda-graph-max-bs-decode": ["80"], - "--max-running-requests": ["80"], - "--mem-fraction-static": ["0.87"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"] + "--cuda-graph-max-bs-decode": ["128"], + "--cuda-graph-max-bs-prefill": ["16384"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.8"], + "--moe-runner-backend": null + }, + "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" } - } + }, + "ref": [34, "sglang", 0] } ] }, - "sglang": { + "dynamo": { "install": [], "files": [], "blocks": [ { - "from": 2, - "cmd": "python3 -m sglang.launch_server", + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null + "--request-plane": ["tcp"] }, "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } } - }, - { - "ref": [26, "sglang", 1] } ] } diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx index 7d466caaa4f0..69507c2a9748 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "Qwen3.5", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" + }, + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" + }, + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } @@ -184,10 +199,6 @@ export const agentx = { "nginx.conf#033bddfb": { "name": "nginx.conf", "content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n server $FRONTEND_1_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}" - }, - "router_policy_config.yaml#005cce35": { - "name": "router_policy_config.yaml", - "content": "worker_selection:\n decode: dynamo-two-tier-cost-fn\n instances:\n - name: dynamo-two-tier-cost-fn\n type: dynamo-two-tier-cost-fn\n" } }, "cells": [ @@ -213,7 +224,7 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -405,7 +416,7 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -525,7 +536,7 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -642,7 +653,7 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -757,8 +768,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1084,8 +1095,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1262,8 +1273,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1416,8 +1427,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1573,8 +1584,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1731,8 +1742,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -1890,8 +1901,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -2052,8 +2063,8 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -2195,25 +2206,27 @@ export const agentx = { "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#0", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Agg TP4", - "detail": "MTP 6-1-7", - "gpus": 4, + "label": "Agg TP2", + "detail": "MTP 3-1-4", + "gpus": 2, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", - "imageStatus": "ok", - "imageCommit": null, + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -2221,276 +2234,61 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" }, "points": [ { - "concs": [8, 9], - "item": 1, - "override": "override_tp4_mrr8_c8", - "goldenAL": "4.8", - "kv": "hicache" + "concs": [1], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" }, { - "concs": [12, 18], - "item": 0, - "override": "override_tp4_mrr8", - "goldenAL": "4.8", + "concs": [24], + "item": 1, + "goldenAL": "3.39", "kv": "hicache", - "patch": { - "sglang": [ - { - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - }, - null - ], - "dynamo": [ - null, - null, - { - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - } - ] - } + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" }, { - "concs": [13], - "item": 5, - "override": "override_tp4_mrr1", - "goldenAL": "4.8", + "concs": [32], + "item": 2, + "goldenAL": "3.39", "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1"], - "--cuda-graph-max-bs-decode": ["1"], - "--max-running-requests": ["1"], - "--pp-max-micro-batch-size": ["1"] - }, - "env": { - "SGLANG_EAGER_INPUT_NO_COPY": "1", - "SGLANG_Q35MF_MKA_PUSH": "1", - "SGLANG_Q35MF_MKA_Q1": "1", - "SGLANG_Q35MF_MKA_SKV2": "1", - "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", - "SGLANG_Q35MF_MKD_HEAD": "1", - "SGLANG_Q35MF_MKD_HOP": "1", - "SGLANG_Q35MF_MKD_HOP_EARLY": "1", - "SGLANG_Q35MF_MKD_HOP_PUSH": "1", - "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1", - "SGLANG_Q35MF_MKH_N3B": "1", - "SGLANG_Q35MF_MKH_N3C": "1", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1" - } - }, - null - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1"], - "--cuda-graph-max-bs-decode": ["1"], - "--max-running-requests": ["1"], - "--pp-max-micro-batch-size": ["1"] - }, - "env": { - "SGLANG_EAGER_INPUT_NO_COPY": "1", - "SGLANG_Q35MF_MKA_PUSH": "1", - "SGLANG_Q35MF_MKA_Q1": "1", - "SGLANG_Q35MF_MKA_SKV2": "1", - "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", - "SGLANG_Q35MF_MKD_HEAD": "1", - "SGLANG_Q35MF_MKD_HOP": "1", - "SGLANG_Q35MF_MKD_HOP_EARLY": "1", - "SGLANG_Q35MF_MKD_HOP_PUSH": "1", - "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1", - "SGLANG_Q35MF_MKH_N3B": "1", - "SGLANG_Q35MF_MKH_N3C": "1", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1" - } - } - ] - } + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" }, { - "concs": [16], + "concs": [40], "item": 3, - "override": "override_tp4_mrr2_k64", - "goldenAL": "4.8", + "goldenAL": "3.39", "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--pp-max-micro-batch-size": ["2"], - "--prefill-decode-interval": ["64"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - }, - null - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--pp-max-micro-batch-size": ["2"], - "--prefill-decode-interval": ["64"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - } - ] - } + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" }, { - "concs": [20], + "concs": [48], "item": 4, - "override": "override_tp4_mrr2_k32", - "goldenAL": "4.8", + "goldenAL": "3.39", "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--pp-max-micro-batch-size": ["2"], - "--prefill-decode-interval": ["32"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - }, - null - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--pp-max-micro-batch-size": ["2"], - "--prefill-decode-interval": ["32"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - } - ] - } + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" }, { - "concs": [23], - "item": 2, - "override": "override_tp4_mrr3_k48", - "goldenAL": "4.8", + "concs": [52], + "item": 5, + "goldenAL": "3.39", "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "3"], - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--pp-max-micro-batch-size": ["3"], - "--prefill-decode-interval": ["48"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - }, - null - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "3"], - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--pp-max-micro-batch-size": ["3"], - "--prefill-decode-interval": ["48"] - }, - "env": { - "SGLANG_GDN_CHUNK_H_BV": "16", - "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", - "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8", - "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1", - "SGLANG_Q35MF_MKM_FUSED_M16": "1", - "SGLANG_Q35MF_MKM_FUSED_M21": "1" - } - } - ] - } + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" + }, + { + "concs": [64], + "item": 6, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" } ], "routers": { @@ -2505,131 +2303,50 @@ export const agentx = { "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", "env": { - "FLASHINFER_DISABLE_VERSION_CHECK": "1", - "MC_FORCE_MNNVL": "1", "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "NCCL_NVLS_ENABLE": "0", - "NVSHMEM_REMOTE_TRANSPORT": "none", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "NCCL_NVLS_ENABLE": "1", "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", - "SGLANG_ENABLE_JIT_DEEPGEMM": "true", - "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "1", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", - "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", - "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", - "SGLANG_MOE_NVFP4_DISPATCH": "1", - "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", - "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", - "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1", - "SGLANG_Q35MF_ARRIVE_D2H_RING": "1", - "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1", - "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1", - "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1", - "SGLANG_Q35MF_ATTN_FRONT": "1", - "SGLANG_Q35MF_ATTN_GATE_QUANT": "1", - "SGLANG_Q35MF_ATTN_SPLIT_KV": "1", - "SGLANG_Q35MF_BK_DRAFT_META": "1", - "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1", - "SGLANG_Q35MF_BK_STASH_SCATTER": "1", - "SGLANG_Q35MF_BK_VEC_COPY": "1", - "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_FC_GEMV": "1", - "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1", - "SGLANG_Q35MF_DRAFT_MNNVL_AR": "1", - "SGLANG_Q35MF_DRAFT_MOE_GEMV": "1", - "SGLANG_Q35MF_DRAFT_MOE_ROUTER": "1", - "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1", - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", - "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1", - "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", - "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", - "SGLANG_Q35MF_GDN_CONV_TILE": "1", - "SGLANG_Q35MF_GDN_LAZY_VALID": "1", - "SGLANG_Q35MF_GDN_NORM_FP8": "1", - "SGLANG_Q35MF_GDN_PDL": "1", - "SGLANG_Q35MF_GDN_TILED_VERIFY": "1", - "SGLANG_Q35MF_GDN_VIEWS": "1", - "SGLANG_Q35MF_K4_LOCAL_VERIFY_SAMPLING": "1", - "SGLANG_Q35MF_MKA_FUSED": "1", - "SGLANG_Q35MF_MKD_EXTEND_FC": "1", - "SGLANG_Q35MF_MKD_MOE1": "1", - "SGLANG_Q35MF_MKD_TTABLE": "1", - "SGLANG_Q35MF_MKD_XCHG": "1", - "SGLANG_Q35MF_MKH_HOP": "1", - "SGLANG_Q35MF_MKH_OPROJ": "1", - "SGLANG_Q35MF_MKM_FUSED": "1", - "SGLANG_Q35MF_ROUTE_FUSED": "1", - "SGLANG_Q35MF_ROUTE_SHARED": "1", - "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", - "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", - "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" }, "args": [ ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], ["--host", "0.0.0.0"], ["--port", "30000"], + ["--allow-auto-truncate"], ["--attention-backend", "trtllm_mha"], ["--chunked-prefill-size", "16384"], - ["--context-length", "262144"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8"], - ["--cuda-graph-max-bs-decode", "8"], - ["--cuda-graph-max-bs-prefill", "16384"], + ["--cuda-graph-max-bs", "80"], ["--data-parallel-size", "1"], - ["--disable-attn-tp-gather"], - ["--disable-shared-experts-fusion"], ["--enable-hierarchical-cache"], - ["--enable-linear-replayssm-spec"], ["--enable-symm-mem"], ["--expert-parallel-size", "1"], - ["--flashinfer-allreduce-fusion-backend", "cutedsl"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"], ["--kv-cache-dtype", "fp8_e4m3"], - ["--linear-attn-decode-backend", "triton"], - ["--load-balance-method", "round_robin"], - ["--mamba-max-states-per-path", "3"], - ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-max-states-per-path", "1"], + ["--mamba-scheduler-strategy", "extra_buffer"], ["--mamba-ssm-dtype", "bfloat16"], ["--mamba-track-interval", "8192"], - ["--max-mamba-cache-size", "1536"], + ["--max-mamba-cache-size", "360"], ["--max-prefill-tokens", "16384"], - ["--max-running-requests", "8"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "none"], - ["--moe-dense-tp-size", "4"], + ["--max-running-requests", "72"], + ["--mem-fraction-static", "0.75"], ["--moe-runner-backend", "flashinfer_trtllm"], ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--pp-max-micro-batch-size", "8"], - ["--prefill-max-requests", "1"], ["--quantization", "modelopt_fp4"], + ["--scheduler-recv-interval", "10"], ["--speculative-algorithm", "NEXTN"], ["--speculative-eagle-topk", "1"], - ["--speculative-moe-a2a-backend", "none"], - ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], - ["--speculative-num-draft-tokens", "7"], - ["--speculative-num-steps", "6"], - ["--stream-interval", "20"], - ["--tensor-parallel-size", "4"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"], - ["--weight-loader-prefetch-num-threads", "4"] + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--stream-interval", "50"], + ["--tensor-parallel-size", "2"], + ["--tokenizer-worker-num", "6"], + ["--trust-remote-code"] ] }, { @@ -2639,14 +2356,14 @@ export const agentx = { "args": [ ["--host", "0.0.0.0"], ["--port", "8000"], - ["--worker-urls", "http://127.0.0.1:30000"] + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] ] } ] }, "dynamo": { "install": [], - "files": [], + "files": ["nats.conf#48a4de48"], "blocks": [ { "kind": "etcd", @@ -2659,13 +2376,21 @@ export const agentx = { ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", "env": { "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ ["--http-port", "8000"], @@ -2681,7 +2406,8 @@ export const agentx = { }, "env": { "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" } } } @@ -2690,28 +2416,30 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#6", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#2", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", "label": "Agg TP8 (2 nodes)", "detail": "MTP 6-1-7", "gpus": 8, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "bb5e6198", "submitted": "sglang", "setup": [], - "disclosures": ["synthetic-acceptance"], + "disclosures": [], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", @@ -2719,19 +2447,17 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" }, "points": [ { - "concs": [11, 18], - "item": 6, - "override": "override_tp8_mrr1", - "goldenAL": "4.8", + "concs": [7], + "item": 2, "kv": "hicache" } ], @@ -2741,40 +2467,93 @@ export const agentx = { "files": [], "blocks": [ { + "kind": "worker", + "role": "agg", + "workers": 1, "nodes_per_worker": 2, - "diff": { - "args": { - "--dist-init-addr": ["$AGG_HEAD_IP:20000"], - "--nnodes": ["2"], - "--node-rank": ["$NODE_RANK"], - "--cuda-graph-bs-decode": ["1"], - "--cuda-graph-max-bs-decode": ["1"], - "--max-running-requests": ["1"], - "--moe-dense-tp-size": ["8"], - "--pp-max-micro-batch-size": ["1"], - "--tensor-parallel-size": ["8"] - }, - "env": { - "SGLANG_EAGER_INPUT_NO_COPY": "1", - "SGLANG_Q35MF_EDGE_MULTIHOST_TCP": "1", - "SGLANG_Q35MF_EDGE_SHM_SYNC_MULTIHOST": "1", - "SGLANG_Q35MF_MKA_PUSH": "1", - "SGLANG_Q35MF_MKA_Q1": "1", - "SGLANG_Q35MF_MKA_SKV2": "1", - "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", - "SGLANG_Q35MF_MKD_GEMV": "1", - "SGLANG_Q35MF_MKD_HEAD": "1", - "SGLANG_Q35MF_MKD_HOP": "1", - "SGLANG_Q35MF_MKD_HOP_EARLY": "1", - "SGLANG_Q35MF_MKG_BA_TC": "1", - "SGLANG_Q35MF_MKG_FUSED": "1", - "SGLANG_Q35MF_MKH_N3B": "1", - "SGLANG_Q35MF_MKH_N3C": "1", - "SGLANG_Q35MF_MKH_PF_GRAPH": "1", - "SGLANG_Q35MF_MKM_GDN_FRONT": "1" - } + "cmd": "python3 -m sglang.launch_server", + "env": { + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" }, - "ref": [12, "sglang", 0] + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--dist-init-addr", "$AGG_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "8192"], + ["--context-length", "262144"], + ["--cuda-graph-bs-decode", "1"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-prefill-cuda-graph"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "32"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "1536"], + ["--max-prefill-tokens", "8192"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "8"], + ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--pp-max-micro-batch-size", "1"], + ["--prefill-max-requests", "1"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "8"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] }, { "kind": "sglang-router", @@ -2790,7 +2569,7 @@ export const agentx = { }, "dynamo": { "install": [], - "files": [], + "files": ["nats.conf#48a4de48"], "blocks": [ { "kind": "etcd", @@ -2803,13 +2582,21 @@ export const agentx = { ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", "env": { "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ ["--http-port", "8000"], @@ -2825,7 +2612,8 @@ export const agentx = { }, "env": { "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" } } } @@ -2834,34 +2622,30 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#5", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#0", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 1P1D · prefill TP1 · decode TP2", - "detail": "MTP 6-1-7", - "gpus": 3, + "label": "Disagg 1P1D · prefill TP4 · decode TP4", + "detail": "no spec", + "gpus": 8, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", - "imageStatus": "ok", - "imageCommit": null, + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "Placement", - "reason": "Workers share nodes: 1 prefill worker on 1 node, 1 decode worker on 1 node. The prefill and decode workers share a node. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." - } - ], + "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -2869,26 +2653,38 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" }, "points": [ { - "concs": [20], - "item": 5, - "override": "override_1p1d_tp2_cap10", - "goldenAL": "4.8", - "kv": "hicache" + "concs": [8], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" }, { - "concs": [26], - "item": 8, - "override": "override_1p1d_tp2_cap4", - "goldenAL": "4.8", + "concs": [16], + "item": 1, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" + }, + { + "concs": [32], + "item": 2, + "goldenAL": "3.39", + "kv": "hicache", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" + }, + { + "concs": [64], + "item": 3, + "goldenAL": "3.39", "kv": "hicache", "patch": { "dynamo": [ @@ -2896,19 +2692,10 @@ export const agentx = { null, null, null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"], - "--pp-max-micro-batch-size": ["4"], - "--cuda-graph-bs-decode": null - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["80"] } } ], @@ -2916,28 +2703,18 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"], - "--pp-max-micro-batch-size": ["4"], - "--cuda-graph-bs-decode": null - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["80"] } }, null ] - } + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" }, { - "concs": [36], - "item": 7, - "override": "override_1p1d_tp2_cap5", - "goldenAL": "4.8", + "concs": [96], + "item": 4, + "goldenAL": "3.39", "kv": "hicache", "patch": { "dynamo": [ @@ -2945,19 +2722,10 @@ export const agentx = { null, null, null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["5"], - "--pp-max-micro-batch-size": ["8"], - "--cuda-graph-bs-decode": null - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["128"] } } ], @@ -2965,28 +2733,18 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["5"], - "--pp-max-micro-batch-size": ["8"], - "--cuda-graph-bs-decode": null - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["128"] } }, null ] - } + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" }, { - "concs": [38], - "item": 6, - "override": "override_1p1d_tp2_cap12", - "goldenAL": "4.8", + "concs": [128], + "item": 5, + "goldenAL": "3.39", "kv": "hicache", "patch": { "dynamo": [ @@ -2994,19 +2752,10 @@ export const agentx = { null, null, null, + null, { "args": { - "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--pp-max-micro-batch-size": ["12"] - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["160"] } } ], @@ -3014,28 +2763,19 @@ export const agentx = { null, { "args": { - "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--pp-max-micro-batch-size": ["12"] - }, - "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "--cuda-graph-max-bs": ["160"] } }, null ] - } + }, + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" } ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"], + "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"], "blocks": [ { "kind": "etcd", @@ -3056,24 +2796,27 @@ export const agentx = { "args": [] }, { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ - ["--http-port", "8000"], + ["--http-port", "8180"], ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"], - ["--router-replica-sync"], - ["--router-queue-threshold", "None"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--router-temperature", "1.0"] + ["--active-prefill-tokens-threshold-frac", "None"] ] }, { @@ -3081,9 +2824,6 @@ export const agentx = { "role": "prefill", "workers": 1, "nodes_per_worker": 1, - "gpu_pinning": [ - ["0"] - ], "cmd": "python3 -m dynamo.sglang", "env": { "DYN_REQUEST_PLANE": "tcp", @@ -3093,29 +2833,20 @@ export const agentx = { "NATS_SERVER": "nats://$NATS_IP:4222", "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", - "NCCL_NVLS_ENABLE": "0", - "NVSHMEM_REMOTE_TRANSPORT": "none", - "PYTHONOPTIMIZE": "1", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "NCCL_NVLS_ENABLE": "1", "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024", "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", - "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1", + "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", - "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", - "SGLANG_Q35MF_K4_MROPE_GPU": "1", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", - "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", - "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" }, "args": [ ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], @@ -3126,49 +2857,30 @@ export const agentx = { ["--disaggregation-bootstrap-port", "8998"], ["--request-plane", "tcp"], ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "8192"], - ["--cuda-graph-backend-prefill", "breakable"], - ["--cuda-graph-max-bs-prefill", "8192"], + ["--chunked-prefill-size", "65536"], ["--data-parallel-size", "1"], ["--disable-cuda-graph"], - ["--disable-overlap-schedule"], - ["--disable-shared-experts-fusion"], - ["--dist-timeout", "1800"], ["--enable-hierarchical-cache"], ["--expert-parallel-size", "1"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"], ["--kv-cache-dtype", "fp8_e4m3"], - ["--linear-attn-decode-backend", "triton"], + ["--linear-attn-decode-backend", "flashinfer"], ["--load-balance-method", "round_robin"], ["--mamba-max-states-per-path", "3"], - ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-scheduler-strategy", "extra_buffer"], ["--mamba-ssm-dtype", "bfloat16"], - ["--mamba-track-interval", "1048576"], - ["--max-mamba-cache-size", "128"], - ["--max-prefill-tokens", "8192"], - ["--max-running-requests", "16"], - ["--mem-fraction-static", "0.94"], - ["--moe-a2a-backend", "none"], - ["--moe-dense-tp-size", "1"], - ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--mamba-track-interval", "2048"], + ["--max-mamba-cache-size", "1100"], + ["--mem-fraction-static", "0.8"], + ["--moe-runner-backend", "flashinfer_trtllm"], ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], ["--quantization", "modelopt_fp4"], - ["--scheduler-recv-interval", "1"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-eagle-topk", "1"], - ["--speculative-moe-a2a-backend", "none"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--speculative-num-draft-tokens", "7"], - ["--speculative-num-steps", "6"], - ["--tensor-parallel-size", "1"], + ["--tensor-parallel-size", "4"], ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"], - ["--weight-loader-prefetch-num-threads", "4"] + ["--watchdog-timeout", "1000000"] ] }, { @@ -3176,151 +2888,77 @@ export const agentx = { "role": "decode", "workers": 1, "nodes_per_worker": 1, - "gpu_pinning": [ - ["1,2"] - ], "cmd": "python3 -m dynamo.sglang", "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", "FLASHINFER_DISABLE_VERSION_CHECK": "1", - "FLASHINFER_ROUTING_FORCE_BLOCK_PER_TOKEN": "0", "MC_FORCE_MNNVL": "1", "NATS_SERVER": "nats://$NATS_IP:4222", "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", - "NCCL_NVLS_ENABLE": "0", - "NVSHMEM_REMOTE_TRANSPORT": "none", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "NCCL_NVLS_ENABLE": "1", "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": "1000", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": "1024", "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", "SGLANG_ENABLE_FLASHINFER_GEMM": "true", "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", "SGLANG_ENABLE_JIT_DEEPGEMM": "true", - "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "0", "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", + "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": "1", "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_HEALTH_STARTING_OK": "1", "SGLANG_MOE_NVFP4_DISPATCH": "1", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", - "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", - "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1", - "SGLANG_Q35MF_ARRIVE_D2H_RING": "1", - "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1", - "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1", - "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1", - "SGLANG_Q35MF_ATTN_FRONT": "1", - "SGLANG_Q35MF_ATTN_GATE_QUANT": "1", - "SGLANG_Q35MF_BK_DRAFT_META": "1", - "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1", - "SGLANG_Q35MF_BK_STASH_SCATTER": "1", - "SGLANG_Q35MF_BK_VEC_COPY": "1", - "SGLANG_Q35MF_CUTEDSL_LL_MAX_M": "96", - "SGLANG_Q35MF_D2R_PVG_SHARDED": "1", - "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1", - "SGLANG_Q35MF_DRAFT_VOCAB_EXCHANGE_MAX_ROWS": "12", - "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1", - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", - "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1", - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "10", - "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", - "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", - "SGLANG_Q35MF_FP8_BMM_TACTICS": "/sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/fp8_bmm_tactics_gb300_tp2.json", - "SGLANG_Q35MF_GDN_CONV_TILE": "1", - "SGLANG_Q35MF_GDN_DIRECT_SOLVE": "0", - "SGLANG_Q35MF_GDN_EMPTY_HISTORY": "0", - "SGLANG_Q35MF_GDN_LAZY_VALID": "1", - "SGLANG_Q35MF_GDN_NORM_FP8": "1", - "SGLANG_Q35MF_GDN_PDL": "1", - "SGLANG_Q35MF_GDN_TILED_VERIFY": "1", - "SGLANG_Q35MF_GDN_VERIFY_BV64": "1", - "SGLANG_Q35MF_GDN_VIEWS": "1", - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "84", - "SGLANG_Q35MF_K4_SIDE_PRIO_MAX_TOKENS": "16", - "SGLANG_Q35MF_KDENSE_DRAFT_ARGMAX_XCHG": "1", - "SGLANG_Q35MF_KDENSE_DRAFT_FC_XCHG": "1", - "SGLANG_Q35MF_KDENSE_DRAFT_META_BRANCH": "1", - "SGLANG_Q35MF_KGDN_VERIFY_REG168": "1", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "84", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MIN_TOKENS": "17", - "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1", - "SGLANG_Q35MF_PDDEC_FENCE": "1", - "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1", - "SGLANG_Q35MF_PDDEC_SHARDED_VERIFY_SAMPLING": "1", - "SGLANG_Q35MF_ROUTE2_EARLY_TRIGGER": "1", - "SGLANG_Q35MF_ROUTE2_SMALL_TABLES": "1", - "SGLANG_Q35MF_ROUTE2_SMALL_TABLES_MIN_TOKENS": "1", - "SGLANG_Q35MF_ROUTE_FUSED": "1", - "SGLANG_Q35MF_ROUTE_PARQ": "1", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "84", - "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "1", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", - "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", - "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" }, "args": [ ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], ["--host", "0.0.0.0"], - ["--port", "30032"], + ["--port", "30000"], ["--disaggregation-mode", "decode"], - ["--disaggregation-bootstrap-port", "8999"], + ["--disaggregation-bootstrap-port", "8998"], ["--request-plane", "tcp"], ["--attention-backend", "trtllm_mha"], ["--chunked-prefill-size", "4096"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], - ["--cuda-graph-max-bs-decode", "10"], + ["--cuda-graph-max-bs", "64"], ["--data-parallel-size", "1"], - ["--disable-attn-tp-gather"], ["--disable-radix-cache"], ["--disable-shared-experts-fusion"], - ["--dist-timeout", "1800"], - ["--enable-linear-replayssm-spec"], + ["--ep-dispatch-algorithm", "static"], + ["--eplb-algorithm", "deepseek"], ["--expert-parallel-size", "1"], - ["--flashinfer-allreduce-fusion-backend", "cutedsl"], ["--kv-cache-dtype", "fp8_e4m3"], - ["--linear-attn-decode-backend", "triton"], - ["--load-balance-method", "round_robin"], - ["--mamba-radix-cache-strategy", "no_buffer"], + ["--linear-attn-decode-backend", "flashinfer"], + ["--mamba-scheduler-strategy", "no_buffer"], ["--mamba-ssm-dtype", "bfloat16"], ["--mamba-track-interval", "128"], - ["--max-mamba-cache-size", "936"], - ["--max-running-requests", "10"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "none"], - ["--moe-dense-tp-size", "2"], + ["--max-mamba-cache-size", "1024"], + ["--max-running-requests", "512"], + ["--mem-fraction-static", "0.75"], ["--moe-runner-backend", "flashinfer_trtllm"], ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--pp-max-micro-batch-size", "10"], + ["--prefill-round-robin-balance"], ["--quantization", "modelopt_fp4"], ["--speculative-algorithm", "NEXTN"], ["--speculative-eagle-topk", "1"], - ["--speculative-moe-a2a-backend", "none"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--speculative-num-draft-tokens", "7"], - ["--speculative-num-steps", "6"], - ["--stream-interval", "30"], - ["--tensor-parallel-size", "2"], + ["--speculative-num-draft-tokens", "4"], + ["--speculative-num-steps", "3"], + ["--tensor-parallel-size", "4"], ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"], - ["--weight-loader-prefetch-num-threads", "4"] + ["--watchdog-timeout", "1000000"] ] } ] @@ -3330,7 +2968,7 @@ export const agentx = { "files": [], "blocks": [ { - "from": 3, + "from": 4, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -3345,7 +2983,7 @@ export const agentx = { } }, { - "from": 4, + "from": 5, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -3364,7 +3002,7 @@ export const agentx = { "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ - ["--decode", "http://$DECODE_0_HEAD_IP:30032"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], ["--pd-disaggregation"], ["--host", "0.0.0.0"], @@ -3377,61 +3015,94 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#9", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#0", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 1P2D · prefill TP1 · decode TP2", - "detail": "MTP 6-1-7", - "gpus": 5, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "label": "Agg TP2", + "detail": "MTP 6-1-7 · --max-running-requests 2", + "gpus": 2, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", "imageStatus": "ok", - "imageCommit": null, - "submitted": "dynamo", - "setup": [ - { - "kind": "note", - "script": "Placement", - "reason": "Workers share nodes: 1 prefill worker on 1 node, 2 decode workers on 1 node. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." - } - ], - "disclosures": ["synthetic-acceptance"], + "imageCommit": "bb5e6198", + "submitted": "sglang", + "setup": [], + "disclosures": [], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" }, "points": [ { - "concs": [48], - "item": 9, - "override": "override_1p2d_tp2_cap3", - "goldenAL": "4.8", + "concs": [44], + "item": 0, "kv": "hicache" } ], "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "nodes_per_worker": 1, + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--hicache-size": ["128"], + "--max-running-requests": ["2"], + "--moe-dense-tp-size": ["2"], + "--moe-runner-backend": ["flashinfer_trtllm"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["0"], + "--prefill-max-requests": ["2"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--tensor-parallel-size": ["2"], + "--dist-init-addr": null, + "--nnodes": null, + "--node-rank": null, + "--disable-prefill-cuda-graph": null + } + }, + "ref": [13, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"], + "install": [], + "files": ["nats.conf#48a4de48"], "blocks": [ { "kind": "etcd", @@ -3451,12 +3122,6 @@ export const agentx = { "env": {}, "args": [] }, - { - "kind": "nginx", - "cmd": "nginx -c nginx.conf -g daemon off;", - "env": {}, - "args": [] - }, { "kind": "dynamo-frontend", "count": 1, @@ -3467,125 +3132,183 @@ export const agentx = { "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ - ["--http-port", "8180"], - ["--router-mode", "kv"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"], - ["--router-replica-sync"], - ["--router-queue-threshold", "None"], - ["--router-temperature", "0.0"], - ["--router-decode-active-request-weight", "8192"] + ["--http-port", "8000"], + ["--router-mode", "kv"] ] }, { - "drop": ["gpu_pinning"], - "ref": [14, "dynamo", 3] - }, - { - "workers": 2, - "gpu_pinning": [ - ["0,1"], - ["2,3"] - ], - "local_ranks": [0, 1], - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "from": 0, + "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--port": ["$PORT"], - "--disaggregation-bootstrap-port": ["$BOOTSTRAP_PORT"], - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--pp-max-micro-batch-size": ["3"], - "--cuda-graph-bs-decode": null + "--request-plane": ["tcp"] }, "env": { - "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", - "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", - "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", - "SGLANG_Q35MF_D2R_PVG_SHARDED": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" } - }, - "ref": [14, "dynamo", 4] + } } ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto#1", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 6-1-7 · --max-running-requests 1", + "gpus": 2, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", + "imageStatus": "ok", + "imageCommit": "bb5e6198", + "submitted": "sglang", + "setup": [], + "disclosures": [], + "kv": { + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" }, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" + }, + "points": [ + { + "concs": [44], + "item": 1, + "kv": "hicache" + } + ], + "routers": { "sglang": { "install": [], "files": [], "blocks": [ { - "from": 4, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--skip-server-warmup": [], - "--request-plane": null - }, - "env": { - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null - } - } - }, - { - "from": 5, - "cmd": "python3 -m sglang.launch_server", "diff": { "args": { - "--skip-server-warmup": [], - "--request-plane": null - }, - "env": { - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null + "--cuda-graph-bs-decode": ["1"], + "--cuda-graph-max-bs-decode": ["1"], + "--hicache-size": ["32"], + "--max-running-requests": ["1"], + "--pp-max-micro-batch-size": ["1"], + "--prefill-max-requests": ["1"] } - } + }, + "ref": [15, "sglang", 0] }, { "kind": "sglang-router", "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ - ["--decode", "http://$DECODE_1_HEAD_IP:30032"], - ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], - ["--pd-disaggregation"], ["--host", "0.0.0.0"], ["--port", "8000"], - ["--policy", "cache_aware"] + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": ["nats.conf#48a4de48"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + } + } } ] } } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#4", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#6", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 2P1D · prefill PP4 · decode DEP16 (4 nodes)", - "detail": "MTP 6-1-7", - "gpus": 24, + "label": "Disagg 1P1D · prefill TP2 · decode TP2", + "detail": "MTP 3-1-4", + "gpus": 4, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", - "imageStatus": "ok", - "imageCommit": null, + "image": "lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1", + "imageStatus": "pruned", + "imageCommit": "433429b1", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -3593,26 +3316,25 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "eb1e4107ac078fa736735066c90b2ef457babab0", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml", + "run": "31042542308/attempts/1", + "pr": "2477", + "status": "verified", + "date": "2026-08-07" }, "points": [ { - "concs": [488], - "item": 4, - "override": "override_2p1d_dep16_cap6", - "goldenAL": "4.8", + "concs": [72], + "item": 6, + "goldenAL": "3.39", "kv": "hicache" } ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/5a638087d82c990d35c69cb8e41c2c2582e9ef9a"], + "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"], "blocks": [ { "kind": "etcd", @@ -3634,164 +3356,53 @@ export const agentx = { }, { "kind": "nginx", - "cmd": "nginx -c nginx.conf -g daemon off;", - "env": {}, - "args": [] - }, - { - "kind": "dynamo-frontend", - "count": 2, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "NATS_SERVER": "nats://$NATS_IP:4222" - }, - "args": [ - ["--http-port", "8180"], - ["--router-mode", "kv"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"], - ["--router-replica-sync"], - ["--router-queue-threshold", "None"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--router-session-affinity-mode", "soft"], - ["--router-temperature", "1.0"], - ["--router-policy-config", "router_policy_config.yaml"] - ] - }, - { - "workers": 2, - "diff": { - "args": { - "--chunked-prefill-size": ["32768"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-ratio": ["0.9"], - "--linear-attn-prefill-backend": ["flashinfer"], - "--max-mamba-cache-size": ["1536"], - "--max-prefill-tokens": ["32768"], - "--max-running-requests": ["128"], - "--mem-fraction-static": ["0.85"], - "--pipeline-parallel-size": ["4"], - "--pp-async-batch-depth": ["1"], - "--cuda-graph-backend-prefill": null, - "--cuda-graph-max-bs-prefill": null - }, - "env": { - "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", - "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": "1", - "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": "1", - "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": "1", - "SGLANG_Q35MF_GDN_PREFILL_SPLIT_CONV": "1", - "SGLANG_Q35MF_GDN_PREFILL_TILES": "1", - "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": "1", - "SGLANG_Q35MF_K4_MROPE_GPU": null - } - }, - "ref": [15, "dynamo", 4] - }, - { - "kind": "worker", - "role": "decode", - "workers": 1, - "nodes_per_worker": 4, - "cmd": "python3 -m dynamo.sglang", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "FLASHINFER_DISABLE_VERSION_CHECK": "1", - "MC_FORCE_MNNVL": "1", - "NATS_SERVER": "nats://$NATS_IP:4222", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "NCCL_NVLS_ENABLE": "0", - "NVSHMEM_REMOTE_TRANSPORT": "none", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", - "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", - "SGLANG_DISAGG_STAGING_BUFFER": "1", - "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", - "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", - "SGLANG_ENABLE_JIT_DEEPGEMM": "true", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", - "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", - "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", - "SGLANG_MOE_NVFP4_DISPATCH": "1", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", - "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": "1", - "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": "2", - "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1", - "SGLANG_Q35MF_PDDEC_FENCE": "1", - "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", - "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", - "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", - "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", - "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ - ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], - ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--disaggregation-mode", "decode"], - ["--disaggregation-bootstrap-port", "8998"], - ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], - ["--nnodes", "4"], - ["--node-rank", "$NODE_RANK"], - ["--request-plane", "tcp"], - ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "4096"], - ["--cuda-graph-max-bs-decode", "6"], - ["--data-parallel-size", "16"], - ["--disable-radix-cache"], - ["--disable-shared-experts-fusion"], - ["--dist-timeout", "1800"], - ["--enable-dp-attention"], - ["--enable-linear-replayssm-spec"], - ["--ep-dispatch-algorithm", "static"], - ["--eplb-algorithm", "deepseek"], - ["--expert-parallel-size", "16"], - ["--gc-threshold", "50000", "50", "1000"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--linear-attn-decode-backend", "triton"], - ["--linear-replayssm-cache-len", "32"], - ["--load-balance-method", "round_robin"], - ["--mamba-radix-cache-strategy", "no_buffer"], - ["--mamba-ssm-dtype", "bfloat16"], - ["--mamba-track-interval", "128"], - ["--max-mamba-cache-size", "1872"], - ["--max-running-requests", "96"], - ["--max-total-tokens", "13000000"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "flashinfer"], - ["--moe-dense-tp-size", "1"], - ["--moe-runner-backend", "flashinfer_cutedsl"], - ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--quantization", "modelopt_fp4"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-eagle-topk", "1"], - ["--speculative-moe-a2a-backend", "flashinfer"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm_routed"], - ["--speculative-num-draft-tokens", "7"], - ["--speculative-num-steps", "6"], - ["--stream-interval", "30"], - ["--tensor-parallel-size", "16"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"], - ["--weight-loader-prefetch-num-threads", "4"] + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] ] + }, + { + "diff": { + "args": { + "--mamba-max-states-per-path": ["1"], + "--max-mamba-cache-size": ["320"], + "--speculative-algorithm": ["NEXTN"], + "--speculative-eagle-topk": ["1"], + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [14, "dynamo", 4] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs": ["128"], + "--mamba-max-states-per-path": ["-1"], + "--max-mamba-cache-size": ["80"], + "--max-running-requests": ["80"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [14, "dynamo", 5] } ] }, @@ -3835,7 +3446,6 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], ["--pd-disaggregation"], ["--host", "0.0.0.0"], @@ -3848,28 +3458,30 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#1", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#1", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 3P1D · prefill PP4 · decode DEP16 (4 nodes)", - "detail": "MTP 6-1-7", + "label": "Disagg 3P4D · prefill PP4 · decode DEP4", + "detail": "MTP 4-1-5", "gpus": 28, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "bb5e6198", "submitted": "dynamo", "setup": [], - "disclosures": ["synthetic-acceptance"], + "disclosures": [], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -3877,120 +3489,24 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" }, "points": [ { - "concs": [530], - "item": 3, - "override": "override_3p1d_dep16_cap9", - "goldenAL": "4.8", - "kv": "hicache" - }, - { - "concs": [660], - "item": 2, - "override": "override_3p1d_dep16_cap12_combine_split", - "goldenAL": "4.8", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - null, - { - "env": { - "SGLANG_Q35MF_PPO": null - } - }, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["192"] - }, - "env": { - "SGLANG_Q35MF_K75P_LMHEAD": null - } - } - ], - "sglang": [ - { - "env": { - "SGLANG_Q35MF_PPO": null - } - }, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["192"] - }, - "env": { - "SGLANG_Q35MF_K75P_LMHEAD": null - } - }, - null - ] - } - }, - { - "concs": [760, 840], + "concs": [565], "item": 1, - "override": "override_3p1d_dep16_cap12", - "goldenAL": "4.8", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - null, - { - "env": { - "SGLANG_Q35MF_PPO": null - } - }, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["192"] - }, - "env": { - "SGLANG_Q35MF_K75P_LMHEAD": null, - "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null - } - } - ], - "sglang": [ - { - "env": { - "SGLANG_Q35MF_PPO": null - } - }, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["192"] - }, - "env": { - "SGLANG_Q35MF_K75P_LMHEAD": null, - "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null - } - }, - null - ] - } + "kv": "hicache" } ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], "blocks": [ { "kind": "etcd", @@ -4028,37 +3544,107 @@ export const agentx = { "args": [ ["--http-port", "8180"], ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"], ["--router-replica-sync"], ["--router-queue-threshold", "None"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--router-session-affinity-mode", "soft"], - ["--router-temperature", "1.0"], - ["--router-policy-config", "router_policy_config.yaml"] + ["--router-temperature", "1.0"] ] }, { "workers": 3, "diff": { + "args": { + "--chunked-prefill-size": ["32768"], + "--disable-overlap-schedule": [], + "--disable-shared-experts-fusion": [], + "--linear-attn-decode-backend": ["triton"], + "--mamba-max-states-per-path": ["3"], + "--mamba-radix-cache-strategy": ["extra_buffer"], + "--mamba-track-interval": ["1048576"], + "--max-mamba-cache-size": ["1536"], + "--max-prefill-tokens": ["32768"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.85"], + "--moe-a2a-backend": ["none"], + "--moe-dense-tp-size": ["1"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--pipeline-parallel-size": ["4"], + "--pp-async-batch-depth": ["1"], + "--scheduler-recv-interval": ["1"], + "--speculative-moe-a2a-backend": ["none"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--speculative-num-draft-tokens": ["5"], + "--speculative-num-steps": ["4"], + "--tensor-parallel-size": ["1"], + "--weight-loader-prefetch-checkpoints": [], + "--weight-loader-prefetch-num-threads": ["4"], + "--mamba-scheduler-strategy": null + }, "env": { - "SGLANG_Q35MF_PPO": "1" + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTHONOPTIMIZE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_AUTOTUNE_EXTEND": "1", + "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null, + "SGLANG_ENABLE_SPEC_V2": null } }, - "ref": [16, "dynamo", 4] + "ref": [17, "dynamo", 4] }, { + "workers": 4, "diff": { "args": { - "--cuda-graph-max-bs-decode": ["9"], - "--max-running-requests": ["144"] + "--cuda-graph-max-bs": ["80"], + "--data-parallel-size": ["4"], + "--disaggregation-decode-extra-slots": ["2"], + "--enable-dp-attention": [], + "--enable-dp-lm-head": [], + "--enable-linear-replayssm-spec": [], + "--expert-parallel-size": ["4"], + "--linear-attn-decode-backend": ["triton"], + "--linear-replayssm-cache-len": ["32"], + "--load-balance-method": ["round_robin"], + "--max-mamba-cache-size": ["468"], + "--max-running-requests": ["320"], + "--mem-fraction-static": ["0.7"], + "--moe-a2a-backend": ["flashinfer"], + "--moe-dense-tp-size": ["1"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--pipeline-parallel-size": ["1"], + "--speculative-draft-model-quantization": ["nvfp4_online"], + "--speculative-moe-a2a-backend": ["flashinfer"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm_routed"], + "--speculative-num-draft-tokens": ["5"], + "--speculative-num-steps": ["4"], + "--stream-interval": ["30"], + "--weight-loader-prefetch-checkpoints": [], + "--weight-loader-prefetch-num-threads": ["4"], + "--prefill-round-robin-balance": null }, "env": { - "SGLANG_Q35MF_K75P_LMHEAD": "1" + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", + "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", + "SGLANG_DECODE_BOOTSTRAP_TIMEOUT": null, + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": null, + "SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB": null, + "SGLANG_HACK_SEQ_BOOTSTRAP_ROOM": null, + "SGLANG_HEALTH_STARTING_OK": null } }, - "ref": [16, "dynamo", 5] + "ref": [14, "dynamo", 5] } ] }, @@ -4101,6 +3687,9 @@ export const agentx = { "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ + ["--decode", "http://$DECODE_3_HEAD_IP:30000"], + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], ["--decode", "http://$DECODE_0_HEAD_IP:30000"], ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], @@ -4116,28 +3705,30 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#0", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto#0", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 3P1D · prefill PP4 · decode DEP8 (2 nodes)", - "detail": "MTP 6-1-7", + "label": "Disagg 3P2D · prefill PP4 · decode DEP4", + "detail": "MTP 5-1-6", "gpus": 20, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "bb5e6198", "submitted": "dynamo", "setup": [], - "disclosures": ["synthetic-acceptance"], + "disclosures": [], "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, + "available": ["none", "hicache"], + "unavailable": { + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + }, "hicache": { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-ratio", "12"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "0.9"], ["--hicache-write-policy", "write_back"] ], "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", @@ -4145,26 +3736,24 @@ export const agentx = { } }, "prov": { - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "39355392627", - "pr": "4333", - "comment": "6334247978", - "status": "verified-pr" + "sha": "9330deaffe72e27f32b40a9e62f2ee445f0cdddc", + "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml", + "run": "34345050756/attempts/1", + "pr": "2812", + "status": "verified", + "date": "2026-09-09" }, "points": [ { "concs": [704], "item": 0, - "override": "override_3p1d_dep8_cap32", - "goldenAL": "4.8", "kv": "hicache" } ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/61c37bc33c3e88da3facaa39fa2b5caf616740ba"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], "blocks": [ { "kind": "etcd", @@ -4202,56 +3791,33 @@ export const agentx = { "args": [ ["--http-port", "8180"], ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"], ["--router-replica-sync"], ["--router-queue-threshold", "None"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--router-session-affinity-mode", "soft"], - ["--router-temperature", "1.0"], - ["--router-policy-config", "router_policy_config.yaml"] + ["--router-temperature", "1.0"] ] }, { - "workers": 3, "diff": { "args": { - "--linear-attn-prefill-backend": null - }, - "env": { - "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": null, - "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": null, - "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": null, - "SGLANG_Q35MF_GDN_PREFILL_TILES": null, - "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": null + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] } }, - "ref": [16, "dynamo", 4] + "ref": [18, "dynamo", 4] }, { - "nodes_per_worker": 2, + "workers": 2, "diff": { "args": { - "--nnodes": ["2"], - "--cuda-graph-max-bs-decode": ["32"], - "--data-parallel-size": ["8"], - "--disaggregation-decode-extra-slots": ["64"], - "--enable-dp-lm-head": [], - "--expert-parallel-size": ["8"], - "--max-mamba-cache-size": ["936"], - "--max-running-requests": ["256"], - "--max-total-tokens": ["12000000"], - "--tensor-parallel-size": ["8"] - }, - "env": { - "SGLANG_Q35MF_PDDEC_SPLIT_HOIST": "1", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", - "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": null, - "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] } }, - "ref": [16, "dynamo", 5] + "ref": [18, "dynamo", 5] } ] }, @@ -4294,6 +3860,7 @@ export const agentx = { "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], ["--decode", "http://$DECODE_0_HEAD_IP:30000"], ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], @@ -4666,7 +4233,7 @@ export const agentx = { "--enable-symm-mem": null } }, - "ref": [19, "dynamo", 3] + "ref": [20, "dynamo", 3] } ] }, @@ -4834,7 +4401,7 @@ export const agentx = { "--tokenizer-worker-num": ["1"] } }, - "ref": [20, "dynamo", 3] + "ref": [21, "dynamo", 3] } ] }, @@ -5691,7 +5258,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [22, "sglang", 0] + "ref": [23, "sglang", 0] } ] }, @@ -6297,7 +5864,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [22, "sglang", 0] + "ref": [23, "sglang", 0] } ] }, @@ -6562,7 +6129,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [24, "dynamo", 2] + "ref": [25, "dynamo", 2] }, { "gpu_pinning": [ @@ -6577,7 +6144,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [24, "dynamo", 3] + "ref": [25, "dynamo", 3] } ] }, @@ -6721,7 +6288,7 @@ export const agentx = { "--expert-parallel-size": ["4"] } }, - "ref": [24, "dynamo", 2] + "ref": [25, "dynamo", 2] }, { "diff": { @@ -6729,7 +6296,7 @@ export const agentx = { "--cuda-graph-max-bs-decode": ["32"] } }, - "ref": [24, "dynamo", 3] + "ref": [25, "dynamo", 3] } ] }, @@ -7603,7 +7170,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [28, "sglang", 0] + "ref": [29, "sglang", 0] } ] }, @@ -8116,7 +7683,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [28, "sglang", 0] + "ref": [29, "sglang", 0] } ] }, @@ -8269,7 +7836,7 @@ export const agentx = { "--disable-prefill-cuda-graph": null } }, - "ref": [30, "sglang", 0] + "ref": [31, "sglang", 0] }, { "kind": "sglang-router", @@ -8390,7 +7957,7 @@ export const agentx = { "--speculative-num-steps": ["5"] } }, - "ref": [32, "sglang", 0] + "ref": [33, "sglang", 0] }, { "kind": "sglang-router", @@ -8538,7 +8105,7 @@ export const agentx = { "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2" } }, - "ref": [32, "sglang", 0] + "ref": [33, "sglang", 0] }, { "kind": "sglang-router", @@ -9556,7 +9123,7 @@ export const agentx = { "--tensor-parallel-size": ["8"] } }, - "ref": [35, "sglang", 0] + "ref": [36, "sglang", 0] } ] }, @@ -10218,7 +9785,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [36, "sglang", 0] + "ref": [37, "sglang", 0] } ] }, @@ -11225,7 +10792,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [39, "sglang", 0] + "ref": [40, "sglang", 0] } ] }, @@ -11465,7 +11032,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [40, "sglang", 0] + "ref": [41, "sglang", 0] } ] }, @@ -11745,7 +11312,7 @@ export const agentx = { "--speculative-num-steps": ["5"] } }, - "ref": [42, "sglang", 0] + "ref": [43, "sglang", 0] } ] }, @@ -11875,7 +11442,7 @@ export const agentx = { "--tokenizer-path": ["Qwen/Qwen3.5-397B-A17B-FP8"] } }, - "ref": [42, "sglang", 0] + "ref": [43, "sglang", 0] } ] }, @@ -12009,7 +11576,7 @@ export const agentx = { "--page-size": null } }, - "ref": [44, "sglang", 0] + "ref": [45, "sglang", 0] } ] }, @@ -12142,7 +11709,7 @@ export const agentx = { "--tensor-parallel-size": ["4"] } }, - "ref": [42, "sglang", 0] + "ref": [43, "sglang", 0] } ] }, @@ -12256,7 +11823,7 @@ export const agentx = { "--speculative-num-steps": ["6"] } }, - "ref": [43, "sglang", 0] + "ref": [44, "sglang", 0] } ] }, @@ -12363,7 +11930,7 @@ export const agentx = { "--speculative-num-steps": ["4"] } }, - "ref": [43, "sglang", 0] + "ref": [44, "sglang", 0] } ] }, @@ -12655,7 +12222,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [49, "sglang", 0] + "ref": [50, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx index 5ab50e132a15..3ef7ca99320e 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "Qwen3.8-Flash-Next", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" + }, + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" + }, + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } @@ -64,16 +79,7 @@ export const agentx = { "id": "nvidia/Qwen3.8-Flash-Next-NVFP4" } ], - "envs": [ - { - "NCCL_CUMEM_ENABLE": "1", - "NCCL_NVLS_ENABLE": "1", - "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGL_ENABLE_JIT_DEEPGEMM": "false", - "TORCH_CUDA_ARCH_LIST": "10.0" - } - ], + "envs": [], "kvMooncake": { "donor": "dsv4-fp4-gb200-dynamo-sglang-agentic-disagg", "donorHardware": "gb200", @@ -353,53 +359,80 @@ export const agentx = { } }, { - "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0", + "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#1", "hw": "b300", - "ckpt": "RadixArk/Qwen3.8-Flash-Next-NVFP4", + "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", "label": "Agg TP1", "detail": "MTP 3-1-4", "gpus": 1, "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "586355a8", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "mooncake"], - "unavailable": { - "hicache": "no verified HiCache settings for this model yet" + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "298"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.8next-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" } }, "prov": { - "sha": "3c0d91ceb2334554aef03a8ddfec7e0377a3f274", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "38635046800/attempts/1", - "pr": "4069", - "status": "verified", - "date": "2026-10-21" + "run": "41117066222", + "pr": "4683", + "comment": "6536482181", + "status": "verified-pr" }, "points": [ { "concs": [1], - "item": 0, + "item": 1, "override": "override_c1", "goldenAL": "2.32", "kv": "none" }, { - "concs": [4], - "item": 0, - "override": "override_c4", + "concs": [2], + "item": 1, + "override": "override_c2", + "goldenAL": "2.32", + "kv": "none" + }, + { + "concs": [12], + "item": 1, + "override": "override_c12", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["8"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-track-interval": ["262144"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ], @@ -408,25 +441,48 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["8"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-track-interval": ["262144"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ] } }, { - "concs": [8], - "item": 0, - "override": "override_c8", + "concs": [24], + "item": 1, + "override": "override_c24", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["16"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["960"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ], @@ -435,25 +491,56 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["16"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["960"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ] } }, { - "concs": [12], - "item": 0, - "override": "override_c12", + "concs": [48], + "item": 2, + "override": "override_c48", "goldenAL": "2.32", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["24"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["298"], + "--hicache-write-policy": ["write_back"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["720"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--tokenizer-worker-num": ["1"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ], @@ -462,25 +549,67 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["24"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "20", "24", "28", "32", "36", "40", "44", "48", "52", "56", "60", "64", "72", "80", "88", "96"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["298"], + "--hicache-write-policy": ["write_back"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["720"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--tokenizer-worker-num": ["1"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ] } }, { - "concs": [16], - "item": 0, - "override": "override_c16", + "concs": [80], + "item": 2, + "override": "override_c80", "goldenAL": "2.32", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["32"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"], + "--cuda-graph-max-bs-decode": ["128"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["298"], + "--hicache-write-policy": ["write_back"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-max-states-per-path": ["2"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["640"], + "--max-running-requests": ["128"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--strip-thinking-cache": [], + "--tokenizer-worker-num": ["1"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ], @@ -489,8 +618,31 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["32"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "88", "92", "96", "100", "104", "108", "112", "116", "120", "124", "128"], + "--cuda-graph-max-bs-decode": ["128"], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["kernel"], + "--hicache-mem-layout": ["page_first"], + "--hicache-size": ["298"], + "--hicache-write-policy": ["write_back"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--mamba-max-states-per-path": ["2"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], + "--mamba-track-interval": ["262144"], + "--max-mamba-cache-size": ["640"], + "--max-running-requests": ["128"], + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--strip-thinking-cache": [], + "--tokenizer-worker-num": ["1"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null } } ] @@ -508,31 +660,210 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": "1", + "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": "1", + "SGLANG_Q35MF_BK_VEC_COPY": "1", + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", + "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", + "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", + "SGLANG_Q35MF_MKH_N3B": "1", + "SGLANG_Q35MF_MKH_N3C": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + }, "args": [ - ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], - ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--cuda-graph-max-bs-decode", "1"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "16384"], + ["--context-length", "262144"], + ["--cuda-graph-backend-decode", "full"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], + ["--cuda-graph-max-bs-decode", "96"], ["--data-parallel-size", "1"], + ["--disable-shared-experts-fusion"], + ["--enable-linear-replayssm-spec"], ["--expert-parallel-size", "1"], + ["--kv-cache-dtype", "bfloat16"], ["--linear-attn-decode-backend", "flashinfer"], ["--linear-attn-prefill-backend", "flashinfer"], + ["--linear-attn-verify-backend", "triton"], ["--mamba-ssm-dtype", "bfloat16"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.8"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "96"], + ["--mem-fraction-static", "0.88"], + ["--moe-a2a-backend", "none"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--ple-offload-backend", "pinned"], + ["--ple-offload-embedding"], + ["--quantization", "modelopt_mixed"], + ["--random-seed", "35"], ["--reasoning-parser", "auto"], - ["--scheduler-recv-interval", "10"], + ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], + ["--scheduler-recv-interval", "1"], ["--speculative-algorithm", "NEXTN"], + ["--speculative-draft-kv-cache-dtype", "bfloat16"], + ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], ["--speculative-eagle-topk", "1"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm"], ["--speculative-num-draft-tokens", "4"], ["--speculative-num-steps", "3"], - ["--stream-interval", "50"], + ["--stream-interval", "20"], ["--tensor-parallel-size", "1"], - ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], - ["--trust-remote-code"] - ], - "envRef": 0 + ["--tokenizer-worker-num", "6"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1800"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0", + "hw": "b300", + "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "label": "Agg TP4·EP4", + "detail": "MTP 3-1-4", + "gpus": 4, + "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "imageStatus": "ok", + "imageCommit": "586355a8", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "298"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.8next-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", + "run": "41117066222", + "pr": "4683", + "comment": "6536482181", + "status": "verified-pr" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "override_tp4_c1", + "goldenAL": "2.32", + "kv": "none" + }, + { + "concs": [2], + "item": 0, + "override": "override_tp4_c2", + "goldenAL": "2.32", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["4"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--expert-parallel-size": ["4"], + "--flashinfer-allreduce-fusion-backend": ["trtllm"], + "--max-running-requests": ["2"], + "--tensor-parallel-size": ["4"] + }, + "env": { + "SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH": null, + "SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH": null, + "SGLANG_Q35MF_BK_VEC_COPY": null, + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, + "SGLANG_Q35MF_EDGE_SHM_SYNC": null, + "SGLANG_Q35MF_MKH_N3B": null, + "SGLANG_Q35MF_MKH_N3C": null + } + }, + "ref": [1, "sglang", 0] } ] }, @@ -715,6 +1046,14 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + }, "args": [ ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], @@ -739,8 +1078,7 @@ export const agentx = { ["--tokenizer-worker-num", "6"], ["--tp", "1"], ["--trust-remote-code"] - ], - "envRef": 0 + ] } ] }, @@ -1103,7 +1441,7 @@ export const agentx = { "--tensor-parallel-size": ["4"] } }, - "ref": [3, "sglang", 0] + "ref": [4, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index d4dbe71f56b3..cb5db60af19d 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" + }, + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" + }, + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } @@ -67,6 +82,19 @@ export const agentx = { } ], "envs": [ + { + "DYN_REQUEST_PLANE": "tcp", + "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", + "DYN_ROUTER_QUEUE_THRESHOLD": "None", + "DYN_ROUTER_TEMPERATURE": "0", + "DYN_TCP_CHANNEL_BUFFER": "128", + "DYN_TCP_REQUEST_TIMEOUT": "60", + "DYN_TOKENIZER": "fastokens", + "DYN_TOKENIZER_CACHE": "1", + "DYN_TOKENIZER_CACHE_BYTES": "8000000000", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, { "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", @@ -174,18 +202,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "40989481011/attempts/1", "pr": "4595", - "status": "in-progress" + "status": "verified", + "date": "2026-10-30" }, "points": [ { @@ -359,19 +389,6 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", - "DYN_ROUTER_QUEUE_THRESHOLD": "None", - "DYN_ROUTER_TEMPERATURE": "0", - "DYN_TCP_CHANNEL_BUFFER": "128", - "DYN_TCP_REQUEST_TIMEOUT": "60", - "DYN_TOKENIZER": "fastokens", - "DYN_TOKENIZER_CACHE": "1", - "DYN_TOKENIZER_CACHE_BYTES": "8000000000", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "NATS_SERVER": "nats://$NATS_IP:4222" - }, "args": [ ["--http-port", "8000"], ["--router-mode", "kv"], @@ -379,7 +396,8 @@ export const agentx = { ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] - ] + ], + "envRef": 0 }, { "kind": "worker", @@ -480,9 +498,9 @@ export const agentx = { "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark", "gpus": 4, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "c10e0e17", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -494,18 +512,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41261902147", "pr": "4603", - "status": "in-progress" + "comment": "6576133363", + "status": "verified-pr" }, "points": [ { @@ -715,16 +735,169 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-sglang-agentic-dspark#0", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg#3", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Agg TP2·EP2", - "detail": "DSpark 5", + "detail": "DSpark 5 · dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", "gpus": 2, - "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", + "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47", "imageStatus": "ok", "imageCommit": "c10e0e17", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.2"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41081085818/attempts/1", + "pr": "4679", + "status": "verified", + "date": "2026-10-30" + }, + "points": [ + { + "concs": [16], + "item": 3, + "override": "override_tp2_c16", + "goldenAL": "3.51", + "kv": "none" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -js -sd /tmp/nats", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "diff": { + "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--expert-parallel-size": ["2"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["1024"], + "--tensor-parallel-size": ["2"], + "--context-length": null, + "--enable-nvidia-megakernel": null, + "--kv-cache-dtype": null, + "--moe-a2a-backend": null, + "--schedule-policy": null, + "--startup-weight-load-mode": null, + "--tokenizer-backend": null + }, + "env": { + "NCCL_CUMEM_ENABLE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_NUMA_BIND_V2": "0", + "NCCL_MNNVL_ENABLE": null, + "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null, + "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null, + "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null, + "SGLANG_SPEC_TP_SYNC": null + } + }, + "ref": [0, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-gb300-sglang-agentic-dspark#0", + "hw": "gb300", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Agg TP2·EP2", + "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark", + "gpus": 2, + "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", + "imageStatus": "ok", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -736,18 +909,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41261902147", "pr": "4603", - "status": "in-progress" + "comment": "6576133363", + "status": "verified-pr" }, "points": [ { @@ -801,34 +976,18 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-backend-prefill": ["disabled"], - "--cuda-graph-max-bs-decode": ["64"], - "--expert-parallel-size": ["2"], - "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.8"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["1024"], - "--tensor-parallel-size": ["2"], - "--tokenizer-worker-num": ["4"], - "--context-length": null, - "--enable-nvidia-megakernel": null, - "--kv-cache-dtype": null, - "--moe-a2a-backend": null, - "--schedule-policy": null, - "--startup-weight-load-mode": null + "--tokenizer-backend": ["fastokens"], + "--tokenizer-worker-num": ["4"] }, "env": { - "NCCL_CUMEM_ENABLE": "1", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_NUMA_BIND_V2": "0", - "NCCL_MNNVL_ENABLE": null, - "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null, - "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null, - "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null, - "SGLANG_SPEC_TP_SYNC": null + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSPARK_FOLDED_SAMPLING": "2", + "SGLANG_DSPARK_OPT_MARKOV_W2_BF16": "True", + "SGLANG_DSV41_REASONING_EFFORT": "high", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" } }, - "ref": [1, "sglang", 0] + "ref": [2, "sglang", 0] } ] }, @@ -885,9 +1044,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 2, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "c10e0e17", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -899,18 +1058,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41261902147", "pr": "4603", - "status": "in-progress" + "comment": "6576133363", + "status": "verified-pr" }, "points": [ { @@ -1012,7 +1173,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [2, "sglang", 0] + "ref": [3, "sglang", 0] }, { "kind": "sglang-router", @@ -1082,9 +1243,9 @@ export const agentx = { "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#0", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "label": "Disagg 3P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", - "gpus": 26, + "gpus": 22, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", "imageStatus": "ok", @@ -1094,7 +1255,7 @@ export const agentx = { { "kind": "note", "script": "Placement", - "reason": "Workers share nodes: 5 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + "reason": "Workers share nodes: 3 prefill workers on 2 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." } ], "disclosures": ["synthetic-acceptance"], @@ -1106,24 +1267,26 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", - "pr": "4595", - "status": "in-progress" + "run": "41081085818/attempts/1", + "pr": "4679", + "status": "verified", + "date": "2026-10-30" }, "points": [ { - "concs": [2884], + "concs": [1228], "item": 0, - "override": "override_5p1d_dep2_dep16_c2884", + "override": "override_3p1d_dep2_dep16_c1228", "goldenAL": "3.51", "kv": "hicache" } @@ -1156,21 +1319,19 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 }, { "kind": "worker", "role": "prefill", - "workers": 5, + "workers": 3, "nodes_per_worker": 1, "gpu_pinning": [ - ["0,1"], - ["2,3"], ["0,1"], ["2,3"], ["0,1"] ], - "local_ranks": [0, 1, 0, 1, 0], + "local_ranks": [0, 1, 0], "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1); 0 for a worker alone on its node.", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], "cmd": "python3 -m dynamo.sglang", "env": { @@ -1285,9 +1446,9 @@ export const agentx = { ["--nnodes", "4"], ["--node-rank", "$NODE_RANK"], ["--request-plane", "tcp"], - ["--cuda-graph-max-bs-decode", "40"], + ["--cuda-graph-max-bs-decode", "24"], ["--data-parallel-size", "16"], - ["--disaggregation-decode-extra-slots", "80"], + ["--disaggregation-decode-extra-slots", "36"], ["--disaggregation-transfer-backend", "mooncake"], ["--dsv4-attn-backend", "flashmla_mega"], ["--enable-decoder-swa-bounded-replay"], @@ -1301,7 +1462,7 @@ export const agentx = { ["--expert-distribution-recorder-buffer-size", "1000"], ["--expert-parallel-size", "16"], ["--load-balance-method", "total_tokens"], - ["--max-running-requests", "640"], + ["--max-running-requests", "384"], ["--mem-fraction-static", "0.75"], ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--moe-a2a-backend", "megamoe"], @@ -1322,6 +1483,177 @@ export const agentx = { } ] }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1); 0 for a worker alone on its node.", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#1", + "hw": "gb300", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "detail": "DSpark 5", + "gpus": 26, + "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "imageStatus": "ok", + "imageCommit": "586355a8", + "submitted": "dynamo", + "setup": [ + { + "kind": "note", + "script": "Placement", + "reason": "Workers share nodes: 5 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + } + ], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.2"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", + "run": "40989481011/attempts/1", + "pr": "4595", + "status": "verified", + "date": "2026-10-30" + }, + "points": [ + { + "concs": [2884], + "item": 1, + "override": "override_5p1d_dep2_dep16_c2884", + "goldenAL": "3.51", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 1 + }, + { + "workers": 5, + "gpu_pinning": [ + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"] + ], + "local_ranks": [0, 1, 0, 1, 0], + "ref": [5, "dynamo", 2] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["40"], + "--disaggregation-decode-extra-slots": ["80"], + "--max-running-requests": ["640"] + } + }, + "ref": [5, "dynamo", 3] + } + ] + }, "sglang": { "install": [], "files": [], @@ -1390,7 +1722,7 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#1", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Disagg 8P1D · prefill DEP2 · decode DEP16 (4 nodes)", @@ -1417,23 +1749,25 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", + "run": "40989481011/attempts/1", "pr": "4595", - "status": "in-progress" + "status": "verified", + "date": "2026-10-30" }, "points": [ { "concs": [4096], - "item": 1, + "item": 2, "override": "override_8p1d_dep2_dep16_c4096", "goldenAL": "3.51", "kv": "hicache" @@ -1467,7 +1801,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 }, { "workers": 8, @@ -1483,7 +1817,7 @@ export const agentx = { ], "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1], "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], - "ref": [4, "dynamo", 2] + "ref": [5, "dynamo", 2] }, { "diff": { @@ -1493,7 +1827,7 @@ export const agentx = { "--max-running-requests": ["1024"] } }, - "ref": [4, "dynamo", 3] + "ref": [5, "dynamo", 3] } ] }, @@ -1568,7 +1902,7 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#3", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Disagg 9P1D · prefill DEP2 · decode DEP16 (4 nodes)", @@ -1595,23 +1929,25 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", + "donorHardware": "gb300" } }, "prov": { "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", + "run": "40989481011/attempts/1", "pr": "4595", - "status": "in-progress" + "status": "verified", + "date": "2026-10-30" }, "points": [ { "concs": [7480], - "item": 2, + "item": 3, "override": "override_9p1d_dep2_dep16_b160_c7480", "goldenAL": "3.51", "kv": "hicache" @@ -1645,7 +1981,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 }, { "workers": 9, @@ -1661,7 +1997,7 @@ export const agentx = { ["0,1"] ], "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1, 0], - "ref": [4, "dynamo", 2] + "ref": [5, "dynamo", 2] }, { "diff": { @@ -1671,7 +2007,7 @@ export const agentx = { "--max-running-requests": ["2560"] } }, - "ref": [4, "dynamo", 3] + "ref": [5, "dynamo", 3] } ] }, @@ -2056,7 +2392,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [7, "sglang", 0] + "ref": [9, "sglang", 0] } ] }, @@ -2292,7 +2628,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [8, "sglang", 0] + "ref": [10, "sglang", 0] }, { "kind": "sglang-router", @@ -2650,7 +2986,7 @@ export const agentx = { "--schedule-policy": null } }, - "ref": [10, "sglang", 0] + "ref": [12, "sglang", 0] } ] }, @@ -2833,7 +3169,7 @@ export const agentx = { "--dsv4-attn-backend": null } }, - "ref": [11, "sglang", 0] + "ref": [13, "sglang", 0] } ] }, @@ -2998,7 +3334,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [10, "sglang", 0] + "ref": [12, "sglang", 0] }, { "kind": "sglang-router", @@ -3403,7 +3739,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [14, "sglang", 0] + "ref": [16, "sglang", 0] } ] }, @@ -3663,7 +3999,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0" } }, - "ref": [15, "sglang", 0] + "ref": [17, "sglang", 0] }, { "kind": "sglang-router", @@ -4104,7 +4440,7 @@ export const agentx = { "--tensor-parallel-size": ["8"] } }, - "ref": [17, "sglang", 0] + "ref": [19, "sglang", 0] } ] }, @@ -4366,7 +4702,7 @@ export const agentx = { ["--trust-remote-code"], ["--watchdog-timeout", "3600"] ], - "envRef": 1 + "envRef": 2 } ] }, @@ -4554,7 +4890,7 @@ export const agentx = { "--swa-prefix-tails": ["64"] } }, - "ref": [19, "sglang", 0] + "ref": [21, "sglang", 0] }, { "kind": "sglang-router", @@ -4571,7 +4907,7 @@ export const agentx = { ["--disable-retries"], ["--worker-urls", "http://127.0.0.1:30000"] ], - "envRef": 1 + "envRef": 2 } ] }, diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx index 9877864d5891..9c6a9d43321e 100644 --- a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx +++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "Muse Spark Open", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" + }, + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" + }, + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } @@ -146,9 +161,9 @@ export const agentx = { "detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash", "gpus": 4, "configKey": "musespark-fp4-gb300-sglang-agentic-dflash", - "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc", "imageStatus": "ok", - "imageCommit": "c1c287a4", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -158,7 +173,7 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "160"], + ["--hicache-size", "150"], ["--hicache-write-policy", "write_back"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", @@ -166,12 +181,12 @@ export const agentx = { } }, "prov": { - "sha": "316905a060115a9a393f6a464d3646b830a3c74e", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "39794121900/attempts/1", - "pr": "4404", - "status": "verified", - "date": "2026-10-23" + "run": "41327192200", + "pr": "4724", + "comment": "6576178636", + "status": "verified-pr" }, "points": [ { @@ -182,19 +197,16 @@ export const agentx = { "kv": "none" }, { - "concs": [5], + "concs": [4], "item": 0, - "override": "override_tp4_c5", + "override": "override_tp4_c4", "goldenAL": "4.05", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--speculative-draft-attention-backend": ["fa4"] + "--schedule-policy": ["shortest-prefill-first"] } } ], @@ -203,10 +215,7 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--speculative-draft-attention-backend": ["fa4"] + "--schedule-policy": ["shortest-prefill-first"] } } ] @@ -222,9 +231,12 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--prefill-decode-interval": ["64"] + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--prefill-decode-interval": ["128"], + "--prefill-decode-interval-min-bs": ["1"], + "--schedule-policy": ["shortest-prefill-first"], + "--speculative-draft-attention-backend": ["fa4"] } } ], @@ -233,9 +245,12 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--prefill-decode-interval": ["64"] + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--prefill-decode-interval": ["128"], + "--prefill-decode-interval-min-bs": ["1"], + "--schedule-policy": ["shortest-prefill-first"], + "--speculative-draft-attention-backend": ["fa4"] } } ] @@ -255,6 +270,7 @@ export const agentx = { "--incremental-streaming-output": [], "--max-running-requests": ["3"], "--prefill-decode-interval": ["64"], + "--schedule-policy": ["shortest-prefill-first"], "--speculative-draft-attention-backend": ["fa4"], "--tokenizer-worker-num": ["4"] } @@ -269,6 +285,7 @@ export const agentx = { "--incremental-streaming-output": [], "--max-running-requests": ["3"], "--prefill-decode-interval": ["64"], + "--schedule-policy": ["shortest-prefill-first"], "--speculative-draft-attention-backend": ["fa4"], "--tokenizer-worker-num": ["4"] } @@ -330,6 +347,7 @@ export const agentx = { "--max-running-requests": ["48"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["96"], + "--prefill-decode-interval-min-bs": ["2"], "--schedule-policy": ["shortest-prefill-first"], "--swa-full-tokens-ratio": ["0.1"] }, @@ -352,57 +370,7 @@ export const agentx = { "--max-running-requests": ["48"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["96"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048" - } - } - ] - } - }, - { - "concs": [28], - "item": 1, - "override": "override_tp4_c28", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["48"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], - "--hicache-write-policy": ["write_back"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048" - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["48"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], - "--hicache-write-policy": ["write_back"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["32"], + "--prefill-decode-interval-min-bs": ["2"], "--schedule-policy": ["shortest-prefill-first"], "--swa-full-tokens-ratio": ["0.1"] }, @@ -426,12 +394,13 @@ export const agentx = { "--cuda-graph-backend-prefill": ["breakable"], "--cuda-graph-max-bs-decode": ["48"], "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["8192"], "--max-running-requests": ["48"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["64"], + "--prefill-decode-interval-min-bs": ["3"], "--schedule-policy": ["shortest-prefill-first"], "--swa-full-tokens-ratio": ["0.1"] }, @@ -448,12 +417,13 @@ export const agentx = { "--cuda-graph-backend-prefill": ["breakable"], "--cuda-graph-max-bs-decode": ["48"], "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["8192"], "--max-running-requests": ["48"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["64"], + "--prefill-decode-interval-min-bs": ["3"], "--schedule-policy": ["shortest-prefill-first"], "--swa-full-tokens-ratio": ["0.1"] }, @@ -465,9 +435,9 @@ export const agentx = { } }, { - "concs": [34], + "concs": [40], "item": 1, - "override": "override_tp4_c34", + "override": "override_tp4_c40", "goldenAL": "4.05", "kv": "hicache", "patch": { @@ -476,16 +446,15 @@ export const agentx = { "args": { "--chunked-prefill-size": ["16384"], "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["96"], + "--cuda-graph-max-bs-decode": ["18"], "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["96"], + "--max-running-requests": ["18"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] + "--schedule-policy": ["shortest-prefill-first"] }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -500,16 +469,15 @@ export const agentx = { "args": { "--chunked-prefill-size": ["16384"], "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["96"], + "--cuda-graph-max-bs-decode": ["18"], "--enable-hierarchical-cache": [], - "--hicache-size": ["160"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["96"], + "--max-running-requests": ["18"], "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] + "--schedule-policy": ["shortest-prefill-first"] }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -529,20 +497,20 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["24"], + "--chunked-prefill-size": ["16384"], + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["20"], "--enable-hierarchical-cache": [], - "--hicache-size": ["149"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], - "--incremental-streaming-output": [], - "--max-running-requests": ["24"], - "--mem-fraction-static": ["0.9"], - "--speculative-draft-attention-backend": ["trtllm_mha"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], - "--swa-full-tokens-ratio": ["0.1"], - "--tokenizer-worker-num": ["4"], - "--chunked-prefill-size": null + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["20"], + "--mem-fraction-static": ["0.95"], + "--prefill-decode-interval": ["48"], + "--schedule-policy": ["shortest-prefill-first"] }, "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" } } @@ -552,20 +520,20 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["24"], + "--chunked-prefill-size": ["16384"], + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["20"], "--enable-hierarchical-cache": [], - "--hicache-size": ["149"], + "--hicache-size": ["150"], "--hicache-write-policy": ["write_back"], - "--incremental-streaming-output": [], - "--max-running-requests": ["24"], - "--mem-fraction-static": ["0.9"], - "--speculative-draft-attention-backend": ["trtllm_mha"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], - "--swa-full-tokens-ratio": ["0.1"], - "--tokenizer-worker-num": ["4"], - "--chunked-prefill-size": null + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["20"], + "--mem-fraction-static": ["0.95"], + "--prefill-decode-interval": ["48"], + "--schedule-policy": ["shortest-prefill-first"] }, "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" } } @@ -723,7 +691,7 @@ export const agentx = { "hw": "gb300", "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", "label": "Agg TP8·EP8 (2 nodes)", - "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash", + "detail": "DFlash 8", "gpus": 8, "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash", "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", @@ -738,7 +706,7 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "160"], + ["--hicache-size", "150"], ["--hicache-write-policy", "write_back"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", @@ -963,17 +931,17 @@ export const agentx = { } }, { - "id": "musespark-fp4-gb300-sglang-agentic-dflash-multinode#0", + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#0", "hw": "gb300", "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", - "label": "Agg TP8·EP8 (2 nodes)", - "detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash-multinode", - "gpus": 8, - "configKey": "musespark-fp4-gb300-sglang-agentic-dflash-multinode", - "image": "lmsysorg/sglang:nightly-dev-20261019-3c560089@sha256:c1f4d9fc6fcf50c2269adbc757ce7b1778883af04155edb888fa9eea3b6648c0", + "label": "Agg TP4·EP4", + "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash", + "gpus": 4, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash", + "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", "imageStatus": "ok", - "imageCommit": "3c560089", - "submitted": "sglang", + "imageCommit": null, + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -982,7 +950,7 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "160"], + ["--hicache-size", "150"], ["--hicache-write-policy", "write_back"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", @@ -990,315 +958,105 @@ export const agentx = { } }, "prov": { - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-tp8-direct-variants.yaml", - "pr": "4454", - "status": "in-progress" + "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "39427144732/attempts/4", + "pr": "4344", + "status": "verified", + "date": "2026-10-22" }, "points": [ { - "concs": [5], - "item": 0, - "override": "override_tp8_c5", + "concs": [8], + "item": 1, + "override": "override_tp4_c8", "goldenAL": "4.05", "kv": "none" }, { - "concs": [10], - "item": 1, - "override": "override_tp8_c10", + "concs": [16], + "item": 0, + "override": "override_tp4_c16", "goldenAL": "4.05", "kv": "none", "patch": { - "sglang": [ + "dynamo": [ + null, + null, + null, { "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["2"], + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"], + "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["128"], - "--page-size": null + "--schedule-policy": ["shortest-prefill-first"], + "--swa-full-tokens-ratio": ["0.1"] } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["2"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["2"], + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"], + "--mem-fraction-static": ["0.95"], "--prefill-decode-interval": ["128"], - "--page-size": null + "--schedule-policy": ["shortest-prefill-first"], + "--swa-full-tokens-ratio": ["0.1"] } - } + }, + null ] } }, { - "concs": [18], - "item": 2, - "override": "override_tp8_c18", + "concs": [20], + "item": 8, + "override": "override_tp4_c20", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { - "sglang": [ + "dynamo": [ + null, + null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--prefill-decode-interval": ["32"] + "--cuda-graph-max-bs-decode": ["10"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["10"], + "--mem-fraction-static": ["0.95"], + "--prefill-decode-interval": ["48"], + "--schedule-policy": ["shortest-prefill-first"] } } ], - "dynamo": [ - null, - null, + "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"], - "--prefill-decode-interval": ["32"] + "--cuda-graph-max-bs-decode": ["10"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["10"], + "--mem-fraction-static": ["0.95"], + "--prefill-decode-interval": ["48"], + "--schedule-policy": ["shortest-prefill-first"] } - } + }, + null ] } }, { "concs": [32], - "item": 3, - "override": "override_tp8_c32", + "item": 7, + "override": "override_tp4_c32", "goldenAL": "4.05", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["6"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["6"], - "--prefill-decode-interval": ["24"], - "--schedule-policy": ["shortest-prefill-first"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["6"], - "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["6"], - "--prefill-decode-interval": ["24"], - "--schedule-policy": ["shortest-prefill-first"] - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--speculative-draft-attention-backend": ["fa4"], - "--swa-full-tokens-ratio": ["0.1"], - "--tokenizer-backend": ["fastokens"], - "--tool-call-parser": ["muse"], - "--model-loader-extra-config": null, - "--preferred-sampling-params": null, - "--prefill-decode-interval": null - }, - "env": { - "SGLANG_PARALLEL_PROMPT_ENCODE": "0" - } - }, - "ref": [1, "sglang", 0] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } - }, - { - "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#0", - "hw": "gb300", - "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", - "label": "Agg TP4·EP4", - "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash", - "gpus": 4, - "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash", - "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "dynamo", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-size", "160"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "musespark-fp4-gb300-sglang-agentic-dflash", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "39427144732/attempts/4", - "pr": "4344", - "status": "verified", - "date": "2026-10-22" - }, - "points": [ - { - "concs": [8], - "item": 1, - "override": "override_tp4_c8", - "goldenAL": "4.05", - "kv": "none" - }, - { - "concs": [16], - "item": 0, - "override": "override_tp4_c16", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["128"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["128"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - } - }, - null - ] - } - }, - { - "concs": [20], - "item": 8, - "override": "override_tp4_c20", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["120"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["10"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["120"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["10"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"] - } - }, - null - ] - } - }, - { - "concs": [32], - "item": 7, - "override": "override_tp4_c32", - "goldenAL": "4.05", - "kv": "hicache", + "kv": "hicache", "patch": { "dynamo": [ null, @@ -1375,241 +1133,23 @@ export const agentx = { "--max-prefill-tokens": ["16384"], "--max-running-requests": ["24"], "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" - } - }, - null - ] - } - } - ], - "routers": { - "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""], - "files": ["nats.conf#13771a85"], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "nats", - "count_nodes": 1, - "cmd": "nats-server -c nats.conf", - "env": {}, - "args": [] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"] - ], - "envRef": 0 - }, - { - "nodes_per_worker": 1, - "diff": { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["8"], - "--expert-parallel-size": ["4"], - "--max-running-requests": ["8"], - "--tensor-parallel-size": ["4"], - "--dist-init-addr": null, - "--nnodes": null, - "--node-rank": null, - "--page-size": null - }, - "env": { - "NCCL_MNNVL_ENABLE": null, - "PYTORCH_CUDA_ALLOC_CONF": null - } - }, - "ref": [1, "dynamo", 3] - } - ] - }, - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "from": 3, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--request-plane": null, - "--kv-events-config": null, - "--dyn-reasoning-parser": null, - "--dyn-tool-call-parser": null - }, - "env": { - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null - } - } - }, - { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, - "args": [ - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--policy", "cache_aware"], - ["--worker-urls", "http://127.0.0.1:30000"] - ] - } - ] - } - } - }, - { - "id": "musespark-fp4-gb300-sglang-agentic-dflash-nightly1019#0", - "hw": "gb300", - "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", - "label": "Agg TP4·EP4", - "detail": "DFlash 8 · musespark-fp4-gb300-sglang-agentic-dflash-nightly1019", - "gpus": 4, - "configKey": "musespark-fp4-gb300-sglang-agentic-dflash-nightly1019", - "image": "lmsysorg/sglang:nightly-dev-20261019-3c560089@sha256:c1f4d9fc6fcf50c2269adbc757ce7b1778883af04155edb888fa9eea3b6648c0", - "imageStatus": "ok", - "imageCommit": "3c560089", - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-size", "160"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "musespark-fp4-gb300-sglang-agentic-dflash", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic-nightly1019.yaml", - "pr": "4454", - "status": "in-progress" - }, - "points": [ - { - "concs": [28], - "item": 0, - "override": "override_tp4_c28", - "goldenAL": "4.05", - "kv": "hicache" - }, - { - "concs": [52], - "item": 1, - "override": "override_tp4_c52", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--flashinfer-autotune-skip-ops": ["flashinfer::trtllm_fp4_block_scale_moe"], - "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["32"], - "--swa-full-tokens-ratio": ["0.15"], - "--chunked-prefill-size": null, - "--max-prefill-tokens": null, - "--prefill-decode-interval": null, - "--schedule-policy": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--flashinfer-autotune-skip-ops": ["flashinfer::trtllm_fp4_block_scale_moe"], - "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["32"], - "--swa-full-tokens-ratio": ["0.15"], - "--chunked-prefill-size": null, - "--max-prefill-tokens": null, - "--prefill-decode-interval": null, - "--schedule-policy": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--enable-hierarchical-cache": [], - "--hicache-size": ["149"], - "--hicache-write-policy": ["write_through"], - "--max-prefill-tokens": ["8192"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"], - "--speculative-draft-attention-backend": ["trtllm_mha"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], - "--tokenizer-backend": ["fastokens"], - "--tool-call-parser": ["muse"], - "--model-loader-extra-config": null, - "--preferred-sampling-params": null, - "--speculative-draft-model-revision": null + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"] }, "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048", - "SGLANG_PARALLEL_PROMPT_ENCODE": "0" + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" } }, - "ref": [3, "sglang", 0] - } - ] - }, + null + ] + } + } + ], + "routers": { "dynamo": { - "install": [], - "files": [], + "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""], + "files": ["nats.conf#13771a85"], "blocks": [ { "kind": "etcd", @@ -1622,31 +1162,81 @@ export const agentx = { ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", + "nodes_per_worker": 1, "diff": { "args": { - "--request-plane": ["tcp"] + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["8"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["8"], + "--tensor-parallel-size": ["4"], + "--dist-init-addr": null, + "--nnodes": null, + "--node-rank": null, + "--page-size": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "NCCL_MNNVL_ENABLE": null, + "PYTORCH_CUDA_ALLOC_CONF": null + } + }, + "ref": [1, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } @@ -1675,15 +1265,16 @@ export const agentx = { ["--hicache-size", "120"] ], "donor": "musespark-fp4-gb200-sglang-agentic-dflash", - "donorHardware": "gb200", - "donorRef": "main" + "donorHardware": "gb200" } }, "prov": { - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "41096285400", "pr": "4670", - "status": "in-progress" + "comment": "6576162113", + "status": "verified-pr" }, "points": [ { @@ -2080,7 +1671,6 @@ export const agentx = { "--mem-fraction-static": ["0.93"], "--prefill-decode-interval": ["64"], "--schedule-policy": ["shortest-prefill-first"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.1"], "--cuda-graph-backend-prefill": null, @@ -2106,7 +1696,6 @@ export const agentx = { "--mem-fraction-static": ["0.93"], "--prefill-decode-interval": ["64"], "--schedule-policy": ["shortest-prefill-first"], - "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.1"], "--cuda-graph-backend-prefill": null, @@ -2224,9 +1813,9 @@ export const agentx = { "detail": "DFlash 8", "gpus": 8, "configKey": "musespark-fp4-b300-sglang-agentic-dflash", - "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc", "imageStatus": "ok", - "imageCommit": "c1c287a4", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2236,19 +1825,19 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "170"] + ["--hicache-size", "250"] ], "donor": "musespark-fp4-b300-sglang-agentic-dflash", "donorHardware": "b300" } }, "prov": { - "sha": "63c4ad272bdbad2150417e9215d0be3409f1e8cb", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", - "run": "40285747600/attempts/4", - "pr": "4417", - "status": "verified", - "date": "2026-10-28" + "run": "41328422543", + "pr": "4725", + "comment": "6576185043", + "status": "verified-pr" }, "points": [ { @@ -2268,43 +1857,8 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--stream-interval": ["20"], - "--speculative-draft-attention-backend": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--stream-interval": ["20"], - "--speculative-draft-attention-backend": null - } - } - ] - } - }, - { - "concs": [8], - "item": 0, - "override": "override_tp8_c8", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--speculative-draft-attention-backend": null + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"] } } ], @@ -2313,19 +1867,17 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["64"], - "--speculative-draft-attention-backend": null + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"] } } ] } }, { - "concs": [10], + "concs": [7], "item": 0, - "override": "override_tp8_c10", + "override": "override_tp8_c7", "goldenAL": "4.05", "kv": "none", "patch": { @@ -2336,6 +1888,7 @@ export const agentx = { "--max-running-requests": ["2"], "--prefill-decode-interval": ["64"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"], "--speculative-draft-attention-backend": null } } @@ -2349,6 +1902,7 @@ export const agentx = { "--max-running-requests": ["2"], "--prefill-decode-interval": ["64"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"], "--speculative-draft-attention-backend": null } } @@ -2356,9 +1910,9 @@ export const agentx = { } }, { - "concs": [14], + "concs": [16], "item": 0, - "override": "override_tp8_c14", + "override": "override_tp8_c16", "goldenAL": "4.05", "kv": "none", "patch": { @@ -2366,9 +1920,11 @@ export const agentx = { { "args": { "--cuda-graph-max-bs-decode": ["3"], + "--incremental-streaming-output": [], "--max-running-requests": ["3"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["88"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"], "--speculative-draft-attention-backend": null } } @@ -2379,42 +1935,11 @@ export const agentx = { { "args": { "--cuda-graph-max-bs-decode": ["3"], + "--incremental-streaming-output": [], "--max-running-requests": ["3"], - "--prefill-decode-interval": ["64"], - "--schedule-policy": ["shortest-prefill-first"], - "--speculative-draft-attention-backend": null - } - } - ] - } - }, - { - "concs": [16], - "item": 0, - "override": "override_tp8_c16", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"], - "--prefill-decode-interval": ["64"], - "--schedule-policy": ["shortest-prefill-first"], - "--speculative-draft-attention-backend": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["88"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["40"], "--speculative-draft-attention-backend": null } } @@ -2433,7 +1958,7 @@ export const agentx = { "args": { "--cuda-graph-max-bs-decode": ["6"], "--max-running-requests": ["6"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["48"], "--schedule-policy": ["shortest-prefill-first"], "--stream-interval": ["20"], "--speculative-draft-attention-backend": null @@ -2447,7 +1972,7 @@ export const agentx = { "args": { "--cuda-graph-max-bs-decode": ["6"], "--max-running-requests": ["6"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["48"], "--schedule-policy": ["shortest-prefill-first"], "--stream-interval": ["20"], "--speculative-draft-attention-backend": null @@ -2468,7 +1993,7 @@ export const agentx = { "args": { "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["80"], "--schedule-policy": ["shortest-prefill-first"], "--stream-interval": ["20"], "--speculative-draft-attention-backend": null @@ -2482,7 +2007,7 @@ export const agentx = { "args": { "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["80"], "--schedule-policy": ["shortest-prefill-first"], "--stream-interval": ["20"], "--speculative-draft-attention-backend": null @@ -2601,9 +2126,9 @@ export const agentx = { "detail": "DFlash 8", "gpus": 4, "configKey": "musespark-fp4-b300-sglang-agentic-dflash", - "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc", "imageStatus": "ok", - "imageCommit": "c1c287a4", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2613,19 +2138,19 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "170"] + ["--hicache-size", "250"] ], "donor": "musespark-fp4-b300-sglang-agentic-dflash", "donorHardware": "b300" } }, "prov": { - "sha": "63c4ad272bdbad2150417e9215d0be3409f1e8cb", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", - "run": "40285747600/attempts/4", - "pr": "4417", - "status": "verified", - "date": "2026-10-28" + "run": "41328422543", + "pr": "4725", + "comment": "6576185043", + "status": "verified-pr" }, "points": [ { @@ -2650,8 +2175,8 @@ export const agentx = { "--hicache-size": ["170"], "--max-prefill-tokens": ["16384"], "--max-running-requests": ["56"], - "--prefill-decode-interval": ["128"], - "--stream-interval": ["20"] + "--prefill-decode-interval": ["104"], + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" @@ -2668,8 +2193,8 @@ export const agentx = { "--hicache-size": ["170"], "--max-prefill-tokens": ["16384"], "--max-running-requests": ["56"], - "--prefill-decode-interval": ["128"], - "--stream-interval": ["20"] + "--prefill-decode-interval": ["104"], + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" @@ -2690,54 +2215,11 @@ export const agentx = { "args": { "--chunked-prefill-size": ["16384"], "--cuda-graph-max-bs-decode": ["64"], - "--hicache-size": ["170"], - "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["64"], - "--prefill-decode-interval": ["128"], - "--stream-interval": ["20"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-max-bs-decode": ["64"], - "--hicache-size": ["170"], + "--hicache-size": ["250"], "--max-prefill-tokens": ["16384"], "--max-running-requests": ["64"], - "--prefill-decode-interval": ["128"], - "--stream-interval": ["20"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" - } - } - ] - } - }, - { - "concs": [40], - "item": 1, - "override": "override_tp4_c40", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-max-bs-decode": ["80"], - "--hicache-size": ["170"], - "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["80"], - "--stream-interval": ["20"], - "--swa-full-tokens-ratio": ["0.2"] + "--swa-full-tokens-ratio": ["0.2"], + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -2751,12 +2233,12 @@ export const agentx = { { "args": { "--chunked-prefill-size": ["16384"], - "--cuda-graph-max-bs-decode": ["80"], - "--hicache-size": ["170"], + "--cuda-graph-max-bs-decode": ["64"], + "--hicache-size": ["250"], "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["80"], - "--stream-interval": ["20"], - "--swa-full-tokens-ratio": ["0.2"] + "--max-running-requests": ["64"], + "--swa-full-tokens-ratio": ["0.2"], + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -2767,9 +2249,9 @@ export const agentx = { } }, { - "concs": [44], + "concs": [38], "item": 1, - "override": "override_tp4_c44", + "override": "override_tp4_c38", "goldenAL": "4.05", "kv": "hicache", "patch": { @@ -2777,16 +2259,16 @@ export const agentx = { { "args": { "--chunked-prefill-size": ["16384"], - "--cuda-graph-max-bs-decode": ["24"], - "--hicache-size": ["170"], - "--hicache-write-policy": ["write_back"], + "--cuda-graph-max-bs-decode": ["76"], + "--hicache-size": ["250"], "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["24"], - "--stream-interval": ["20"], - "--prefill-decode-interval": null, - "--schedule-policy": null + "--max-running-requests": ["16"], + "--prefill-decode-interval": ["56"], + "--swa-full-tokens-ratio": ["0.2"], + "--scheduler-recv-interval": null }, "env": { + "SGLANG_DFLASH_EAGER_DRAFT_SAMPLER": "0", "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" } @@ -2797,17 +2279,17 @@ export const agentx = { null, { "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-max-bs-decode": ["24"], - "--hicache-size": ["170"], - "--hicache-write-policy": ["write_back"], - "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["24"], - "--stream-interval": ["20"], - "--prefill-decode-interval": null, - "--schedule-policy": null + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["76"], + "--hicache-size": ["250"], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["16"], + "--prefill-decode-interval": ["56"], + "--swa-full-tokens-ratio": ["0.2"], + "--scheduler-recv-interval": null }, "env": { + "SGLANG_DFLASH_EAGER_DRAFT_SAMPLER": "0", "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" } @@ -2831,9 +2313,9 @@ export const agentx = { "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["16384"], "--max-running-requests": ["26"], - "--stream-interval": ["20"], "--prefill-decode-interval": null, - "--schedule-policy": null + "--schedule-policy": null, + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -2852,9 +2334,9 @@ export const agentx = { "--hicache-write-policy": ["write_back"], "--max-prefill-tokens": ["16384"], "--max-running-requests": ["26"], - "--stream-interval": ["20"], "--prefill-decode-interval": null, - "--schedule-policy": null + "--schedule-policy": null, + "--scheduler-recv-interval": null }, "env": { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", @@ -2875,6 +2357,7 @@ export const agentx = { "args": { "--chunked-prefill-size": ["8192"], "--cuda-graph-max-bs-decode": ["12"], + "--disable-prefill-cuda-graph": [], "--enable-hierarchical-cache": [], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], @@ -2882,11 +2365,14 @@ export const agentx = { "--hicache-size": ["120"], "--max-prefill-tokens": ["8192"], "--max-running-requests": ["12"], - "--mem-fraction-static": ["0.95"], + "--mem-fraction-static": ["0.92"], "--prefill-decode-interval": ["96"], "--schedule-policy": ["shortest-prefill-first"], + "--scheduler-recv-interval": ["20"], + "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.1"], "--tensor-parallel-size": ["4"], + "--cuda-graph-backend-prefill": null, "--speculative-draft-attention-backend": null }, "env": { @@ -2894,7 +2380,7 @@ export const agentx = { "PYTORCH_CUDA_ALLOC_CONF": null } }, - "ref": [6, "sglang", 0] + "ref": [4, "sglang", 0] } ] }, @@ -2951,9 +2437,9 @@ export const agentx = { "detail": "DFlash 8", "gpus": 8, "configKey": "musespark-fp4-b200-sglang-agentic-dflash", - "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc", "imageStatus": "ok", - "imageCommit": "c1c287a4", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2971,12 +2457,12 @@ export const agentx = { } }, "prov": { - "sha": "11358f4843ec3e086014d3bba1a102e400944a59", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", - "run": "39756010759/attempts/1", - "pr": "4400", - "status": "verified", - "date": "2026-10-23" + "run": "41297696082", + "pr": "4707", + "comment": "6576173162", + "status": "verified-pr" }, "points": [ { @@ -2986,13 +2472,6 @@ export const agentx = { "goldenAL": "4.05", "kv": "none" }, - { - "concs": [2], - "item": 0, - "override": "override_tp8_c2", - "goldenAL": "4.05", - "kv": "none" - }, { "concs": [4], "item": 0, @@ -3003,59 +2482,12 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--enable-layernorm-sp": [], - "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["4"], - "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], - "--swa-full-tokens-ratio": ["0.4"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192", - "PYTORCH_CUDA_ALLOC_CONF": null, - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--enable-layernorm-sp": [], - "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["4"], - "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], - "--swa-full-tokens-ratio": ["0.4"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192", - "PYTORCH_CUDA_ALLOC_CONF": null, - "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": null - } - } - ] - } - }, - { - "concs": [6], - "item": 0, - "override": "override_tp8_c6", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["6"], + "--cuda-graph-max-bs-decode": ["2"], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["6"], + "--max-running-requests": ["2"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["128"], "--swa-full-tokens-ratio": ["0.4"] }, "env": { @@ -3070,12 +2502,12 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["6"], + "--cuda-graph-max-bs-decode": ["2"], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["6"], + "--max-running-requests": ["2"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["128"], "--swa-full-tokens-ratio": ["0.4"] }, "env": { @@ -3087,33 +2519,6 @@ export const agentx = { ] } }, - { - "concs": [8], - "item": 0, - "override": "override_tp8_c8", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["3"], - "--max-running-requests": ["3"] - } - } - ] - } - }, { "concs": [10], "item": 0, @@ -3124,12 +2529,12 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["10"], + "--cuda-graph-max-bs-decode": ["3"], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["10"], + "--max-running-requests": ["3"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["128"], "--swa-full-tokens-ratio": ["0.4"] }, "env": { @@ -3144,12 +2549,12 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["10"], + "--cuda-graph-max-bs-decode": ["3"], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], - "--max-running-requests": ["10"], + "--max-running-requests": ["3"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["128"], "--swa-full-tokens-ratio": ["0.4"] }, "env": { @@ -3162,9 +2567,9 @@ export const agentx = { } }, { - "concs": [56], + "concs": [64], "item": 1, - "override": "override_tp8_c56", + "override": "override_tp8_c64", "goldenAL": "4.05", "kv": "hicache", "patch": { @@ -3180,8 +2585,9 @@ export const agentx = { "--max-prefill-tokens": ["32768"], "--max-running-requests": ["56"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], + "--prefill-decode-interval": ["48"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.25"], "--speculative-draft-attention-backend": null }, @@ -3206,67 +2612,9 @@ export const agentx = { "--max-prefill-tokens": ["32768"], "--max-running-requests": ["56"], "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["64"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.25"], - "--speculative-draft-attention-backend": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", - "PYTORCH_CUDA_ALLOC_CONF": null - } - } - ] - } - }, - { - "concs": [64], - "item": 1, - "override": "override_tp8_c64", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--chunked-prefill-size": ["32768"], - "--cuda-graph-max-bs-decode": ["64"], - "--enable-hierarchical-cache": [], - "--enable-layernorm-sp": [], - "--enable-megamoe-for-layernorm-sp": [], - "--hicache-size": ["160"], - "--max-prefill-tokens": ["32768"], - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.9"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.25"], - "--speculative-draft-attention-backend": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "8192", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", - "PYTORCH_CUDA_ALLOC_CONF": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--chunked-prefill-size": ["32768"], - "--cuda-graph-max-bs-decode": ["64"], - "--enable-hierarchical-cache": [], - "--enable-layernorm-sp": [], - "--enable-megamoe-for-layernorm-sp": [], - "--hicache-size": ["160"], - "--max-prefill-tokens": ["32768"], - "--max-running-requests": ["64"], - "--mem-fraction-static": ["0.9"], "--prefill-decode-interval": ["48"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.25"], "--speculative-draft-attention-backend": null }, @@ -3324,6 +2672,7 @@ export const agentx = { ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], ["--speculative-num-draft-tokens", "8"], ["--speculative-use-block-verification"], + ["--stream-interval", "40"], ["--swa-full-tokens-ratio", "0.3"], ["--tensor-parallel-size", "8"], ["--tokenizer-worker-num", "4"], @@ -3387,9 +2736,9 @@ export const agentx = { "detail": "DFlash 8", "gpus": 4, "configKey": "musespark-fp4-b200-sglang-agentic-dflash", - "image": "lmsysorg/sglang:nightly-dev-20261021-c1c287a4", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc", "imageStatus": "ok", - "imageCommit": "c1c287a4", + "imageCommit": "302b78cc", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -3407,39 +2756,35 @@ export const agentx = { } }, "prov": { - "sha": "11358f4843ec3e086014d3bba1a102e400944a59", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", - "run": "39756010759/attempts/1", - "pr": "4400", - "status": "verified", - "date": "2026-10-23" + "run": "41297696082", + "pr": "4707", + "comment": "6576173162", + "status": "verified-pr" }, "points": [ { - "concs": [8], + "concs": [7], "item": 2, - "override": "override_tp4_c8", + "override": "override_tp4_c7", "goldenAL": "4.05", "kv": "hicache" }, { - "concs": [10], + "concs": [16], "item": 2, - "override": "override_tp4_c10", + "override": "override_tp4_c16", "goldenAL": "4.05", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--max-running-requests": ["10"], - "--prefill-decode-interval": ["64"], - "--enable-layernorm-sp": null, - "--enable-megamoe-for-layernorm-sp": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": null + "--cuda-graph-max-bs-decode": ["6"], + "--incremental-streaming-output": [], + "--max-running-requests": ["6"], + "--prefill-decode-interval": ["44"] } } ], @@ -3448,32 +2793,33 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--max-running-requests": ["10"], - "--prefill-decode-interval": ["64"], - "--enable-layernorm-sp": null, - "--enable-megamoe-for-layernorm-sp": null - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": null + "--cuda-graph-max-bs-decode": ["6"], + "--incremental-streaming-output": [], + "--max-running-requests": ["6"], + "--prefill-decode-interval": ["44"] } } ] } }, { - "concs": [16], + "concs": [24], "item": 2, - "override": "override_tp4_c16", + "override": "override_tp4_c24", "goldenAL": "4.05", "kv": "hicache", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"], + "--cuda-graph-max-bs-decode": ["48"], + "--hicache-size": ["180"], + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.98"], "--prefill-decode-interval": ["48"] + }, + "env": { + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" } } ], @@ -3482,32 +2828,41 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"], + "--cuda-graph-max-bs-decode": ["48"], + "--hicache-size": ["180"], + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.98"], "--prefill-decode-interval": ["48"] + }, + "env": { + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" } } ] } }, { - "concs": [24], + "concs": [40], "item": 2, - "override": "override_tp4_c24", + "override": "override_tp4_c40", "goldenAL": "4.05", "kv": "hicache", "patch": { "sglang": [ { "args": { + "--chunked-prefill-size": ["16384"], "--cuda-graph-max-bs-decode": ["48"], - "--hicache-size": ["180"], + "--disable-prefill-cuda-graph": [], + "--hicache-size": ["160"], + "--max-prefill-tokens": ["16384"], "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.98"], - "--prefill-decode-interval": ["48"] + "--mem-fraction-static": ["0.93"], + "--cuda-graph-backend-prefill": null }, "env": { - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" } } ], @@ -3516,14 +2871,18 @@ export const agentx = { null, { "args": { + "--chunked-prefill-size": ["16384"], "--cuda-graph-max-bs-decode": ["48"], - "--hicache-size": ["180"], + "--disable-prefill-cuda-graph": [], + "--hicache-size": ["160"], + "--max-prefill-tokens": ["16384"], "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.98"], - "--prefill-decode-interval": ["48"] + "--mem-fraction-static": ["0.93"], + "--cuda-graph-backend-prefill": null }, "env": { - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096" } } ] @@ -3539,7 +2898,7 @@ export const agentx = { "diff": { "args": { "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["8"], + "--cuda-graph-max-bs-decode": ["2"], "--enable-hierarchical-cache": [], "--enable-layernorm-sp": [], "--enable-megamoe-for-layernorm-sp": [], @@ -3547,10 +2906,11 @@ export const agentx = { "--hicache-size": ["120"], "--hicache-write-policy": ["write_through"], "--max-prefill-tokens": ["8192"], - "--max-running-requests": ["8"], + "--max-running-requests": ["2"], "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["96"], + "--prefill-decode-interval": ["64"], "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"], "--swa-full-tokens-ratio": ["0.1"], "--tensor-parallel-size": ["4"], "--speculative-draft-attention-backend": null @@ -3560,7 +2920,7 @@ export const agentx = { "PYTORCH_CUDA_ALLOC_CONF": null } }, - "ref": [8, "sglang", 0] + "ref": [6, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index 31d0cfc258b4..e3e02fe358bc 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" + }, + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" + }, + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } } diff --git a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx index 1f6551be7b7d..72904b8ea8a1 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx @@ -1,36 +1,51 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@ea8b34394a60 plus open PRs #4333, #4454, #4595, #4603, #4670), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. export const agentx = { "model": "GLM-5.3", - "generatedOn": "2026-10-30", + "generatedOn": "2026-11-02", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "ea8b34394a603396d681e3c3d466d81cc885779b", + "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", "prs": { - "4670": { + "4726": { "run": null, - "sha": "5692839d90345486f1f90a1ea48b1d15ef755169", + "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", "status": "in-progress" }, - "4603": { - "run": null, - "sha": "9bd047ac75b8275723fd864987639e29d03efc91", - "status": "in-progress" + "4725": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", + "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", + "status": "verified-pr" }, - "4595": { - "run": null, - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", - "status": "in-progress" + "4724": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", + "sha": "98ea294c289546dca624e1f07d9d3d2106715741", + "status": "verified-pr" }, - "4454": { - "run": null, - "sha": "a73b274a6007c981eab97123d29fd03b74fe0b8b", - "status": "in-progress" + "4707": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", + "sha": "c965ca2eb81db733ad280ea693025389ac11014b", + "status": "verified-pr" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", + "status": "verified-pr" + }, + "4683": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", + "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", + "status": "verified-pr" }, - "4333": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", - "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "4670": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", + "sha": "c9eb455f32e7f026d7424de723756de2e719d868", + "status": "verified-pr" + }, + "4603": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", + "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "status": "verified-pr" } }