From 558e62c95f125209a218d5d1f22da77d656db7d2 Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" Date: Thu, 15 Oct 2026 20:51:57 -0700 Subject: [PATCH] [Doc] cookbook: Add SemiAnalysis AgentX launches to Muse Spark Open, refresh DeepSeek-V4.1-Flash Add the Agentic Long-Context Deployment section to the Muse Spark Open page (after the Playground, with the Deployment panel's agenticLink footer), mirroring SemiAnalysis InferenceX AgentX's Muse Spark Open SGLang lanes on B200 / GB200 (InferenceX #4147, #4140, merged) and B300 / GB300 (#4146, #4141, in progress). Refresh the generated AgentX data to InferenceX main c08eb51d with InferenceX #4111 / #4094 (DeepSeek-V4.1-Flash B200 / B300, /use-pinned) and #4112 (GB200, in progress) overlaid; main adds the Qwen3.5 NVFP4 B300 Dynamo+SGLang lane (#2978) and moves GLM-5.2 B300 TP4 (#3953) to a merged lane. Co-Authored-By: Claude Opus 5.5 --- .../autoregressive/Meta/MuseSparkOpen.mdx | 15 +- docs/src/snippets/agentx/Qwen/qwen3.5.jsx | 799 ++++++++- .../agentx/Qwen/qwen3.8-flash-next.jsx | 33 +- .../agentx/deepseek-ai/deepseek-v4.jsx | 33 +- .../agentx/deepseek-ai/deepseek-v4_1.jsx | 1392 +++++++++++---- .../agentx/meta-models/muse-spark-open.jsx | 1584 +++++++++++++++++ docs/src/snippets/agentx/zai-org/glm-5.2.jsx | 45 +- 7 files changed, 3411 insertions(+), 490 deletions(-) create mode 100644 docs/src/snippets/agentx/meta-models/muse-spark-open.jsx diff --git a/docs/cookbook/autoregressive/Meta/MuseSparkOpen.mdx b/docs/cookbook/autoregressive/Meta/MuseSparkOpen.mdx index 91ac20a677e0..2d2ba90de1cc 100644 --- a/docs/cookbook/autoregressive/Meta/MuseSparkOpen.mdx +++ b/docs/cookbook/autoregressive/Meta/MuseSparkOpen.mdx @@ -25,7 +25,7 @@ Choose your **Hardware Platform**, **Strategy**, and **Quantization** below. The import { Deployment } from "/src/snippets/_deployment.jsx"; import { config } from "/src/snippets/configs/meta-models/muse-spark.jsx"; - + ## Playground @@ -35,6 +35,19 @@ import { Playground } from "/src/snippets/_playground.jsx"; +## Agentic Long-Context Deployment + +Agentic workloads, such as coding agents and other multi-turn, tool-using assistants, send long prompts whose prefixes grow and repeat from turn to turn. Serving them well depends on reusing the KV cache across turns, offloading it beyond GPU memory, and keeping each session on the workers that already hold its prefix. This section gives complete launch commands for such deployments, each benchmarked in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) and linked to the run that measured it. Pick a hardware platform, checkpoint, deployment shape, concurrency, KV cache offloading mechanism and router to get every process of the deployment as a raw command, one tab per process in launch order. + +- **Router.** Each configuration was submitted behind one front door: Dynamo (`dynamo.frontend` with `dynamo.sglang` workers) or SGLang (`sglang.launch_server`, behind `sglang_router` when there are several workers or DP-attention ranks). +- **KV offload.** Choose among these KV cache offloading mechanisms: none (GPU KV only), HiCache (host DRAM), or a Mooncake store attached through the external linker. The Mooncake option needs an image whose SGLang build includes `--enable-unified-cache-external-linker`. +- **Container image.** Run the commands inside the image listed for each configuration; flag names follow that image. + +import { AgentX } from "/src/snippets/_agentx.jsx"; +import { agentx } from "/src/snippets/agentx/meta-models/muse-spark-open.jsx"; + + + ## 1. Model introduction This cookbook provides deployment recipes for the Muse Spark Open 816B-A42B checkpoints: diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx index 820944310b61..787a5ebec3fe 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -1,21 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. export const agentx = { "model": "Qwen3.5", - "generatedOn": "2026-10-13", + "generatedOn": "2026-10-15", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", "prs": { - "3953": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105", - "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "status": "verified-pr" }, - "3927": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261", - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "status": "verified-pr" } } @@ -7393,6 +7408,188 @@ export const agentx = { } } }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#4", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP4", + "detail": "MTP 6-1-7", + "gpus": 4, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml", + "run": "38390415472/attempts/1", + "pr": "2978", + "status": "verified", + "date": "2026-10-14" + }, + "points": [ + { + "concs": [4], + "item": 4, + "override": "base", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "8192"], + ["--context-length", "262144"], + ["--cuda-graph-bs-decode", "1"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-prefill-cuda-graph"], + ["--disable-shared-experts-fusion"], + ["--enable-hierarchical-cache"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "32"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--mamba-max-states-per-path", "3"], + ["--mamba-radix-cache-strategy", "extra_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "8192"], + ["--max-mamba-cache-size", "512"], + ["--max-prefill-tokens", "8192"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--pp-max-micro-batch-size", "1"], + ["--prefill-max-requests", "1"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "20"], + ["--tensor-parallel-size", "4"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, { "id": "qwen3.5-fp4-b300-sglang-agentic-mtp#2", "hw": "b300", @@ -7725,16 +7922,16 @@ export const agentx = { } }, { - "id": "qwen3.5-fp8-b200-sglang-agentic-mtp#2", - "hw": "b200", - "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", - "label": "Agg TP4", - "detail": "MTP 3-1-4", - "gpus": 4, - "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", - "imageStatus": "pruned", - "imageCommit": "4358a161", + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#1", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 7-1-8", + "gpus": 2, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -7749,91 +7946,523 @@ export const agentx = { ["--hicache-size", "144"], ["--hicache-write-policy", "write_through_selective"] ], - "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", - "donorHardware": "b200" + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" } }, "prov": { - "sha": "21a63c471574e0cda725d4a61d3837e91405b522", - "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", - "run": "35032235397/attempts/1", - "pr": "3142", + "sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml", + "run": "38390415472/attempts/1", + "pr": "2978", "status": "verified", - "date": "2026-09-16" + "date": "2026-10-14" }, "points": [ { - "concs": [1], - "item": 2, - "goldenAL": "3.39", - "kv": "none", - "job": "35032235397/job/104593683554" - }, - { - "concs": [4], + "concs": [36], "item": 2, - "goldenAL": "3.39", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["8"] - } - } - ] - }, - "job": "35032235397/job/104593683369" + "override": "zip_override_running_cap[0]", + "goldenAL": "5.04", + "kv": "hicache" }, { - "concs": [8], - "item": 2, - "goldenAL": "3.39", - "kv": "none", + "concs": [44], + "item": 1, + "override": "zip_override_running_cap[2]", + "goldenAL": "5.04", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"], "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["16"] + "--max-running-requests": ["8"], + "--pp-max-micro-batch-size": ["8"], + "--prefill-max-requests": ["8"] } - } + }, + null ], "dynamo": [ null, null, { "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"], "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["16"] + "--max-running-requests": ["8"], + "--pp-max-micro-batch-size": ["8"], + "--prefill-max-requests": ["8"] } } ] - }, - "job": "35032235397/job/104593684311" - }, - { - "concs": [12], - "item": 2, - "goldenAL": "3.39", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["24"] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4"], + "--cuda-graph-max-bs-decode": ["4"], + "--hicache-size": ["128"], + "--max-mamba-cache-size": ["1536"], + "--max-running-requests": ["4"], + "--moe-runner-backend": ["flashinfer_trtllm"], + "--pp-max-micro-batch-size": ["4"], + "--prefill-decode-interval": ["0"], + "--prefill-max-requests": ["4"], + "--speculative-moe-runner-backend": ["flashinfer_trtllm"], + "--speculative-num-draft-tokens": ["8"], + "--speculative-num-steps": ["7"], + "--tensor-parallel-size": ["2"], + "--disable-prefill-cuda-graph": null + } + }, + "ref": [33, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#3", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2", + "detail": "MTP 5-1-6", + "gpus": 2, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml", + "run": "38390415472/attempts/1", + "pr": "2978", + "status": "verified", + "date": "2026-10-14" + }, + "points": [ + { + "concs": [40], + "item": 3, + "goldenAL": "4.41", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-max-requests": ["2"], + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"] + } + }, + "ref": [35, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#0", + "hw": "b300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Agg TP2·EP2", + "detail": "MTP 3-1-4", + "gpus": 2, + "configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto", + "image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b300-sglang-agentic-mtp", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml", + "run": "38390415472/attempts/1", + "pr": "2978", + "status": "verified", + "date": "2026-10-14" + }, + "points": [ + { + "concs": [64], + "item": 0, + "goldenAL": "3.39", + "kv": "hicache" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--allow-auto-truncate": [], + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["224"], + "--expert-parallel-size": ["2"], + "--hicache-ratio": ["1.8"], + "--linear-attn-prefill-backend": ["flashinfer"], + "--mamba-max-states-per-path": ["1"], + "--mamba-track-interval": ["1048576"], + "--max-mamba-cache-size": ["512"], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["224"], + "--moe-runner-backend": ["flashinfer_cutedsl"], + "--scheduler-recv-interval": ["10"], + "--speculative-num-draft-tokens": ["4"], + "--speculative-num-steps": ["3"], + "--stream-interval": ["50"], + "--tokenizer-worker-num": ["6"], + "--weight-loader-drop-cache-after-load": [], + "--cuda-graph-bs-decode": null, + "--disable-attn-tp-gather": null, + "--disable-shared-experts-fusion": null, + "--hicache-size": null, + "--linear-attn-decode-backend": null, + "--moe-a2a-backend": null, + "--pp-max-micro-batch-size": null, + "--prefill-decode-interval": null, + "--prefill-max-requests": null, + "--speculative-moe-a2a-backend": null, + "--weight-loader-prefetch-checkpoints": null, + "--weight-loader-prefetch-num-threads": null + }, + "env": { + "SGLANG_ENABLE_JIT_DEEPGEMM": "false", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2" + } + }, + "ref": [35, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "qwen3.5-fp8-b200-sglang-agentic-mtp#2", + "hw": "b200", + "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", + "label": "Agg TP4", + "detail": "MTP 3-1-4", + "gpus": 4, + "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", + "imageStatus": "pruned", + "imageCommit": "4358a161", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "144"], + ["--hicache-write-policy", "write_through_selective"] + ], + "donor": "qwen3.5-fp4-b200-sglang-agentic-mtp", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "21a63c471574e0cda725d4a61d3837e91405b522", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", + "run": "35032235397/attempts/1", + "pr": "3142", + "status": "verified", + "date": "2026-09-16" + }, + "points": [ + { + "concs": [1], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "job": "35032235397/job/104593683554" + }, + { + "concs": [4], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["8"] + } + } + ] + }, + "job": "35032235397/job/104593683369" + }, + { + "concs": [8], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["16"] + } + } + ] + }, + "job": "35032235397/job/104593684311" + }, + { + "concs": [12], + "item": 2, + "goldenAL": "3.39", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["24"] } } ], @@ -8684,7 +9313,7 @@ export const agentx = { "--tp": ["8"] } }, - "ref": [34, "sglang", 0] + "ref": [38, "sglang", 0] } ] }, @@ -9339,7 +9968,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [35, "sglang", 0] + "ref": [39, "sglang", 0] } ] }, @@ -10346,7 +10975,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [38, "sglang", 0] + "ref": [42, "sglang", 0] } ] }, @@ -10586,7 +11215,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [39, "sglang", 0] + "ref": [43, "sglang", 0] } ] }, @@ -10902,7 +11531,7 @@ export const agentx = { "--expert-parallel-size": ["8"] } }, - "ref": [41, "sglang", 0] + "ref": [45, "sglang", 0] } ] }, @@ -11036,7 +11665,7 @@ export const agentx = { "--page-size": null } }, - "ref": [42, "sglang", 0] + "ref": [46, "sglang", 0] } ] }, @@ -11328,7 +11957,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [44, "sglang", 0] + "ref": [48, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx index 315769f09161..298f1361ad0e 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -1,21 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. export const agentx = { "model": "Qwen3.8-Flash-Next", - "generatedOn": "2026-10-13", + "generatedOn": "2026-10-15", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", "prs": { - "3953": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105", - "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "status": "verified-pr" }, - "3927": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261", - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "status": "verified-pr" } } diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx index ac14931f70fd..c5e689be5645 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx @@ -1,21 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4", - "generatedOn": "2026-10-13", + "generatedOn": "2026-10-15", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", "prs": { - "3953": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105", - "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "status": "verified-pr" }, - "3927": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261", - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "status": "verified-pr" } } diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index 54059c3c478b..10f4b5d400f1 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,21 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-10-13", + "generatedOn": "2026-10-15", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", "prs": { - "3953": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105", - "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "status": "verified-pr" }, - "3927": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261", - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "status": "verified-pr" } } @@ -58,14 +73,6 @@ export const agentx = { "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", "NATS_SERVER": "nats://$NATS_IP:4222" }, - { - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", - "SGLANG_DSV41_REASONING_EFFORT": "high", - "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900" - }, { "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "SGLANG_DEFAULT_THINKING": "1", @@ -1556,13 +1563,13 @@ export const agentx = { "id": "dsv41flash-fp4-gb200-sglang-agentic-dspark#0", "hw": "gb200", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP4·EP4", + "label": "Agg TP4", "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-gb200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0@sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "d32d8fa0", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -1573,41 +1580,43 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml", - "run": "38266927261", - "pr": "3927", - "comment": "6137929315", - "status": "verified-pr" + "pr": "4112", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 0, - "override": "override_tp4_c1", + "override": "override_tp4ep1_c1", "goldenAL": "3.51", "kv": "none" }, { "concs": [4], "item": 0, - "override": "override_tp4_c4", + "override": "override_tp4ep1_c4", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["384"] + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["64"], + "--schedule-policy": ["shortest-prefill-first"] } } ], @@ -1616,8 +1625,11 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["384"] + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["64"], + "--schedule-policy": ["shortest-prefill-first"] } } ] @@ -1626,15 +1638,20 @@ export const agentx = { { "concs": [8], "item": 0, - "override": "override_tp4_c8", + "override": "override_tp4ep1_c8", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["64"], + "--enable-decoder-swa-bounded-replay": [], "--max-running-requests": ["16"], - "--prefill-decode-interval": ["384"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["96"], + "--schedule-policy": ["shortest-prefill-first"], "--swa-prefix-tails": ["256"] } } @@ -1644,8 +1661,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["64"], + "--enable-decoder-swa-bounded-replay": [], "--max-running-requests": ["16"], - "--prefill-decode-interval": ["384"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["96"], + "--schedule-policy": ["shortest-prefill-first"], "--swa-prefix-tails": ["256"] } } @@ -1653,18 +1675,22 @@ export const agentx = { } }, { - "concs": [16], + "concs": [32], "item": 0, - "override": "override_tp4_c16", + "override": "override_tp4ep1_c32", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["32"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["512"] + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["80"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["1024"] } } ], @@ -1673,9 +1699,13 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["32"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["512"] + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["64"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["80"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["1024"] } } ] @@ -1693,27 +1723,36 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", + "SGLANG_DSV41_REASONING_EFFORT": "high", + "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], ["--host", "0.0.0.0"], ["--port", "30000"], ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs-decode", "64"], - ["--expert-parallel-size", "4"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.76"], + ["--cuda-graph-max-bs-decode", "1"], + ["--expert-parallel-size", "1"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.88"], ["--prefill-decode-interval", "16"], ["--reasoning-parser", "auto"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "5"], ["--swa-prefix-tails", "128"], ["--tensor-parallel-size", "4"], + ["--tokenizer-backend", "fastokens"], ["--tool-call-parser", "auto"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"] - ], - "envRef": 1 + ] } ] }, @@ -1766,13 +1805,13 @@ export const agentx = { "id": "dsv41flash-fp4-gb200-sglang-agentic-dspark#1", "hw": "gb200", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg DEP4", + "label": "Agg TP4·EP4", "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-gb200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0@sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "d32d8fa0", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -1783,41 +1822,152 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml", - "run": "38266927261", - "pr": "3927", - "comment": "6137929315", - "status": "verified-pr" + "pr": "4112", + "status": "in-progress" }, "points": [ { - "concs": [32], + "concs": [16], "item": 1, - "override": "override_dep4_c32", + "override": "override_tp4_c16", "goldenAL": "3.51", "kv": "none" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["96"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["512"] + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null + } + }, + "ref": [8, "sglang", 0] + } + ] }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-gb200-sglang-agentic-dspark#2", + "hw": "gb200", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Agg DEP4", + "detail": "DSpark 5", + "gpus": 4, + "configKey": "dsv41flash-fp4-gb200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "imageStatus": "ok", + "imageCommit": "684fea8a", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml", + "pr": "4112", + "status": "in-progress" + }, + "points": [ { "concs": [64], - "item": 1, - "override": "override_dep4_c64", + "item": 2, + "override": "override_dep4_hicache_c64", "goldenAL": "3.51", - "kv": "none", + "kv": "hicache" + }, + { + "concs": [96], + "item": 2, + "override": "override_dep4_hicache_c96", + "goldenAL": "3.51", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--chunked-prefill-size": ["65536"], - "--prefill-decode-interval": ["72"] + "--prefill-decode-interval": ["56"] } }, null @@ -1827,8 +1977,7 @@ export const agentx = { null, { "args": { - "--chunked-prefill-size": ["65536"], - "--prefill-decode-interval": ["72"] + "--prefill-decode-interval": ["56"] } } ] @@ -1836,16 +1985,15 @@ export const agentx = { }, { "concs": [128], - "item": 1, - "override": "override_dep4_c128", + "item": 2, + "override": "override_dep4_hicache_c128", "goldenAL": "3.51", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--chunked-prefill-size": ["65536"], - "--prefill-decode-interval": ["24"] + "--prefill-decode-interval": ["48"] } }, null @@ -1855,27 +2003,23 @@ export const agentx = { null, { "args": { - "--chunked-prefill-size": ["65536"], - "--prefill-decode-interval": ["24"] + "--prefill-decode-interval": ["48"] } } ] } }, { - "concs": [192], - "item": 1, - "override": "override_dep4_c192", + "concs": [160], + "item": 2, + "override": "override_dep4_hicache_c160", "goldenAL": "3.51", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--chunked-prefill-size": ["65536"], - "--max-running-requests": ["384"], - "--prefill-decode-interval": ["12"], - "--swa-prefix-tails": ["3072"] + "--prefill-decode-interval": ["40"] } }, null @@ -1885,10 +2029,35 @@ export const agentx = { null, { "args": { - "--chunked-prefill-size": ["65536"], - "--max-running-requests": ["384"], - "--prefill-decode-interval": ["12"], - "--swa-prefix-tails": ["3072"] + "--prefill-decode-interval": ["40"] + } + } + ] + } + }, + { + "concs": [256], + "item": 2, + "override": "override_dep4_hicache_c256", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--moe-runner-backend": ["flashinfer_mxfp4"], + "--prefill-decode-interval": ["24"] + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--moe-runner-backend": ["flashinfer_mxfp4"], + "--prefill-decode-interval": ["24"] } } ] @@ -1903,19 +2072,34 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["32768"], + "--chunked-prefill-size": ["65536"], + "--cuda-graph-backend-prefill": ["disabled"], "--data-parallel-size": ["4"], + "--dsv4-attn-backend": ["trtllm"], "--enable-dp-attention": [], "--enable-dp-attention-local-control-broadcast": [], "--enable-dp-lm-head": [], + "--enable-hierarchical-cache": [], + "--enable-symm-mem": [], + "--hicache-io-backend": ["direct"], + "--hicache-mem-layout": ["page_first_direct"], + "--hicache-ratio": ["2"], + "--hicache-write-policy": ["write_back"], "--load-balance-method": ["total_requests"], - "--max-running-requests": ["256"], - "--prefill-decode-interval": ["88"], - "--swa-prefix-tails": ["2048"], + "--max-running-requests": ["384"], + "--moe-a2a-backend": ["megamoe"], + "--prefill-decode-interval": ["72"], + "--swa-prefix-tails": ["3072"], "--tokenizer-worker-num": ["8"] + }, + "env": { + "SGLANG_DP_USE_GATHERV": "1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0", + "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [8, "sglang", 0] + "ref": [9, "sglang", 0] }, { "kind": "sglang-router", @@ -1982,16 +2166,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#1", + "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#0", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP2", + "label": "Agg TP4", "detail": "DSpark 5", - "gpus": 2, + "gpus": 4, "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "295a53c9", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2002,42 +2186,45 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "4"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300" } }, "prov": { - "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", - "run": "37816190252/attempts/1", - "pr": "3744", - "status": "verified", - "date": "2026-10-09" + "run": "38705241139", + "pr": "4094", + "comment": "6206114636", + "status": "verified-pr" }, "points": [ { "concs": [1], - "item": 1, - "override": "override_tp2_c1", + "item": 0, + "override": "override_tp4_c1", "goldenAL": "3.51", "kv": "none" }, { "concs": [4], - "item": 1, - "override": "override_tp2_c4", + "item": 0, + "override": "override_tp4_c4", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["256"] + "--prefill-decode-interval": ["192"], + "--swa-prefix-tails": ["256"], + "--schedule-policy": null } } ], @@ -2046,9 +2233,48 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["8"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["256"] + "--prefill-decode-interval": ["192"], + "--swa-prefix-tails": ["256"], + "--schedule-policy": null + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "override_tp4_c8", + "goldenAL": "3.51", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["16"], + "--prefill-decode-interval": ["192"], + "--swa-prefix-tails": ["512"], + "--schedule-policy": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["16"], + "--prefill-decode-interval": ["192"], + "--swa-prefix-tails": ["512"], + "--schedule-policy": null } } ] @@ -2056,18 +2282,21 @@ export const agentx = { }, { "concs": [16], - "item": 1, - "override": "override_tp2_c16", + "item": 0, + "override": "override_tp4_c16", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["64"], + "--enable-decoder-swa-bounded-replay": [], "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.8"], - "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["1024"] + "--prefill-decode-interval": ["128"], + "--swa-prefix-tails": ["512"], + "--schedule-policy": null } } ], @@ -2076,29 +2305,35 @@ export const agentx = { null, { "args": { + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["64"], + "--enable-decoder-swa-bounded-replay": [], "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.8"], - "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["1024"] + "--prefill-decode-interval": ["128"], + "--swa-prefix-tails": ["512"], + "--schedule-policy": null } } ] } }, { - "concs": [24], - "item": 1, - "override": "override_tp2_c24", + "concs": [20], + "item": 0, + "override": "override_tp4_c20", "goldenAL": "3.51", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.8"], + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["40"], "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["1536"] + "--swa-prefix-tails": ["512"], + "--schedule-policy": null } } ], @@ -2107,14 +2342,153 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.8"], + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--max-running-requests": ["40"], "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["1536"] + "--swa-prefix-tails": ["512"], + "--schedule-policy": null } } ] } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", + "SGLANG_DSV41_REASONING_EFFORT": "high", + "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], + ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--chunked-prefill-size", "16384"], + ["--cuda-graph-max-bs-decode", "1"], + ["--expert-parallel-size", "1"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.7"], + ["--prefill-decode-interval", "16"], + ["--reasoning-parser", "auto"], + ["--schedule-policy", "lpm"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-dspark-block-size", "5"], + ["--swa-prefix-tails", "128"], + ["--tensor-parallel-size", "4"], + ["--tokenizer-backend", "fastokens"], + ["--tool-call-parser", "auto"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"], + ["--weight-loader-prefetch-checkpoints"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#1", + "hw": "b300", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Agg TP2", + "detail": "DSpark 5", + "gpus": 2, + "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "imageStatus": "ok", + "imageCommit": "684fea8a", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "4"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300" + } + }, + "prov": { + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", + "run": "38705241139", + "pr": "4094", + "comment": "6206114636", + "status": "verified-pr" + }, + "points": [ + { + "concs": [16], + "item": 1, + "override": "override_tp2_c16", + "goldenAL": "3.51", + "kv": "none" }, { "concs": [32], @@ -2128,8 +2502,9 @@ export const agentx = { "args": { "--max-running-requests": ["64"], "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["64"], - "--swa-prefix-tails": ["2048"] + "--swa-prefix-tails": ["2048"], + "--cuda-graph-backend-prefill": null, + "--enable-decoder-swa-bounded-replay": null } } ], @@ -2140,8 +2515,9 @@ export const agentx = { "args": { "--max-running-requests": ["64"], "--mem-fraction-static": ["0.85"], - "--prefill-decode-interval": ["64"], - "--swa-prefix-tails": ["2048"] + "--swa-prefix-tails": ["2048"], + "--cuda-graph-backend-prefill": null, + "--enable-decoder-swa-bounded-replay": null } } ] @@ -2157,10 +2533,13 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--dsv4-attn-backend": ["trtllm"], "--max-running-requests": ["64"], "--mem-fraction-static": ["0.8"], "--prefill-decode-interval": ["64"], - "--swa-prefix-tails": ["3072"] + "--swa-prefix-tails": ["3072"], + "--enable-decoder-swa-bounded-replay": null } } ], @@ -2169,10 +2548,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--dsv4-attn-backend": ["trtllm"], "--max-running-requests": ["64"], "--mem-fraction-static": ["0.8"], "--prefill-decode-interval": ["64"], - "--swa-prefix-tails": ["3072"] + "--swa-prefix-tails": ["3072"], + "--enable-decoder-swa-bounded-replay": null } } ] @@ -2185,33 +2567,19 @@ export const agentx = { "files": [], "blocks": [ { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "args": [ - ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], - ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--chunked-prefill-size", "16384"], - ["--cuda-graph-max-bs-decode", "64"], - ["--expert-parallel-size", "1"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.7"], - ["--prefill-decode-interval", "16"], - ["--reasoning-parser", "auto"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-dspark-block-size", "5"], - ["--swa-prefix-tails", "128"], - ["--tensor-parallel-size", "2"], - ["--tool-call-parser", "auto"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"], - ["--weight-loader-prefetch-checkpoints"] - ], - "envRef": 1 + "diff": { + "args": { + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["64"], + "--enable-decoder-swa-bounded-replay": [], + "--max-running-requests": ["32"], + "--prefill-decode-interval": ["96"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["1024"], + "--tensor-parallel-size": ["2"] + } + }, + "ref": [11, "sglang", 0] } ] }, @@ -2261,16 +2629,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#0", + "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#2", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP4", + "label": "Agg DEP2", "detail": "DSpark 5", - "gpus": 4, + "gpus": 2, "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "295a53c9", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2281,53 +2649,55 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "4"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300" } }, "prov": { - "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", - "run": "37816190252/attempts/1", - "pr": "3744", - "status": "verified", - "date": "2026-10-09" + "run": "38705241139", + "pr": "4094", + "comment": "6206114636", + "status": "verified-pr" }, "points": [ { - "concs": [1], - "item": 0, - "override": "override_tp4_c1", + "concs": [64], + "item": 2, + "override": "override_dep2_c64", "goldenAL": "3.51", - "kv": "none" + "kv": "hicache" }, { - "concs": [4], - "item": 0, - "override": "override_tp4_c4", + "concs": [256], + "item": 2, + "override": "override_dep2_c256", "goldenAL": "3.51", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["8"], - "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["256"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["256"], + "--prefill-decode-interval": ["16"] } - } + }, + null ], "dynamo": [ null, null, { "args": { - "--max-running-requests": ["8"], - "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["256"] + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["256"], + "--prefill-decode-interval": ["16"] } } ] @@ -2342,10 +2712,55 @@ export const agentx = { { "diff": { "args": { - "--tensor-parallel-size": ["4"] + "--chunked-prefill-size": ["32768"], + "--cuda-graph-max-bs-decode": ["96"], + "--data-parallel-size": ["2"], + "--disable-prefill-cuda-graph": [], + "--dsv4-attn-backend": ["trtllm"], + "--enable-dp-attention": [], + "--enable-dp-attention-local-control-broadcast": [], + "--enable-dp-lm-head": [], + "--enable-hierarchical-cache": [], + "--expert-parallel-size": ["2"], + "--hicache-io-backend": ["direct"], + "--hicache-mem-layout": ["page_first_direct"], + "--hicache-ratio": ["4"], + "--hicache-write-policy": ["write_back"], + "--load-balance-method": ["total_requests"], + "--max-running-requests": ["192"], + "--mem-fraction-static": ["0.82"], + "--moe-a2a-backend": ["megamoe"], + "--moe-runner-backend": ["flashinfer_mxfp4"], + "--prefill-decode-interval": ["48"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["3072"], + "--tensor-parallel-size": ["2"], + "--tokenizer-worker-num": ["8"] + }, + "env": { + "SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION": "1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", + "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [10, "sglang", 0] + "ref": [11, "sglang", 0] + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "consistent_hashing"], + ["--request-id-headers", "x-correlation-id"], + ["--dp-aware"], + ["--connect-timeout-secs", "900"], + ["--request-timeout-secs", "14400"], + ["--disable-health-check"], + ["--disable-retries"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] }, @@ -2395,16 +2810,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#2", + "id": "dsv41flash-fp4-b300-sglang-agentic-dspark#3", "hw": "b300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Agg DEP4", "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "295a53c9", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2415,28 +2830,29 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "4"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b300-sglang-agentic-dspark", + "donorHardware": "b300" } }, "prov": { - "sha": "ec20ed5660994bc20a94d68a04488aa5bc2b33ea", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml", - "run": "37816190252/attempts/1", - "pr": "3744", - "status": "verified", - "date": "2026-10-09" + "run": "38705241139", + "pr": "4094", + "comment": "6206114636", + "status": "verified-pr" }, "points": [ { - "concs": [192], - "item": 2, - "override": "override_dp4_c192", + "concs": [256], + "item": 3, + "override": "override_dep4_c256", "goldenAL": "3.51", - "kv": "none" + "kv": "hicache" } ], "routers": { @@ -2447,20 +2863,18 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["32768"], + "--chunked-prefill-size": ["65536"], + "--cuda-graph-max-bs-decode": ["64"], "--data-parallel-size": ["4"], - "--enable-dp-attention": [], - "--enable-dp-attention-local-control-broadcast": [], - "--enable-dp-lm-head": [], "--expert-parallel-size": ["4"], - "--load-balance-method": ["total_requests"], - "--max-running-requests": ["384"], - "--mem-fraction-static": ["0.76"], - "--swa-prefix-tails": ["3072"], - "--tokenizer-worker-num": ["8"] + "--hicache-ratio": ["2"], + "--max-running-requests": ["256"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["32"], + "--tensor-parallel-size": ["4"] } }, - "ref": [11, "sglang", 0] + "ref": [13, "sglang", 0] }, { "kind": "sglang-router", @@ -2469,7 +2883,7 @@ export const agentx = { "args": [ ["--host", "0.0.0.0"], ["--port", "8000"], - ["--policy", "cache_aware"], + ["--policy", "consistent_hashing"], ["--request-id-headers", "x-correlation-id"], ["--dp-aware"], ["--connect-timeout-secs", "900"], @@ -2530,13 +2944,13 @@ export const agentx = { "id": "dsv41flash-fp4-b200-sglang-agentic-dspark#0", "hw": "b200", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP4·EP4", + "label": "Agg TP4", "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "295a53c9", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2547,20 +2961,21 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "322bc37b368c85828357afb57dce6739d94ef955", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "37420473448/attempts/1", - "pr": "3745", - "status": "verified", - "date": "2026-10-08" + "run": "38749770474", + "pr": "4111", + "comment": "6206108905", + "status": "verified-pr" }, "points": [ { @@ -2580,8 +2995,20 @@ export const agentx = { "sglang": [ { "args": { - "--max-running-requests": ["8"], - "--prefill-decode-interval": ["192"] + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--max-running-requests": ["2"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["256"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["256"], + "--data-parallel-size": null, + "--moe-a2a-backend": null + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1" } } ], @@ -2590,8 +3017,20 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["8"], - "--prefill-decode-interval": ["192"] + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--kv-cache-dtype": ["fp8_e4m3"], + "--max-running-requests": ["2"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["256"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["256"], + "--data-parallel-size": null, + "--moe-a2a-backend": null + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1" } } ] @@ -2607,9 +3046,20 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--kv-cache-dtype": ["fp8_e4m3"], "--max-running-requests": ["16"], + "--mem-fraction-static": ["0.76"], "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["256"] + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["256"], + "--data-parallel-size": null, + "--moe-a2a-backend": null + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1" } } ], @@ -2618,48 +3068,167 @@ export const agentx = { null, { "args": { + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--dsv4-attn-backend": ["trtllm"], + "--kv-cache-dtype": ["fp8_e4m3"], "--max-running-requests": ["16"], + "--mem-fraction-static": ["0.76"], "--prefill-decode-interval": ["192"], - "--swa-prefix-tails": ["256"] + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["256"], + "--data-parallel-size": null, + "--moe-a2a-backend": null + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1" } } ] } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", + "SGLANG_DSV41_REASONING_EFFORT": "high", + "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], + ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--chunked-prefill-size", "8192"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--dsv4-attn-backend", "flashmla"], + ["--expert-parallel-size", "1"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.88"], + ["--moe-a2a-backend", "none"], + ["--prefill-decode-interval", "16"], + ["--reasoning-parser", "auto"], + ["--schedule-policy", "lpm"], + ["--speculative-algorithm", "DSPARK"], + ["--speculative-dspark-block-size", "5"], + ["--swa-prefix-tails", "128"], + ["--tensor-parallel-size", "4"], + ["--tokenizer-backend", "fastokens"], + ["--tool-call-parser", "auto"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-b200-sglang-agentic-dspark#1", + "hw": "b200", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Agg TP4·EP4", + "detail": "DSpark 5", + "gpus": 4, + "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "imageStatus": "ok", + "imageCommit": "684fea8a", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", + "run": "38749770474", + "pr": "4111", + "comment": "6206108905", + "status": "verified-pr" + }, + "points": [ { "concs": [16], - "item": 0, + "item": 1, "override": "override_tp4_c16", "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["32"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["512"], - "--schedule-policy": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["32"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["512"], - "--schedule-policy": null - } - } - ] - } + "kv": "none" }, { "concs": [24], - "item": 0, + "item": 1, "override": "override_tp4_c24", "goldenAL": "3.51", "kv": "none", @@ -2668,9 +3237,8 @@ export const agentx = { { "args": { "--max-running-requests": ["48"], - "--prefill-decode-interval": ["48"], - "--swa-prefix-tails": ["768"], - "--schedule-policy": null + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["768"] } } ], @@ -2680,40 +3248,8 @@ export const agentx = { { "args": { "--max-running-requests": ["48"], - "--prefill-decode-interval": ["48"], - "--swa-prefix-tails": ["768"], - "--schedule-policy": null - } - } - ] - } - }, - { - "concs": [32], - "item": 0, - "override": "override_tp4_c32", - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"], - "--prefill-decode-interval": ["32"], - "--swa-prefix-tails": ["1024"], - "--schedule-policy": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"], - "--prefill-decode-interval": ["32"], - "--swa-prefix-tails": ["1024"], - "--schedule-policy": null + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["768"] } } ] @@ -2726,33 +3262,24 @@ export const agentx = { "files": [], "blocks": [ { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "args": [ - ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], - ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs-decode", "64"], - ["--expert-parallel-size", "4"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.76"], - ["--prefill-decode-interval", "16"], - ["--reasoning-parser", "auto"], - ["--schedule-policy", "lpm"], - ["--speculative-algorithm", "DSPARK"], - ["--speculative-dspark-block-size", "5"], - ["--swa-prefix-tails", "128"], - ["--tensor-parallel-size", "4"], - ["--tool-call-parser", "auto"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ], - "envRef": 1 + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--expert-parallel-size": ["4"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.76"], + "--prefill-decode-interval": ["192"], + "--schedule-policy": ["shortest-prefill-first"], + "--swa-prefix-tails": ["512"], + "--data-parallel-size": null, + "--dsv4-attn-backend": null, + "--moe-a2a-backend": null + }, + "env": { + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null + } + }, + "ref": [15, "sglang", 0] } ] }, @@ -2802,16 +3329,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-b200-sglang-agentic-dspark#1", + "id": "dsv41flash-fp4-b200-sglang-agentic-dspark#2", "hw": "b200", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Agg DEP4", "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", "imageStatus": "ok", - "imageCommit": "295a53c9", + "imageCommit": "684fea8a", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2822,40 +3349,93 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "322bc37b368c85828357afb57dce6739d94ef955", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "37420473448/attempts/1", - "pr": "3745", - "status": "verified", - "date": "2026-10-08" + "run": "38749770474", + "pr": "4111", + "comment": "6206108905", + "status": "verified-pr" }, "points": [ { "concs": [48], - "item": 1, + "item": 2, "override": "override_dep4_c48", "goldenAL": "3.51", - "kv": "none" + "kv": "hicache" + }, + { + "concs": [64], + "item": 2, + "override": "override_dep4_c64", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["96"] + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--prefill-decode-interval": ["96"] + } + } + ] + } + }, + { + "concs": [96], + "item": 2, + "override": "override_dep4_c96", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["64"] + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--prefill-decode-interval": ["64"] + } + } + ] + } }, { "concs": [128], - "item": 1, + "item": 2, "override": "override_dep4_c128", "goldenAL": "3.51", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { - "--prefill-decode-interval": ["16"] + "--prefill-decode-interval": ["48"] } }, null @@ -2865,7 +3445,59 @@ export const agentx = { null, { "args": { - "--prefill-decode-interval": ["16"] + "--prefill-decode-interval": ["48"] + } + } + ] + } + }, + { + "concs": [192], + "item": 2, + "override": "override_dep4_c192", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["40"] + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--prefill-decode-interval": ["40"] + } + } + ] + } + }, + { + "concs": [256], + "item": 2, + "override": "override_dep4_c256", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["24"] + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--prefill-decode-interval": ["24"] } } ] @@ -2880,20 +3512,34 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["32768"], + "--chunked-prefill-size": ["65536"], + "--cuda-graph-backend-prefill": ["disabled"], "--data-parallel-size": ["4"], + "--dsv4-attn-backend": ["trtllm"], "--enable-dp-attention": [], "--enable-dp-attention-local-control-broadcast": [], "--enable-dp-lm-head": [], + "--enable-hierarchical-cache": [], + "--hicache-io-backend": ["direct"], + "--hicache-mem-layout": ["page_first_direct"], + "--hicache-ratio": ["2"], + "--hicache-write-policy": ["write_back"], + "--kv-cache-dtype": ["fp8_e4m3"], "--load-balance-method": ["total_requests"], "--max-running-requests": ["256"], - "--prefill-decode-interval": ["64"], - "--swa-prefix-tails": ["2048"], - "--tokenizer-worker-num": ["8"], - "--schedule-policy": null + "--moe-a2a-backend": ["megamoe"], + "--moe-runner-backend": ["flashinfer_mxfp4"], + "--prefill-decode-interval": ["128"], + "--swa-prefix-tails": ["3072"], + "--tokenizer-backend": ["huggingface"], + "--tokenizer-worker-num": ["1"] + }, + "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0" } }, - "ref": [13, "sglang", 0] + "ref": [16, "sglang", 0] }, { "kind": "sglang-router", @@ -2902,13 +3548,13 @@ export const agentx = { "args": [ ["--host", "0.0.0.0"], ["--port", "8000"], - ["--policy", "cache_aware"], - ["--request-id-headers", "x-correlation-id"], - ["--dp-aware"], ["--connect-timeout-secs", "900"], - ["--request-timeout-secs", "14400"], ["--disable-health-check"], ["--disable-retries"], + ["--dp-aware"], + ["--policy", "cache_aware"], + ["--request-id-headers", "x-correlation-id"], + ["--request-timeout-secs", "14400"], ["--worker-urls", "http://127.0.0.1:30000"] ] } @@ -2980,11 +3626,12 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -3289,11 +3936,12 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -3499,7 +4147,7 @@ export const agentx = { "--tp": ["8"] } }, - "ref": [15, "sglang", 0] + "ref": [18, "sglang", 0] } ] }, @@ -3569,11 +4217,12 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -3760,7 +4409,7 @@ export const agentx = { ["--trust-remote-code"], ["--watchdog-timeout", "3600"] ], - "envRef": 2 + "envRef": 1 } ] }, @@ -3830,11 +4479,12 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-ratio", "1"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -3947,7 +4597,7 @@ export const agentx = { "--swa-prefix-tails": ["64"] } }, - "ref": [17, "sglang", 0] + "ref": [20, "sglang", 0] }, { "kind": "sglang-router", @@ -3964,7 +4614,7 @@ export const agentx = { ["--disable-retries"], ["--worker-urls", "http://127.0.0.1:30000"] ], - "envRef": 2 + "envRef": 1 } ] }, diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx new file mode 100644 index 000000000000..678da5470769 --- /dev/null +++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx @@ -0,0 +1,1584 @@ +// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. +// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. +export const agentx = { + "model": "Muse Spark Open", + "generatedOn": "2026-10-15", + "source": { + "repo": "https://github.com/SemiAnalysisAI/InferenceX", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", + "prs": { + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", + "status": "verified-pr" + }, + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", + "status": "verified-pr" + } + } + }, + "hardware": [ + { + "id": "gb300", + "label": "GB300" + }, + { + "id": "gb200", + "label": "GB200" + }, + { + "id": "b300", + "label": "B300" + }, + { + "id": "b200", + "label": "B200" + } + ], + "checkpoints": [ + { + "id": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4" + } + ], + "envs": [], + "kvMooncake": { + "donor": "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", + "donorHardware": "gb300", + "master": { + "dynamo": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + }, + "sglang": { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--rpc_thread_num", "16"], + ["--eviction_high_watermark_ratio", "0.90"] + ] + } + }, + "args": [ + ["--enable-unified-cache-external-linker"], + ["--hicache-storage-backend-extra-config", "{\"enable_group_semantics\":true}"], + ["--unified-cache-external-linker-backend", "mooncake"] + ], + "env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "140gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + }, + "decode_env": { + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata" + } + }, + "files": {}, + "cells": [ + { + "id": "musespark-fp4-gb300-sglang-agentic-dflash#0", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Agg TP4·EP4", + "detail": "DFlash 8", + "gpus": 4, + "configKey": "musespark-fp4-gb300-sglang-agentic-dflash", + "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "mooncake"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet" + } + }, + "prov": { + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml", + "pr": "4141", + "status": "in-progress" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "zip_override_tp4[0]", + "goldenAL": "4.05", + "kv": "none" + }, + { + "concs": [2], + "item": 0, + "override": "zip_override_tp4[1]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["2"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["2"] + } + } + ] + } + }, + { + "concs": [3], + "item": 0, + "override": "zip_override_tp4[2]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["3"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["3"] + } + } + ] + } + }, + { + "concs": [4], + "item": 0, + "override": "zip_override_tp4[3]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["4"] + } + } + ] + } + }, + { + "concs": [6], + "item": 0, + "override": "zip_override_tp4[4]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["6"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["6"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "zip_override_tp4[5]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [10], + "item": 0, + "override": "zip_override_tp4[6]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["10"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["10"] + } + } + ] + } + }, + { + "concs": [12], + "item": 0, + "override": "zip_override_tp4[7]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["12"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["12"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "zip_override_tp4[8]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [20], + "item": 0, + "override": "zip_override_tp4[9]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["20"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["20"] + } + } + ] + } + }, + { + "concs": [24], + "item": 0, + "override": "zip_override_tp4[10]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["24"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "zip_override_tp4[11]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--max-running-requests": ["32"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--enable-layernorm-sp"], + ["--enable-megamoe-for-layernorm-sp"], + ["--expert-parallel-size", "4"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-running-requests", "1"], + ["--mem-fraction-static", "0.85"], + ["--mm-feature-transport", "cpu"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "fa4"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-num-draft-tokens", "8"], + ["--swa-full-tokens-ratio", "0.1"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "muse"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "musespark-fp4-gb200-sglang-agentic-dflash#0", + "hw": "gb200", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Agg TP4·EP4", + "detail": "DFlash 8", + "gpus": 4, + "configKey": "musespark-fp4-gb200-sglang-agentic-dflash", + "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "120"] + ], + "donor": "musespark-fp4-gb200-sglang-agentic-dflash", + "donorHardware": "gb200" + } + }, + "prov": { + "sha": "5b0708723cf37931a80eccf1b830c7519b48f24e", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "38834513220/attempts/1", + "pr": "4140", + "status": "verified", + "date": "2026-10-16" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "override_tp4_c1", + "goldenAL": "4.05", + "kv": "hicache" + }, + { + "concs": [2], + "item": 0, + "override": "override_tp4_c2", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ] + } + }, + { + "concs": [4], + "item": 0, + "override": "override_tp4_c4", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [6], + "item": 0, + "override": "override_tp4_c6", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "override_tp4_c8", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [10], + "item": 0, + "override": "override_tp4_c10", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--max-running-requests": ["20"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--max-running-requests": ["20"] + } + } + ] + } + }, + { + "concs": [12], + "item": 0, + "override": "override_tp4_c12", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "override_tp4_c16", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [24], + "item": 0, + "override": "override_tp4_c24", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "override_tp4_c32", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--cuda-graph-max-bs-decode", "2"], + ["--disable-prefill-cuda-graph"], + ["--enable-hierarchical-cache"], + ["--enable-layernorm-sp"], + ["--enable-megamoe-for-layernorm-sp"], + ["--expert-parallel-size", "4"], + ["--hicache-size", "120"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.88"], + ["--mm-feature-transport", "cpu"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "fa4"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-num-draft-tokens", "8"], + ["--swa-full-tokens-ratio", "0.1"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "muse"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "musespark-fp4-b300-sglang-agentic-dflash#0", + "hw": "b300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Agg TP4·EP4", + "detail": "DFlash 8", + "gpus": 4, + "configKey": "musespark-fp4-b300-sglang-agentic-dflash", + "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "mooncake"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet" + } + }, + "prov": { + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", + "pr": "4146", + "status": "in-progress" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "zip_override_tp4[0]", + "goldenAL": "4.05", + "kv": "none" + }, + { + "concs": [2], + "item": 0, + "override": "zip_override_tp4[1]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ] + } + }, + { + "concs": [4], + "item": 0, + "override": "zip_override_tp4[2]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "zip_override_tp4[3]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "zip_override_tp4[4]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [24], + "item": 0, + "override": "zip_override_tp4[5]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "zip_override_tp4[6]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + }, + { + "concs": [48], + "item": 0, + "override": "zip_override_tp4[7]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["96"], + "--max-running-requests": ["96"] + } + } + ] + } + }, + { + "concs": [64], + "item": 0, + "override": "zip_override_tp4[8]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["128"], + "--max-running-requests": ["128"] + } + } + ] + } + }, + { + "concs": [128], + "item": 0, + "override": "zip_override_tp4[9]", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["256"], + "--max-running-requests": ["256"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["256"], + "--max-running-requests": ["256"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--cuda-graph-max-bs-decode", "2"], + ["--enable-layernorm-sp"], + ["--enable-megamoe-for-layernorm-sp"], + ["--expert-parallel-size", "4"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.8"], + ["--mm-feature-transport", "cpu"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "fa4"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-num-draft-tokens", "8"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "muse"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "musespark-fp4-b200-sglang-agentic-dflash#0", + "hw": "b200", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Agg TP4·EP4", + "detail": "DFlash 8", + "gpus": 4, + "configKey": "musespark-fp4-b200-sglang-agentic-dflash", + "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "mooncake"], + "unavailable": { + "hicache": "no verified HiCache settings for this model yet" + } + }, + "prov": { + "sha": "7587c01b8df9ea77877a97815442f6f776a951c4", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", + "run": "38825822770/attempts/1", + "pr": "4147", + "status": "verified", + "date": "2026-10-16" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "override_tp4_c1", + "goldenAL": "4.05", + "kv": "none" + }, + { + "concs": [2], + "item": 0, + "override": "override_tp4_c2", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"] + } + } + ] + } + }, + { + "concs": [4], + "item": 0, + "override": "override_tp4_c4", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "override_tp4_c8", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [12], + "item": 0, + "override": "override_tp4_c12", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "override_tp4_c16", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [24], + "item": 0, + "override": "override_tp4_c24", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["48"], + "--max-running-requests": ["48"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "override_tp4_c32", + "goldenAL": "4.05", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["64"] + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "worker", + "role": "agg", + "workers": 1, + "nodes_per_worker": 1, + "cmd": "python3 -m sglang.launch_server", + "env": { + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--cuda-graph-max-bs-decode", "2"], + ["--disable-prefill-cuda-graph"], + ["--enable-layernorm-sp"], + ["--enable-megamoe-for-layernorm-sp"], + ["--expert-parallel-size", "4"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-running-requests", "2"], + ["--mem-fraction-static", "0.88"], + ["--mm-feature-transport", "cpu"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "fa4"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-num-draft-tokens", "8"], + ["--swa-full-tokens-ratio", "0.1"], + ["--tensor-parallel-size", "4"], + ["--tool-call-parser", "muse"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + } + ] +}; diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index 397dd20b31f1..579d3751e5da 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,21 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-10-13", + "generatedOn": "2026-10-15", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d", + "main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff", "prs": { - "3953": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105", - "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", + "4146": { + "run": null, + "sha": "a23546bea6a5f1d34b75261728ac110403a86540", + "status": "in-progress" + }, + "4141": { + "run": null, + "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "status": "in-progress" + }, + "4112": { + "run": null, + "sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72", + "status": "in-progress" + }, + "4111": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474", + "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", "status": "verified-pr" }, - "3927": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261", - "sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78", + "4094": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139", + "sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac", "status": "verified-pr" } } @@ -2479,10 +2494,10 @@ export const agentx = { "prov": { "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4/agentx/agg-tp4-variants.yaml", - "run": "38356572105", + "run": "38356572105/attempts/2", "pr": "3953", - "comment": "6161711288", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-14" }, "points": [ { @@ -3210,10 +3225,10 @@ export const agentx = { "prov": { "sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4/agentx/agg-tp4-variants.yaml", - "run": "38356572105", + "run": "38356572105/attempts/2", "pr": "3953", - "comment": "6161711288", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-14" }, "points": [ {