From c39acc7599c0a97ec9a3a48625b096a01e76d8ba Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" Date: Mon, 19 Oct 2026 21:10:20 -0700 Subject: [PATCH] [Doc] cookbook: Refresh SemiAnalysis AgentX launches with five open InferenceX PRs Regenerate the six Agentic Long-Context Deployment data files from InferenceX main (660b5a8d) with five open SemiAnalysisAI/InferenceX pull requests overlaid: 4333, 4327, 4300, 4334 and 3817. - Qwen3.5: NVFP4 GB300 Dynamo+SGLang refresh on a new image (agg TP4/TP8 c8-c23, disagg 1P1D/1P2D TP2 and 2P1D/3P1D DEP16 points); two superseded pareto keys leave the page. - DeepSeek-V4.1-Flash: H200 SGLang moves to nightly-20261013-7224fce1 (c1-c16). GB300 lanes merged on main are verified; derived HiCache rows use the measured GB300 HiCache worker. - Muse Spark Open: GB300 HiCache above c6; B200 on nightly-20261019-3c560089; B300/H200/GB200 now verified on main; H100 left out (the dashboard lost its changelog entry). - GLM-5.2: B200 agg verified on main; the knob-tuned FP8 B300 lane is no longer overlaid. - Qwen3.8-Flash-Next: B200/B300 back to the measured main runs. Co-Authored-By: Claude Opus 5.5 --- docs/src/snippets/agentx/Qwen/qwen3.5.jsx | 1966 ++++++++++++----- .../agentx/Qwen/qwen3.8-flash-next.jsx | 500 +---- .../agentx/deepseek-ai/deepseek-v4.jsx | 59 +- .../agentx/deepseek-ai/deepseek-v4_1.jsx | 382 +--- .../agentx/meta-models/muse-spark-open.jsx | 374 +--- docs/src/snippets/agentx/zai-org/glm-5.2.jsx | 583 +---- 6 files changed, 1803 insertions(+), 2061 deletions(-) diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx index c0ae0bc479d2..51f5b12d15c3 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "Qwen3.5", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { - "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", - "status": "in-progress" - }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", - "status": "verified-pr" - }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { + "4333": { "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "4279": { - "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "status": "in-progress" - }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, @@ -212,13 +177,17 @@ export const agentx = { "name": "nats.conf", "content": "max_payload: 8388608\njetstream { store_dir: \"/tmp/nats\" }\n" }, + "nginx.conf#748c9006": { + "name": "nginx.conf", + "content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}" + }, "nginx.conf#033bddfb": { "name": "nginx.conf", "content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n server $FRONTEND_1_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}" }, - "nginx.conf#748c9006": { - "name": "nginx.conf", - "content": "# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n# SPDX-License-Identifier: Apache-2.0\n# Defines the group of servers to which NGINX will proxy requests.\n# NGINX will cycle through these servers in a round-robin fashion by default.\nworker_processes auto;\n\n\nhttp {\n # Raise the request-body cap (nginx default is 1 MiB) so the load balancer\n # does not reject large client requests with 413 before they reach the\n # frontend. Agentic replay traces (e.g. multi-turn cc-traces warmup bodies)\n # exceed 1 MiB; this mirrors the dynamo frontend's own body limit so nginx\n # is never the bottleneck and is never more permissive than the backend.\n client_max_body_size 45m;\n access_log off;\n\n # Keep a Dynamo session on one frontend. Anonymous requests use NGINX's\n # generated request ID so an empty session header cannot pin all traffic.\n\n map $http_x_dynamo_session_id $frontend_hash_key {\n default $http_x_dynamo_session_id;\n \"\" $request_id;\n }\n\n\n # Agentic replay reproduces recorded think-time, so a session's connection can\n # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s\n # closes it underneath a pooled client connection, and the client's next write\n # on that socket surfaces as \"broken pipe\" / \"server disconnected\" with nothing\n # logged server-side.\n keepalive_timeout 600s;\n\n upstream backend_servers {\n\n hash $frontend_hash_key consistent;\n\n \n server $FRONTEND_0_IP:8180;\n \n # Reuse upstream connections instead of a fresh TCP handshake per request.\n keepalive 512;\n keepalive_timeout 600s;\n }\n\n # The main server block that listens for incoming traffic.\n server {\n listen 8000; # Public-facing port for incoming HTTP requests.\n\n location / {\n # Pass the request to the upstream group defined above.\n proxy_pass http://backend_servers;\n\n proxy_buffering off;\n proxy_read_timeout 24h;\n proxy_send_timeout 24h;\n\n # Required for the upstream keepalive pool above: without these nginx\n # talks HTTP/1.0 to the backend and opens a new connection per request.\n proxy_http_version 1.1;\n proxy_set_header Connection \"\";\n }\n }\n}\n\nevents {\n\n #\n # Determines how many clients will be served by each worker process.\n # (Max clients = worker_connections * worker_processes)\n # Should be equal to `ulimit -n / worker_processes`\n #\n worker_connections 65535;\n\n #\n # Let each process accept multiple connections.\n # Accept as many connections as possible, after nginx gets notification\n # about a new connection.\n # May flood worker_connections, if that option is set too low.\n #\n multi_accept on;\n\n #\n # Preferred connection method for newer linux versions.\n # Essential for linux, optimized to serve many clients with each thread.\n #\n use epoll;\n}" + "router_policy_config.yaml#005cce35": { + "name": "router_policy_config.yaml", + "content": "worker_selection:\n decode: dynamo-two-tier-cost-fn\n instances:\n - name: dynamo-two-tier-cost-fn\n type: dynamo-two-tier-cost-fn\n" } }, "cells": [ @@ -2461,11 +2430,11 @@ export const agentx = { "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#0", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Agg TP2", + "label": "Agg TP4", "detail": "MTP 6-1-7", - "gpus": 2, + "gpus": 4, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-ccebaceeeca3", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", "imageStatus": "ok", "imageCommit": null, "submitted": "sglang", @@ -2487,19 +2456,274 @@ export const agentx = { } }, "prov": { - "sha": "6d447b6ae695bfa973eda6d1416436623130c578", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", - "pr": "4162", - "comment": "6214273964", + "pr": "4333", "status": "in-progress" }, "points": [ { - "concs": [5, 7, 10], - "item": 0, - "override": "override_tp2_c10", + "concs": [8, 9], + "item": 1, + "override": "override_tp4_mrr8_c8", "goldenAL": "4.8", "kv": "hicache" + }, + { + "concs": [12, 18], + "item": 0, + "override": "override_tp4_mrr8", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "sglang": [ + { + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + }, + null + ], + "dynamo": [ + null, + null, + { + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + } + ] + } + }, + { + "concs": [13], + "item": 5, + "override": "override_tp4_mrr1", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-bs-decode": ["1"], + "--cuda-graph-max-bs-decode": ["1"], + "--max-running-requests": ["1"], + "--pp-max-micro-batch-size": ["1"] + }, + "env": { + "SGLANG_EAGER_INPUT_NO_COPY": "1", + "SGLANG_Q35MF_MKA_PUSH": "1", + "SGLANG_Q35MF_MKA_Q1": "1", + "SGLANG_Q35MF_MKA_SKV2": "1", + "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", + "SGLANG_Q35MF_MKD_HEAD": "1", + "SGLANG_Q35MF_MKD_HOP": "1", + "SGLANG_Q35MF_MKD_HOP_EARLY": "1", + "SGLANG_Q35MF_MKD_HOP_PUSH": "1", + "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1", + "SGLANG_Q35MF_MKH_N3B": "1", + "SGLANG_Q35MF_MKH_N3C": "1", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1" + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1"], + "--cuda-graph-max-bs-decode": ["1"], + "--max-running-requests": ["1"], + "--pp-max-micro-batch-size": ["1"] + }, + "env": { + "SGLANG_EAGER_INPUT_NO_COPY": "1", + "SGLANG_Q35MF_MKA_PUSH": "1", + "SGLANG_Q35MF_MKA_Q1": "1", + "SGLANG_Q35MF_MKA_SKV2": "1", + "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", + "SGLANG_Q35MF_MKD_HEAD": "1", + "SGLANG_Q35MF_MKD_HOP": "1", + "SGLANG_Q35MF_MKD_HOP_EARLY": "1", + "SGLANG_Q35MF_MKD_HOP_PUSH": "1", + "SGLANG_Q35MF_MKG_TP4_MIDDLE": "1", + "SGLANG_Q35MF_MKH_N3B": "1", + "SGLANG_Q35MF_MKH_N3C": "1", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1" + } + } + ] + } + }, + { + "concs": [16], + "item": 3, + "override": "override_tp4_mrr2_k64", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["64"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["64"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + } + ] + } + }, + { + "concs": [20], + "item": 4, + "override": "override_tp4_mrr2_k32", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["32"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1", "2"], + "--cuda-graph-max-bs-decode": ["2"], + "--max-running-requests": ["2"], + "--pp-max-micro-batch-size": ["2"], + "--prefill-decode-interval": ["32"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + } + ] + } + }, + { + "concs": [23], + "item": 2, + "override": "override_tp4_mrr3_k48", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3"], + "--cuda-graph-max-bs-decode": ["3"], + "--max-running-requests": ["3"], + "--pp-max-micro-batch-size": ["3"], + "--prefill-decode-interval": ["48"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + }, + null + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3"], + "--cuda-graph-max-bs-decode": ["3"], + "--max-running-requests": ["3"], + "--pp-max-micro-batch-size": ["3"], + "--prefill-decode-interval": ["48"] + }, + "env": { + "SGLANG_GDN_CHUNK_H_BV": "16", + "SGLANG_GDN_CHUNK_H_NUM_STAGES": "3", + "SGLANG_Q35MF_DRAFT_EXCHANGE_STEP_ROWS": "8", + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "8", + "SGLANG_Q35MF_MKH_HT_REDUCTION_WARPS": "4", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKH_PREP_NOSYNC": "1", + "SGLANG_Q35MF_MKM_FUSED_M16": "1", + "SGLANG_Q35MF_MKM_FUSED_M21": "1" + } + } + ] + } } ], "routers": { @@ -2530,6 +2754,7 @@ export const agentx = { "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", "SGLANG_MOE_NVFP4_DISPATCH": "1", "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", @@ -2543,9 +2768,18 @@ export const agentx = { "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1", "SGLANG_Q35MF_ATTN_FRONT": "1", "SGLANG_Q35MF_ATTN_GATE_QUANT": "1", + "SGLANG_Q35MF_ATTN_SPLIT_KV": "1", + "SGLANG_Q35MF_BK_DRAFT_META": "1", + "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1", + "SGLANG_Q35MF_BK_STASH_SCATTER": "1", + "SGLANG_Q35MF_BK_VEC_COPY": "1", "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1", "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1", + "SGLANG_Q35MF_DRAFT_FC_GEMV": "1", "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1", + "SGLANG_Q35MF_DRAFT_MNNVL_AR": "1", + "SGLANG_Q35MF_DRAFT_MOE_GEMV": "1", + "SGLANG_Q35MF_DRAFT_MOE_ROUTER": "1", "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1", "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1", @@ -2553,13 +2787,22 @@ export const agentx = { "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1", "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", "SGLANG_Q35MF_GDN_CONV_TILE": "1", - "SGLANG_Q35MF_GDN_DIRECT_SOLVE": "0", - "SGLANG_Q35MF_GDN_EMPTY_HISTORY": "0", "SGLANG_Q35MF_GDN_LAZY_VALID": "1", "SGLANG_Q35MF_GDN_NORM_FP8": "1", + "SGLANG_Q35MF_GDN_PDL": "1", "SGLANG_Q35MF_GDN_TILED_VERIFY": "1", "SGLANG_Q35MF_GDN_VIEWS": "1", + "SGLANG_Q35MF_K4_LOCAL_VERIFY_SAMPLING": "1", + "SGLANG_Q35MF_MKA_FUSED": "1", + "SGLANG_Q35MF_MKD_EXTEND_FC": "1", + "SGLANG_Q35MF_MKD_MOE1": "1", + "SGLANG_Q35MF_MKD_TTABLE": "1", + "SGLANG_Q35MF_MKD_XCHG": "1", + "SGLANG_Q35MF_MKH_HOP": "1", + "SGLANG_Q35MF_MKH_OPROJ": "1", + "SGLANG_Q35MF_MKM_FUSED": "1", "SGLANG_Q35MF_ROUTE_FUSED": "1", + "SGLANG_Q35MF_ROUTE_SHARED": "1", "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", @@ -2571,15 +2814,18 @@ export const agentx = { ["--host", "0.0.0.0"], ["--port", "30000"], ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "8192"], + ["--chunked-prefill-size", "16384"], ["--context-length", "262144"], - ["--cuda-graph-bs-decode", "1"], - ["--cuda-graph-max-bs-decode", "1"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8"], + ["--cuda-graph-max-bs-decode", "8"], + ["--cuda-graph-max-bs-prefill", "16384"], ["--data-parallel-size", "1"], ["--disable-attn-tp-gather"], ["--disable-shared-experts-fusion"], ["--enable-hierarchical-cache"], ["--enable-linear-replayssm-spec"], + ["--enable-symm-mem"], ["--expert-parallel-size", "1"], ["--flashinfer-allreduce-fusion-backend", "cutedsl"], ["--hicache-io-backend", "kernel"], @@ -2594,26 +2840,25 @@ export const agentx = { ["--mamba-ssm-dtype", "bfloat16"], ["--mamba-track-interval", "8192"], ["--max-mamba-cache-size", "1536"], - ["--max-prefill-tokens", "8192"], - ["--max-running-requests", "1"], + ["--max-prefill-tokens", "16384"], + ["--max-running-requests", "8"], ["--mem-fraction-static", "0.85"], ["--moe-a2a-backend", "none"], - ["--moe-dense-tp-size", "2"], + ["--moe-dense-tp-size", "4"], ["--moe-runner-backend", "flashinfer_trtllm"], ["--page-size", "64"], ["--pipeline-parallel-size", "1"], - ["--pp-max-micro-batch-size", "1"], - ["--prefill-decode-interval", "0"], + ["--pp-max-micro-batch-size", "8"], ["--prefill-max-requests", "1"], ["--quantization", "modelopt_fp4"], ["--speculative-algorithm", "NEXTN"], ["--speculative-eagle-topk", "1"], ["--speculative-moe-a2a-backend", "none"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], + ["--speculative-moe-runner-backend", "flashinfer_cutedsl"], ["--speculative-num-draft-tokens", "7"], ["--speculative-num-steps", "6"], ["--stream-interval", "20"], - ["--tensor-parallel-size", "2"], + ["--tensor-parallel-size", "4"], ["--trust-remote-code"], ["--watchdog-timeout", "1000000"], ["--weight-loader-prefetch-checkpoints"], @@ -2678,14 +2923,14 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#2", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#6", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Agg TP4", + "label": "Agg TP8 (2 nodes)", "detail": "MTP 6-1-7", - "gpus": 4, + "gpus": 8, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-ccebaceeeca3", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", "imageStatus": "ok", "imageCommit": null, "submitted": "sglang", @@ -2707,17 +2952,16 @@ export const agentx = { } }, "prov": { - "sha": "6d447b6ae695bfa973eda6d1416436623130c578", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", - "pr": "4162", - "comment": "6214273964", + "pr": "4333", "status": "in-progress" }, "points": [ { - "concs": [7], - "item": 2, - "override": "override_tp4_c7", + "concs": [11, 18], + "item": 6, + "override": "override_tp8_mrr1", "goldenAL": "4.8", "kv": "hicache" } @@ -2728,26 +2972,37 @@ export const agentx = { "files": [], "blocks": [ { + "nodes_per_worker": 2, "diff": { "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-prefill": ["16384"], - "--enable-symm-mem": [], - "--max-prefill-tokens": ["16384"], - "--moe-dense-tp-size": ["4"], - "--speculative-moe-runner-backend": ["flashinfer_cutedsl"], - "--tensor-parallel-size": ["4"], - "--prefill-decode-interval": null + "--dist-init-addr": ["$AGG_HEAD_IP:20000"], + "--nnodes": ["2"], + "--node-rank": ["$NODE_RANK"], + "--cuda-graph-bs-decode": ["1"], + "--cuda-graph-max-bs-decode": ["1"], + "--max-running-requests": ["1"], + "--moe-dense-tp-size": ["8"], + "--pp-max-micro-batch-size": ["1"], + "--tensor-parallel-size": ["8"] }, "env": { - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", - "SGLANG_Q35MF_DRAFT_FC_GEMV": "1", - "SGLANG_Q35MF_DRAFT_MOE_GEMV": "1", - "SGLANG_Q35MF_DRAFT_MOE_ROUTER": "1", - "SGLANG_Q35MF_GDN_PDL": "1", - "SGLANG_Q35MF_GDN_DIRECT_SOLVE": null, - "SGLANG_Q35MF_GDN_EMPTY_HISTORY": null + "SGLANG_EAGER_INPUT_NO_COPY": "1", + "SGLANG_Q35MF_EDGE_MULTIHOST_TCP": "1", + "SGLANG_Q35MF_EDGE_SHM_SYNC_MULTIHOST": "1", + "SGLANG_Q35MF_MKA_PUSH": "1", + "SGLANG_Q35MF_MKA_Q1": "1", + "SGLANG_Q35MF_MKA_SKV2": "1", + "SGLANG_Q35MF_MKA_SKV2_VERIFY_MIN_LENGTH": "8192", + "SGLANG_Q35MF_MKD_GEMV": "1", + "SGLANG_Q35MF_MKD_HEAD": "1", + "SGLANG_Q35MF_MKD_HOP": "1", + "SGLANG_Q35MF_MKD_HOP_EARLY": "1", + "SGLANG_Q35MF_MKG_BA_TC": "1", + "SGLANG_Q35MF_MKG_FUSED": "1", + "SGLANG_Q35MF_MKH_N3B": "1", + "SGLANG_Q35MF_MKH_N3C": "1", + "SGLANG_Q35MF_MKH_PF_GRAPH": "1", + "SGLANG_Q35MF_MKM_GDN_FRONT": "1" } }, "ref": [14, "sglang", 0] @@ -2759,7 +3014,7 @@ export const agentx = { "args": [ ["--host", "0.0.0.0"], ["--port", "8000"], - ["--worker-urls", "http://127.0.0.1:30000"] + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] ] } ] @@ -2810,17 +3065,17 @@ export const agentx = { } }, { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg#1", + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#5", "hw": "gb300", "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Agg TP8 (2 nodes)", + "label": "Disagg 1P1D · prefill TP1 · decode TP2", "detail": "MTP 6-1-7", - "gpus": 8, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-ccebaceeeca3", + "gpus": 3, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", "imageStatus": "ok", "imageCommit": null, - "submitted": "sglang", + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -2829,162 +3084,83 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], + ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "32"], + ["--hicache-ratio", "1.0"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", "donorHardware": "gb300" } }, "prov": { - "sha": "6d447b6ae695bfa973eda6d1416436623130c578", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", - "pr": "4162", - "comment": "6214273964", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4333", "status": "in-progress" }, "points": [ { - "concs": [7, 10], - "item": 1, - "override": "override_tp8_c7", + "concs": [20], + "item": 5, + "override": "override_1p1d_tp2_cap10", "goldenAL": "4.8", "kv": "hicache" - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "nodes_per_worker": 2, - "diff": { - "args": { - "--dist-init-addr": ["$AGG_HEAD_IP:20000"], - "--nnodes": ["2"], - "--node-rank": ["$NODE_RANK"], - "--moe-dense-tp-size": ["8"], - "--tensor-parallel-size": ["8"] - }, - "env": { - "SGLANG_Q35MF_ATTN_SPLIT_KV": "1", - "SGLANG_Q35MF_EDGE_MULTIHOST_TCP": "1", - "SGLANG_Q35MF_EDGE_SHM_SYNC_MULTIHOST": "1", - "SGLANG_Q35MF_ROUTE_SHARED": "1", - "SGLANG_Q35MF_GDN_PDL": null - } - }, - "ref": [15, "sglang", 0] - }, - { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, - "args": [ - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] - ] - } - ] }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { + { + "concs": [26], + "item": 8, + "override": "override_1p1d_tp2_cap4", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { "args": { - "--request-plane": ["tcp"] + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"], + "--pp-max-micro-batch-size": ["4"], + "--cuda-graph-bs-decode": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null } } - } - ] - } - } - }, - { - "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#1", - "hw": "gb300", - "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", - "label": "Disagg 1P1D · prefill TP1 · decode TP2", - "detail": "MTP 6-1-7", - "gpus": 3, - "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-ccebaceeeca3", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "dynamo", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], - ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "6d447b6ae695bfa973eda6d1416436623130c578", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "pr": "4162", - "comment": "6214273964", - "status": "in-progress" - }, - "points": [ - { - "concs": [38], - "item": 2, - "override": "override_1p1d_b6_c38", - "goldenAL": "4.8", - "kv": "hicache" + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["4"], + "--max-running-requests": ["4"], + "--pp-max-micro-batch-size": ["4"], + "--cuda-graph-bs-decode": null + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + } + }, + null + ] + } }, { - "concs": [56], - "item": 1, - "override": "override_1p1d_b10_c56", + "concs": [36], + "item": 7, + "override": "override_1p1d_tp2_cap5", "goldenAL": "4.8", "kv": "hicache", "patch": { @@ -2992,30 +3168,87 @@ export const agentx = { null, null, null, + null, { "args": { - "--hicache-ratio": ["12"] - } - }, - { - "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--max-running-requests": ["10"], - "--pp-max-micro-batch-size": ["10"] + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["5"], + "--pp-max-micro-batch-size": ["8"], + "--cuda-graph-bs-decode": null + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null } } ], "sglang": [ + null, { "args": { - "--hicache-ratio": ["12"] + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["5"], + "--pp-max-micro-batch-size": ["8"], + "--cuda-graph-bs-decode": null + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null } }, + null + ] + } + }, + { + "concs": [38], + "item": 6, + "override": "override_1p1d_tp2_cap12", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"], + "--pp-max-micro-batch-size": ["12"] + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + } + } + ], + "sglang": [ + null, { "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--max-running-requests": ["10"], - "--pp-max-micro-batch-size": ["10"] + "--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["12"], + "--pp-max-micro-batch-size": ["12"] + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null } }, null @@ -3058,12 +3291,12 @@ export const agentx = { "args": [ ["--http-port", "8000"], ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"], ["--router-replica-sync"], ["--router-queue-threshold", "None"], + ["--router-session-affinity-ttl-secs", "3600"], ["--router-temperature", "1.0"] ] }, @@ -3103,6 +3336,7 @@ export const agentx = { "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "8192", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_Q35MF_K4_MROPE_GPU": "1", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" @@ -3117,6 +3351,8 @@ export const agentx = { ["--request-plane", "tcp"], ["--attention-backend", "trtllm_mha"], ["--chunked-prefill-size", "8192"], + ["--cuda-graph-backend-prefill", "breakable"], + ["--cuda-graph-max-bs-prefill", "8192"], ["--data-parallel-size", "1"], ["--disable-cuda-graph"], ["--disable-overlap-schedule"], @@ -3125,8 +3361,8 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--expert-parallel-size", "1"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "6"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"], ["--kv-cache-dtype", "fp8_e4m3"], ["--linear-attn-decode-backend", "triton"], @@ -3160,126 +3396,880 @@ export const agentx = { ] }, { - "kind": "worker", - "role": "decode", - "workers": 1, - "nodes_per_worker": 1, - "gpu_pinning": [ - ["1,2"] - ], - "cmd": "python3 -m dynamo.sglang", + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 1, + "gpu_pinning": [ + ["1,2"] + ], + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "FLASHINFER_ROUTING_FORCE_BLOCK_PER_TOKEN": "0", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "0", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", + "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1", + "SGLANG_Q35MF_ARRIVE_D2H_RING": "1", + "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1", + "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1", + "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1", + "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1", + "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1", + "SGLANG_Q35MF_ATTN_FRONT": "1", + "SGLANG_Q35MF_ATTN_GATE_QUANT": "1", + "SGLANG_Q35MF_BK_DRAFT_META": "1", + "SGLANG_Q35MF_BK_EXTEND_ARGMAX": "1", + "SGLANG_Q35MF_BK_STASH_SCATTER": "1", + "SGLANG_Q35MF_BK_VEC_COPY": "1", + "SGLANG_Q35MF_CUTEDSL_LL_MAX_M": "96", + "SGLANG_Q35MF_D2R_PVG_SHARDED": "1", + "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1", + "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1", + "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1", + "SGLANG_Q35MF_DRAFT_VOCAB_EXCHANGE_MAX_ROWS": "12", + "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1", + "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", + "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1", + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": "10", + "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", + "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1", + "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", + "SGLANG_Q35MF_FP8_BMM_TACTICS": "/sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/fp8_bmm_tactics_gb300_tp2.json", + "SGLANG_Q35MF_GDN_CONV_TILE": "1", + "SGLANG_Q35MF_GDN_DIRECT_SOLVE": "0", + "SGLANG_Q35MF_GDN_EMPTY_HISTORY": "0", + "SGLANG_Q35MF_GDN_LAZY_VALID": "1", + "SGLANG_Q35MF_GDN_NORM_FP8": "1", + "SGLANG_Q35MF_GDN_PDL": "1", + "SGLANG_Q35MF_GDN_TILED_VERIFY": "1", + "SGLANG_Q35MF_GDN_VERIFY_BV64": "1", + "SGLANG_Q35MF_GDN_VIEWS": "1", + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "84", + "SGLANG_Q35MF_K4_SIDE_PRIO_MAX_TOKENS": "16", + "SGLANG_Q35MF_KDENSE_DRAFT_ARGMAX_XCHG": "1", + "SGLANG_Q35MF_KDENSE_DRAFT_FC_XCHG": "1", + "SGLANG_Q35MF_KDENSE_DRAFT_META_BRANCH": "1", + "SGLANG_Q35MF_KGDN_VERIFY_REG168": "1", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "84", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MIN_TOKENS": "17", + "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1", + "SGLANG_Q35MF_PDDEC_FENCE": "1", + "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1", + "SGLANG_Q35MF_PDDEC_SHARDED_VERIFY_SAMPLING": "1", + "SGLANG_Q35MF_ROUTE2_EARLY_TRIGGER": "1", + "SGLANG_Q35MF_ROUTE2_SMALL_TABLES": "1", + "SGLANG_Q35MF_ROUTE2_SMALL_TABLES_MIN_TOKENS": "1", + "SGLANG_Q35MF_ROUTE_FUSED": "1", + "SGLANG_Q35MF_ROUTE_PARQ": "1", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "84", + "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", + "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--disaggregation-bootstrap-port", "8998"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "4096"], + ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], + ["--cuda-graph-max-bs-decode", "10"], + ["--data-parallel-size", "1"], + ["--disable-attn-tp-gather"], + ["--disable-radix-cache"], + ["--disable-shared-experts-fusion"], + ["--dist-timeout", "1800"], + ["--enable-linear-replayssm-spec"], + ["--expert-parallel-size", "1"], + ["--flashinfer-allreduce-fusion-backend", "cutedsl"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--load-balance-method", "round_robin"], + ["--mamba-radix-cache-strategy", "no_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "128"], + ["--max-mamba-cache-size", "936"], + ["--max-running-requests", "10"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "none"], + ["--moe-dense-tp-size", "2"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--pp-max-micro-batch-size", "10"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "none"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "30"], + ["--tensor-parallel-size", "2"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#9", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 1P2D · prefill TP1 · decode TP2", + "detail": "MTP 6-1-7", + "gpus": 5, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-host-memory-mode", "cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.0"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4333", + "status": "in-progress" + }, + "points": [ + { + "concs": [48], + "item": 9, + "override": "override_1p2d_tp2_cap3", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], + "files": ["nats.conf#48a4de48", "nginx.conf#748c9006"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-temperature", "0.0"], + ["--router-decode-active-request-weight", "8192"] + ] + }, + { + "drop": ["gpu_pinning"], + "ref": [16, "dynamo", 3] + }, + { + "workers": 2, + "gpu_pinning": [ + ["0,1"], + ["2,3"] + ], + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["3"], + "--max-running-requests": ["3"], + "--pp-max-micro-batch-size": ["3"], + "--cuda-graph-bs-decode": null + }, + "env": { + "SGLANG_Q35MF_K4_PARQ_MAX_TOKENS": "70", + "SGLANG_Q35MF_KTP2_SHARED_FUSED_MAX_TOKENS": "70", + "SGLANG_Q35MF_ROUTE_TABLES_MAX_TOKENS": "70", + "SGLANG_Q35MF_D2R_PVG_SHARDED": null, + "SGLANG_Q35MF_EDGE_POST_VERIFY_MAX_BS": null + } + }, + "ref": [16, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#4", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 2P1D · prefill PP4 · decode DEP16 (4 nodes)", + "detail": "MTP 6-1-7", + "gpus": 24, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-host-memory-mode", "cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.0"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4333", + "status": "in-progress" + }, + "points": [ + { + "concs": [488], + "item": 4, + "override": "override_2p1d_dep16_cap6", + "goldenAL": "4.8", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, + "args": [ + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--router-session-affinity-mode", "soft"], + ["--router-temperature", "1.0"], + ["--router-policy-config", "router_policy_config.yaml"] + ] + }, + { + "workers": 2, + "diff": { + "args": { + "--chunked-prefill-size": ["32768"], + "--hicache-mem-layout": ["page_first_direct"], + "--hicache-ratio": ["0.9"], + "--linear-attn-prefill-backend": ["flashinfer"], + "--max-mamba-cache-size": ["1536"], + "--max-prefill-tokens": ["32768"], + "--max-running-requests": ["128"], + "--mem-fraction-static": ["0.85"], + "--pipeline-parallel-size": ["4"], + "--pp-async-batch-depth": ["1"], + "--cuda-graph-backend-prefill": null, + "--cuda-graph-max-bs-prefill": null + }, + "env": { + "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", + "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": "1", + "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": "1", + "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": "1", + "SGLANG_Q35MF_GDN_PREFILL_SPLIT_CONV": "1", + "SGLANG_Q35MF_GDN_PREFILL_TILES": "1", + "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": "1", + "SGLANG_Q35MF_K4_MROPE_GPU": null + } + }, + "ref": [17, "dynamo", 4] + }, + { + "kind": "worker", + "role": "decode", + "workers": 1, + "nodes_per_worker": 4, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NVSHMEM_REMOTE_TRANSPORT": "none", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", + "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", + "SGLANG_ENABLE_JIT_DEEPGEMM": "true", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", + "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", + "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", + "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", + "SGLANG_MOE_NVFP4_DISPATCH": "1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", + "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": "1", + "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": "2", + "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1", + "SGLANG_Q35MF_PDDEC_FENCE": "1", + "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", + "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", + "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", + "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + }, + "args": [ + ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--disaggregation-bootstrap-port", "8998"], + ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], + ["--nnodes", "4"], + ["--node-rank", "$NODE_RANK"], + ["--request-plane", "tcp"], + ["--attention-backend", "trtllm_mha"], + ["--chunked-prefill-size", "4096"], + ["--cuda-graph-max-bs-decode", "6"], + ["--data-parallel-size", "16"], + ["--disable-radix-cache"], + ["--disable-shared-experts-fusion"], + ["--dist-timeout", "1800"], + ["--enable-dp-attention"], + ["--enable-linear-replayssm-spec"], + ["--ep-dispatch-algorithm", "static"], + ["--eplb-algorithm", "deepseek"], + ["--expert-parallel-size", "16"], + ["--gc-threshold", "50000", "50", "1000"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--linear-attn-decode-backend", "triton"], + ["--linear-replayssm-cache-len", "32"], + ["--load-balance-method", "round_robin"], + ["--mamba-radix-cache-strategy", "no_buffer"], + ["--mamba-ssm-dtype", "bfloat16"], + ["--mamba-track-interval", "128"], + ["--max-mamba-cache-size", "1872"], + ["--max-running-requests", "96"], + ["--max-total-tokens", "13000000"], + ["--mem-fraction-static", "0.85"], + ["--moe-a2a-backend", "flashinfer"], + ["--moe-dense-tp-size", "1"], + ["--moe-runner-backend", "flashinfer_cutedsl"], + ["--page-size", "64"], + ["--pipeline-parallel-size", "1"], + ["--quantization", "modelopt_fp4"], + ["--speculative-algorithm", "NEXTN"], + ["--speculative-eagle-topk", "1"], + ["--speculative-moe-a2a-backend", "flashinfer"], + ["--speculative-moe-runner-backend", "flashinfer_trtllm_routed"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--stream-interval", "30"], + ["--tensor-parallel-size", "16"], + ["--trust-remote-code"], + ["--watchdog-timeout", "1000000"], + ["--weight-loader-prefetch-checkpoints"], + ["--weight-loader-prefetch-num-threads", "4"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 5, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg#1", + "hw": "gb300", + "ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4", + "label": "Disagg 3P1D · prefill PP4 · decode DEP16 (4 nodes)", + "detail": "MTP 6-1-7", + "gpus": 28, + "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-host-memory-mode", "cache"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "1.0"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4333", + "status": "in-progress" + }, + "points": [ + { + "concs": [530], + "item": 3, + "override": "override_3p1d_dep16_cap9", + "goldenAL": "4.8", + "kv": "hicache" + }, + { + "concs": [660], + "item": 2, + "override": "override_3p1d_dep16_cap12_combine_split", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "env": { + "SGLANG_Q35MF_PPO": null + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["192"] + }, + "env": { + "SGLANG_Q35MF_K75P_LMHEAD": null + } + } + ], + "sglang": [ + { + "env": { + "SGLANG_Q35MF_PPO": null + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["192"] + }, + "env": { + "SGLANG_Q35MF_K75P_LMHEAD": null + } + }, + null + ] + } + }, + { + "concs": [760, 840], + "item": 1, + "override": "override_3p1d_dep16_cap12", + "goldenAL": "4.8", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "env": { + "SGLANG_Q35MF_PPO": null + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["192"] + }, + "env": { + "SGLANG_Q35MF_K75P_LMHEAD": null, + "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null + } + } + ], + "sglang": [ + { + "env": { + "SGLANG_Q35MF_PPO": null + } + }, + { + "args": { + "--cuda-graph-max-bs-decode": ["12"], + "--max-running-requests": ["192"] + }, + "env": { + "SGLANG_Q35MF_K75P_LMHEAD": null, + "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null + } + }, + null + ] + } + } + ], + "routers": { + "dynamo": { + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "nginx", + "cmd": "nginx -c nginx.conf -g daemon off;", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 2, + "cmd": "python3 -m dynamo.frontend", "env": { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "FLASHINFER_DISABLE_VERSION_CHECK": "1", - "MC_FORCE_MNNVL": "1", - "NATS_SERVER": "nats://$NATS_IP:4222", - "NCCL_CUMEM_ENABLE": "1", - "NCCL_MNNVL_ENABLE": "1", - "NCCL_NVLS_ENABLE": "0", - "NVSHMEM_REMOTE_TRANSPORT": "none", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "1", - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", - "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", - "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", - "SGLANG_DISAGG_STAGING_BUFFER": "1", - "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "8192", - "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0", - "SGLANG_ENABLE_JIT_DEEPGEMM": "true", - "SGLANG_ENABLE_METADATA_GLUE_GRAPH": "0", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1", - "SGLANG_FLASHINFER_FP4_GEMM_BACKEND": "cutlass", - "SGLANG_FLASHINFER_MOE_FUSED_FINALIZE": "1", - "SGLANG_HEALTH_CHECK_TIMEOUT": "1800", - "SGLANG_MOE_NVFP4_DISPATCH": "1", - "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", - "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1", - "SGLANG_Q35MF_ARRIVE_COPY_PLAN": "1", - "SGLANG_Q35MF_ARRIVE_D2H_RING": "1", - "SGLANG_Q35MF_ARRIVE_DECODE_PREP": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": "1", - "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": "1", - "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": "1", - "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": "1", - "SGLANG_Q35MF_ATTN_FRONT": "1", - "SGLANG_Q35MF_ATTN_GATE_QUANT": "1", - "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": "1", - "SGLANG_Q35MF_DRAFT_FC_SPLIT": "1", - "SGLANG_Q35MF_EDGE_CHAIN_TREE": "1", - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": "1", - "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": "1", - "SGLANG_Q35MF_EDGE_SHM_SYNC": "1", - "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": "1", - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": "1", - "SGLANG_Q35MF_GDN_CONV_TILE": "1", - "SGLANG_Q35MF_GDN_DIRECT_SOLVE": "0", - "SGLANG_Q35MF_GDN_EMPTY_HISTORY": "0", - "SGLANG_Q35MF_GDN_LAZY_VALID": "1", - "SGLANG_Q35MF_GDN_NORM_FP8": "1", - "SGLANG_Q35MF_GDN_TILED_VERIFY": "1", - "SGLANG_Q35MF_GDN_VIEWS": "1", - "SGLANG_Q35MF_ROUTE_FUSED": "1", - "SGLANG_SCHEDULER_SKIP_ALL_GATHER": "1", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", - "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", - "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", - "TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC": "1800" + "NATS_SERVER": "nats://$NATS_IP:4222" }, "args": [ - ["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"], - ["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--disaggregation-mode", "decode"], - ["--disaggregation-bootstrap-port", "8998"], - ["--request-plane", "tcp"], - ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "4096"], - ["--cuda-graph-max-bs-decode", "6"], - ["--data-parallel-size", "1"], - ["--disable-attn-tp-gather"], - ["--disable-radix-cache"], - ["--disable-shared-experts-fusion"], - ["--dist-timeout", "1800"], - ["--enable-linear-replayssm-spec"], - ["--expert-parallel-size", "1"], - ["--flashinfer-allreduce-fusion-backend", "cutedsl"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--linear-attn-decode-backend", "triton"], - ["--load-balance-method", "round_robin"], - ["--mamba-radix-cache-strategy", "no_buffer"], - ["--mamba-ssm-dtype", "bfloat16"], - ["--mamba-track-interval", "128"], - ["--max-mamba-cache-size", "936"], - ["--max-running-requests", "6"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "none"], - ["--moe-dense-tp-size", "2"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--pp-max-micro-batch-size", "6"], - ["--quantization", "modelopt_fp4"], - ["--speculative-algorithm", "NEXTN"], - ["--speculative-eagle-topk", "1"], - ["--speculative-moe-a2a-backend", "none"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], - ["--speculative-num-draft-tokens", "7"], - ["--speculative-num-steps", "6"], - ["--stream-interval", "30"], - ["--tensor-parallel-size", "2"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1000000"], - ["--weight-loader-prefetch-checkpoints"], - ["--weight-loader-prefetch-num-threads", "4"] + ["--http-port", "8180"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-replica-sync"], + ["--router-queue-threshold", "None"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--router-session-affinity-mode", "soft"], + ["--router-temperature", "1.0"], + ["--router-policy-config", "router_policy_config.yaml"] ] + }, + { + "workers": 3, + "diff": { + "env": { + "SGLANG_Q35MF_PPO": "1" + } + }, + "ref": [18, "dynamo", 4] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["9"], + "--max-running-requests": ["144"] + }, + "env": { + "SGLANG_Q35MF_K75P_LMHEAD": "1" + } + }, + "ref": [18, "dynamo", 5] } ] }, @@ -3288,7 +4278,7 @@ export const agentx = { "files": [], "blocks": [ { - "from": 3, + "from": 4, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -3303,7 +4293,7 @@ export const agentx = { } }, { - "from": 4, + "from": 5, "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -3323,6 +4313,8 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], ["--pd-disaggregation"], ["--host", "0.0.0.0"], @@ -3342,7 +4334,7 @@ export const agentx = { "detail": "MTP 6-1-7", "gpus": 20, "configKey": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-ccebaceeeca3", + "image": "lmsysorg/sglang:dev-qwen35-agentx-gb300-42ad537de600@sha256:9b167808908231cfbebc7f62cc7ce776a4f47546cc8aba7eeb6658486a7e6116", "imageStatus": "ok", "imageCommit": null, "submitted": "dynamo", @@ -3365,17 +4357,16 @@ export const agentx = { } }, "prov": { - "sha": "6d447b6ae695bfa973eda6d1416436623130c578", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "pr": "4162", - "comment": "6214273964", + "pr": "4333", "status": "in-progress" }, "points": [ { "concs": [704], "item": 0, - "override": "override_3p1d_c704", + "override": "override_3p1d_dep8_cap32", "goldenAL": "4.8", "kv": "hicache" } @@ -3383,7 +4374,7 @@ export const agentx = { "routers": { "dynamo": { "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b208989fbe1cfbfde64d11334d8af4ff04fe1a99"], - "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb"], + "files": ["nats.conf#48a4de48", "nginx.conf#033bddfb", "router_policy_config.yaml#005cce35"], "blocks": [ { "kind": "etcd", @@ -3421,104 +4412,56 @@ export const agentx = { "args": [ ["--http-port", "8180"], ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"], ["--router-replica-sync"], ["--router-queue-threshold", "None"], - ["--router-temperature", "1.0"] + ["--router-session-affinity-ttl-secs", "3600"], + ["--router-session-affinity-mode", "soft"], + ["--router-temperature", "1.0"], + ["--router-policy-config", "router_policy_config.yaml"] ] }, { - "drop": ["gpu_pinning"], "workers": 3, "diff": { "args": { - "--chunked-prefill-size": ["32768"], - "--hicache-ratio": ["0.9"], - "--max-mamba-cache-size": ["1536"], - "--max-prefill-tokens": ["32768"], - "--max-running-requests": ["128"], - "--mem-fraction-static": ["0.85"], - "--pipeline-parallel-size": ["4"], - "--pp-async-batch-depth": ["1"] + "--linear-attn-prefill-backend": null }, "env": { - "SGLANG_PP_LAYER_PARTITION": "16,16,16,12", - "SGLANG_Q35MF_GDN_PREFILL_SPLIT_CONV": "1" + "SGLANG_Q35MF_GDN_PREFILL_CONV_VEC": null, + "SGLANG_Q35MF_GDN_PREFILL_NORM_FP8": null, + "SGLANG_Q35MF_GDN_PREFILL_NO_A_MEMSET": null, + "SGLANG_Q35MF_GDN_PREFILL_TILES": null, + "SGLANG_Q35MF_K4_FI_CKPT_SPARSE": null } }, - "ref": [17, "dynamo", 3] + "ref": [18, "dynamo", 4] }, { - "drop": ["gpu_pinning"], "nodes_per_worker": 2, "diff": { "args": { - "--dist-init-addr": ["$DECODE_HEAD_IP:20000"], "--nnodes": ["2"], - "--node-rank": ["$NODE_RANK"], "--cuda-graph-max-bs-decode": ["32"], "--data-parallel-size": ["8"], "--disaggregation-decode-extra-slots": ["64"], - "--enable-dp-attention": [], "--enable-dp-lm-head": [], - "--ep-dispatch-algorithm": ["static"], - "--eplb-algorithm": ["deepseek"], "--expert-parallel-size": ["8"], - "--linear-replayssm-cache-len": ["32"], + "--max-mamba-cache-size": ["936"], "--max-running-requests": ["256"], "--max-total-tokens": ["12000000"], - "--moe-a2a-backend": ["flashinfer"], - "--moe-dense-tp-size": ["1"], - "--moe-runner-backend": ["flashinfer_cutedsl"], - "--speculative-moe-a2a-backend": ["flashinfer"], - "--speculative-moe-runner-backend": ["flashinfer_trtllm_routed"], - "--tensor-parallel-size": ["8"], - "--disable-attn-tp-gather": null, - "--flashinfer-allreduce-fusion-backend": null, - "--pp-max-micro-batch-size": null + "--tensor-parallel-size": ["8"] }, "env": { - "SGLANG_Q35MF_PDDEC_DEFER_PREALLOC": "1", - "SGLANG_Q35MF_PDDEC_FENCE": "1", - "SGLANG_Q35MF_PDDEC_NONBLOCKING_ADMIT": "1", "SGLANG_Q35MF_PDDEC_SPLIT_HOIST": "1", "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4", - "SGLANG_USE_SYMM_MEM_DP_SYNC": "true", - "SGLANG_ENABLE_METADATA_GLUE_GRAPH": null, - "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": null, - "SGLANG_Q35MF_ARRIVE_COPY_PLAN": null, - "SGLANG_Q35MF_ARRIVE_D2H_RING": null, - "SGLANG_Q35MF_ARRIVE_DECODE_PREP": null, - "SGLANG_Q35MF_ARRIVE_DRAFT_COPY": null, - "SGLANG_Q35MF_ARRIVE_DRAFT_PREP": null, - "SGLANG_Q35MF_ARRIVE_SEQ_LENS_SELECT": null, - "SGLANG_Q35MF_ARRIVE_SKIP_REGATHER": null, - "SGLANG_Q35MF_ATTN_FRONT": null, - "SGLANG_Q35MF_ATTN_GATE_QUANT": null, - "SGLANG_Q35MF_DRAFT_ARGMAX_EXCHANGE": null, - "SGLANG_Q35MF_DRAFT_EXTEND_EXCHANGE": null, - "SGLANG_Q35MF_DRAFT_FC_SPLIT": null, - "SGLANG_Q35MF_EDGE_CHAIN_TREE": null, - "SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST": null, - "SGLANG_Q35MF_EDGE_POST_VERIFY_GRAPH": null, - "SGLANG_Q35MF_EDGE_SHM_SYNC": null, - "SGLANG_Q35MF_EDGE_SIDE_STREAM_COMMIT": null, - "SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING": null, - "SGLANG_Q35MF_GDN_CONV_TILE": null, - "SGLANG_Q35MF_GDN_DIRECT_SOLVE": null, - "SGLANG_Q35MF_GDN_EMPTY_HISTORY": null, - "SGLANG_Q35MF_GDN_LAZY_VALID": null, - "SGLANG_Q35MF_GDN_NORM_FP8": null, - "SGLANG_Q35MF_GDN_TILED_VERIFY": null, - "SGLANG_Q35MF_GDN_VIEWS": null, - "SGLANG_Q35MF_ROUTE_FUSED": null, - "SGLANG_SCHEDULER_SKIP_ALL_GATHER": null + "SGLANG_Q35MF_K4_MLP_SYNC_SIDE_STREAM": null, + "SGLANG_Q35MF_MIDQ_COMBINE_SPLIT": null } }, - "ref": [17, "dynamo", 4] + "ref": [18, "dynamo", 5] } ] }, @@ -3932,7 +4875,7 @@ export const agentx = { "--mamba-scheduler-strategy": null } }, - "ref": [19, "dynamo", 3] + "ref": [21, "dynamo", 3] } ] }, @@ -4099,7 +5042,7 @@ export const agentx = { "--tokenizer-worker-num": ["1"] } }, - "ref": [20, "dynamo", 3] + "ref": [22, "dynamo", 3] } ] }, @@ -4956,7 +5899,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [22, "sglang", 0] + "ref": [24, "sglang", 0] } ] }, @@ -5555,7 +6498,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [22, "sglang", 0] + "ref": [24, "sglang", 0] } ] }, @@ -5813,7 +6756,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [24, "dynamo", 2] + "ref": [26, "dynamo", 2] }, { "gpu_pinning": [ @@ -5828,7 +6771,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [24, "dynamo", 3] + "ref": [26, "dynamo", 3] } ] }, @@ -5965,7 +6908,7 @@ export const agentx = { "--expert-parallel-size": ["4"] } }, - "ref": [24, "dynamo", 2] + "ref": [26, "dynamo", 2] }, { "diff": { @@ -5973,7 +6916,7 @@ export const agentx = { "--cuda-graph-max-bs-decode": ["32"] } }, - "ref": [24, "dynamo", 3] + "ref": [26, "dynamo", 3] } ] }, @@ -6847,7 +7790,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [28, "sglang", 0] + "ref": [30, "sglang", 0] } ] }, @@ -7360,7 +8303,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [28, "sglang", 0] + "ref": [30, "sglang", 0] } ] }, @@ -7513,7 +8456,7 @@ export const agentx = { "--disable-prefill-cuda-graph": null } }, - "ref": [30, "sglang", 0] + "ref": [32, "sglang", 0] }, { "kind": "sglang-router", @@ -7634,7 +8577,7 @@ export const agentx = { "--speculative-num-steps": ["5"] } }, - "ref": [32, "sglang", 0] + "ref": [34, "sglang", 0] }, { "kind": "sglang-router", @@ -7782,7 +8725,7 @@ export const agentx = { "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2" } }, - "ref": [32, "sglang", 0] + "ref": [34, "sglang", 0] }, { "kind": "sglang-router", @@ -8801,7 +9744,7 @@ export const agentx = { "--tp": ["8"] } }, - "ref": [35, "sglang", 0] + "ref": [37, "sglang", 0] } ] }, @@ -9456,7 +10399,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [36, "sglang", 0] + "ref": [38, "sglang", 0] } ] }, @@ -10463,7 +11406,7 @@ export const agentx = { "--tp": ["4"] } }, - "ref": [39, "sglang", 0] + "ref": [41, "sglang", 0] } ] }, @@ -10703,7 +11646,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [40, "sglang", 0] + "ref": [42, "sglang", 0] } ] }, @@ -10772,20 +11715,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2.5"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], ["--hicache-write-policy", "write_through_selective"] ], - "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "donorHardware": "gb200" + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" } }, "prov": { "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", + "run": "38965319292/attempts/1", "pr": "4209", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -10958,49 +11903,51 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2.5"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], ["--hicache-write-policy", "write_through_selective"] ], - "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "donorHardware": "gb200" + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" } }, "prov": { - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", - "pr": "4209", - "status": "in-progress" + "sha": "69d978f24846de079e8cdcbbdcdc5c29629ca90e", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh", + "run": "35146004544/attempts/2", + "pr": "3198", + "status": "verified", + "date": "2026-09-17" }, "points": [ { "concs": [4], "item": 1, - "override": "override_hicache", "goldenAL": "3.39", - "kv": "hicache" + "kv": "hicache", + "job": "35146004544/job/104988562901" }, { "concs": [8], "item": 1, - "override": "override_hicache", "goldenAL": "3.39", - "kv": "hicache" + "kv": "hicache", + "job": "35146004544/job/104988564135" }, { "concs": [12], "item": 1, - "override": "override_hicache", "goldenAL": "3.39", - "kv": "hicache" + "kv": "hicache", + "job": "35146004544/job/104988564737" }, { "concs": [16], "item": 1, - "override": "override_hicache", "goldenAL": "3.39", - "kv": "hicache" + "kv": "hicache", + "job": "35146004544/job/104988563015" } ], "routers": { @@ -11011,10 +11958,11 @@ export const agentx = { { "diff": { "args": { - "--expert-parallel-size": ["8"] + "--expert-parallel-size": ["8"], + "--tokenizer-path": ["Qwen/Qwen3.5-397B-A17B-FP8"] } }, - "ref": [42, "sglang", 0] + "ref": [44, "sglang", 0] } ] }, @@ -11085,49 +12033,51 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2.5"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], ["--hicache-write-policy", "write_through_selective"] ], - "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "donorHardware": "gb200" + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" } }, "prov": { - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", - "pr": "4209", - "status": "in-progress" + "sha": "69d978f24846de079e8cdcbbdcdc5c29629ca90e", + "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh", + "run": "35146004544/attempts/2", + "pr": "3198", + "status": "verified", + "date": "2026-09-17" }, "points": [ { "concs": [4], "item": 0, - "override": "override_gpu_kv", "goldenAL": "3.39", - "kv": "none" + "kv": "none", + "job": "35146004544/job/104988563924" }, { "concs": [8], "item": 0, - "override": "override_gpu_kv", "goldenAL": "3.39", - "kv": "none" + "kv": "none", + "job": "35146004544/job/104988563857" }, { "concs": [12], "item": 0, - "override": "override_gpu_kv", "goldenAL": "3.39", - "kv": "none" + "kv": "none", + "job": "35146004544/job/104988562507" }, { "concs": [16], "item": 0, - "override": "override_gpu_kv", "goldenAL": "3.39", - "kv": "none" + "kv": "none", + "job": "35146004544/job/104988563025" } ], "routers": { @@ -11146,7 +12096,7 @@ export const agentx = { "--page-size": null } }, - "ref": [43, "sglang", 0] + "ref": [45, "sglang", 0] } ] }, @@ -11215,20 +12165,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2.5"], + ["--hicache-io-backend", "kernel"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-size", "77"], ["--hicache-write-policy", "write_through_selective"] ], - "donor": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "donorHardware": "gb200" + "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", + "donorHardware": "h200" } }, "prov": { "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", + "run": "38965319292/attempts/1", "pr": "4209", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -11277,7 +12229,7 @@ export const agentx = { "--tensor-parallel-size": ["4"] } }, - "ref": [42, "sglang", 0] + "ref": [44, "sglang", 0] } ] }, @@ -11569,7 +12521,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [46, "sglang", 0] + "ref": [48, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx index 49328f90ca7a..bf88c8d50e6d 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "Qwen3.8-Flash-Next", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { - "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", - "status": "in-progress" - }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", - "status": "verified-pr" - }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { + "4333": { "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "4279": { - "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "status": "in-progress" - }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, @@ -381,48 +346,51 @@ export const agentx = { { "id": "qwen3.8next-fp4-b300-sglang-agentic-mtp#0", "hw": "b300", - "ckpt": "nvidia/Qwen3.8-Flash-Next-NVFP4", + "ckpt": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "label": "Agg TP1", "detail": "MTP 3-1-4", "gpus": 1, "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "image": "lmsysorg/sglang:qwen38flashnext", "imageStatus": "ok", "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "mooncake"], + "available": ["none"], "unavailable": { - "hicache": "no verified HiCache settings for this model yet" + "hicache": "no verified HiCache settings for this model yet", + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" } }, "prov": { - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "pr": "4280", - "status": "in-progress" + "sha": "7caed5a556ee1ae97f072328fb2c1befa338caa4", + "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh", + "run": "33039186365/attempts/1", + "pr": "2758", + "status": "verified", + "date": "2026-08-27" }, "points": [ { "concs": [1], "item": 0, - "override": "override_c1", "goldenAL": "2.32", - "kv": "none" + "kv": "none", + "job": "33039186365/job/98408899989" }, { "concs": [4], "item": 0, - "override": "override_c4", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] } } ], @@ -431,23 +399,25 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["4"], + "--max-running-requests": ["8"] } } ] - } + }, + "job": "33039186365/job/98408899982" }, { "concs": [8], "item": 0, - "override": "override_c8", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] } } ], @@ -456,23 +426,25 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["16"] } } ] - } + }, + "job": "33039186365/job/98408899939" }, { "concs": [12], "item": 0, - "override": "override_c12", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] } } ], @@ -481,103 +453,25 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["96"] + "--cuda-graph-max-bs": ["12"], + "--max-running-requests": ["24"] } } ] - } + }, + "job": "33039186365/job/98408899965" }, { "concs": [16], "item": 0, - "override": "override_c16", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["96"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["96"] - } - } - ] - } - }, - { - "concs": [32], - "item": 1, - "override": "override_c32", - "goldenAL": "2.32", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["298"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["480"], - "--max-running-requests": ["96"], - "--tokenizer-worker-num": ["1"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["298"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["480"], - "--max-running-requests": ["96"], - "--tokenizer-worker-num": ["1"] - } - } - ] - } - }, - { - "concs": [48], - "item": 1, - "override": "override_c48", - "goldenAL": "2.32", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["298"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["480"], - "--max-running-requests": ["96"], - "--tokenizer-worker-num": ["1"] + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] } } ], @@ -586,21 +480,13 @@ export const agentx = { null, { "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["298"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["480"], - "--max-running-requests": ["96"], - "--tokenizer-worker-num": ["1"] + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["32"] } } ] - } + }, + "job": "33039186365/job/98408899999" } ], "routers": { @@ -615,57 +501,35 @@ export const agentx = { "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", "env": { - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" }, "args": [ - ["--model-path", "nvidia/Qwen3.8-Flash-Next-NVFP4"], - ["--served-model-name", "nvidia/Qwen3.8-Flash-Next-NVFP4"], + ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "16384"], - ["--context-length", "262144"], - ["--cuda-graph-backend-decode", "full"], - ["--cuda-graph-backend-prefill", "disabled"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96"], - ["--cuda-graph-max-bs-decode", "96"], - ["--data-parallel-size", "1"], - ["--disable-shared-experts-fusion"], - ["--enable-linear-replayssm-spec"], - ["--expert-parallel-size", "1"], - ["--kv-cache-dtype", "bfloat16"], + ["--cuda-graph-max-bs", "1"], + ["--dp", "1"], + ["--ep-size", "1"], ["--linear-attn-decode-backend", "flashinfer"], ["--linear-attn-prefill-backend", "flashinfer"], - ["--linear-attn-verify-backend", "triton"], ["--mamba-ssm-dtype", "bfloat16"], - ["--max-prefill-tokens", "16384"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.88"], - ["--moe-a2a-backend", "none"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--ple-offload-backend", "pinned"], - ["--ple-offload-embedding"], - ["--quantization", "modelopt_mixed"], - ["--random-seed", "35"], + ["--mem-fraction-static", "0.80"], ["--reasoning-parser", "auto"], - ["--revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], - ["--scheduler-recv-interval", "1"], + ["--scheduler-recv-interval", "10"], ["--speculative-algorithm", "NEXTN"], - ["--speculative-draft-kv-cache-dtype", "bfloat16"], - ["--speculative-draft-model-revision", "fc694b54fb0174e0913e6adf86691ef85a4ead47"], ["--speculative-eagle-topk", "1"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], ["--speculative-num-draft-tokens", "4"], ["--speculative-num-steps", "3"], - ["--stream-interval", "20"], - ["--tensor-parallel-size", "1"], - ["--tokenizer-worker-num", "6"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1800"] + ["--stream-interval", "50"], + ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], + ["--tp", "1"], + ["--trust-remote-code"] ] } ] @@ -723,50 +587,46 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 1, "configKey": "qwen3.8next-fp4-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:2dbe4c7f53230b09aaa8fa41891c13c51d9d2e253f486a70822aaf08783fff75", + "image": "lmsysorg/sglang:qwen38flashnext", "imageStatus": "ok", "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "mooncake"], + "available": ["none"], "unavailable": { - "hicache": "no verified HiCache settings for this model yet" + "hicache": "no verified HiCache settings for this model yet", + "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" } }, "prov": { - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml", - "pr": "4279", - "status": "in-progress" + "sha": "4b9dddd090c913e0fc305bb37d88c28b32113b60", + "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh", + "run": "34879996995/attempts/1", + "pr": "2751", + "status": "verified", + "date": "2026-09-15" }, "points": [ { "concs": [1], "item": 0, - "override": "override_c1", "goldenAL": "2.32", - "kv": "none" - }, - { - "concs": [2], - "item": 0, - "override": "override_c2", - "goldenAL": "2.32", - "kv": "none" + "kv": "none", + "job": "34879996995/job/104097491275" }, { "concs": [4], "item": 0, - "override": "override_c4", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["64"] + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["8"] } } ], @@ -775,23 +635,25 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["64"] + "--cuda-graph-max-bs": ["8"], + "--max-running-requests": ["8"] } } ] - } + }, + "job": "34879996995/job/104097491564" }, { "concs": [8], "item": 0, - "override": "override_c8", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--max-running-requests": ["64"] + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["16"] } } ], @@ -800,123 +662,27 @@ export const agentx = { null, { "args": { - "--max-running-requests": ["64"] + "--cuda-graph-max-bs": ["16"], + "--max-running-requests": ["16"] } } ] - } + }, + "job": "34879996995/job/104097491581" }, { "concs": [16], - "item": 1, - "override": "override_c16", - "goldenAL": "2.32", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] - } - } - ] - } - }, - { - "concs": [24], - "item": 1, - "override": "override_c24", - "goldenAL": "2.32", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] - } - } - ] - } - }, - { - "concs": [32], - "item": 1, - "override": "override_c32", + "item": 0, "goldenAL": "2.32", - "kv": "hicache", + "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] + "--cuda-graph-max-bs": ["32"], + "--max-mamba-cache-size": ["160"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.90"] } } ], @@ -925,21 +691,15 @@ export const agentx = { null, { "args": { - "--cuda-graph-bs-decode": ["1", "2", "4", "8", "16", "24", "32", "40", "48", "56", "64"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["kernel"], - "--hicache-mem-layout": ["page_first"], - "--hicache-size": ["213"], - "--hicache-write-policy": ["write_back"], - "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], - "--mamba-track-interval": ["256"], - "--max-mamba-cache-size": ["320"], - "--max-running-requests": ["64"], - "--tokenizer-worker-num": ["1"] + "--cuda-graph-max-bs": ["32"], + "--max-mamba-cache-size": ["160"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.90"] } } ] - } + }, + "job": "34879996995/job/104097491534" } ], "routers": { @@ -955,54 +715,36 @@ export const agentx = { "cmd": "python3 -m sglang.launch_server", "env": { "NCCL_CUMEM_ENABLE": "1", - "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800" + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" }, "args": [ ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--attention-backend", "trtllm_mha"], - ["--chunked-prefill-size", "16384"], - ["--context-length", "262144"], - ["--cuda-graph-backend-decode", "full"], - ["--cuda-graph-backend-prefill", "disabled"], - ["--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "16", "24", "32", "40", "48", "56", "64"], - ["--cuda-graph-max-bs-decode", "64"], - ["--data-parallel-size", "1"], - ["--disable-shared-experts-fusion"], - ["--enable-linear-replayssm-spec"], - ["--expert-parallel-size", "1"], - ["--kv-cache-dtype", "bfloat16"], + ["--cuda-graph-max-bs", "2"], + ["--dp", "1"], + ["--ep-size", "1"], ["--linear-attn-decode-backend", "flashinfer"], ["--linear-attn-prefill-backend", "flashinfer"], - ["--linear-attn-verify-backend", "triton"], ["--mamba-ssm-dtype", "bfloat16"], - ["--max-prefill-tokens", "16384"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.85"], - ["--moe-a2a-backend", "none"], - ["--moe-runner-backend", "flashinfer_trtllm"], - ["--page-size", "64"], - ["--pipeline-parallel-size", "1"], - ["--ple-offload-backend", "pinned"], - ["--ple-offload-embedding"], - ["--quantization", "modelopt_fp4"], - ["--random-seed", "35"], + ["--mem-fraction-static", "0.80"], ["--reasoning-parser", "auto"], - ["--scheduler-recv-interval", "1"], + ["--scheduler-recv-interval", "10"], ["--speculative-algorithm", "NEXTN"], - ["--speculative-draft-kv-cache-dtype", "bfloat16"], ["--speculative-eagle-topk", "1"], - ["--speculative-moe-runner-backend", "flashinfer_trtllm"], ["--speculative-num-draft-tokens", "4"], ["--speculative-num-steps", "3"], - ["--stream-interval", "20"], - ["--tensor-parallel-size", "1"], + ["--stream-interval", "50"], + ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--tokenizer-worker-num", "6"], - ["--trust-remote-code"], - ["--watchdog-timeout", "1800"] + ["--tp", "1"], + ["--trust-remote-code"] ] } ] @@ -1075,10 +817,10 @@ export const agentx = { "prov": { "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml", - "run": "39109720946", + "run": "39109720946/attempts/1", "pr": "4272", - "comment": "6252968837", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -1313,10 +1055,10 @@ export const agentx = { "prov": { "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml", - "run": "39109720946", + "run": "39109720946/attempts/1", "pr": "4272", - "comment": "6252968837", - "status": "verified-pr" + "status": "verified", + "date": "2026-10-19" }, "points": [ { diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx index 4b94ccdd5c34..859c52c64954 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { + "4333": { "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { - "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", - "status": "in-progress" - }, - "4279": { - "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "status": "in-progress" - }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", - "status": "verified-pr" - }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index e27ba35330fa..c17ea7719a80 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { + "4333": { "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { - "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", - "status": "in-progress" - }, - "4279": { - "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "status": "in-progress" - }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", - "status": "verified-pr" - }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, @@ -209,18 +174,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "38911006463/attempts/1", "pr": "4180", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -450,20 +417,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "a61b7eba2e536cce11ffe7f2620bbea4e8fbed85", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "38614048439", + "run": "38614048439/attempts/2", "pr": "4033", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-14" }, "points": [ { @@ -653,18 +620,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "38911006463/attempts/1", "pr": "4180", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -787,20 +756,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "a61b7eba2e536cce11ffe7f2620bbea4e8fbed85", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "38614048439", + "run": "38614048439/attempts/2", "pr": "4033", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-14" }, "points": [ { @@ -936,20 +905,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "a61b7eba2e536cce11ffe7f2620bbea4e8fbed85", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "38614048439", + "run": "38614048439/attempts/2", "pr": "4033", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-14" }, "points": [ { @@ -1138,18 +1107,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "38911006463/attempts/1", "pr": "4180", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -1396,18 +1367,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "2"], + ["--hicache-ratio", "1.02"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "donorHardware": "b200" + "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "donorHardware": "gb300" } }, "prov": { "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "38911006463/attempts/1", "pr": "4180", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -3602,9 +3575,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-h200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17", - "imageStatus": "pruned", - "imageCommit": "06008c17", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -3624,49 +3597,32 @@ export const agentx = { } }, "prov": { - "sha": "8f0557a3602d48cad916b9e37c11acf7452c283f", - "recipe": "benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh", - "run": "35893977246/attempts/1", - "pr": "3392", - "status": "verified", - "date": "2026-09-24" + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml", + "run": "39318221321", + "pr": "4327", + "comment": "6281433847", + "status": "verified-pr" }, "points": [ { "concs": [1], "item": 0, + "override": "override_tp4_c1", "goldenAL": "3.51", - "kv": "none", - "job": "35893977246/job/107293899458" + "kv": "none" }, { "concs": [2], "item": 0, + "override": "override_tp4_c2", "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["4"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["4"] - } - } - ] - }, - "job": "35893977246/job/107293900133" + "kv": "none" }, { "concs": [4], "item": 0, + "override": "override_tp4_c4", "goldenAL": "3.51", "kv": "none", "patch": { @@ -3686,12 +3642,12 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293899676" + } }, { "concs": [8], "item": 0, + "override": "override_tp4_c8", "goldenAL": "3.51", "kv": "none", "patch": { @@ -3711,12 +3667,12 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293899409" + } }, { "concs": [16], "item": 0, + "override": "override_tp4_c16", "goldenAL": "3.51", "kv": "none", "patch": { @@ -3736,83 +3692,7 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293900085" - }, - { - "concs": [32], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293899493" - }, - { - "concs": [64], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293900130" - }, - { - "concs": [128], - "item": 0, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293900276" + } } ], "routers": { @@ -3841,18 +3721,19 @@ export const agentx = { ["--attention-backend", "dsv4"], ["--chunked-prefill-size", "4096"], ["--cuda-graph-max-bs-decode", "64"], + ["--data-parallel-size", "1"], ["--enable-decoder-swa-bounded-replay"], - ["--ep-size", "1"], + ["--expert-parallel-size", "1"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.70"], + ["--mem-fraction-static", "0.7"], ["--moe-runner-backend", "marlin"], ["--prefill-decode-interval", "16"], ["--reasoning-parser", "auto"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "5"], ["--swa-prefix-tails", "1024"], + ["--tensor-parallel-size", "4"], ["--tool-call-parser", "auto"], - ["--tp", "4"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"] ] @@ -3912,9 +3793,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 8, "configKey": "dsv41flash-fp4-h200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17", - "imageStatus": "pruned", - "imageCommit": "06008c17", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1", + "imageStatus": "ok", + "imageCommit": "7224fce1", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -3934,24 +3815,25 @@ export const agentx = { } }, "prov": { - "sha": "8f0557a3602d48cad916b9e37c11acf7452c283f", - "recipe": "benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh", - "run": "35893977246/attempts/1", - "pr": "3392", - "status": "verified", - "date": "2026-09-24" + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml", + "run": "39318221321", + "pr": "4327", + "comment": "6281433847", + "status": "verified-pr" }, "points": [ { "concs": [1], "item": 1, + "override": "override_tp8_c1", "goldenAL": "3.51", - "kv": "none", - "job": "35893977246/job/107293900132" + "kv": "none" }, { "concs": [2], "item": 1, + "override": "override_tp8_c2", "goldenAL": "3.51", "kv": "none", "patch": { @@ -3971,12 +3853,12 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293900277" + } }, { "concs": [4], "item": 1, + "override": "override_tp8_c4", "goldenAL": "3.51", "kv": "none", "patch": { @@ -3996,12 +3878,12 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293900225" + } }, { "concs": [8], "item": 1, + "override": "override_tp8_c8", "goldenAL": "3.51", "kv": "none", "patch": { @@ -4021,12 +3903,12 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293899538" + } }, { "concs": [16], "item": 1, + "override": "override_tp8_c16", "goldenAL": "3.51", "kv": "none", "patch": { @@ -4046,83 +3928,7 @@ export const agentx = { } } ] - }, - "job": "35893977246/job/107293899316" - }, - { - "concs": [32], - "item": 1, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293899340" - }, - { - "concs": [64], - "item": 1, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293899314" - }, - { - "concs": [128], - "item": 1, - "goldenAL": "3.51", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--max-running-requests": ["64"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--max-running-requests": ["64"] - } - } - ] - }, - "job": "35893977246/job/107293900360" + } } ], "routers": { @@ -4133,7 +3939,7 @@ export const agentx = { { "diff": { "args": { - "--tp": ["8"] + "--tensor-parallel-size": ["8"] } }, "ref": [17, "sglang", 0] diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx index a75bfdd33c00..7298d894817c 100644 --- a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx +++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "Muse Spark Open", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { - "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", - "status": "in-progress" - }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", - "status": "verified-pr" - }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { - "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", - "status": "in-progress" - }, - "4279": { + "4333": { "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, @@ -90,10 +55,6 @@ export const agentx = { { "id": "h200", "label": "H200" - }, - { - "id": "h100", - "label": "H100" } ], "checkpoints": [ @@ -180,18 +141,24 @@ export const agentx = { "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "mooncake"], - "unavailable": { - "hicache": "no verified HiCache settings for this model yet" + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "120"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" } }, "prov": { - "sha": "d3850c4581d03e908bb0a27c39830185a4591465", + "sha": "71cd615068e450b29c8806accdca624874cfd3b4", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "38835086328/attempts/1", - "pr": "4141", + "run": "39258994981/attempts/1", + "pr": "4317", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-19" }, "points": [ { @@ -306,11 +273,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[5]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["8"] } } @@ -320,6 +289,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["8"] } } @@ -331,11 +302,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[6]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["10"] } } @@ -345,6 +318,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["10"] } } @@ -356,11 +331,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[7]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["12"] } } @@ -370,6 +347,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["12"] } } @@ -381,11 +360,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[8]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["16"] } } @@ -395,6 +376,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["16"] } } @@ -406,11 +389,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[9]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["20"] } } @@ -420,6 +405,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["20"] } } @@ -431,11 +418,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[10]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["24"] } } @@ -445,6 +434,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["24"] } } @@ -456,11 +447,13 @@ export const agentx = { "item": 0, "override": "zip_override_tp4[11]", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache", "patch": { "sglang": [ { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["32"] } } @@ -470,6 +463,8 @@ export const agentx = { null, { "args": { + "--enable-hierarchical-cache": [], + "--hicache-size": ["120"], "--max-running-requests": ["32"] } } @@ -588,15 +583,16 @@ export const agentx = { ["--hicache-size", "120"] ], "donor": "musespark-fp4-gb200-sglang-agentic-dflash", - "donorHardware": "gb200", - "donorRef": "main" + "donorHardware": "gb200" } }, "prov": { "sha": "511afb819454a40704f661d34011cbdff6907cc8", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "39195111790/attempts/1", "pr": "4302", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -1191,12 +1187,12 @@ export const agentx = { } }, "prov": { - "sha": "3586b3bccece7fa1bb8feb1df6bd0a96fc5776fe", + "sha": "69836fd1c56aebe5dccecf69e96f11a7056acc24", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", - "run": "38897356089/attempts/2", - "pr": "4156", + "run": "39181803276/attempts/1", + "pr": "4292", "status": "verified", - "date": "2026-10-17" + "date": "2026-10-19" }, "points": [ { @@ -1556,9 +1552,9 @@ export const agentx = { "detail": "DFlash 8", "gpus": 8, "configKey": "musespark-fp4-b200-sglang-agentic-dflash", - "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "image": "lmsysorg/sglang:nightly-dev-20261019-3c560089", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "3c560089", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -1576,9 +1572,9 @@ export const agentx = { } }, "prov": { - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", - "pr": "4221", + "pr": "4334", "status": "in-progress" }, "points": [ @@ -1926,12 +1922,12 @@ export const agentx = { } }, "prov": { - "sha": "b5b61acbd577f2524f9a23fca05063e6812144ba", + "sha": "51ad89483da916866ef51ac47ac2834855c97196", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml", - "run": "38941442268/attempts/1", - "pr": "4199", + "run": "39180248743/attempts/4", + "pr": "4290", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-19" }, "points": [ { @@ -2246,224 +2242,6 @@ export const agentx = { ] } } - }, - { - "id": "musespark-fp4-h100-sglang-agentic-dflash#0", - "hw": "h100", - "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", - "label": "Agg TP8·EP8", - "detail": "DFlash 8", - "gpus": 8, - "configKey": "musespark-fp4-h100-sglang-agentic-dflash", - "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-size", "160"] - ], - "donor": "musespark-fp4-h100-sglang-agentic-dflash", - "donorHardware": "h100" - } - }, - "prov": { - "sha": "a0df1d4558d925e1ed8d230a0c296bd9ef208a91", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml", - "run": "38925660934/attempts/1", - "pr": "4182", - "status": "verified", - "date": "2026-10-17" - }, - "points": [ - { - "concs": [1], - "item": 0, - "override": "zip_override_tp8_hicache[0]", - "goldenAL": "4.05", - "kv": "hicache" - }, - { - "concs": [2], - "item": 0, - "override": "zip_override_tp8_hicache[1]", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["4"], - "--max-running-requests": ["4"] - } - } - ] - } - }, - { - "concs": [4], - "item": 0, - "override": "zip_override_tp8_hicache[2]", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ] - } - }, - { - "concs": [8], - "item": 0, - "override": "zip_override_tp8_hicache[3]", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_DFLASH_EAGER_DRAFT_SAMPLER": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900" - }, - "args": [ - ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], - ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--attention-backend", "fa3"], - ["--chunked-prefill-size", "1024"], - ["--cuda-graph-max-bs-decode", "2"], - ["--disable-prefill-cuda-graph"], - ["--enable-hierarchical-cache"], - ["--expert-parallel-size", "8"], - ["--fp4-gemm-backend", "marlin"], - ["--hicache-size", "160"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.965"], - ["--mm-feature-transport", "cpu"], - ["--moe-runner-backend", "marlin"], - ["--quantization", "modelopt_fp4"], - ["--reasoning-parser", "muse"], - ["--speculative-algorithm", "DFLASH"], - ["--speculative-draft-attention-backend", "fa4"], - ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], - ["--speculative-num-draft-tokens", "8"], - ["--swa-full-tokens-ratio", "0.1"], - ["--tensor-parallel-size", "8"], - ["--tool-call-parser", "muse"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"] - ] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } } ] }; diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index 33ff6969ea1f..1beb21be57c6 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,72 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@b9c6fc3020f9 plus open PRs #3817, #3933, #4162, #4180, #4209, #4221, #4272, #4279, #4280, #4286, #4300, #4302), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@660b5a8dbda9 plus open PRs #3817, #4300, #4327, #4333, #4334), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-10-18", + "generatedOn": "2026-10-19", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "b9c6fc3020f90b2be827b1df5ee35aa7f80bac68", + "main": "660b5a8dbda948f1042f1f11da354b2d2a39e169", "prs": { - "4300": { - "run": null, - "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", - "status": "in-progress" - }, - "4286": { - "run": null, - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", - "status": "in-progress" - }, - "3933": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38610716806", - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "status": "verified-pr" - }, - "4162": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38860741275", - "sha": "cd91760593aee18ce10a9331e68415e815236eba", - "status": "verified-pr" - }, - "4209": { - "run": null, - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "status": "in-progress" - }, - "4280": { + "4333": { "run": null, - "sha": "078be423235958c4ad19dfb5acca39a5134c99b1", + "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "status": "in-progress" }, - "4279": { - "run": null, - "sha": "d9f52576248a7d6b4edeffe1df8a21cfe091552f", - "status": "in-progress" - }, - "4272": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39109720946", - "sha": "0d749be128354a7403c6bf97e55734afc2cdde51", + "4327": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39318221321", + "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "status": "verified-pr" }, - "4221": { + "4300": { "run": null, - "sha": "822d95799e4f5875b8f4f294500b7a537fc856bc", + "sha": "8f965ad4226d59ee22d5f70f382f9c2eff7c874e", "status": "in-progress" }, - "4180": { + "4334": { "run": null, - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "6a8c8179da2f7528912ccba51e3461c728a910ad", "status": "in-progress" }, "3817": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37715955859", "sha": "b8bafb340b8be6369e5f4df272cd60d5f7e3c587", "status": "verified-pr" - }, - "4302": { - "run": null, - "sha": "511afb819454a40704f661d34011cbdff6907cc8", - "status": "in-progress" } } }, @@ -3759,147 +3724,6 @@ export const agentx = { } } }, - { - "id": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned#2", - "hw": "b300", - "ckpt": "zai-org/GLM-5.2-FP8", - "label": "Agg TP4", - "detail": "MTP 4-1-5", - "gpus": 4, - "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned", - "image": "lmsysorg/sglang@sha256:239ae983b1bc6ead51e3fe88f21b7ab795ba7ef1a0666384924dac87d855005d", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "270"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", - "donorHardware": "b300" - } - }, - "prov": { - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic-tuned.yaml", - "run": "38610716806", - "pr": "3933", - "comment": "6178700398", - "status": "verified-pr" - }, - "points": [ - { - "concs": [1], - "item": 2, - "override": "override_tp4_c1", - "goldenAL": "3.33", - "kv": "none" - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "worker", - "role": "agg", - "workers": 1, - "nodes_per_worker": 1, - "cmd": "python3 -m sglang.launch_server", - "env": { - "NCCL_CUMEM_ENABLE": "1", - "SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION": "0.05", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900", - "TORCH_CUDA_ARCH_LIST": "10.0" - }, - "args": [ - ["--model-path", "zai-org/GLM-5.2-FP8"], - ["--served-model-name", "zai-org/GLM-5.2-FP8"], - ["--host", "0.0.0.0"], - ["--port", "30000"], - ["--chunked-prefill-size", "8192"], - ["--context-length", "1048576"], - ["--cuda-graph-max-bs-decode", "32"], - ["--default-chat-template-kwargs", "{\"enable_thinking\":true}"], - ["--dsa-decode-backend", "trtllm"], - ["--dsa-prefill-backend", "trtllm"], - ["--expert-parallel-size", "1"], - ["--kv-cache-dtype", "fp8_e4m3"], - ["--max-prefill-tokens", "8192"], - ["--max-running-requests", "32"], - ["--mem-fraction-static", "0.92"], - ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], - ["--page-size", "64"], - ["--quantization", "fp8"], - ["--reasoning-parser", "glm45"], - ["--speculative-algorithm", "EAGLE"], - ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "5"], - ["--speculative-num-steps", "4"], - ["--tensor-parallel-size", "4"], - ["--tool-call-parser", "glm47"], - ["--trust-remote-code"], - ["--watchdog-timeout", "3600"], - ["--weight-loader-prefetch-checkpoints"] - ] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } - }, { "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c1", "hw": "b300", @@ -4130,198 +3954,6 @@ export const agentx = { "--speculative-num-steps": ["4"] } }, - "ref": [18, "sglang", 0] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } - }, - { - "id": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned#0", - "hw": "b300", - "ckpt": "zai-org/GLM-5.2-FP8", - "label": "Agg TP8", - "detail": "MTP 3-1-4 · glm5.2-fp8-b300-sglang-agentic-mtp-tuned", - "gpus": 8, - "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned", - "image": "lmsysorg/sglang@sha256:239ae983b1bc6ead51e3fe88f21b7ab795ba7ef1a0666384924dac87d855005d", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "270"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", - "donorHardware": "b300" - } - }, - "prov": { - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic-tuned.yaml", - "run": "38610716806", - "pr": "3933", - "comment": "6178700398", - "status": "verified-pr" - }, - "points": [ - { - "concs": [4], - "item": 0, - "override": "override_tp8_c4", - "goldenAL": "2.99", - "kv": "none" - }, - { - "concs": [8], - "item": 0, - "override": "override_tp8_c8", - "goldenAL": "2.99", - "kv": "none" - }, - { - "concs": [16], - "item": 1, - "override": "override_tp8_c16", - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-prefill": ["4096"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["direct"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-size": ["270"], - "--hicache-write-policy": ["write_back"], - "--prefill-decode-interval": ["32"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-prefill": ["4096"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["direct"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-size": ["270"], - "--hicache-write-policy": ["write_back"], - "--prefill-decode-interval": ["32"] - } - } - ] - } - }, - { - "concs": [32], - "item": 1, - "override": "override_tp8_c32", - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-prefill": ["4096"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["direct"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-size": ["270"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.92"], - "--prefill-decode-interval": ["16"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-prefill": ["4096"], - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["direct"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-size": ["270"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.92"], - "--prefill-decode-interval": ["16"] - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--mem-fraction-static": ["0.85"], - "--speculative-num-draft-tokens": ["4"], - "--speculative-num-steps": ["3"], - "--tensor-parallel-size": ["8"] - } - }, "ref": [17, "sglang", 0] } ] @@ -4371,151 +4003,12 @@ export const agentx = { } } }, - { - "id": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned#3", - "hw": "b300", - "ckpt": "zai-org/GLM-5.2-FP8", - "label": "Agg TP4", - "detail": "MTP 3-1-4", - "gpus": 4, - "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp-tuned", - "image": "lmsysorg/sglang@sha256:239ae983b1bc6ead51e3fe88f21b7ab795ba7ef1a0666384924dac87d855005d", - "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-size", "270"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "glm5.2-fp4-b300-sglang-agentic-mtp", - "donorHardware": "b300" - } - }, - "prov": { - "sha": "04e78ef164087fa6133746b2cad5d2a7f7cfa93e", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic-tuned.yaml", - "run": "38610716806", - "pr": "3933", - "comment": "6178700398", - "status": "verified-pr" - }, - "points": [ - { - "concs": [8], - "item": 3, - "override": "override_tp4_c8", - "goldenAL": "2.99", - "kv": "hicache" - }, - { - "concs": [12], - "item": 3, - "override": "override_tp4_c12", - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--mem-fraction-static": ["0.92"], - "--prefill-decode-interval": ["96"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--mem-fraction-static": ["0.92"], - "--prefill-decode-interval": ["96"] - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--enable-hierarchical-cache": [], - "--hicache-io-backend": ["direct"], - "--hicache-mem-layout": ["page_first_direct"], - "--hicache-size": ["270"], - "--hicache-write-policy": ["write_back"], - "--mem-fraction-static": ["0.9"], - "--tensor-parallel-size": ["4"] - } - }, - "ref": [20, "sglang", 0] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } - }, { "id": "glm5.2-fp8-b300-sglang-agentic-mtp#0@c8", "hw": "b300", "ckpt": "zai-org/GLM-5.2-FP8", "label": "Agg TP8", - "detail": "MTP 3-1-4 · glm5.2-fp8-b300-sglang-agentic-mtp", + "detail": "MTP 3-1-4", "gpus": 8, "configKey": "glm5.2-fp8-b300-sglang-agentic-mtp", "image": "lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df", @@ -4651,7 +4144,7 @@ export const agentx = { "--speculative-num-steps": ["3"] } }, - "ref": [18, "sglang", 0] + "ref": [17, "sglang", 0] } ] }, @@ -4760,7 +4253,7 @@ export const agentx = { "--speculative-num-steps": ["5"] } }, - "ref": [18, "sglang", 0] + "ref": [17, "sglang", 0] } ] }, @@ -4833,18 +4326,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "0.75"], + ["--hicache-size", "170"], ["--hicache-write-policy", "write_back"] ], - "donor": "glm5.2-fp4-b200-sglang-agentic-mtp", + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", "donorHardware": "b200" } }, "prov": { "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", + "run": "39166018772/attempts/1", "pr": "4286", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -5082,18 +4577,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "0.75"], + ["--hicache-size", "170"], ["--hicache-write-policy", "write_back"] ], - "donor": "glm5.2-fp4-b200-sglang-agentic-mtp", + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", "donorHardware": "b200" } }, "prov": { "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", + "run": "39166018772/attempts/1", "pr": "4286", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -5153,7 +4650,7 @@ export const agentx = { "--tensor-parallel-size": ["8"] } }, - "ref": [24, "dynamo", 3] + "ref": [21, "dynamo", 3] } ] }, @@ -5214,10 +4711,10 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "0.75"], + ["--hicache-size", "170"], ["--hicache-write-policy", "write_back"] ], - "donor": "glm5.2-fp4-b200-sglang-agentic-mtp", + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", "donorHardware": "b200" } }, @@ -5379,7 +4876,7 @@ export const agentx = { "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": null } }, - "ref": [25, "sglang", 0] + "ref": [22, "sglang", 0] } ] }, @@ -5452,18 +4949,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "0.75"], + ["--hicache-size", "170"], ["--hicache-write-policy", "write_back"] ], - "donor": "glm5.2-fp4-b200-sglang-agentic-mtp", + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", "donorHardware": "b200" } }, "prov": { "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", + "run": "39166018772/attempts/1", "pr": "4286", - "status": "in-progress" + "status": "verified", + "date": "2026-10-19" }, "points": [ { @@ -5528,7 +5027,7 @@ export const agentx = { "--mem-fraction-static": ["0.82"] } }, - "ref": [25, "dynamo", 3] + "ref": [22, "dynamo", 3] } ] }, @@ -5948,7 +5447,7 @@ export const agentx = { "SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT": "1800" } }, - "ref": [28, "dynamo", 4] + "ref": [25, "dynamo", 4] }, { "drop": ["gpu_pinning"], @@ -5977,7 +5476,7 @@ export const agentx = { "SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT": "1800" } }, - "ref": [28, "dynamo", 5] + "ref": [25, "dynamo", 5] } ] }, @@ -6055,10 +5554,10 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "0.75"], + ["--hicache-size", "170"], ["--hicache-write-policy", "write_back"] ], - "donor": "glm5.2-fp4-b200-sglang-agentic-mtp", + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", "donorHardware": "b200" } }, @@ -6928,7 +6427,7 @@ export const agentx = { "--page-size": null } }, - "ref": [32, "dynamo", 4] + "ref": [29, "dynamo", 4] }, { "workers": 2, @@ -6938,7 +6437,7 @@ export const agentx = { "--speculative-num-steps": ["3"] } }, - "ref": [32, "dynamo", 5] + "ref": [29, "dynamo", 5] } ] },