diff --git a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx index 238ed8101d61..72da1c224fd2 100644 --- a/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx +++ b/docs/src/snippets/agentx/MiniMaxAI/minimax-m3.1.jsx @@ -1,56 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "MiniMax-M3.1-Flash", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4750": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41528865560", - "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", - "status": "verified-pr" - }, - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } @@ -212,7 +192,7 @@ export const agentx = { "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7", + "detail": "DSpark 7 · measured with no KV offload · --max-running-requests 2", "gpus": 4, "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", @@ -235,12 +215,12 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, "points": [ { @@ -249,134 +229,6 @@ export const agentx = { "override": "override_tp4_c1", "goldenAL": "4.04", "kv": "none" - }, - { - "concs": [4], - "item": 2, - "override": "override_tp4_c4", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - }, - null - ] - } - }, - { - "concs": [8], - "item": 3, - "override": "override_tp4_c8", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - }, - null - ] - } - }, - { - "concs": [16], - "item": 4, - "override": "override_tp4_c16", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - }, - null - ] - } - }, - { - "concs": [24], - "item": 5, - "override": "override_tp4_hicache_spf_c24", - "goldenAL": "4.04", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["48"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.87"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"] - } - } - ], - "sglang": [ - { - "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["48"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"], - "--max-running-requests": ["48"], - "--mem-fraction-static": ["0.87"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"] - } - }, - null - ] - } } ], "routers": { @@ -484,17 +336,17 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#1", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", + "detail": "DSpark 7 · measured with no KV offload · --max-running-requests 8", "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", - "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", - "imageCommit": null, - "submitted": "sglang", + "imageCommit": "e371881", + "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { @@ -511,112 +363,293 @@ export const agentx = { } }, "prov": { - "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "41054395691/attempts/2", - "pr": "4676", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-10-31" + "date": "2026-11-03" }, "points": [ { "concs": [1], - "item": 0, - "override": "zip_override_tp4[0]", + "item": 1, + "override": "override_tp4_monokernel_c1", "goldenAL": "4.04", "kv": "none" }, { "concs": [4], - "item": 0, - "override": "zip_override_tp4[1]", + "item": 2, + "override": "override_tp4_c4", "goldenAL": "4.04", "kv": "none", "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ], "dynamo": [ null, null, { "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"] - } - } - ] - } - }, - { - "concs": [8], - "item": 0, - "override": "zip_override_tp4[2]", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--disable-attn-tp-gather": null, + "--enable-nvidia-megakernel": null, + "--page-size": null, + "--speculative-eagle-topk": null, + "--speculative-num-draft-tokens": null, + "--speculative-num-steps": null + }, + "env": { + "ALLOW_JIT": null, + "SGLANG_FORCE_COARSE_WAR_BARRIER": null } } ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["16"], - "--max-running-requests": ["16"] - } - } - ] - } - }, - { - "concs": [16], - "item": 0, - "override": "zip_override_tp4[3]", - "goldenAL": "4.04", - "kv": "none", - "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--max-running-requests": ["32"] + "--speculative-draft-kv-cache-dtype": ["fp8_e4m3"], + "--disable-attn-tp-gather": null, + "--enable-nvidia-megakernel": null, + "--page-size": null, + "--speculative-eagle-topk": null, + "--speculative-num-draft-tokens": null, + "--speculative-num-steps": null + }, + "env": { + "ALLOW_JIT": null, + "SGLANG_FORCE_COARSE_WAR_BARRIER": null } - } + }, + null ] } - }, - { - "concs": [32], - "item": 0, - "override": "zip_override_tp4[4]", - "goldenAL": "4.04", - "kv": "none", - "patch": { - "sglang": [ - { - "args": { + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--disable-attn-tp-gather": [], + "--enable-nvidia-megakernel": [], + "--max-running-requests": ["8"], + "--page-size": ["128"], + "--speculative-draft-kv-cache-dtype": ["auto"], + "--speculative-eagle-topk": ["1"], + "--speculative-num-draft-tokens": ["8"], + "--speculative-num-steps": ["1"] + }, + "env": { + "ALLOW_JIT": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" + } + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#0", + "hw": "gb300", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --chunked-prefill-size 32768", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", + "imageStatus": "ok", + "imageCommit": null, + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "51c91c3cfd2ad631fdb3fa9dd8a2c8d94f000194", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "41054395691/attempts/2", + "pr": "4676", + "status": "verified", + "date": "2026-10-31" + }, + "points": [ + { + "concs": [1], + "item": 0, + "override": "zip_override_tp4[0]", + "goldenAL": "4.04", + "kv": "none" + }, + { + "concs": [4], + "item": 0, + "override": "zip_override_tp4[1]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, + { + "concs": [8], + "item": 0, + "override": "zip_override_tp4[2]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [16], + "item": 0, + "override": "zip_override_tp4[3]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + } + ] + } + }, + { + "concs": [32], + "item": 0, + "override": "zip_override_tp4[4]", + "goldenAL": "4.04", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { "--cuda-graph-max-bs-decode": ["64"], "--max-running-requests": ["64"] } @@ -732,42 +765,247 @@ export const agentx = { }, "points": [ { - "concs": [1], - "item": 2, - "override": "override_tp4_c1", + "concs": [1], + "item": 2, + "override": "override_tp4_c1", + "goldenAL": "4.04", + "kv": "none" + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--monokernel-backend": ["msa"], + "--chunked-prefill-size": null, + "--cuda-graph-backend-prefill": null, + "--cuda-graph-max-bs-decode": null, + "--enable-nvidia-megakernel": null, + "--speculative-draft-kv-cache-dtype": null + } + }, + "ref": [1, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#3", + "hw": "gb300", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --max-running-requests 16", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41528865560/attempts/2", + "pr": "4750", + "status": "verified", + "date": "2026-11-03" + }, + "points": [ + { + "concs": [8], + "item": 3, + "override": "override_tp4_c8", + "goldenAL": "4.04", + "kv": "none" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#4", + "hw": "gb300", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with no KV offload · --max-running-requests 32", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41528865560/attempts/2", + "pr": "4750", + "status": "verified", + "date": "2026-11-03" + }, + "points": [ + { + "concs": [16], + "item": 4, + "override": "override_tp4_c16", "goldenAL": "4.04", "kv": "none" } ], "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--disable-attn-tp-gather": [], - "--max-running-requests": ["8"], - "--monokernel-backend": ["msa"], - "--page-size": ["128"], - "--speculative-eagle-topk": ["1"], - "--speculative-num-draft-tokens": ["8"], - "--speculative-num-steps": ["1"], - "--chunked-prefill-size": null, - "--cuda-graph-backend-prefill": null, - "--cuda-graph-max-bs-decode": null, - "--speculative-draft-kv-cache-dtype": null - }, - "env": { - "ALLOW_JIT": "1", - "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" - } - }, - "ref": [0, "sglang", 0] - } - ] - }, "dynamo": { "install": [], "files": [], @@ -787,27 +1025,61 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, "args": [ ["--http-port", "8000"], - ["--router-mode", "kv"] - ] + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 }, { - "from": 0, - "cmd": "python3 -m dynamo.sglang", "diff": { "args": { - "--request-plane": ["tcp"] + "--cuda-graph-max-bs-decode": ["32"], + "--max-running-requests": ["32"] + } + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null } } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] } ] } @@ -1042,7 +1314,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [1, "sglang", 0] + "ref": [2, "sglang", 0] } ] }, @@ -1284,7 +1556,7 @@ export const agentx = { "--stream-interval": ["20"] } }, - "ref": [3, "sglang", 0] + "ref": [6, "sglang", 0] } ] }, @@ -1333,12 +1605,144 @@ export const agentx = { } } }, + { + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#5", + "hw": "gb300", + "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", + "label": "Agg TP4", + "detail": "DSpark 7 · measured with HiCache · minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "gpus": 4, + "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", + "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", + "imageStatus": "ok", + "imageCommit": "e371881", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "180"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "41528865560/attempts/2", + "pr": "4750", + "status": "verified", + "date": "2026-11-03" + }, + "points": [ + { + "concs": [24], + "item": 5, + "override": "override_tp4_hicache_spf_c24", + "goldenAL": "4.04", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "diff": { + "args": { + "--chunked-prefill-size": ["8192"], + "--cuda-graph-max-bs-decode": ["48"], + "--enable-hierarchical-cache": [], + "--hicache-size": ["180"], + "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["48"], + "--mem-fraction-static": ["0.87"], + "--prefill-decode-interval": ["32"], + "--schedule-policy": ["shortest-prefill-first"], + "--stream-interval": ["20"] + } + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 2, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, + "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, + "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, { "id": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark#3", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP4", - "detail": "DSpark 7 · measured with HiCache", + "detail": "DSpark 7 · measured with HiCache · minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "gpus": 4, "configKey": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -1477,12 +1881,10 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["48"], - "--max-running-requests": ["48"], - "--tensor-parallel-size": ["4"] + "--tokenizer-worker-num": ["8"] } }, - "ref": [4, "sglang", 0] + "ref": [8, "sglang", 0] } ] }, @@ -1654,7 +2056,7 @@ export const agentx = { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" } }, - "ref": [1, "sglang", 0] + "ref": [2, "sglang", 0] } ] }, @@ -1731,12 +2133,12 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, "points": [ { @@ -1780,20 +2182,12 @@ export const agentx = { { "diff": { "args": { - "--chunked-prefill-size": ["8192"], "--cuda-graph-max-bs-decode": ["80"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["180"], - "--hicache-write-policy": ["write_back"], "--max-running-requests": ["80"], - "--mem-fraction-static": ["0.87"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"], - "--stream-interval": ["20"], "--tensor-parallel-size": ["2"] } }, - "ref": [0, "dynamo", 2] + "ref": [8, "dynamo", 2] } ] }, @@ -1864,12 +2258,12 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, "points": [ { @@ -1947,7 +2341,7 @@ export const agentx = { "--stream-interval": null } }, - "ref": [7, "dynamo", 2] + "ref": [11, "dynamo", 2] } ] }, @@ -1991,12 +2385,12 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#3", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)", + "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 7", - "gpus": 16, + "gpus": 28, "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", @@ -2006,7 +2400,7 @@ export const agentx = { { "kind": "note", "script": "Placement", - "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." } ], "disclosures": ["synthetic-acceptance"], @@ -2024,18 +2418,18 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, - "points": [ - { - "concs": [224, 256], - "item": 1, - "override": "override_4p_dep2_dep8", + "points": [ + { + "concs": [220, 240, 260, 300, 360, 400], + "item": 3, + "override": "override_6p_dep2_dep16", "goldenAL": "4.04", "kv": "hicache" } @@ -2073,15 +2467,17 @@ export const agentx = { { "kind": "worker", "role": "prefill", - "workers": 4, + "workers": 6, "nodes_per_worker": 1, "gpu_pinning": [ + ["0,1"], + ["2,3"], ["0,1"], ["2,3"], ["0,1"], ["2,3"] ], - "local_ranks": [0, 1, 0, 1], + "local_ranks": [0, 1, 0, 1, 0, 1], "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], "cmd": "python3 -m dynamo.sglang", "args": [ @@ -2124,7 +2520,7 @@ export const agentx = { "kind": "worker", "role": "decode", "workers": 1, - "nodes_per_worker": 2, + "nodes_per_worker": 4, "cmd": "python3 -m dynamo.sglang", "args": [ ["--model-path", "MiniMaxAI/MiniMax-M3.1-Flash"], @@ -2133,17 +2529,17 @@ export const agentx = { ["--port", "30000"], ["--disaggregation-mode", "decode"], ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], - ["--nnodes", "2"], + ["--nnodes", "4"], ["--node-rank", "$NODE_RANK"], ["--request-plane", "tcp"], ["--cuda-graph-max-bs-decode", "48"], - ["--data-parallel-size", "8"], + ["--data-parallel-size", "16"], ["--disaggregation-transfer-backend", "mooncake"], ["--enable-dp-attention"], ["--enable-dp-lm-head"], - ["--expert-parallel-size", "8"], + ["--expert-parallel-size", "16"], ["--load-balance-method", "total_tokens"], - ["--max-running-requests", "384"], + ["--max-running-requests", "768"], ["--mem-fraction-static", "0.85"], ["--moe-a2a-backend", "megamoe"], ["--reasoning-parser", "minimax-m3"], @@ -2152,7 +2548,7 @@ export const agentx = { ["--speculative-draft-kv-cache-dtype", "fp8_e4m3"], ["--speculative-dspark-block-size", "7"], ["--speculative-use-block-verification"], - ["--tensor-parallel-size", "8"], + ["--tensor-parallel-size", "16"], ["--tool-call-parser", "minimax-m3"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"] @@ -2213,6 +2609,8 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], @@ -2228,12 +2626,12 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)", + "label": "Disagg 4P1D · prefill DEP2 · decode DEP8 (2 nodes)", "detail": "DSpark 7", - "gpus": 20, + "gpus": 16, "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", @@ -2243,7 +2641,7 @@ export const agentx = { { "kind": "note", "script": "Placement", - "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + "reason": "Workers share nodes: 4 prefill workers on 2 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." } ], "disclosures": ["synthetic-acceptance"], @@ -2261,18 +2659,18 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, "points": [ { - "concs": [280, 320, 340, 360, 400], - "item": 0, - "override": "override_6p_dep2_dep8", + "concs": [224, 256], + "item": 1, + "override": "override_4p_dep2_dep8", "goldenAL": "4.04", "kv": "hicache" } @@ -2308,20 +2706,28 @@ export const agentx = { "envRef": 0 }, { - "workers": 6, + "workers": 4, "gpu_pinning": [ - ["0,1"], - ["2,3"], ["0,1"], ["2,3"], ["0,1"], ["2,3"] ], - "local_ranks": [0, 1, 0, 1, 0, 1], - "ref": [9, "dynamo", 2] + "local_ranks": [0, 1, 0, 1], + "ref": [13, "dynamo", 2] }, { - "ref": [9, "dynamo", 3] + "nodes_per_worker": 2, + "diff": { + "args": { + "--nnodes": ["2"], + "--data-parallel-size": ["8"], + "--expert-parallel-size": ["8"], + "--max-running-requests": ["384"], + "--tensor-parallel-size": ["8"] + } + }, + "ref": [13, "dynamo", 3] } ] }, @@ -2377,8 +2783,6 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], - ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_3_HEAD_IP:30032", "8999"], ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_1_HEAD_IP:30032", "8999"], @@ -2394,12 +2798,12 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#3", + "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0", "hw": "gb300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "label": "Disagg 6P1D · prefill DEP2 · decode DEP8 (2 nodes)", "detail": "DSpark 7", - "gpus": 28, + "gpus": 20, "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", "imageStatus": "ok", @@ -2409,7 +2813,7 @@ export const agentx = { { "kind": "note", "script": "Placement", - "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + "reason": "Workers share nodes: 6 prefill workers on 3 nodes, 1 decode worker on 2 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." } ], "disclosures": ["synthetic-acceptance"], @@ -2429,16 +2833,16 @@ export const agentx = { "prov": { "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41528865560", + "run": "41528865560/attempts/2", "pr": "4750", "status": "verified", "date": "2026-11-03" }, "points": [ { - "concs": [220, 240, 260, 300, 360, 400], - "item": 3, - "override": "override_6p_dep2_dep16", + "concs": [280, 320, 340, 360, 400], + "item": 0, + "override": "override_6p_dep2_dep8", "goldenAL": "4.04", "kv": "hicache" } @@ -2474,20 +2878,10 @@ export const agentx = { "envRef": 0 }, { - "ref": [10, "dynamo", 2] + "ref": [13, "dynamo", 2] }, { - "nodes_per_worker": 4, - "diff": { - "args": { - "--nnodes": ["4"], - "--data-parallel-size": ["16"], - "--expert-parallel-size": ["16"], - "--max-running-requests": ["768"], - "--tensor-parallel-size": ["16"] - } - }, - "ref": [9, "dynamo", 3] + "ref": [14, "dynamo", 3] } ] }, @@ -2593,12 +2987,12 @@ export const agentx = { } }, "prov": { - "sha": "b1af0163907bb3bd238986c4377be6ad81aec9ff", + "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "41271829525/attempts/1", - "pr": "4698", + "run": "41528865560/attempts/2", + "pr": "4750", "status": "verified", - "date": "2026-11-02" + "date": "2026-11-03" }, "points": [ { @@ -2652,10 +3046,10 @@ export const agentx = { ["2,3"] ], "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1], - "ref": [9, "dynamo", 2] + "ref": [13, "dynamo", 2] }, { - "ref": [9, "dynamo", 3] + "ref": [14, "dynamo", 3] } ] }, @@ -3274,7 +3668,7 @@ export const agentx = { "--tokenizer-worker-num": ["8"] } }, - "ref": [13, "sglang", 0] + "ref": [17, "sglang", 0] } ] }, @@ -3392,7 +3786,7 @@ export const agentx = { "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" } }, - "ref": [13, "sglang", 0] + "ref": [17, "sglang", 0] } ] }, @@ -3609,7 +4003,7 @@ export const agentx = { "--mem-fraction-static": ["0.81"] } }, - "ref": [14, "sglang", 0] + "ref": [18, "sglang", 0] } ] }, @@ -3728,7 +4122,7 @@ export const agentx = { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" } }, - "ref": [16, "sglang", 0] + "ref": [20, "sglang", 0] } ] }, @@ -4085,10 +4479,10 @@ export const agentx = { }, { "workers": 3, - "ref": [18, "dynamo", 2] + "ref": [22, "dynamo", 2] }, { - "ref": [18, "dynamo", 3] + "ref": [22, "dynamo", 3] } ] }, @@ -4237,7 +4631,7 @@ export const agentx = { "envRef": 0 }, { - "ref": [19, "dynamo", 2] + "ref": [23, "dynamo", 2] }, { "nodes_per_worker": 4, @@ -4251,7 +4645,7 @@ export const agentx = { "--tensor-parallel-size": ["16"] } }, - "ref": [18, "dynamo", 3] + "ref": [22, "dynamo", 3] } ] }, @@ -4343,15 +4737,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -4580,15 +4975,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-low-latency.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -4625,7 +5021,7 @@ export const agentx = { "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" } }, - "ref": [21, "sglang", 0] + "ref": [25, "sglang", 0] } ] }, @@ -4698,14 +5094,15 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -4961,7 +5358,7 @@ export const agentx = { "--stream-interval": ["20"] } }, - "ref": [21, "sglang", 0] + "ref": [25, "sglang", 0] } ] }, @@ -5034,15 +5431,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -5068,7 +5466,7 @@ export const agentx = { "--hicache-write-policy": null } }, - "ref": [23, "sglang", 0] + "ref": [27, "sglang", 0] } ] }, @@ -5141,14 +5539,15 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -5371,7 +5770,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [23, "sglang", 0] + "ref": [27, "sglang", 0] } ] }, @@ -5444,15 +5843,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -5621,7 +6021,7 @@ export const agentx = { "--stream-interval": null } }, - "ref": [25, "sglang", 0] + "ref": [29, "sglang", 0] } ] }, @@ -5694,15 +6094,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -5806,7 +6207,7 @@ export const agentx = { "--tensor-parallel-size": ["4"] } }, - "ref": [26, "sglang", 0] + "ref": [30, "sglang", 0] } ] }, @@ -5879,15 +6280,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -6019,7 +6421,7 @@ export const agentx = { "--prefill-decode-interval": ["16"] } }, - "ref": [25, "sglang", 0] + "ref": [29, "sglang", 0] } ] }, @@ -6073,7 +6475,7 @@ export const agentx = { "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.92", + "detail": "DSpark 7 · --prefill-decode-interval 24", "gpus": 2, "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -6092,15 +6494,16 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -6124,7 +6527,7 @@ export const agentx = { "--prefill-decode-interval": ["24"] } }, - "ref": [25, "sglang", 0] + "ref": [29, "sglang", 0] } ] }, @@ -6174,11 +6577,11 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#30", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#27", "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 24 · --mem-fraction-static 0.93", + "detail": "DSpark 7 · --prefill-decode-interval 20 · --mem-fraction-static 0.92", "gpus": 2, "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -6197,21 +6600,22 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { - "concs": [48], - "item": 30, - "override": "override_tp2_c48_spf_interval24_mem093", + "concs": [50], + "item": 27, + "override": "override_tp2_c50_spf_interval20", "goldenAL": "4.04", "kv": "hicache" } @@ -6224,7 +6628,9 @@ export const agentx = { { "diff": { "args": { - "--mem-fraction-static": ["0.93"] + "--cuda-graph-max-bs-decode": ["100"], + "--max-running-requests": ["100"], + "--prefill-decode-interval": ["20"] } }, "ref": [29, "sglang", 0] @@ -6277,11 +6683,11 @@ export const agentx = { } }, { - "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#27", + "id": "minimaxm31flash-fp4-b300-sglang-agentic-dspark#29", "hw": "b300", "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", "label": "Agg TP2", - "detail": "DSpark 7 · --prefill-decode-interval 20", + "detail": "DSpark 7 · --prefill-decode-interval 20 · --mem-fraction-static 0.93", "gpus": 2, "configKey": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", "image": "lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87", @@ -6300,27 +6706,28 @@ export const agentx = { ["--hicache-write-policy", "write_back"] ], "donor": "minimaxm31flash-fp4-b300-sglang-agentic-dspark", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "ab42462db15e7937ba868d11c520c123479703a5", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml", + "run": "41488743791/attempts/1", "pr": "4726", - "status": "in-progress" + "status": "verified", + "date": "2026-11-03" }, "points": [ { "concs": [50], - "item": 27, - "override": "override_tp2_c50_spf_interval20", + "item": 29, + "override": "override_tp2_c50_spf_interval20_mem093", "goldenAL": "4.04", - "kv": "hicache", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-interval.yaml" + "kv": "hicache" }, { "concs": [52], - "item": 29, + "item": 30, "override": "override_tp2_c52_spf_interval20_mem093", "goldenAL": "4.04", "kv": "hicache", @@ -6329,8 +6736,7 @@ export const agentx = { { "args": { "--cuda-graph-max-bs-decode": ["104"], - "--max-running-requests": ["104"], - "--mem-fraction-static": ["0.93"] + "--max-running-requests": ["104"] } } ], @@ -6340,13 +6746,11 @@ export const agentx = { { "args": { "--cuda-graph-max-bs-decode": ["104"], - "--max-running-requests": ["104"], - "--mem-fraction-static": ["0.93"] + "--max-running-requests": ["104"] } } ] - }, - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml" + } } ], "routers": { @@ -6357,12 +6761,10 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["100"], - "--max-running-requests": ["100"], - "--prefill-decode-interval": ["20"] + "--mem-fraction-static": ["0.93"] } }, - "ref": [25, "sglang", 0] + "ref": [34, "sglang", 0] } ] }, @@ -6719,7 +7121,7 @@ export const agentx = { "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" } }, - "ref": [32, "sglang", 0] + "ref": [36, "sglang", 0] } ] }, @@ -6936,7 +7338,7 @@ export const agentx = { "--mem-fraction-static": ["0.81"] } }, - "ref": [32, "sglang", 0] + "ref": [36, "sglang", 0] } ] }, @@ -7055,7 +7457,7 @@ export const agentx = { "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768" } }, - "ref": [34, "sglang", 0] + "ref": [38, "sglang", 0] } ] }, @@ -7103,114 +7505,6 @@ export const agentx = { ] } } - }, - { - "id": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg#1", - "hw": "gb300", - "ckpt": "MiniMaxAI/MiniMax-M3.1-Flash", - "label": "Agg TP4", - "detail": "DSpark 7 · MSA MonoKernel", - "gpus": 4, - "configKey": "minimaxm31flash-fp4-gb300-dynamo-sglang-agentic-dspark-agg", - "image": "lmsysorg/sglang:nightly-cu134-20261030-e371881@sha256:a82fff791d7dc1e5cfdfde9022487c60c079286fd54684950d55b787eb2e673a", - "imageStatus": "ok", - "imageCommit": "e371881", - "submitted": "dynamo", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-size", "180"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "minimaxm31flash-fp4-gb300-sglang-agentic-dspark", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "6528aa7c4341e9027ed152fa280fa22f32363b1f", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41528865560", - "pr": "4750", - "status": "verified", - "date": "2026-11-03" - }, - "points": [ - { - "concs": [1], - "item": 1, - "override": "override_tp4_monokernel_c1", - "goldenAL": "4.04", - "kv": "none" - } - ], - "routers": { - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "ref": [0, "dynamo", 0] - }, - { - "ref": [0, "dynamo", 1] - }, - { - "ref": [0, "dynamo", 2], - "diff": { - "args": { - "--cuda-graph-max-bs-decode": ["8"], - "--max-running-requests": ["8"], - "--disable-attn-tp-gather": [], - "--enable-nvidia-megakernel": [], - "--page-size": ["128"], - "--speculative-draft-kv-cache-dtype": ["auto"], - "--speculative-eagle-topk": ["1"], - "--speculative-num-draft-tokens": ["8"], - "--speculative-num-steps": ["1"] - }, - "env": { - "ALLOW_JIT": "1", - "SGLANG_FORCE_COARSE_WAR_BARRIER": "1" - } - } - } - ] - }, - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "from": 2, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--request-plane": null, - "--kv-events-config": null, - "--router-mode": null, - "--router-session-affinity-ttl-secs": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_EVENT_PLANE": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null - } - } - }, - { - "ref": [0, "sglang", 1] - } - ] - } - } } ] }; diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx index 69507c2a9748..0890f5f8b695 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "Qwen3.5", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx index 3ef7ca99320e..9196797cab1b 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "Qwen3.8-Flash-Next", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } @@ -173,12 +158,12 @@ export const agentx = { } }, "prov": { - "sha": "b3c469ec2cdfc4279e678aacadc4b89973e9b2f3", + "sha": "0950b48d5e4060a5e50413ac09bdcc5e81924178", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "40221796954/attempts/4", - "pr": "4512", + "run": "40864354242/attempts/1", + "pr": "4552", "status": "verified", - "date": "2026-10-26" + "date": "2026-11-03" }, "points": [ { @@ -390,10 +375,10 @@ export const agentx = { "prov": { "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41117066222", + "run": "41117066222/attempts/1", "pr": "4683", - "comment": "6536482181", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -801,10 +786,10 @@ export const agentx = { "prov": { "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41117066222", + "run": "41117066222/attempts/1", "pr": "4683", - "comment": "6536482181", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-03" }, "points": [ { diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index cb5db60af19d..e1c708917e73 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } @@ -82,19 +67,6 @@ export const agentx = { } ], "envs": [ - { - "DYN_REQUEST_PLANE": "tcp", - "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", - "DYN_ROUTER_QUEUE_THRESHOLD": "None", - "DYN_ROUTER_TEMPERATURE": "0", - "DYN_TCP_CHANNEL_BUFFER": "128", - "DYN_TCP_REQUEST_TIMEOUT": "60", - "DYN_TOKENIZER": "fastokens", - "DYN_TOKENIZER_CACHE": "1", - "DYN_TOKENIZER_CACHE_BYTES": "8000000000", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", - "NATS_SERVER": "nats://$NATS_IP:4222" - }, { "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", @@ -188,9 +160,9 @@ export const agentx = { "detail": "DSpark 5 · dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", "gpus": 4, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "c10e0e17", + "imageCommit": "302b78cc", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -202,20 +174,18 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "40989481011/attempts/1", - "pr": "4595", - "status": "verified", - "date": "2026-10-30" + "pr": "4673", + "status": "in-progress" }, "points": [ { @@ -389,6 +359,19 @@ export const agentx = { "kind": "dynamo-frontend", "count": 1, "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", + "DYN_ROUTER_QUEUE_THRESHOLD": "None", + "DYN_ROUTER_TEMPERATURE": "0", + "DYN_TCP_CHANNEL_BUFFER": "128", + "DYN_TCP_REQUEST_TIMEOUT": "60", + "DYN_TOKENIZER": "fastokens", + "DYN_TOKENIZER_CACHE": "1", + "DYN_TOKENIZER_CACHE_BYTES": "8000000000", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" + }, "args": [ ["--http-port", "8000"], ["--router-mode", "kv"], @@ -396,8 +379,7 @@ export const agentx = { ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] - ], - "envRef": 0 + ] }, { "kind": "worker", @@ -512,20 +494,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "41261902147", + "run": "41261902147/attempts/2", "pr": "4603", - "comment": "6576133363", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -734,165 +716,12 @@ export const agentx = { } } }, - { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg#3", - "hw": "gb300", - "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Agg TP2·EP2", - "detail": "DSpark 5 · dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", - "gpus": 2, - "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47", - "imageStatus": "ok", - "imageCommit": "c10e0e17", - "submitted": "dynamo", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], - ["--hicache-write-policy", "write_back"] - ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" - } - }, - "prov": { - "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "41081085818/attempts/1", - "pr": "4679", - "status": "verified", - "date": "2026-10-30" - }, - "points": [ - { - "concs": [16], - "item": 3, - "override": "override_tp2_c16", - "goldenAL": "3.51", - "kv": "none" - } - ], - "routers": { - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "nats", - "count_nodes": 1, - "cmd": "nats-server -js -sd /tmp/nats", - "env": {}, - "args": [] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "14400"], - ["--active-decode-blocks-threshold", "None"], - ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"] - ], - "envRef": 0 - }, - { - "diff": { - "args": { - "--cuda-graph-backend-prefill": ["disabled"], - "--cuda-graph-max-bs-decode": ["64"], - "--expert-parallel-size": ["2"], - "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.8"], - "--prefill-decode-interval": ["96"], - "--swa-prefix-tails": ["1024"], - "--tensor-parallel-size": ["2"], - "--context-length": null, - "--enable-nvidia-megakernel": null, - "--kv-cache-dtype": null, - "--moe-a2a-backend": null, - "--schedule-policy": null, - "--startup-weight-load-mode": null, - "--tokenizer-backend": null - }, - "env": { - "NCCL_CUMEM_ENABLE": "1", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_NUMA_BIND_V2": "0", - "NCCL_MNNVL_ENABLE": null, - "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null, - "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null, - "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null, - "SGLANG_SPEC_TP_SYNC": null - } - }, - "ref": [0, "dynamo", 3] - } - ] - }, - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "from": 3, - "cmd": "python3 -m sglang.launch_server", - "diff": { - "args": { - "--request-plane": null - }, - "env": { - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT": null, - "DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT": null, - "DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT": null, - "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null - } - } - }, - { - "kind": "sglang-router", - "cmd": "python3 -m sglang_router.launch_router", - "env": {}, - "args": [ - ["--host", "0.0.0.0"], - ["--port", "8000"], - ["--policy", "cache_aware"], - ["--worker-urls", "http://127.0.0.1:30000"] - ] - } - ] - } - } - }, { "id": "dsv41flash-fp4-gb300-sglang-agentic-dspark#0", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Agg TP2·EP2", - "detail": "DSpark 5 · dsv41flash-fp4-gb300-sglang-agentic-dspark", + "detail": "DSpark 5", "gpus": 2, "configKey": "dsv41flash-fp4-gb300-sglang-agentic-dspark", "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", @@ -909,20 +738,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "41261902147", + "run": "41261902147/attempts/2", "pr": "4603", - "comment": "6576133363", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -976,18 +805,34 @@ export const agentx = { { "diff": { "args": { - "--tokenizer-backend": ["fastokens"], - "--tokenizer-worker-num": ["4"] + "--cuda-graph-backend-prefill": ["disabled"], + "--cuda-graph-max-bs-decode": ["64"], + "--expert-parallel-size": ["2"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.8"], + "--prefill-decode-interval": ["96"], + "--swa-prefix-tails": ["1024"], + "--tensor-parallel-size": ["2"], + "--tokenizer-worker-num": ["4"], + "--context-length": null, + "--enable-nvidia-megakernel": null, + "--kv-cache-dtype": null, + "--moe-a2a-backend": null, + "--schedule-policy": null, + "--startup-weight-load-mode": null }, "env": { - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DSPARK_FOLDED_SAMPLING": "2", - "SGLANG_DSPARK_OPT_MARKOV_W2_BF16": "True", - "SGLANG_DSV41_REASONING_EFFORT": "high", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + "NCCL_CUMEM_ENABLE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_NUMA_BIND_V2": "0", + "NCCL_MNNVL_ENABLE": null, + "SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION": null, + "SGLANG_NVIDIA_MEGAKERNEL_SLOTS": null, + "SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD": null, + "SGLANG_SPEC_TP_SYNC": null } }, - "ref": [2, "sglang", 0] + "ref": [1, "sglang", 0] } ] }, @@ -1058,20 +903,20 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "41261902147", + "run": "41261902147/attempts/2", "pr": "4603", - "comment": "6576133363", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-03" }, "points": [ { @@ -1173,7 +1018,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [3, "sglang", 0] + "ref": [2, "sglang", 0] }, { "kind": "sglang-router", @@ -1247,9 +1092,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 22, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "586355a8", + "imageCommit": "302b78cc", "submitted": "dynamo", "setup": [ { @@ -1267,28 +1112,58 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "d43831083511ec8d9cf1340e0bb3811fa683692c", + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", - "run": "41081085818/attempts/1", - "pr": "4679", - "status": "verified", - "date": "2026-10-30" + "pr": "4673", + "status": "in-progress" }, "points": [ { - "concs": [1228], + "concs": [700], "item": 0, - "override": "override_3p1d_dep2_dep16_c1228", + "override": "override_3p1d_dep2_dep16_c700", "goldenAL": "3.51", "kv": "hicache" + }, + { + "concs": [1228], + "item": 1, + "override": "override_3p1d_dep2_dep16_c1228", + "goldenAL": "3.51", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--disaggregation-decode-extra-slots": ["36"], + "--max-running-requests": ["384"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--disaggregation-decode-extra-slots": ["36"], + "--max-running-requests": ["384"] + } + }, + null + ] + } } ], "routers": { @@ -1319,7 +1194,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 1 + "envRef": 0 }, { "kind": "worker", @@ -1446,9 +1321,9 @@ export const agentx = { ["--nnodes", "4"], ["--node-rank", "$NODE_RANK"], ["--request-plane", "tcp"], - ["--cuda-graph-max-bs-decode", "24"], + ["--cuda-graph-max-bs-decode", "18"], ["--data-parallel-size", "16"], - ["--disaggregation-decode-extra-slots", "36"], + ["--disaggregation-decode-extra-slots", "27"], ["--disaggregation-transfer-backend", "mooncake"], ["--dsv4-attn-backend", "flashmla_mega"], ["--enable-decoder-swa-bounded-replay"], @@ -1462,7 +1337,7 @@ export const agentx = { ["--expert-distribution-recorder-buffer-size", "1000"], ["--expert-parallel-size", "16"], ["--load-balance-method", "total_tokens"], - ["--max-running-requests", "384"], + ["--max-running-requests", "288"], ["--mem-fraction-static", "0.75"], ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--moe-a2a-backend", "megamoe"], @@ -1549,16 +1424,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#1", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", "gpus": 26, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "586355a8", + "imageCommit": "302b78cc", "submitted": "dynamo", "setup": [ { @@ -1576,25 +1451,23 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", - "run": "40989481011/attempts/1", - "pr": "4595", - "status": "verified", - "date": "2026-10-30" + "pr": "4673", + "status": "in-progress" }, "points": [ { "concs": [2884], - "item": 1, + "item": 2, "override": "override_5p1d_dep2_dep16_c2884", "goldenAL": "3.51", "kv": "hicache" @@ -1628,7 +1501,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 1 + "envRef": 0 }, { "workers": 5, @@ -1640,7 +1513,7 @@ export const agentx = { ["0,1"] ], "local_ranks": [0, 1, 0, 1, 0], - "ref": [5, "dynamo", 2] + "ref": [4, "dynamo", 2] }, { "diff": { @@ -1650,7 +1523,7 @@ export const agentx = { "--max-running-requests": ["640"] } }, - "ref": [5, "dynamo", 3] + "ref": [4, "dynamo", 3] } ] }, @@ -1722,22 +1595,22 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#2", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#3", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Disagg 8P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "label": "Disagg 7P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", - "gpus": 32, + "gpus": 30, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "586355a8", + "imageCommit": "302b78cc", "submitted": "dynamo", "setup": [ { "kind": "note", "script": "Placement", - "reason": "Workers share nodes: 8 prefill workers on 4 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." + "reason": "Workers share nodes: 7 prefill workers on 4 nodes, 1 decode worker on 4 nodes. LOCAL_WORKER_RANK gives each worker on a node its own GPUs and ports. The GPU count above counts the GPUs the workers use; reserve whole nodes when planning capacity." } ], "disclosures": ["synthetic-acceptance"], @@ -1749,26 +1622,24 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", - "run": "40989481011/attempts/1", - "pr": "4595", - "status": "verified", - "date": "2026-10-30" + "pr": "4673", + "status": "in-progress" }, "points": [ { - "concs": [4096], - "item": 2, - "override": "override_8p1d_dep2_dep16_c4096", + "concs": [5540], + "item": 3, + "override": "override_7p1d_dep2_dep16_b96_c5540", "goldenAL": "3.51", "kv": "hicache" } @@ -1801,10 +1672,10 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 1 + "envRef": 0 }, { - "workers": 8, + "workers": 7, "gpu_pinning": [ ["0,1"], ["2,3"], @@ -1812,22 +1683,20 @@ export const agentx = { ["2,3"], ["0,1"], ["2,3"], - ["0,1"], - ["2,3"] + ["0,1"] ], - "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1], - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))", "KV_EVENTS_CONFIG=\"{\\\"publisher\\\":\\\"zmq\\\",\\\"topic\\\":\\\"kv-events\\\",\\\"endpoint\\\":\\\"tcp://*:$((5557 + 2 * LOCAL_WORKER_RANK))\\\"}\""], - "ref": [5, "dynamo", 2] + "local_ranks": [0, 1, 0, 1, 0, 1, 0], + "ref": [4, "dynamo", 2] }, { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["64"], - "--disaggregation-decode-extra-slots": ["128"], - "--max-running-requests": ["1024"] + "--cuda-graph-max-bs-decode": ["96"], + "--disaggregation-decode-extra-slots": ["256"], + "--max-running-requests": ["1536"] } }, - "ref": [5, "dynamo", 3] + "ref": [4, "dynamo", 3] } ] }, @@ -1837,7 +1706,7 @@ export const agentx = { "blocks": [ { "from": 2, - "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1).", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], + "preamble": ["# LOCAL_WORKER_RANK is this worker's index on its node (0 or 1); 0 for a worker alone on its node.", "case \"$LOCAL_WORKER_RANK\" in 0) export CUDA_VISIBLE_DEVICES=0,1 ;; 1) export CUDA_VISIBLE_DEVICES=2,3 ;; *) echo \"Set LOCAL_WORKER_RANK to 0 or 1\" >&2; exit 1 ;; esac", "PORT=$((30000 + 32 * LOCAL_WORKER_RANK))", "BOOTSTRAP_PORT=$((8998 + LOCAL_WORKER_RANK))"], "cmd": "python3 -m sglang.launch_server", "diff": { "args": { @@ -1883,7 +1752,6 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_7_HEAD_IP:30032", "8999"], ["--prefill", "http://$PREFILL_6_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_5_HEAD_IP:30032", "8999"], ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], @@ -1902,16 +1770,16 @@ export const agentx = { } }, { - "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#3", + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache#4", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", "label": "Disagg 9P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", "gpus": 34, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579", "imageStatus": "ok", - "imageCommit": "586355a8", + "imageCommit": "302b78cc", "submitted": "dynamo", "setup": [ { @@ -1929,26 +1797,24 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.2"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "4145e263b55bbb071885a6c17bb40c5463f6f70e", + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml", - "run": "40989481011/attempts/1", - "pr": "4595", - "status": "verified", - "date": "2026-10-30" + "pr": "4673", + "status": "in-progress" }, "points": [ { "concs": [7480], - "item": 3, - "override": "override_9p1d_dep2_dep16_b160_c7480", + "item": 4, + "override": "override_9p1d_dep2_dep16_b192_c7480", "goldenAL": "3.51", "kv": "hicache" } @@ -1981,7 +1847,7 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 1 + "envRef": 0 }, { "workers": 9, @@ -1997,17 +1863,17 @@ export const agentx = { ["0,1"] ], "local_ranks": [0, 1, 0, 1, 0, 1, 0, 1, 0], - "ref": [5, "dynamo", 2] + "ref": [4, "dynamo", 2] }, { "diff": { "args": { - "--cuda-graph-max-bs-decode": ["160"], - "--disaggregation-decode-extra-slots": ["256"], - "--max-running-requests": ["2560"] + "--cuda-graph-max-bs-decode": ["192"], + "--load-balance-method": ["total_requests"], + "--max-running-requests": ["3072"] } }, - "ref": [5, "dynamo", 3] + "ref": [6, "dynamo", 3] } ] }, @@ -2392,7 +2258,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [9, "sglang", 0] + "ref": [8, "sglang", 0] } ] }, @@ -2628,7 +2494,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [10, "sglang", 0] + "ref": [9, "sglang", 0] }, { "kind": "sglang-router", @@ -2986,7 +2852,7 @@ export const agentx = { "--schedule-policy": null } }, - "ref": [12, "sglang", 0] + "ref": [11, "sglang", 0] } ] }, @@ -3169,7 +3035,7 @@ export const agentx = { "--dsv4-attn-backend": null } }, - "ref": [13, "sglang", 0] + "ref": [12, "sglang", 0] } ] }, @@ -3334,7 +3200,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [12, "sglang", 0] + "ref": [11, "sglang", 0] }, { "kind": "sglang-router", @@ -3739,7 +3605,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [16, "sglang", 0] + "ref": [15, "sglang", 0] } ] }, @@ -3999,7 +3865,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0" } }, - "ref": [17, "sglang", 0] + "ref": [16, "sglang", 0] }, { "kind": "sglang-router", @@ -4440,7 +4306,7 @@ export const agentx = { "--tensor-parallel-size": ["8"] } }, - "ref": [19, "sglang", 0] + "ref": [18, "sglang", 0] } ] }, @@ -4702,7 +4568,7 @@ export const agentx = { ["--trust-remote-code"], ["--watchdog-timeout", "3600"] ], - "envRef": 2 + "envRef": 1 } ] }, @@ -4890,7 +4756,7 @@ export const agentx = { "--swa-prefix-tails": ["64"] } }, - "ref": [21, "sglang", 0] + "ref": [20, "sglang", 0] }, { "kind": "sglang-router", @@ -4907,7 +4773,7 @@ export const agentx = { ["--disable-retries"], ["--worker-urls", "http://127.0.0.1:30000"] ], - "envRef": 2 + "envRef": 1 } ] }, diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx index 9c6a9d43321e..021227dc3201 100644 --- a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx +++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "Muse Spark Open", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } @@ -83,6 +68,14 @@ export const agentx = { "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", "NATS_SERVER": "nats://$NATS_IP:4222" + }, + { + "DYN_NATS_REQUEST_TIMEOUT_SECS": "1800", + "DYN_REQUEST_PLANE": "tcp", + "DYN_ROUTER_TTL_SECS": "14400", + "DYN_ROUTER_USE_KV_EVENTS": "false", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222" } ], "kvMooncake": { @@ -183,10 +176,10 @@ export const agentx = { "prov": { "sha": "98ea294c289546dca624e1f07d9d3d2106715741", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml", - "run": "41327192200", + "run": "41327192200/attempts/1", "pr": "4724", - "comment": "6576178636", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-04" }, "points": [ { @@ -687,16 +680,16 @@ export const agentx = { } }, { - "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#2", + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg#0", "hw": "gb300", "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", "label": "Agg TP8·EP8 (2 nodes)", "detail": "DFlash 8", "gpus": 8, - "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash", - "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "797af717", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -714,25 +707,23 @@ export const agentx = { } }, "prov": { - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "39427144732/attempts/4", - "pr": "4344", - "status": "verified", - "date": "2026-10-22" + "pr": "4757", + "status": "in-progress" }, "points": [ { - "concs": [3], - "item": 5, - "override": "override_tp8_c3", + "concs": [1], + "item": 1, + "override": "override_tp8_c1", "goldenAL": "4.05", "kv": "none" }, { - "concs": [4], - "item": 4, - "override": "override_tp8_c4", + "concs": [36], + "item": 0, + "override": "override_tp8_c36", "goldenAL": "4.05", "kv": "none", "patch": { @@ -742,82 +733,210 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["256"] + "--cuda-graph-max-bs-decode": ["36"], + "--max-running-requests": ["36"], + "--prefill-decode-interval": ["128"], + "--schedule-policy": ["shortest-prefill-first"], + "--speculative-draft-attention-backend": ["fa4"] } } ], "sglang": [ { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--prefill-decode-interval": ["256"] + "--cuda-graph-max-bs-decode": ["36"], + "--max-running-requests": ["36"], + "--prefill-decode-interval": ["128"], + "--schedule-policy": ["shortest-prefill-first"], + "--speculative-draft-attention-backend": ["fa4"] } }, null ] } - }, - { - "concs": [10], - "item": 3, - "override": "override_tp8_c10", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - null, - { + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "nodes_per_worker": 2, + "diff": { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--page-size": null + "--dist-init-addr": ["$AGG_HEAD_IP:20000"], + "--nnodes": ["2"], + "--node-rank": ["$NODE_RANK"], + "--kv-events-config": ["{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], + "--dyn-reasoning-parser": ["muse"], + "--dyn-tool-call-parser": ["muse"], + "--expert-parallel-size": ["8"], + "--model-loader-extra-config": ["{\"enable_multithread_load\":true}"], + "--preferred-sampling-params": ["{\"temperature\":1.0,\"top_p\":1.0,\"top_k\":64}"], + "--prefill-decode-interval": ["64"], + "--speculative-draft-attention-backend": ["trtllm_mha"], + "--speculative-draft-model-revision": ["46c90f412549c472941a68382d91a99e5e6372a0"], + "--tensor-parallel-size": ["8"], + "--chunked-prefill-size": null, + "--tool-call-parser": null + }, + "env": { + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_MNNVL_ENABLE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS": "16", + "SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS_MIN_BS": "4" } - } - ], - "sglang": [ - { + }, + "ref": [0, "dynamo", 2] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { "args": { - "--cuda-graph-max-bs-decode": ["2"], - "--max-running-requests": ["2"], - "--page-size": null + "--request-plane": null, + "--kv-events-config": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null } - }, - null - ] - } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#5", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Disagg 1P4D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 40, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [32], + "item": 8, + "override": "override_1p4d_tp8_tp8_c32", + "goldenAL": "4.05", + "kv": "hicache" }, { - "concs": [14], - "item": 2, - "override": "override_tp8_c14", + "concs": [56], + "item": 7, + "override": "override_1p4d_tp8_tp8_c56", "goldenAL": "4.05", - "kv": "none", + "kv": "hicache" + }, + { + "concs": [112], + "item": 5, + "override": "override_1p4d_tp8_tp8_c112", + "goldenAL": "4.05", + "kv": "hicache", "patch": { "dynamo": [ null, null, null, + null, { "args": { - "--cuda-graph-max-bs-decode": ["14"], - "--max-running-requests": ["14"], - "--speculative-draft-attention-backend": ["fa4"], - "--page-size": null + "--cuda-graph-max-bs-decode": ["6"], + "--disaggregation-decode-extra-slots": ["6"], + "--max-running-requests": ["6"] } } ], "sglang": [ + null, { "args": { - "--cuda-graph-max-bs-decode": ["14"], - "--max-running-requests": ["14"], - "--speculative-draft-attention-backend": ["fa4"], - "--page-size": null + "--cuda-graph-max-bs-decode": ["6"], + "--disaggregation-decode-extra-slots": ["6"], + "--max-running-requests": ["6"] } }, null @@ -827,7 +946,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""], + "install": [], "files": ["nats.conf#13771a85"], "blocks": [ { @@ -855,42 +974,987 @@ export const agentx = { "args": [ ["--http-port", "8000"], ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 + }, + { + "kind": "worker", + "role": "prefill", + "workers": 1, + "nodes_per_worker": 2, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "14400", + "SGLANG_DISAGGREGATION_QUEUE_SIZE": "8", + "SGLANG_DISAGGREGATION_THREAD_POOL_SIZE": "32", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "14400", + "SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX": "0", + "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "prefill"], + ["--disaggregation-bootstrap-port", "8998"], + ["--dist-init-addr", "$PREFILL_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--request-plane", "tcp"], + ["--chunked-prefill-size", "32768"], + ["--context-length", "1048576"], + ["--cuda-graph-max-bs-decode", "64"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--dyn-reasoning-parser", "muse"], + ["--dyn-tool-call-parser", "muse"], + ["--enable-hierarchical-cache"], + ["--enable-layernorm-sp"], + ["--enable-megamoe-for-layernorm-sp"], + ["--expert-parallel-size", "8"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-prefill-tokens", "32768"], + ["--max-running-requests", "256"], + ["--mem-fraction-static", "0.85"], + ["--mm-feature-transport", "cpu"], + ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--optimistic-prefill-attempts", "0"], + ["--preferred-sampling-params", "{\"temperature\":1.0,\"top_p\":1.0,\"top_k\":64}"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--schedule-policy", "shortest-prefill-first"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "trtllm_mha"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-draft-model-revision", "46c90f412549c472941a68382d91a99e5e6372a0"], + ["--speculative-num-draft-tokens", "8"], + ["--stream-interval", "20"], + ["--swa-full-tokens-ratio", "0.16"], + ["--tensor-parallel-size", "8"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + }, + { + "kind": "worker", + "role": "decode", + "workers": 4, + "nodes_per_worker": 2, + "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_FORCE_MNNVL": "1", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_MNNVL_ENABLE": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "14400", + "SGLANG_DISAGGREGATION_QUEUE_SIZE": "8", + "SGLANG_DISAGGREGATION_THREAD_POOL_SIZE": "32", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "14400", + "SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX": "0", + "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "0", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", + "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", + "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1", + "SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS": "16", + "SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS_MIN_BS": "12" + }, + "args": [ + ["--model-path", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--served-model-name", "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"], + ["--host", "0.0.0.0"], + ["--port", "30000"], + ["--disaggregation-mode", "decode"], + ["--dist-init-addr", "$DECODE_HEAD_IP:20000"], + ["--nnodes", "2"], + ["--node-rank", "$NODE_RANK"], + ["--request-plane", "tcp"], + ["--context-length", "1048576"], + ["--cuda-graph-max-bs-decode", "3"], + ["--disaggregation-decode-extra-slots", "3"], + ["--disaggregation-transfer-backend", "mooncake"], + ["--dyn-reasoning-parser", "muse"], + ["--dyn-tool-call-parser", "muse"], + ["--expert-parallel-size", "8"], + ["--kv-cache-dtype", "fp8_e4m3"], + ["--max-running-requests", "3"], + ["--mem-fraction-static", "0.92"], + ["--mm-feature-transport", "cpu"], + ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], + ["--moe-runner-backend", "flashinfer_trtllm"], + ["--preferred-sampling-params", "{\"temperature\":1.0,\"top_p\":1.0,\"top_k\":64}"], + ["--quantization", "modelopt_fp4"], + ["--reasoning-parser", "muse"], + ["--speculative-algorithm", "DFLASH"], + ["--speculative-draft-attention-backend", "trtllm_mha"], + ["--speculative-draft-model-path", "RadixArk/Muse-Spark-Open-DFlash2"], + ["--speculative-draft-model-revision", "46c90f412549c472941a68382d91a99e5e6372a0"], + ["--speculative-num-draft-tokens", "8"], + ["--stream-interval", "20"], + ["--swa-full-tokens-ratio", "0.1"], + ["--tensor-parallel-size", "8"], + ["--trust-remote-code"], + ["--watchdog-timeout", "3600"] + ] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_3_HEAD_IP:30000"], + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg#2", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Agg TP4·EP4", + "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg", + "gpus": 4, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [46], + "item": 4, + "override": "override_tp4_c46", + "goldenAL": "4.05", + "kv": "hicache" + }, + { + "concs": [50], + "item": 3, + "override": "override_tp4_c50", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--prefill-decode-interval": ["20"] + } + } + ], + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["20"] + } + }, + null + ] + } + }, + { + "concs": [56], + "item": 2, + "override": "override_tp4_c56", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + { + "args": { + "--prefill-decode-interval": ["8"] + } + } + ], + "sglang": [ + { + "args": { + "--prefill-decode-interval": ["8"] + } + }, + null + ] + } + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "3600"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 0 + }, + { + "nodes_per_worker": 1, + "diff": { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-backend-prefill": ["breakable"], + "--cuda-graph-max-bs-decode": ["32"], + "--enable-hierarchical-cache": [], + "--expert-parallel-size": ["4"], + "--hicache-size": ["140"], + "--hicache-write-policy": ["write_back"], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.95"], + "--prefill-decode-interval": ["24"], + "--schedule-policy": ["fcfs"], + "--swa-full-tokens-ratio": ["0.2"], + "--tensor-parallel-size": ["4"], + "--dist-init-addr": null, + "--nnodes": null, + "--node-rank": null + }, + "env": { + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", + "NCCL_MNNVL_ENABLE": null, + "PYTORCH_CUDA_ALLOC_CONF": null + } + }, + "ref": [1, "dynamo", 3] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--request-plane": null, + "--kv-events-config": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"], + ["--worker-urls", "http://127.0.0.1:30000"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#6", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Disagg 1P5D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 48, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [120], + "item": 6, + "override": "override_1p5d_tp8_tp8_c120", + "goldenAL": "4.05", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 1 + }, + { + "ref": [2, "dynamo", 3] + }, + { + "workers": 5, + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["5"], + "--disaggregation-decode-extra-slots": ["5"], + "--max-running-requests": ["5"] + } + }, + "ref": [2, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_4_HEAD_IP:30000"], + ["--decode", "http://$DECODE_3_HEAD_IP:30000"], + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#2", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Disagg 1P2D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 24, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [160], + "item": 3, + "override": "override_1p2d_tp8_tp8_c160", + "goldenAL": "4.05", + "kv": "hicache" + }, + { + "concs": [224], + "item": 2, + "override": "override_1p2d_tp8_tp8_c224", + "goldenAL": "4.05", + "kv": "hicache", + "patch": { + "dynamo": [ + null, + null, + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["36"], + "--disaggregation-decode-extra-slots": ["36"], + "--max-running-requests": ["36"] + } + } + ], + "sglang": [ + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["36"], + "--disaggregation-decode-extra-slots": ["36"], + "--max-running-requests": ["36"] + } + }, + null + ] + } + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 1 + }, + { + "ref": [2, "dynamo", 3] + }, + { + "workers": 2, + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--disaggregation-decode-extra-slots": ["24"], + "--max-running-requests": ["24"] + } + }, + "ref": [2, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#4", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Disagg 1P3D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 32, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [224], + "item": 4, + "override": "override_1p3d_tp8_tp8_c224", + "goldenAL": "4.05", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 1 + }, + { + "ref": [2, "dynamo", 3] + }, + { + "workers": 3, + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["20"], + "--disaggregation-decode-extra-slots": ["20"], + "--max-running-requests": ["20"] + } + }, + "ref": [2, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#1", + "hw": "gb300", + "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", + "label": "Disagg 2P3D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 40, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", + "imageStatus": "ok", + "imageCommit": "797af717", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-size", "150"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "musespark-fp4-gb300-sglang-agentic-dflash", + "donorHardware": "gb300" + } + }, + "prov": { + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" + }, + "points": [ + { + "concs": [352], + "item": 1, + "override": "override_2p3d_tp8_tp8_c352", + "goldenAL": "4.05", + "kv": "hicache" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": ["nats.conf#13771a85"], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "nats", + "count_nodes": 1, + "cmd": "nats-server -c nats.conf", + "env": {}, + "args": [] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-session-affinity-mode", "soft"], + ["--router-session-affinity-ttl-secs", "14400"] + ], + "envRef": 1 + }, + { + "workers": 2, + "ref": [2, "dynamo", 3] }, { - "nodes_per_worker": 2, "diff": { "args": { - "--dist-init-addr": ["$AGG_HEAD_IP:20000"], - "--nnodes": ["2"], - "--node-rank": ["$NODE_RANK"], - "--kv-events-config": ["{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:5557\"}"], - "--dyn-reasoning-parser": ["muse"], - "--dyn-tool-call-parser": ["muse"], - "--expert-parallel-size": ["8"], - "--model-loader-extra-config": ["{\"enable_multithread_load\":true}"], - "--page-size": ["128"], - "--preferred-sampling-params": ["{\"temperature\":1.0,\"top_p\":1.0,\"top_k\":64}"], - "--prefill-decode-interval": ["64"], - "--speculative-draft-model-revision": ["46c90f412549c472941a68382d91a99e5e6372a0"], - "--tensor-parallel-size": ["8"], - "--chunked-prefill-size": null, - "--stream-interval": null, - "--tool-call-parser": null - }, - "env": { - "NATS_SERVER": "nats://$NATS_IP:4222", - "NCCL_MNNVL_ENABLE": "1", - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": null + "--cuda-graph-max-bs-decode": ["40"], + "--disaggregation-decode-extra-slots": ["40"], + "--max-running-requests": ["40"] } }, - "ref": [0, "dynamo", 2] + "ref": [6, "dynamo", 4] } ] }, @@ -903,8 +1967,25 @@ export const agentx = { "cmd": "python3 -m sglang.launch_server", "diff": { "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], "--request-plane": null, - "--kv-events-config": null, "--dyn-reasoning-parser": null, "--dyn-tool-call-parser": null }, @@ -920,10 +2001,15 @@ export const agentx = { "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], ["--host", "0.0.0.0"], ["--port", "8000"], - ["--policy", "cache_aware"], - ["--worker-urls", "http://$AGG_0_HEAD_IP:30000"] + ["--policy", "cache_aware"] ] } ] @@ -931,16 +2017,16 @@ export const agentx = { } }, { - "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash#0", + "id": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg#0", "hw": "gb300", "ckpt": "meta-models/Muse-Spark-Open-816B-A42B-NVFP4", - "label": "Agg TP4·EP4", - "detail": "DFlash 8 · musespark-fp4-gb300-dynamo-sglang-agentic-dflash", - "gpus": 4, - "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash", - "image": "lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a", + "label": "Disagg 3P4D · prefill TP8·EP8 (2 nodes) · decode TP8·EP8 (2 nodes)", + "detail": "DFlash 8", + "gpus": 56, + "configKey": "musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd", "imageStatus": "ok", - "imageCommit": null, + "imageCommit": "797af717", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -958,197 +2044,23 @@ export const agentx = { } }, "prov": { - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", - "run": "39427144732/attempts/4", - "pr": "4344", - "status": "verified", - "date": "2026-10-22" + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4757", + "status": "in-progress" }, "points": [ { - "concs": [8], - "item": 1, - "override": "override_tp4_c8", - "goldenAL": "4.05", - "kv": "none" - }, - { - "concs": [16], + "concs": [672], "item": 0, - "override": "override_tp4_c16", - "goldenAL": "4.05", - "kv": "none", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["128"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["12"], - "--max-running-requests": ["12"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["128"], - "--schedule-policy": ["shortest-prefill-first"], - "--swa-full-tokens-ratio": ["0.1"] - } - }, - null - ] - } - }, - { - "concs": [20], - "item": 8, - "override": "override_tp4_c20", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["120"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["10"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["10"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["120"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["10"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["48"], - "--schedule-policy": ["shortest-prefill-first"] - } - }, - null - ] - } - }, - { - "concs": [32], - "item": 7, - "override": "override_tp4_c32", - "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["170"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["56"], - "--schedule-policy": ["shortest-prefill-first"] - } - } - ], - "sglang": [ - { - "args": { - "--cuda-graph-max-bs-decode": ["32"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["170"], - "--hicache-write-policy": ["write_through"], - "--max-running-requests": ["32"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["56"], - "--schedule-policy": ["shortest-prefill-first"] - } - }, - null - ] - } - }, - { - "concs": [40], - "item": 6, - "override": "override_tp4_c40", + "override": "override_3p4d_tp8_tp8_c672", "goldenAL": "4.05", - "kv": "hicache", - "patch": { - "dynamo": [ - null, - null, - null, - { - "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["24"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["24"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" - } - } - ], - "sglang": [ - { - "args": { - "--chunked-prefill-size": ["16384"], - "--cuda-graph-backend-prefill": ["breakable"], - "--cuda-graph-max-bs-decode": ["24"], - "--enable-hierarchical-cache": [], - "--hicache-size": ["175"], - "--max-prefill-tokens": ["16384"], - "--max-running-requests": ["24"], - "--mem-fraction-static": ["0.95"], - "--prefill-decode-interval": ["32"], - "--schedule-policy": ["shortest-prefill-first"] - }, - "env": { - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", - "SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "4" - } - }, - null - ] - } + "kv": "hicache" } ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.6.0.dev20261016\" \"ai-dynamo==1.6.0.dev20261016\""], + "install": [], "files": ["nats.conf#13771a85"], "blocks": [ { @@ -1176,33 +2088,27 @@ export const agentx = { "args": [ ["--http-port", "8000"], ["--router-mode", "kv"], - ["--router-session-affinity-ttl-secs", "3600"], ["--active-decode-blocks-threshold", "None"], ["--active-prefill-tokens-threshold", "None"], - ["--active-prefill-tokens-threshold-frac", "None"] + ["--active-prefill-tokens-threshold-frac", "None"], + ["--router-session-affinity-mode", "soft"], + ["--router-session-affinity-ttl-secs", "14400"] ], - "envRef": 0 + "envRef": 1 + }, + { + "workers": 3, + "ref": [2, "dynamo", 3] }, { - "nodes_per_worker": 1, "diff": { "args": { - "--chunked-prefill-size": ["8192"], - "--cuda-graph-max-bs-decode": ["8"], - "--expert-parallel-size": ["4"], - "--max-running-requests": ["8"], - "--tensor-parallel-size": ["4"], - "--dist-init-addr": null, - "--nnodes": null, - "--node-rank": null, - "--page-size": null - }, - "env": { - "NCCL_MNNVL_ENABLE": null, - "PYTORCH_CUDA_ALLOC_CONF": null + "--cuda-graph-max-bs-decode": ["56"], + "--disaggregation-decode-extra-slots": ["56"], + "--max-running-requests": ["56"] } }, - "ref": [1, "dynamo", 3] + "ref": [2, "dynamo", 4] } ] }, @@ -1215,8 +2121,25 @@ export const agentx = { "cmd": "python3 -m sglang.launch_server", "diff": { "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--dyn-reasoning-parser": null, + "--dyn-tool-call-parser": null + }, + "env": { + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null, + "NATS_SERVER": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], "--request-plane": null, - "--kv-events-config": null, "--dyn-reasoning-parser": null, "--dyn-tool-call-parser": null }, @@ -1232,10 +2155,17 @@ export const agentx = { "cmd": "python3 -m sglang_router.launch_router", "env": {}, "args": [ + ["--decode", "http://$DECODE_3_HEAD_IP:30000"], + ["--decode", "http://$DECODE_2_HEAD_IP:30000"], + ["--decode", "http://$DECODE_1_HEAD_IP:30000"], + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], ["--host", "0.0.0.0"], ["--port", "8000"], - ["--policy", "cache_aware"], - ["--worker-urls", "http://127.0.0.1:30000"] + ["--policy", "cache_aware"] ] } ] @@ -1834,10 +2764,10 @@ export const agentx = { "prov": { "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41328422543", + "run": "41328422543/attempts/1", "pr": "4725", - "comment": "6576185043", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-04" }, "points": [ { @@ -2147,10 +3077,10 @@ export const agentx = { "prov": { "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", - "run": "41328422543", + "run": "41328422543/attempts/1", "pr": "4725", - "comment": "6576185043", - "status": "verified-pr" + "status": "verified", + "date": "2026-11-04" }, "points": [ { @@ -2380,7 +3310,7 @@ export const agentx = { "PYTORCH_CUDA_ALLOC_CONF": null } }, - "ref": [4, "sglang", 0] + "ref": [10, "sglang", 0] } ] }, @@ -2920,7 +3850,7 @@ export const agentx = { "PYTORCH_CUDA_ALLOC_CONF": null } }, - "ref": [6, "sglang", 0] + "ref": [12, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index e3e02fe358bc..30b5d91a1a64 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } } @@ -4061,12 +4046,12 @@ export const agentx = { } }, "prov": { - "sha": "3120bc1b90db358d8bca1031225c110f3c60e579", + "sha": "3b3b616206e90a95102f312bf114b6b934caa2e8", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", - "run": "38304664439/attempts/2", - "pr": "3836", + "run": "41347984144/attempts/1", + "pr": "4702", "status": "verified", - "date": "2026-10-14" + "date": "2026-11-03" }, "points": [ { @@ -4200,12 +4185,12 @@ export const agentx = { } }, "prov": { - "sha": "3120bc1b90db358d8bca1031225c110f3c60e579", + "sha": "3b3b616206e90a95102f312bf114b6b934caa2e8", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", - "run": "38304664439/attempts/2", - "pr": "3836", + "run": "41347984144/attempts/1", + "pr": "4702", "status": "verified", - "date": "2026-10-14" + "date": "2026-11-03" }, "points": [ { @@ -4340,12 +4325,12 @@ export const agentx = { } }, "prov": { - "sha": "3120bc1b90db358d8bca1031225c110f3c60e579", + "sha": "3b3b616206e90a95102f312bf114b6b934caa2e8", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", - "run": "38304664439/attempts/2", - "pr": "3836", + "run": "41347984144/attempts/1", + "pr": "4702", "status": "verified", - "date": "2026-10-14" + "date": "2026-11-03" }, "points": [ { @@ -4530,12 +4515,12 @@ export const agentx = { } }, "prov": { - "sha": "3120bc1b90db358d8bca1031225c110f3c60e579", + "sha": "3b3b616206e90a95102f312bf114b6b934caa2e8", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml", - "run": "38304664439/attempts/2", - "pr": "3836", + "run": "41347984144/attempts/1", + "pr": "4702", "status": "verified", - "date": "2026-10-14" + "date": "2026-11-03" }, "points": [ { @@ -4639,12 +4624,12 @@ export const agentx = { } }, "prov": { - "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", + "sha": "ae71693bf2a806b1ce2bf480b500ee6bc8bd3f8c", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39626904306/attempts/1", - "pr": "4370", + "run": "41003299710/attempts/1", + "pr": "4656", "status": "verified", - "date": "2026-10-22" + "date": "2026-11-03" }, "points": [ { @@ -4888,12 +4873,12 @@ export const agentx = { } }, "prov": { - "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", + "sha": "ae71693bf2a806b1ce2bf480b500ee6bc8bd3f8c", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39626904306/attempts/1", - "pr": "4370", + "run": "41003299710/attempts/1", + "pr": "4656", "status": "verified", - "date": "2026-10-22" + "date": "2026-11-03" }, "points": [ { @@ -5252,12 +5237,12 @@ export const agentx = { } }, "prov": { - "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", + "sha": "ae71693bf2a806b1ce2bf480b500ee6bc8bd3f8c", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39626904306/attempts/1", - "pr": "4370", + "run": "41003299710/attempts/1", + "pr": "4656", "status": "verified", - "date": "2026-10-22" + "date": "2026-11-03" }, "points": [ { @@ -5367,9 +5352,9 @@ export const agentx = { "detail": "MTP 2-1-3", "gpus": 24, "configKey": "glm5.2-fp4-b200-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-20260910-00840301", - "imageStatus": "pruned", - "imageCommit": "00840301", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -5394,12 +5379,12 @@ export const agentx = { } }, "prov": { - "sha": "53483461cd7aace0c8712c2d5cf7ec4a2b3e819a", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml", - "run": "36304573738/attempts/4", - "pr": "3400", + "sha": "ae71693bf2a806b1ce2bf480b500ee6bc8bd3f8c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml", + "run": "41003299710/attempts/1", + "pr": "4656", "status": "verified", - "date": "2026-10-02" + "date": "2026-11-03" }, "points": [ { @@ -5657,9 +5642,9 @@ export const agentx = { "detail": "MTP 2-1-3", "gpus": 16, "configKey": "glm5.2-fp4-b200-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-20260910-00840301", - "imageStatus": "pruned", - "imageCommit": "00840301", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -5678,12 +5663,12 @@ export const agentx = { } }, "prov": { - "sha": "53483461cd7aace0c8712c2d5cf7ec4a2b3e819a", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml", - "run": "36304573738/attempts/4", - "pr": "3400", + "sha": "ae71693bf2a806b1ce2bf480b500ee6bc8bd3f8c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml", + "run": "41003299710/attempts/1", + "pr": "4656", "status": "verified", - "date": "2026-10-02" + "date": "2026-11-03" }, "points": [ { diff --git a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx index 72904b8ea8a1..de421b8372c8 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx @@ -1,51 +1,36 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@c5e8b9b3e9b9 plus open PRs #4603, #4670, #4683, #4695, #4707, #4724, #4725, #4726), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@57116fff418d plus open PRs #4670, #4673, #4695, #4707, #4757), with SA-specific settings removed. export const agentx = { "model": "GLM-5.3", - "generatedOn": "2026-11-02", + "generatedOn": "2026-11-04", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "c5e8b9b3e9b90511fa5c7774d83e4663ca4db553", + "main": "57116fff418d7bb651567e722907253a12b75a67", "prs": { - "4726": { + "4757": { "run": null, - "sha": "0979dfd84e3fe408e9c36752b69da7f5915ce5ab", + "sha": "b37254d0ccded4a7cf9a8ebd90370e658a5c4b5c", "status": "in-progress" }, - "4725": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41328422543", - "sha": "7516df00ac37c119055834dbc6d48811ae6a0b99", - "status": "verified-pr" - }, - "4724": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41327192200", - "sha": "98ea294c289546dca624e1f07d9d3d2106715741", - "status": "verified-pr" - }, "4707": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41297696082", "sha": "c965ca2eb81db733ad280ea693025389ac11014b", "status": "verified-pr" }, - "4695": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", - "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", - "status": "verified-pr" - }, - "4683": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41117066222", - "sha": "d06564bc533cc54584ef6a35837336a2b4b84421", - "status": "verified-pr" - }, "4670": { "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41096285400", "sha": "c9eb455f32e7f026d7424de723756de2e719d868", "status": "verified-pr" }, - "4603": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41261902147", - "sha": "41d29b2834a85fcee09db6f9f52653b7c2f30ef3", + "4673": { + "run": null, + "sha": "be7bc6df6bb01ad03aed970aa962bc002424694e", + "status": "in-progress" + }, + "4695": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41271615328", + "sha": "fa292b9045dbf8c18df4c8d9c183b33cd8f0351e", "status": "verified-pr" } }