From 9ef3ff3bda92c7282c022ef0fdcdca38328d2416 Mon Sep 17 00:00:00 2001 From: cquil11 Date: Thu, 15 Oct 2026 17:40:44 -0500 Subject: [PATCH 1/5] feat(agentx): add Muse Spark Open GB200 SGLang DFlash2 AgentX config Serve meta-models/Muse-Spark-Open-816B-A42B-NVFP4 on one GB200 node at TP4/EP4 from the staged Lustre checkpoint with the DFlash2 draft, and let the srt connector derive DFlash's draft length from its verify width. --- .../sglang/gb200-fp4-mtp/agentic.yaml | 141 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 14 ++ inferencex-e2e/configs/runners.yaml | 2 + inferencex-e2e/docs/MODELS.md | 1 + inferencex-e2e/infx/srt_slurm/single_node.py | 4 +- .../infx/srt_slurm/synthetic_acceptance.py | 10 ++ .../srt_slurm/test_synthetic_acceptance.py | 15 ++ inferencex-e2e/perf-changelog.yaml | 11 ++ 8 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..50c215c55c --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml @@ -0,0 +1,141 @@ +base: + schema: 2 + name: musespark-fp4-gb200-sglang-agentic + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + trust-remote-code: true + tensor-parallel-size: 4 + expert-parallel-size: 4 + quantization: modelopt_fp4 + moe-runner-backend: flashinfer_trtllm + kv-cache-dtype: fp8_e4m3 + enable-layernorm-sp: true + enable-megamoe-for-layernorm-sp: true + # 133 GB of weights per GPU leave the KV pool what prefill graphs would take. + mem-fraction-static: 0.88 + disable-prefill-cuda-graph: true + swa-full-tokens-ratio: 0.1 + # DFlash2 verifies 8 tokens per step: the current token plus 7 drafts. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Muse-Spark-Open-DFlash2 + speculative-draft-attention-backend: fa4 + speculative-num-draft-tokens: 8 + mm-feature-transport: cpu + reasoning-parser: muse + tool-call-parser: muse + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # 533 GB of weights load from Lustre before graph capture. + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + # The gated Hub repo is unreadable to the client; use the staged checkpoint. + TOKENIZER: /model + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + + +override_tp4_c1: + roles: + agg: + args: + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + benchmark: + env: + CONC: '1' + +override_tp4_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '2' + +override_tp4_c4: + roles: + agg: + args: + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '4' + +override_tp4_c8: + roles: + agg: + args: + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '8' + +override_tp4_c12: + roles: + agg: + args: + max-running-requests: 24 + cuda-graph-max-bs-decode: 24 + benchmark: + env: + CONC: '12' + +override_tp4_c16: + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + +override_tp4_c24: + roles: + agg: + args: + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + benchmark: + env: + CONC: '24' + +override_tp4_c32: + roles: + agg: + args: + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '32' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2ff820f5b9..3dde84708d 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8365,6 +8365,20 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128, 192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } +musespark-fp4-gb200-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:gb200-nv + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: musespark/sglang/gb200-fp4-mtp + scenarios: + agentic-coding: + - search-space: + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 12, 16, 24, 32], srt-recipe: agentic.yaml } + dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 4d25914d7e..846fa8e857 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -577,6 +577,7 @@ clusters: MiniMax-M3-NVFP4: {root: lustre-models, dir: MiniMax-M3-NVFP4} Qwen3.5-397B-A17B-FP8: {root: lustre-models, dir: Qwen3.5-397B-A17B-FP8} Qwen3.5-397B-A17B-NVFP4-V2: {root: lustre-models, dir: Qwen3.5-397B-A17B-NVFP4-V2} + Muse-Spark-Open-816B-A42B-NVFP4: {root: lustre-models, dir: Muse-Spark-Open-816B-A42B-NVFP4} GLM-5.2-NVFP4: {root: sa-shared-models, dir: GLM-5.2-NVFP4} DeepSeek-R1-0528: {root: numa1, dir: DeepSeek-R1-0528} DeepSeek-R1-0528-NVFP4-v2: {root: numa1, dir: DeepSeek-R1-0528-NVFP4-v2} @@ -613,6 +614,7 @@ clusters: rdma-gid-index: 3 job-tag: gb200 default-time-limit: "6:00:00" + single-node-models: staged preflight: false segment-directive: false container-aliases: [dynamo-trtllm, dynamo-sglang] diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index 0733de44f4..bb04039310 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -110,6 +110,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| +| Muse Spark Open 816B-A42B | `musespark` | 2026-10-15 | Agentic coding (SGLang with DFlash2 draft `RadixArk/Muse-Spark-Open-DFlash2`; golden AL curve `musespark_dflash.yaml`) | | | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463, restored on srt-slurm 2026-09-30; GPU validation pending) | | | GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the deprecation policy) | Single-turn 1k1k (deprecated for all models before this model was added; never run) | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | | diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index c9f8b6c8ac..3e7fc8c6c1 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -113,9 +113,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark", "dflash"}: raise ValueError( - "Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation" + "Single-node SRT supports only native MTP, EAGLE3, DSpark, DFlash or no speculation" ) speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") agentic = environment["IS_AGENTIC"] == "1" diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 2de8687c22..9884fe1a07 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -77,6 +77,16 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: algorithm = next(iter(algorithms), "") if not algorithm: return {} + if algorithm == "dflash": + # DFlash verifies a block of the current token plus its drafts. + width = args.get("speculative-num-draft-tokens") + return { + "method": "dflash", + "num_speculative_tokens": width - 1 + if isinstance(width, int) and not isinstance(width, bool) + else None, + "model": args.get("speculative-draft-model-path", ""), + } return { "method": "dspark" if algorithm == "dspark" else algorithm, "num_speculative_tokens": args.get( diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index b51e80bfd1..18645901ff 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -51,6 +51,7 @@ def golden_dir(tmp_path: Path) -> Path: ("minimaxm3_eagle3.yaml", "minimax-m3", 2.5), ("minimaxm3_eagle3_gqa.yaml", "minimax-m3", 2.6), ("glm5.3_mtp.yaml", "glm-5.3", 3.2), + ("musespark_dflash.yaml", "muse-spark-open", 3.4), ]: (directory / filename).write_text( yaml.safe_dump( @@ -177,6 +178,20 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", }, ), + ( + "sglang", + { + "speculative-algorithm": "DFLASH", + "speculative-draft-model-path": "RadixArk/Muse-Spark-Open-DFlash2", + "speculative-num-draft-tokens": 4, + }, + {"MODEL_PREFIX": "musespark"}, + { + "SGLANG_SIMULATE_ACC_LEN": "3.4", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + }, + ), ( "dynamo-sglang", {"speculative-algorithm": "EAGLE", "speculative-num-steps": 3}, diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f66d1177c7..5db84f123b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9892,3 +9892,14 @@ - "The new image brings Kimi-K3 ROCm changes vllm-project/vllm#59591 (latent MoE up-proj stores only the local rank's shards), #59069 (AttnRes output fused with per-token FP8 input quantization), #60717 (vLLM's FP8 max in that fused quantization), #60446 (KDA autotune keys), #59523 (cuMem CUDA-graph pool offload on ROCm) and #59880 (speculative block reservations across KV cache groups with a KV connector)." - "No change to the Inferact/Kimi-K3-DSpark draft's precision: K3DSparkModel and get_draft_quant_config() are unchanged between the two images, so the draft keeps BF16 weights and an fp8 KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3951 + +- config-keys: + - musespark-fp4-gb200-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Muse Spark Open (meta-models/Muse-Spark-Open-816B-A42B-NVFP4) agentic coding on one GB200 node via SGLang (lmsysorg/sglang:dev-muse-spark-open, multiarch digest sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a) at TP4/EP4, concurrency 1, 2, 4, 8, 12, 16, 24 and 32, GPU-resident KV." + - "Serve flags follow the SGLang cookbook's NVFP4 cell: modelopt_fp4, flashinfer_trtllm MoE, fp8_e4m3 KV, layernorm SP with MegaMoE prefill, DFlash2 draft RadixArk/Muse-Spark-Open-DFlash2 with fa4 draft attention and 8 verify tokens (7 drafts), muse reasoning and tool-call parsers. Prefill CUDA graphs are off and mem-fraction-static is 0.88 to leave room for the KV pool; admission and the decode graph batch are 2x concurrency." + - "Synthetic acceptance follows the committed musespark_dflash golden curve: AL 4.05 at 7 draft tokens, thinking_on." + - "The checkpoint is served from the staged Lustre copy; the AgentX client reads its tokenizer from it." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From a107f976ca2dc3a3603b8e43436822a1bd6c78bc Mon Sep 17 00:00:00 2001 From: cquil11 Date: Thu, 15 Oct 2026 17:41:49 -0500 Subject: [PATCH 2/5] chore(changelog): link Muse Spark Open GB200 PR #4140 --- inferencex-e2e/perf-changelog.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5db84f123b..eb8f39133c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4140 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4140 - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -9902,4 +9902,4 @@ - "Serve flags follow the SGLang cookbook's NVFP4 cell: modelopt_fp4, flashinfer_trtllm MoE, fp8_e4m3 KV, layernorm SP with MegaMoE prefill, DFlash2 draft RadixArk/Muse-Spark-Open-DFlash2 with fa4 draft attention and 8 verify tokens (7 drafts), muse reasoning and tool-call parsers. Prefill CUDA graphs are off and mem-fraction-static is 0.88 to leave room for the KV pool; admission and the decode graph batch are 2x concurrency." - "Synthetic acceptance follows the committed musespark_dflash golden curve: AL 4.05 at 7 draft tokens, thinking_on." - "The checkpoint is served from the staged Lustre copy; the AgentX client reads its tokenizer from it." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4140 From 6cc71dbb95fb3b39c45d5c38fa068858fca3f08e Mon Sep 17 00:00:00 2001 From: cquil11 Date: Thu, 15 Oct 2026 18:33:57 -0500 Subject: [PATCH 3/5] feat(agentx): add Muse Spark Open GB200 concurrency 6 and 10 --- .../sglang/gb200-fp4-mtp/agentic.yaml | 20 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml index 50c215c55c..f1f972a5d8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml @@ -90,6 +90,16 @@ override_tp4_c4: env: CONC: '4' +override_tp4_c6: + roles: + agg: + args: + max-running-requests: 12 + cuda-graph-max-bs-decode: 12 + benchmark: + env: + CONC: '6' + override_tp4_c8: roles: agg: @@ -100,6 +110,16 @@ override_tp4_c8: env: CONC: '8' +override_tp4_c10: + roles: + agg: + args: + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + benchmark: + env: + CONC: '10' + override_tp4_c12: roles: agg: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 3dde84708d..58a4dfa3e8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8377,7 +8377,7 @@ musespark-fp4-gb200-sglang-agentic-dflash: scenarios: agentic-coding: - search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 12, 16, 24, 32], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 24, 32], srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index eb8f39133c..3b551a8928 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9898,7 +9898,7 @@ scenario-type: - agentic-coding description: - - "Add Muse Spark Open (meta-models/Muse-Spark-Open-816B-A42B-NVFP4) agentic coding on one GB200 node via SGLang (lmsysorg/sglang:dev-muse-spark-open, multiarch digest sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a) at TP4/EP4, concurrency 1, 2, 4, 8, 12, 16, 24 and 32, GPU-resident KV." + - "Add Muse Spark Open (meta-models/Muse-Spark-Open-816B-A42B-NVFP4) agentic coding on one GB200 node via SGLang (lmsysorg/sglang:dev-muse-spark-open, multiarch digest sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a) at TP4/EP4, concurrency 1, 2, 4, 6, 8, 10, 12, 16, 24 and 32, GPU-resident KV." - "Serve flags follow the SGLang cookbook's NVFP4 cell: modelopt_fp4, flashinfer_trtllm MoE, fp8_e4m3 KV, layernorm SP with MegaMoE prefill, DFlash2 draft RadixArk/Muse-Spark-Open-DFlash2 with fa4 draft attention and 8 verify tokens (7 drafts), muse reasoning and tool-call parsers. Prefill CUDA graphs are off and mem-fraction-static is 0.88 to leave room for the KV pool; admission and the decode graph batch are 2x concurrency." - "Synthetic acceptance follows the committed musespark_dflash golden curve: AL 4.05 at 7 draft tokens, thinking_on." - "The checkpoint is served from the staged Lustre copy; the AgentX client reads its tokenizer from it." From 5efcd734bdf6626ede61b09889ede46b414e3551 Mon Sep 17 00:00:00 2001 From: cquil11 Date: Thu, 15 Oct 2026 18:40:03 -0500 Subject: [PATCH 4/5] fix(changelog): restore two unrelated placeholder PR links --- inferencex-e2e/perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3b551a8928..62b269b6a2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4140 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4140 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic From 5b0708723cf37931a80eccf1b830c7519b48f24e Mon Sep 17 00:00:00 2001 From: cquil11 Date: Thu, 15 Oct 2026 19:28:17 -0500 Subject: [PATCH 5/5] perf(agentx): offload Muse Spark Open GB200 KV to HiCache --- .../musespark/sglang/gb200-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/configs/nvidia-master.yaml | 5 +++-- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml index f1f972a5d8..ac8d2c9a7d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml @@ -37,6 +37,8 @@ base: speculative-draft-attention-backend: fa4 speculative-num-draft-tokens: 8 mm-feature-transport: cpu + enable-hierarchical-cache: true + hicache-size: 120 reasoning-parser: muse tool-call-parser: muse watchdog-timeout: 3600 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 58a4dfa3e8..5a1f31e4c0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8376,8 +8376,9 @@ musespark-fp4-gb200-sglang-agentic-dflash: srt-recipe-dir: musespark/sglang/gb200-fp4-mtp scenarios: agentic-coding: - - search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 24, 32], srt-recipe: agentic.yaml } + - dram-utilization: 0.80 + search-space: + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 24, 32], srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 62b269b6a2..c396247c11 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9898,7 +9898,7 @@ scenario-type: - agentic-coding description: - - "Add Muse Spark Open (meta-models/Muse-Spark-Open-816B-A42B-NVFP4) agentic coding on one GB200 node via SGLang (lmsysorg/sglang:dev-muse-spark-open, multiarch digest sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a) at TP4/EP4, concurrency 1, 2, 4, 6, 8, 10, 12, 16, 24 and 32, GPU-resident KV." + - "Add Muse Spark Open (meta-models/Muse-Spark-Open-816B-A42B-NVFP4) agentic coding on one GB200 node via SGLang (lmsysorg/sglang:dev-muse-spark-open, multiarch digest sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a) at TP4/EP4, concurrency 1, 2, 4, 6, 8, 10, 12, 16, 24 and 32, with HiCache host KV offload (120 GB per rank)." - "Serve flags follow the SGLang cookbook's NVFP4 cell: modelopt_fp4, flashinfer_trtllm MoE, fp8_e4m3 KV, layernorm SP with MegaMoE prefill, DFlash2 draft RadixArk/Muse-Spark-Open-DFlash2 with fa4 draft attention and 8 verify tokens (7 drafts), muse reasoning and tool-call parsers. Prefill CUDA graphs are off and mem-fraction-static is 0.88 to leave room for the KV pool; admission and the decode graph batch are 2x concurrency." - "Synthetic acceptance follows the committed musespark_dflash golden curve: AL 4.05 at 7 draft tokens, thinking_on." - "The checkpoint is served from the staged Lustre copy; the AgentX client reads its tokenizer from it."