From 83e2635a1822d8ce9b7c4e2d06beb7d1e6820182 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:26:23 +0800 Subject: [PATCH 01/10] perf(agentx): refresh GB300 DSV4 with DSpark6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The GB300 DeepSeek-V4-Pro AgentX matrix still used the older EAGLE/MTP serving path and an outdated SGLang nightly. - The previous draft targeted vLLM recipes instead of the existing SGLang topology ladder. Fix: - Replace the SGLang aggregate and disaggregated GB300 recipes with DSpark block size 6 on DeepSeek-V4-Pro-DSpark. - Use nightly-dev-cu13-20260816-4a6dc267, which already includes the required DSpark performance and MegaMoE support, without a runtime patch overlay. - Force thinking-on AL 3.82 with static ragged verification, retain MegaMoE on DEP workers, and set the 8P4D throughput endpoint to concurrency 1536. Validation: - Passed srtctl dry-run for all seven recipes. - Passed 224 matrix and schema tests. - Generated the eight targeted matrix jobs at concurrency 1, 4, 8, 32, 192, 400, 640, and 1536. 中文：将 GB300 DeepSeek-V4-Pro AgentX 的 SGLang 配置从 EAGLE/MTP 更新为 DSpark6，使用已包含所需上游修复的 20260816 nightly，不再注入补丁；固定 AL 3.82 与 static ragged verify，并将 8P4D 吞吐端点设为并发 1536。七份 recipe dry-run、224 项矩阵与 schema 测试以及八个目标矩阵任务生成均已通过。 --- .../agentic/agg-gb300-tp4-mtp-kvoffload.yaml | 156 ------------------ .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 121 ++++++++++++++ .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 121 ++++++++++++++ ...b300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} | 50 ++++-- ...0-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} | 46 ++++-- ...0-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} | 46 ++++-- ...0-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} | 46 ++++-- ...-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} | 54 +++--- configs/nvidia-master.yaml | 71 ++++---- perf-changelog.yaml | 12 ++ 10 files changed, 450 insertions(+), 273 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml => disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} (82%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml deleted file mode 100644 index 7bb82d2b5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "agg-gb300-tp4-mtp-kvoffload" - -# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 -# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). -# -# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT -# a recipe field here: the GHA matrix fans out one job per concurrency from -# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on -# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) -# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - # Dedicated etcd/nats node — matches the vllm agentic recipes; under the - # dynamo router + multiple frontends the infra services need their own node. - etcd_nats_dedicated_node: true - # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS - # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom - # over the largest observed payload); schema recommends 24+ for long ISL. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo_wheels.py / source build) runs pip and fails with - # "externally-managed-environment" without this. Lets pip install into - # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: '1' - # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend - # graph. Keep the target/decode graphs and use the upstream eager fallback. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - # from submission for B300 - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - # for kvcache offload - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - chunked-prefill-size: 8192 # can only support 11000 for TP - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 128 - cuda-graph-max-bs: 128 - - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - # kv-cache offload - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - # mtp section - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting - # is the single-worker form num_gpus = TP (not the disagg prefill+decode - # sum). The multinode post-processing path assumes disagg and would divide - # throughput by prefill_gpus + decode_gpus; force the single-node code path - # in process_agentic_result.py instead. TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's - # router keep the session on the worker that owns its KV prefix. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, - # aiperf re-tokenizes + re-writes the dataset mmap on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml new file mode 100644 index 0000000000..7addf32aef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml new file mode 100644 index 0000000000..287a5d1fec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml index e414387655..f86d91f390 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" +name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. +# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,13 +100,14 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative # draft-extend graph capture while retaining the target/decode graphs. SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 128 cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -187,10 +199,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -214,3 +227,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 74b371e94e..2e783f7e20 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index 6b585fba80..fc2ca94b40 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -9,13 +9,19 @@ name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -56,6 +62,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -89,10 +99,11 @@ backend: SGLANG_DSV4_MHC_PREWARM: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index b2cac21c78..04a9de4da7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 512 cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml similarity index 82% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 709962936f..7a2b5d20b1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1536. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -27,9 +33,9 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 12 + prefill_nodes: 8 decode_nodes: 4 - prefill_workers: 6 + prefill_workers: 4 decode_workers: 1 gpus_per_prefill: 8 gpus_per_decode: 16 @@ -60,7 +66,11 @@ backend: type: sglang prefill_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -93,10 +103,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -184,14 +196,15 @@ backend: mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 1024 + cuda-graph-max-bs: 192 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -215,3 +228,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 62ba9cff33..4934805365 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7692,33 +7692,42 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: mtp - conc-list: [2, 4, 8, 16] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - search-space: + - spec-decoding: draft_model + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - search-space: + - spec-decoding: draft_model + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 tp: 4 @@ -7726,13 +7735,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7740,8 +7749,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - conc-list: [80] + - spec-decoding: draft_model + conc-list: [32] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7750,14 +7759,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - conc-list: [256] + - spec-decoding: draft_model + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7766,14 +7775,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [512] + - spec-decoding: draft_model + conc-list: [400] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7782,14 +7791,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [768] + - spec-decoding: draft_model + conc-list: [640] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7798,23 +7807,23 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp + - spec-decoding: draft_model conc-list: [1536] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 6 + num-worker: 4 tp: 8 ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index da3f8d6dd5..b431fe0ff7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6018,3 +6018,15 @@ description: - "Refresh the MiniMax-M3 B200 AgentX submission: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, plus a TP4 SimpleCPU KV-offload sweep at C15–C40 using the full-capacity DRAM budget." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2611 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" + - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" + - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From d9a6672dc55a8d9f19b588670fd5bc3bfa3880d3 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:33:18 +0800 Subject: [PATCH 02/10] fix(agentx): keep DSpark synthetic AL out of eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The InferenceX port hard-coded SGLANG_SIMULATE_ACC_* in every recipe instead of carrying NVIDIA InferenceMAX PR #214 throughput-only master-config injection. - Eval-only launch therefore skipped the injector but still inherited synthetic acceptance from the checked-in recipe. - The previous AL 3.82 curve was measured for a different DeepSeek-V4-Pro variant and is not the model-specific golden value for the 0813 checkpoint. Fix: - Keep SGLANG_RAGGED_VERIFY_MODE=static in all six recipes while moving synthetic AL into each throughput search-space entry. - Preserve the earlier DSpark curve and commit a separate DeepSeek-V4-Pro-0813 probabilistic curve, selecting thinking-on DSpark6 AL 3.77 for throughput. - Add tests covering all six recipes, throughput injection, eval preservation, launcher ordering, DSpark6 parameters, tokenizer paths, and the model-specific golden value. Validation: - Passed 235 matrix, schema, runner, and synthetic-acceptance tests. - Generated seven throughput points and four eval-only points at the intended concurrencies. - Passed YAML parsing, formatting, and git diff checks. 中文：模拟接受长度仅在吞吐测试启动时注入，精度评测继续使用真实 DSpark 验证；保留旧模型曲线并新增 DeepSeek-V4-Pro-0813 概率曲线，DSpark6 吞吐 AL 改为 3.77。 --- .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 3 - .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 3 - ...gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml | 230 ------------------ ...00-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml | 6 - ...00-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml | 6 - ...00-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml | 6 - ...0-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml | 6 - configs/nvidia-master.yaml | 28 +-- golden_al_distribution/README.md | 2 + golden_al_distribution/README_zh.md | 2 + .../dsv4_0813_dspark_probabilistic.yaml | 24 ++ .../dsv4_dspark_probabilistic.yaml | 28 +++ perf-changelog.yaml | 6 +- runners/launch_gb300-nv.sh | 6 + runners/test_synthetic_acceptance.py | 134 ++++++++++ 15 files changed, 211 insertions(+), 279 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml create mode 100644 golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml create mode 100644 golden_al_distribution/dsv4_dspark_probabilistic.yaml create mode 100644 runners/test_synthetic_acceptance.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml index 7addf32aef..7cfd733f93 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -55,9 +55,6 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml index 287a5d1fec..e6bc127801 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -55,9 +55,6 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml deleted file mode 100644 index f86d91f390..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml +++ /dev/null @@ -1,230 +0,0 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-dspark" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - -dynamo: - install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative - # draft-extend graph capture while retaining the target/decode graphs. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 4 - dp-size: 1 - ep-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - moe-runner-backend: flashinfer_mxfp4 - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 2e783f7e20..976a6c9989 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index fc2ca94b40..364cd582d2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -63,9 +63,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -100,9 +97,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index 04a9de4da7..79ecbb704f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 7a2b5d20b1..d587144c8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -67,9 +67,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -104,9 +101,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4934805365..e9cc7e6f8c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7712,6 +7712,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7727,6 +7729,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7749,22 +7753,6 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: draft_model - conc-list: [32] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - spec-decoding: draft_model conc-list: [192] kv-offloading: dram @@ -7775,6 +7763,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7791,6 +7781,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7807,6 +7799,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7823,6 +7817,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 95e4a56223..2993f9c1b6 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -109,6 +109,8 @@ Before accepting an updated curve, reviewers should verify: | Model | Method | Golden YAML | Source run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark (probabilistic drafting) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark (probabilistic drafting) | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | Provided 2026-08-17; same SpeedBench AL method | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index a56a005f26..18b38be808 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -109,6 +109,8 @@ gh workflow run speedbench-al.yml \ | 模型 | 方法 | 黄金 YAML | 源 run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark（概率式草稿采样） | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark（概率式草稿采样） | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | 2026-08-17 提供；使用相同 SpeedBench AL 测法 | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml new file mode 100644 index 0000000000..1333b4de82 --- /dev/null +++ b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml @@ -0,0 +1,24 @@ +# Acceptance Length (AL) reference values measured with the same SpeedBench +# setup as the existing DeepSeek-V4-Pro DSpark probabilistic curve, using the +# DeepSeek-V4-Pro-0813 model. Values provided on 2026-08-17. +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-0813: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 3.01 + 4: 3.36 + 5: 3.61 + 6: 3.77 + 7: 3.73 + 8: 3.47 + thinking_off: + 1: 1.94 + 2: 2.81 + 3: 3.58 + 4: 4.17 + 5: 4.74 + 6: 5.12 + 7: 5.24 + 8: 4.95 diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 0000000000..516aeb2ab7 --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,28 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355 +# Acceptance Length (AL) reference values measured with SPEED-Bench. +# dataset: coding | temperature: 1.0 | output_len: 4096 +# thinking_on chat_template_kwargs: {"thinking": true, "reasoning_effort": "high"} +# speculative-config: method=dspark | draft_sample_method=probabilistic +# Measured on deepseek-v4-pro-dspark (B300, vLLM DSpark), per num_speculative_tokens. +# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml). +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-dspark: + thinking_on: + 1: 1.86 + 2: 2.55 + 3: 3.08 + 4: 3.43 + 5: 3.74 + 6: 3.82 + 7: 3.86 + 8: 3.75 + thinking_off: + 1: 1.93 + 2: 2.76 + 3: 3.47 + 4: 4.10 + 5: 4.52 + 6: 4.87 + 7: 4.94 + 8: 4.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b431fe0ff7..952631aefe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6025,8 +6025,8 @@ scenario-type: - agentic-coding description: - - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" - - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and four disaggregated recipes" + - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and throughput-only thinking-on acceptance length 3.77 from the committed DeepSeek-V4-Pro-0813 probabilistic DSpark curve (golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml); eval keeps real DSpark verification" - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" - - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 83252461d0..cc583fb7f9 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -44,6 +44,12 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export SERVED_MODEL_NAME="deepseek-r1-fp8" export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-DSpark" ]]; then + # The DSpark checkpoint is staged in the shared runner cache visible from + # the GB300 compute nodes. Keep a distinct alias from the legacy MTP + # checkpoint so other DSv4 lanes retain their node-local model mapping. + export MODEL_PATH="$HF_HUB_CACHE_HOST_PATH/inferencex-models/DeepSeek-V4-Pro-DSpark" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-dspark" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Use the node-local /scratch SSD for the 806 GB DSv4-Pro # checkpoint. Faster than the Vast NFS path, but this dir only diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py new file mode 100644 index 0000000000..a8fe7827a7 --- /dev/null +++ b/runners/test_synthetic_acceptance.py @@ -0,0 +1,134 @@ +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" +MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" +RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" +GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4_0813_dspark_probabilistic.yaml" +CONFIG_KEYS = ( + "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", + "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", +) + + +def run_injector( + recipe: Path, *, eval_only: bool, acceptance_length: str = "3.77" +) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "EVAL_ONLY": str(eval_only).lower(), + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, + } + return subprocess.run( + [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + +def _configured_recipe_paths(master: dict) -> set[Path]: + recipe_paths = set() + for config_key in CONFIG_KEYS: + assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-DSpark" + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + assert point["spec-decoding"] == "draft_model" + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE=true" in settings + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + config = next( + item for item in settings if item.startswith("CONFIG_FILE=") + ) + recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) + return recipe_paths + + +def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: + original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" + + throughput_recipe = tmp_path / "throughput.yaml" + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 + + eval_recipe = tmp_path / "eval.yaml" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + assert eval_recipe.read_text() == original + + +def test_gb300_dsv4_configs_inject_only_at_launch() -> None: + master = yaml.safe_load(MASTER.read_text()) + recipe_paths = _configured_recipe_paths(master) + + assert len(recipe_paths) == 6 + for recipe in recipe_paths: + text = recipe.read_text() + parsed = yaml.safe_load(text) + worker_count = 1 if recipe.name.startswith("agg-") else 2 + assert "SGLANG_SIMULATE_ACC_" not in text + assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + assert text.count("speculative-algorithm: DSPARK") == worker_count + assert text.count("speculative-dspark-block-size: 6") == worker_count + assert text.count("speculative-num-draft-tokens: 7") == worker_count + assert "speculative-algorithm: EAGLE" not in text + assert 'path: "deepseek-v4-pro-dspark"' in text + assert parsed["benchmark"]["env"]["INFMAX_AIPERF_TOKENIZER"] == "/model" + assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] + + launcher = LAUNCHER.read_text() + source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' + inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' + apply = "SRTCTL_OUTPUT=$(srtctl apply" + assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) + + +def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: + golden = yaml.safe_load(GOLDEN_AL.read_text()) + assert golden["deepseek-v4-pro-0813"]["thinking_on"][6] == 3.77 + + master = yaml.safe_load(MASTER.read_text()) + for config_key in CONFIG_KEYS: + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + + +def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( + tmp_path: Path, +) -> None: + master = yaml.safe_load(MASTER.read_text()) + for recipe in _configured_recipe_paths(master): + original = recipe.read_text() + worker_count = 1 if recipe.name.startswith("agg-") else 2 + + throughput_recipe = tmp_path / recipe.name + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + throughput_text = throughput_recipe.read_text() + assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.77"') == worker_count + assert ( + throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + ) + + eval_recipe = tmp_path / f"eval-{recipe.name}" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + eval_text = eval_recipe.read_text() + assert eval_text == original + assert "SGLANG_SIMULATE_ACC_" not in eval_text + assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count From e1c77addc81a7bf02bdc9fb115c412bf4a3dd6c2 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:26:23 +0800 Subject: [PATCH 03/10] perf(agentx): refresh GB300 DSV4 with DSpark6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The GB300 DeepSeek-V4-Pro AgentX matrix still used the older EAGLE/MTP serving path and an outdated SGLang nightly. - The previous draft targeted vLLM recipes instead of the existing SGLang topology ladder. Fix: - Replace the SGLang aggregate and disaggregated GB300 recipes with DSpark block size 6 on DeepSeek-V4-Pro-DSpark. - Use nightly-dev-cu13-20260816-4a6dc267, which already includes the required DSpark performance and MegaMoE support, without a runtime patch overlay. - Force thinking-on AL 3.82 with static ragged verification, retain MegaMoE on DEP workers, and set the 8P4D throughput endpoint to concurrency 1536. Validation: - Passed srtctl dry-run for all seven recipes. - Passed 224 matrix and schema tests. - Generated the eight targeted matrix jobs at concurrency 1, 4, 8, 32, 192, 400, 640, and 1536. 中文：将 GB300 DeepSeek-V4-Pro AgentX 的 SGLang 配置从 EAGLE/MTP 更新为 DSpark6，使用已包含所需上游修复的 20260816 nightly，不再注入补丁；固定 AL 3.82 与 static ragged verify，并将 8P4D 吞吐端点设为并发 1536。七份 recipe dry-run、224 项矩阵与 schema 测试以及八个目标矩阵任务生成均已通过。 --- .../agentic/agg-gb300-tp4-mtp-kvoffload.yaml | 156 ------------------ .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 121 ++++++++++++++ .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 121 ++++++++++++++ ...b300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} | 50 ++++-- ...0-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} | 46 ++++-- ...0-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} | 46 ++++-- ...0-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} | 46 ++++-- ...-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} | 54 +++--- configs/nvidia-master.yaml | 71 ++++---- perf-changelog.yaml | 13 +- 10 files changed, 450 insertions(+), 274 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml => disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml} (84%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml} (82%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml deleted file mode 100644 index 7bb82d2b5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "agg-gb300-tp4-mtp-kvoffload" - -# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 -# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). -# -# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT -# a recipe field here: the GHA matrix fans out one job per concurrency from -# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on -# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) -# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - # Dedicated etcd/nats node — matches the vllm agentic recipes; under the - # dynamo router + multiple frontends the infra services need their own node. - etcd_nats_dedicated_node: true - # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS - # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom - # over the largest observed payload); schema recommends 24+ for long ISL. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo_wheels.py / source build) runs pip and fails with - # "externally-managed-environment" without this. Lets pip install into - # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: '1' - # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend - # graph. Keep the target/decode graphs and use the upstream eager fallback. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - # from submission for B300 - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - # for kvcache offload - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.90 - chunked-prefill-size: 8192 # can only support 11000 for TP - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 128 - cuda-graph-max-bs: 128 - - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - # kv-cache offload - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - # mtp section - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting - # is the single-worker form num_gpus = TP (not the disagg prefill+decode - # sum). The multinode post-processing path assumes disagg and would divide - # throughput by prefill_gpus + decode_gpus; force the single-node code path - # in process_agentic_result.py instead. TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's - # router keep the session on the worker that owns its KV prefix. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, - # aiperf re-tokenizes + re-writes the dataset mmap on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml new file mode 100644 index 0000000000..7addf32aef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml new file mode 100644 index 0000000000..287a5d1fec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -0,0 +1,121 @@ +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-dspark" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + +dynamo: + install: true + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + aggregated_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + enable-metrics: true + trust-remote-code: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_WARMUP_REQUESTS_PER_LANE: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml index e414387655..f86d91f390 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" +name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 80. +# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,13 +100,14 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative # draft-extend graph capture while retaining the target/decode graphs. SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - SGLANG_SIMULATE_ACC_LEN: '2.49' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 128 cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -187,10 +199,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -214,3 +227,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 74b371e94e..2e783f7e20 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index 6b585fba80..fc2ca94b40 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. @@ -9,13 +9,19 @@ name: "disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -56,6 +62,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -89,10 +99,11 @@ backend: SGLANG_DSV4_MHC_PREWARM: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 256 cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml similarity index 84% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index b2cac21c78..04a9de4da7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 768. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -57,6 +63,10 @@ backend: type: sglang prefill_environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -90,10 +100,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -127,7 +138,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -143,18 +154,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 512 cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -162,7 +174,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -185,10 +197,11 @@ backend: disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -212,3 +225,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml similarity index 82% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml index 709962936f..7a2b5d20b1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1536. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1536. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,19 @@ name: "disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro" + path: "deepseek-v4-pro-dspark" container: "dynamo-sglang" precision: "fp4" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" + dynamo: install: true - wheel: "1.3.0.dev20260718" + hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" slurm: time_limit: "8:00:00" @@ -27,9 +33,9 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 12 + prefill_nodes: 8 decode_nodes: 4 - prefill_workers: 6 + prefill_workers: 4 decode_workers: 1 gpus_per_prefill: 8 gpus_per_decode: 16 @@ -60,7 +66,11 @@ backend: type: sglang prefill_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -93,10 +103,11 @@ backend: SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' decode_environment: - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_SIMULATE_ACC_LEN: "3.82" SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' @@ -130,7 +141,7 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -146,18 +157,19 @@ backend: disaggregation-transfer-backend: mooncake disaggregation-mode: prefill load-balance-method: total_tokens - mem-fraction-static: 0.9 + mem-fraction-static: 0.85 swa-full-tokens-ratio: 0.02 max-running-requests: 1024 cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 + chunked-prefill-size: 65536 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 1 @@ -165,7 +177,7 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro + served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark enable-metrics: true model-path: /model/ trust-remote-code: true @@ -184,14 +196,15 @@ backend: mem-fraction-static: 0.9 swa-full-tokens-ratio: 0.02 max-running-requests: 3072 - cuda-graph-max-bs: 1024 + cuda-graph-max-bs: 192 disable-flashinfer-autotune: true model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 + speculative-num-draft-tokens: 7 sbatch_directives: @@ -215,3 +228,4 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" + INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2f73b1c64a..a8b6d9cb3d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7784,33 +7784,42 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } multinode: true disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: mtp - conc-list: [2, 4, 8, 16] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - search-space: + - spec-decoding: draft_model + conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - search-space: + - spec-decoding: draft_model + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 tp: 4 @@ -7818,13 +7827,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 - model: deepseek-ai/DeepSeek-V4-Pro + image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 + model: deepseek-ai/DeepSeek-V4-Pro-DSpark model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev20260718" } + router: { name: dynamo-router, version: "7a4e47ead90ec6b479e397996f6479ce7d251510" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7832,8 +7841,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - conc-list: [80] + - spec-decoding: draft_model + conc-list: [32] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7842,14 +7851,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c80-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - conc-list: [256] + - spec-decoding: draft_model + conc-list: [192] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7858,14 +7867,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [512] + - spec-decoding: draft_model + conc-list: [400] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7874,14 +7883,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp - conc-list: [768] + - spec-decoding: draft_model + conc-list: [640] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7890,23 +7899,23 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c768-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: mtp + - spec-decoding: draft_model conc-list: [1536] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 6 + num-worker: 4 tp: 8 ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 085e9cb889..3d07cffcf6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6039,7 +6039,6 @@ - "EVAL_ONLY runs switch to real block verification by removing synthetic acceptance (TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS stripped from all agentx recipes before srtctl)." - "Add SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 to all 6 agentx recipe env blocks and make build_replay_cmd prefer SERVED_MODEL_NAME over $MODEL for the aiperf --model flag." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2612 - - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg @@ -6049,3 +6048,15 @@ - "Add GB200 GLM-5.2 FP4 Dynamo-SGLang AgentX with EAGLE MTP, HiCache DRAM offload, and KV-aware correlation-ID affinity." - "Measure the TP8 aggregate curve and tuned 1P1D TP4 HiCache c10/c12 disaggregated frontier with every logical SGLang metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2620 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" + - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" + - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From 86a9eaa8f428c26fdd452d91bb52e59b1c967034 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:06:34 +0800 Subject: [PATCH 04/10] fix(agentx): harden GB300 DSpark sweep execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: - The sweep still referenced a nonstandard model/tokenizer path, included a low-value concurrency point, and allowed synthetic acceptance controls to leak into evaluation. - Aggregate TP checkpoint loading and long-lived AgentX replay connections could stall or expire during startup and multi-turn idle gaps. - Worker cache metrics were not exposed consistently across the recipe ladder. Fix: - Use the standard DeepSeek-V4-Pro-0813 model layout and probabilistic DSpark acceptance data, preserve real evaluation behavior, and keep the validated 1152-concurrency endpoint. - Enable cache reporting on every DSpark worker and checkpoint prefetch on pure-TP aggregate workers. - Use srt-slurm v1.0.40 and a 900-second nginx/AIPerf connection timeout for the 1152-concurrency recipe. - Retain the aggregate startup barrier and focused regression coverage. Validation: - Passed YAML and shell syntax validation, git diff checks, and the synthetic-acceptance launcher test. - Passed exact-key NVIDIA matrix expansion. - Passed srtctl dry-run for all six generated recipes. 中文：统一使用标准 DeepSeek-V4-Pro-0813 模型布局和 0813 DSpark 接收长度；所有 worker 开启 cache report，纯 TP 配置开启 checkpoint prefetch，并用 srt-slurm v1.0.40 与 900 秒连接保活修复高并发多轮回放连接超时。相关矩阵展开、六份 recipe dry-run、YAML/Shell 校验和 launcher 测试均已通过。 --- .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 15 +- .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 15 +- ...gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml | 230 ------------------ ...00-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml | 17 +- ...00-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml | 17 +- ...00-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml | 17 +- ...-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml} | 23 +- configs/nvidia-master.yaml | 36 ++- golden_al_distribution/README.md | 2 + golden_al_distribution/README_zh.md | 2 + .../dsv4_0813_dspark_probabilistic.yaml | 24 ++ .../dsv4_dspark_probabilistic.yaml | 28 +++ perf-changelog.yaml | 26 +- runners/launch_gb300-nv.sh | 12 +- runners/test_synthetic_acceptance.py | 136 +++++++++++ 15 files changed, 282 insertions(+), 318 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml} (92%) create mode 100644 golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml create mode 100644 golden_al_distribution/dsv4_dspark_probabilistic.yaml create mode 100644 runners/test_synthetic_acceptance.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml index 7addf32aef..5f125e330e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -4,13 +4,13 @@ name: "agg-gb300-tp4-mtp-lowlatency" # four-GPU GB300 node and serves both prefill and decode with DSpark K=6. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -55,13 +55,12 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + # Avoid the TP4 DSpark startup stall in the shared-read-done fastpath. + SGLANG_FORCE_COARSE_WAR_BARRIER: "1" SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" @@ -72,9 +71,11 @@ backend: sglang_config: aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" enable-metrics: true + enable-cache-report: true trust-remote-code: true + weight-loader-prefetch-checkpoints: true stream-interval: 10 watchdog-timeout: 1000000 mem-fraction-static: 0.94 @@ -115,7 +116,5 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_WARMUP_REQUESTS_PER_LANE: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml index 287a5d1fec..da765fd1e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -4,13 +4,13 @@ name: "agg-gb300-tp8-mtp-lowlatency" # four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -55,9 +55,8 @@ backend: type: sglang aggregated_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + # Avoid aggregate DSpark startup stalls in the shared-read-done fastpath. + SGLANG_FORCE_COARSE_WAR_BARRIER: "1" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV4_REASONING_EFFORT: high PIP_BREAK_SYSTEM_PACKAGES: "1" @@ -72,9 +71,11 @@ backend: sglang_config: aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" enable-metrics: true + enable-cache-report: true trust-remote-code: true + weight-loader-prefetch-checkpoints: true stream-interval: 10 watchdog-timeout: 1000000 mem-fraction-static: 0.94 @@ -115,7 +116,5 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_WARMUP_REQUESTS_PER_LANE: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml deleted file mode 100644 index f86d91f390..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml +++ /dev/null @@ -1,230 +0,0 @@ -name: "disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x TP4, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 32. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-dspark" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" - -dynamo: - install: true - hash: "7a4e47ead90ec6b479e397996f6479ce7d251510" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - # Match the TP4 aggregate worker: avoid the pathological DSV4 speculative - # draft-extend graph capture while retaining the target/decode graphs. - SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark - enable-metrics: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 4 - dp-size: 1 - ep-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - moe-runner-backend: flashinfer_mxfp4 - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - swa-full-tokens-ratio: 0.02 - max-running-requests: 128 - cuda-graph-max-bs: 128 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml index 2e783f7e20..eb1ef5313b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml @@ -9,13 +9,13 @@ name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -138,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -174,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -225,4 +221,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml index fc2ca94b40..df487dd820 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml @@ -9,13 +9,13 @@ name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" # from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -63,9 +63,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -100,9 +97,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -138,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -174,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -225,4 +221,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml index 04a9de4da7..d78f361533 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -9,13 +9,13 @@ name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -64,9 +64,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -101,9 +98,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -138,8 +132,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -174,8 +169,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -225,4 +221,3 @@ benchmark: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml similarity index 92% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml index 7a2b5d20b1..798d963452 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1536. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1152. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized @@ -9,13 +9,13 @@ name: "disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload" # Concurrency is exported into agentic_srt.sh from the master-config conc-list. model: - path: "deepseek-v4-pro-dspark" + path: "deepseek-v4-pro-0813" container: "dynamo-sglang" precision: "fp4" identity: model: - repo: "deepseek-ai/DeepSeek-V4-Pro-DSpark" + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" container: image: "lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267" @@ -48,6 +48,7 @@ frontend: type: dynamo nginx_session_affinity: true nginx_session_affinity_header: X-Dynamo-Session-ID + nginx_keepalive_timeout: "900s" enable_multiple_frontends: true num_additional_frontends: 4 env: @@ -67,9 +68,6 @@ backend: prefill_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSV4_MHC_PREWARM: '1' PIP_BREAK_SYSTEM_PACKAGES: '1' @@ -104,9 +102,6 @@ backend: decode_environment: SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_SIMULATE_ACC_LEN: "3.82" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" PIP_BREAK_SYSTEM_PACKAGES: '1' PYTHONUNBUFFERED: '1' @@ -141,8 +136,9 @@ backend: sglang_config: prefill: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -177,8 +173,9 @@ backend: decode: host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-DSpark + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 enable-metrics: true + enable-cache-report: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -224,8 +221,8 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - INFMAX_AIPERF_TOKENIZER: "/model" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a8b6d9cb3d..7be3444ccb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7785,7 +7785,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 - model: deepseek-ai/DeepSeek-V4-Pro-DSpark + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 @@ -7804,6 +7804,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7819,6 +7821,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" decode: num-worker: 0 @@ -7828,7 +7832,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260816-4a6dc267 - model: deepseek-ai/DeepSeek-V4-Pro-DSpark + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 @@ -7841,22 +7845,6 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: draft_model - conc-list: [32] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c32-mtp-kvoffload.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - spec-decoding: draft_model conc-list: [192] kv-offloading: dram @@ -7867,6 +7855,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7883,6 +7873,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7899,6 +7891,8 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 @@ -7906,7 +7900,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true - spec-decoding: draft_model - conc-list: [1536] + conc-list: [1152] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -7915,7 +7909,9 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1536-mtp-kvoffload.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 95e4a56223..2993f9c1b6 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -109,6 +109,8 @@ Before accepting an updated curve, reviewers should verify: | Model | Method | Golden YAML | Source run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark (probabilistic drafting) | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark (probabilistic drafting) | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | Provided 2026-08-17; same SpeedBench AL method | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index a56a005f26..18b38be808 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -109,6 +109,8 @@ gh workflow run speedbench-al.yml \ | 模型 | 方法 | 黄金 YAML | 源 run | | --- | --- | --- | --- | | DeepSeek V4 Pro | MTP | [`dsv4_mtp.yaml`](dsv4_mtp.yaml) | [27180633016](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27180633016) | +| DeepSeek V4 Pro DSpark | DSpark（概率式草稿采样） | [`dsv4_dspark_probabilistic.yaml`](dsv4_dspark_probabilistic.yaml) | [31279568355](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355) | +| DeepSeek V4 Pro 0813 | DSpark（概率式草稿采样） | [`dsv4_0813_dspark_probabilistic.yaml`](dsv4_0813_dspark_probabilistic.yaml) | 2026-08-17 提供；使用相同 SpeedBench AL 测法 | | Qwen3.5 397B-A17B | MTP | [`qwen3.5_mtp.yaml`](qwen3.5_mtp.yaml) | [27317114007](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/27317114007) | | Kimi K2.5 | EAGLE3 | [`kimik2.5_eagle3.yaml`](kimik2.5_eagle3.yaml) | [28122195822](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28122195822) | | MiniMax-M3 | EAGLE3 | [`minimaxm3_eagle3.yaml`](minimaxm3_eagle3.yaml) | [28061204145](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28061204145) | diff --git a/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml new file mode 100644 index 0000000000..1333b4de82 --- /dev/null +++ b/golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml @@ -0,0 +1,24 @@ +# Acceptance Length (AL) reference values measured with the same SpeedBench +# setup as the existing DeepSeek-V4-Pro DSpark probabilistic curve, using the +# DeepSeek-V4-Pro-0813 model. Values provided on 2026-08-17. +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-0813: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 3.01 + 4: 3.36 + 5: 3.61 + 6: 3.77 + 7: 3.73 + 8: 3.47 + thinking_off: + 1: 1.94 + 2: 2.81 + 3: 3.58 + 4: 4.17 + 5: 4.74 + 6: 5.12 + 7: 5.24 + 8: 4.95 diff --git a/golden_al_distribution/dsv4_dspark_probabilistic.yaml b/golden_al_distribution/dsv4_dspark_probabilistic.yaml new file mode 100644 index 0000000000..516aeb2ab7 --- /dev/null +++ b/golden_al_distribution/dsv4_dspark_probabilistic.yaml @@ -0,0 +1,28 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31279568355 +# Acceptance Length (AL) reference values measured with SPEED-Bench. +# dataset: coding | temperature: 1.0 | output_len: 4096 +# thinking_on chat_template_kwargs: {"thinking": true, "reasoning_effort": "high"} +# speculative-config: method=dspark | draft_sample_method=probabilistic +# Measured on deepseek-v4-pro-dspark (B300, vLLM DSpark), per num_speculative_tokens. +# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml). +# +# key = num_speculative_tokens (DSpark level); value = golden AL +deepseek-v4-pro-dspark: + thinking_on: + 1: 1.86 + 2: 2.55 + 3: 3.08 + 4: 3.43 + 5: 3.74 + 6: 3.82 + 7: 3.86 + 8: 3.75 + thinking_off: + 1: 1.93 + 2: 2.76 + 3: 3.47 + 4: 4.10 + 5: 4.52 + 6: 4.87 + 7: 4.94 + 8: 4.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3d07cffcf6..b8ea19f206 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6055,8 +6055,28 @@ scenario-type: - agentic-coding description: - - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and five disaggregated recipes" - - "Use DeepSeek-V4-Pro-DSpark with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and forced thinking-on acceptance length 3.82" + - "Replace the GB300 DeepSeek-V4-Pro AgentX EAGLE/MTP serving path with DSpark block size 6 across two aggregate and four disaggregated recipes" + - "Use the official DeepSeek-V4-Pro-0813 checkpoint with SGLANG_RAGGED_VERIFY_MODE=static, MegaMoE for DEP workers, and throughput-only thinking-on acceptance length 3.77 from the committed DeepSeek-V4-Pro-0813 probabilistic DSpark curve (golden_al_distribution/dsv4_0813_dspark_probabilistic.yaml); eval keeps real DSpark verification" - "Refresh SGLang to nightly-dev-cu13-20260816-4a6dc267, which already contains the required DSpark performance and MegaMoE support" - - "Retune the Pareto ladder to concurrency 1/4/8, 32, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + - "Retune the Pareto ladder to concurrency 1/4/8, 192, 400, 640, and 1536; the 8P4D concurrency-1536 point is the throughput endpoint" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Restore concurrency 1152 as the validated 8P4D DSpark throughput endpoint after the concurrency-1536 formal AgentX warmup exceeded prefill activation-memory headroom" + - "Keep the 8P4D topology, DSpark block size 6, prefill mem-fraction-static 0.85, decode CUDA Graph batch size 192, and all other serving parameters unchanged" + - "Use the coarse WAR barrier only for the TP4 aggregate point to avoid the shared-read-done startup stall observed with the 20260816 nightly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Enable SGLang cache reporting on every GB300 DeepSeek-V4-Pro DSpark worker and prefetch checkpoint shards on the two pure-TP aggregate recipes" + - "Move the DSv4 AgentX runner from srt-slurm v1.0.38 to v1.0.40; set the concurrency-1152 recipe's nginx client/upstream keepalive to 900 seconds and AIPerf TCP user timeout to 900 seconds so a pooled multi-turn connection is not closed between turns" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8ad..bea9f742b5 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -44,6 +44,11 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export SERVED_MODEL_NAME="deepseek-r1-fp8" export MODEL_PATH=/scratch/models/DeepSeek-R1-0528 export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" +elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then + # The official 0813 checkpoint, including its attached DSpark weights, is + # staged on every GB300 node in batch_1. + export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # Use the node-local /scratch SSD for the 806 GB DSv4-Pro # checkpoint. Faster than the Vast NFS path, but this dir only @@ -224,15 +229,16 @@ elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX = cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5 elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.38. In addition to + # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.40. In addition to # the nginx body-size fix, session-affinity frontend, and custom benchmark # schema required by these recipes, this release injects every logical - # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS. + # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS and + # supports long-lived nginx keepalive for multi-turn AgentX replay. # AgentX forwards that list to aiperf's --server-metrics argument so its # trace artifacts include backend metrics for every engine. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.38 + git checkout v1.0.40 mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py new file mode 100644 index 0000000000..6a5b910a64 --- /dev/null +++ b/runners/test_synthetic_acceptance.py @@ -0,0 +1,136 @@ +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" +MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" +RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" +GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4_0813_dspark_probabilistic.yaml" +CONFIG_KEYS = ( + "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", + "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", +) + + +def run_injector( + recipe: Path, *, eval_only: bool, acceptance_length: str = "3.77" +) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "EVAL_ONLY": str(eval_only).lower(), + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, + } + return subprocess.run( + [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + +def _configured_recipe_paths(master: dict) -> set[Path]: + recipe_paths = set() + for config_key in CONFIG_KEYS: + assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-0813" + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + assert point["spec-decoding"] == "draft_model" + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE=true" in settings + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + config = next( + item for item in settings if item.startswith("CONFIG_FILE=") + ) + recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) + return recipe_paths + + +def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: + original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" + + throughput_recipe = tmp_path / "throughput.yaml" + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 + + eval_recipe = tmp_path / "eval.yaml" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + assert eval_recipe.read_text() == original + + +def test_gb300_dsv4_configs_inject_only_at_launch() -> None: + master = yaml.safe_load(MASTER.read_text()) + recipe_paths = _configured_recipe_paths(master) + + assert len(recipe_paths) == 6 + for recipe in recipe_paths: + text = recipe.read_text() + parsed = yaml.safe_load(text) + worker_count = 1 if recipe.name.startswith("agg-") else 2 + assert "SGLANG_SIMULATE_ACC_" not in text + assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + assert text.count("speculative-algorithm: DSPARK") == worker_count + assert text.count("speculative-dspark-block-size: 6") == worker_count + assert text.count("speculative-num-draft-tokens: 7") == worker_count + assert "speculative-algorithm: EAGLE" not in text + assert 'path: "deepseek-v4-pro-0813"' in text + assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] + + launcher = LAUNCHER.read_text() + assert 'MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813"' in launcher + assert 'MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813"' in launcher + assert 'SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813"' in launcher + source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' + inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' + apply = "SRTCTL_OUTPUT=$(srtctl apply" + assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) + + +def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: + golden = yaml.safe_load(GOLDEN_AL.read_text()) + assert golden["deepseek-v4-pro-0813"]["thinking_on"][6] == 3.77 + + master = yaml.safe_load(MASTER.read_text()) + for config_key in CONFIG_KEYS: + for group in master[config_key]["scenarios"]["agentic-coding"]: + for point in group["search-space"]: + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings + + +def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( + tmp_path: Path, +) -> None: + master = yaml.safe_load(MASTER.read_text()) + for recipe in _configured_recipe_paths(master): + original = recipe.read_text() + worker_count = 1 if recipe.name.startswith("agg-") else 2 + + throughput_recipe = tmp_path / recipe.name + throughput_recipe.write_text(original) + result = run_injector(throughput_recipe, eval_only=False) + assert result.returncode == 0, result.stderr + throughput_text = throughput_recipe.read_text() + assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.77"') == worker_count + assert ( + throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count + ) + + eval_recipe = tmp_path / f"eval-{recipe.name}" + eval_recipe.write_text(original) + result = run_injector(eval_recipe, eval_only=True) + assert result.returncode == 0, result.stderr + eval_text = eval_recipe.read_text() + assert eval_text == original + assert "SGLANG_SIMULATE_ACC_" not in eval_text + assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count From 9366b46fe1bcd129d73c10898b03480f6ccd5b63 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:29:57 +0800 Subject: [PATCH 05/10] Fix GB300 runner inventory for sweep scheduling --- configs/runners.yaml | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/configs/runners.yaml b/configs/runners.yaml index 32fe3bdef8..e52e9a854e 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -118,9 +118,24 @@ labels: - b300-nv_16 - b300-nv_17 gb300: - - gb300-nv_0 - - gb300-nv_1 - - gb300-nv_2 + - gb300-nv_00 + - gb300-nv_01 + - gb300-nv_02 + - gb300-nv_03 + - gb300-nv_04 + - gb300-nv_05 + - gb300-nv_06 + - gb300-nv_07 + - gb300-nv_08 + - gb300-nv_09 + - gb300-nv_10 + - gb300-nv_11 + - gb300-nv_12 + - gb300-nv_13 + - gb300-nv_14 + - gb300-nv_15 + - gb300-nv_16 + - gb300-nv_17 rtx6000pro: - rtx6000pro-lat_00 rtx6000pro-lat: @@ -202,9 +217,24 @@ labels: - gb200-nv_2 - gb200-nv_3 cluster:gb300-nv: - - gb300-nv_0 - - gb300-nv_1 - - gb300-nv_2 + - gb300-nv_00 + - gb300-nv_01 + - gb300-nv_02 + - gb300-nv_03 + - gb300-nv_04 + - gb300-nv_05 + - gb300-nv_06 + - gb300-nv_07 + - gb300-nv_08 + - gb300-nv_09 + - gb300-nv_10 + - gb300-nv_11 + - gb300-nv_12 + - gb300-nv_13 + - gb300-nv_14 + - gb300-nv_15 + - gb300-nv_16 + - gb300-nv_17 cluster:rtx6000pro-lat: - rtx6000pro-lat_00 cluster:mi300x-amd: From 2530e5adb726e391c0d5b25dfddda81391bd9be8 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:54:44 +0800 Subject: [PATCH 06/10] perf(agentx): retune GB300 DSpark concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GB300 DSpark AgentX 并发档位调整为本地完整时长测试验证的 320、640、960 和 1280，并保留 DEP12 的 c1152 效率点。 --- ...b300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml} | 4 ++-- ...b300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml} | 4 ++-- ...b300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml} | 4 ++-- ...300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml} | 4 ++-- configs/nvidia-master.yaml | 16 ++++++++-------- perf-changelog.yaml | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml} (98%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml index 64c0cf5928..67a60c9c00 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml @@ -1,8 +1,8 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload" setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 192. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 320. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml index d4be9a674a..042022f303 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,8 +1,8 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload" setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 400. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml index c687bc43c0..0fee7ae266 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -1,8 +1,8 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload" setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml index aa2094038d..58cb022853 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml @@ -1,8 +1,8 @@ -name: "disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload" setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1152. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1280. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 061fe12041..52f062ab46 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8713,7 +8713,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - dram-utilization: 0.80 search-space: - spec-decoding: draft_model - conc-list: [192] + conc-list: [320] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8724,14 +8724,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c192-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: draft_model - conc-list: [400] + conc-list: [640] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8742,14 +8742,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c400-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: draft_model - conc-list: [640] + conc-list: [960] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8760,7 +8760,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c640-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -8785,7 +8785,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 12 dp-attn: true - spec-decoding: draft_model - conc-list: [1152] + conc-list: [1280] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8796,7 +8796,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1152-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e079b81e40..613aae6b0b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6830,5 +6830,5 @@ description: - "Replace the GB300 DeepSeek-V4-Pro AgentX MTP serving path with DSpark block size 6 across aggregate and disaggregated recipes." - "Use the DeepSeek-V4-Pro-0813 checkpoint, SGLang 20260829 CUDA 13 nightly, Dynamo 1.4.0, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." - - "Retune the Pareto ladder to concurrency 1, 4, 8, 192, 400, 640, and two concurrency-1152 topologies, including the 2P DEP8 prefill plus DEP12 decode efficiency point." + - "Retune the Pareto ladder to concurrency 1, 4, 8, 320, 640, 960, and 1280, while retaining the concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From 4b8c3f323d169c83965a9d6c8079edcd5e90c672 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:50:19 +0800 Subject: [PATCH 07/10] fix(agentx): replace the SGLang parser patch with public Dynamo wheels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: Dynamo 1.4.0 still imported the legacy SGLang config-parser path, so the GB300 recipes wrote a compatibility module into the installed SGLang package at runtime. Fix: Pin all GB300 DSV4 AgentX recipes and router metadata to the public Dynamo 1.5.0.dev20260902 wheels containing ai-dynamo/dynamo#14054, then remove the setup hook, launcher copy, and site-packages patch. Validation: Confirmed the public aarch64 CPython 3.12 ai-dynamo and ai-dynamo-runtime wheels are downloadable and contain the new compatibility import; all seven recipes pass srtctl dry-run; 250 focused matrix and synthetic-acceptance tests pass; YAML, Bash syntax, changelog generation, and diff checks pass. 根因：Dynamo 1.4.0 仍导入旧的 SGLang 配置解析器路径，因此 GB300 配方会在运行时向已安装的 SGLang 包写入兼容模块。修复：固定到包含上游 #14054 的公开 Dynamo 1.5.0.dev20260902 wheels，并删除 setup hook、launcher 复制逻辑和 site-packages patch。验证：公开 aarch64 wheels、7 个 dry-run、250 项测试及静态检查均通过。 --- .../agentic/agg-gb300-tp4-mtp-lowlatency.yaml | 3 +-- .../agentic/agg-gb300-tp8-mtp-lowlatency.yaml | 3 +-- ...0-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml | 3 +-- ...00-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml | 3 +-- ...00-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml | 3 +-- ...00-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml | 3 +-- ...0-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml | 3 +-- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- runners/launch_gb300-nv.sh | 2 -- ...sglang-server-args-config-parser-compat.sh | 23 ------------------- 11 files changed, 10 insertions(+), 42 deletions(-) delete mode 100644 runners/patches/sglang-server-args-config-parser-compat.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml index 9adcb6cf6a..0c2554a0e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml @@ -1,5 +1,4 @@ name: "agg-gb300-tp4-mtp-lowlatency" -setup_script: sglang-server-args-config-parser-compat.sh # Low-latency AgentX aggregate topology: one TP4 worker occupies one # four-GPU GB300 node and serves both prefill and decode with DSpark K=6. @@ -17,7 +16,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "4:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml index cc78c27124..5c27bc78ab 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml @@ -1,5 +1,4 @@ name: "agg-gb300-tp8-mtp-lowlatency" -setup_script: sglang-server-args-config-parser-compat.sh # Low-latency AgentX aggregate topology: one TP8 worker spans two # four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. @@ -17,7 +16,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "4:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml index e5c02f644a..1138d8f390 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml @@ -1,5 +1,4 @@ name: "disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload" -setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 # (2P x DEP8 / 1D x DEP12, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1152. @@ -21,7 +20,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml index 67a60c9c00..07775d80a3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml @@ -1,5 +1,4 @@ name: "disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload" -setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 # (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 320. @@ -22,7 +21,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml index 042022f303..64d6819677 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml @@ -1,5 +1,4 @@ name: "disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload" -setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 # (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. @@ -22,7 +21,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml index 0fee7ae266..22d80a4cf6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -1,5 +1,4 @@ name: "disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload" -setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 # (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. @@ -22,7 +21,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml index 58cb022853..47e09c2a3c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml @@ -1,5 +1,4 @@ name: "disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload" -setup_script: sglang-server-args-config-parser-compat.sh # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 # (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1280. @@ -22,7 +21,7 @@ identity: dynamo: install: true - wheel: "1.4.0" + wheel: "1.5.0.dev20260902" slurm: time_limit: "8:00:00" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9fe7a796cf..dea7d0940a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8666,7 +8666,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.5.0.dev20260902" } multinode: true disagg: false scenarios: @@ -8704,7 +8704,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.5.0.dev20260902" } kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7de43995b6..be68242f75 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6839,6 +6839,6 @@ - agentic-coding description: - "Replace the GB300 DeepSeek-V4-Pro AgentX MTP serving path with DSpark block size 6 across aggregate and disaggregated recipes." - - "Use the DeepSeek-V4-Pro-0813 checkpoint, SGLang 20260829 CUDA 13 nightly, Dynamo 1.4.0, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use the DeepSeek-V4-Pro-0813 checkpoint, SGLang 20260829 CUDA 13 nightly, public Dynamo 1.5.0.dev20260902 wheels containing the upstream SGLang config-parser compatibility fix, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." - "Retune the Pareto ladder to concurrency 1, 4, 8, 320, 640, 960, and 1280, while retaining the concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 2e6ca9292b..edf2aa6360 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -265,8 +265,6 @@ elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX = git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout v1.0.40 - cp "$GITHUB_WORKSPACE/runners/patches/sglang-server-args-config-parser-compat.sh" \ - configs/sglang-server-args-config-parser-compat.sh mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic diff --git a/runners/patches/sglang-server-args-config-parser-compat.sh b/runners/patches/sglang-server-args-config-parser-compat.sh deleted file mode 100644 index 09d1ed0893..0000000000 --- a/runners/patches/sglang-server-args-config-parser-compat.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -python3 - <<'PY' -from importlib.util import find_spec -from pathlib import Path - -legacy_spec = find_spec("sglang.srt.server_args_config_parser") -if legacy_spec is not None: - print("SGLang legacy server-args config parser is already available") - raise SystemExit(0) - -current_spec = find_spec("sglang.srt.utils.server_args_config_parser") -if current_spec is None or current_spec.origin is None: - raise SystemExit("SGLang server-args config parser is unavailable") - -legacy_path = Path(current_spec.origin).parents[1] / "server_args_config_parser.py" -legacy_path.write_text( - "from sglang.srt.utils.server_args_config_parser import * # noqa: F403\n" -) -print(f"Installed SGLang server-args compatibility module at {legacy_path}") -PY From 8c2a015b6650854137259e578f767bfea39135e6 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:05:53 +0800 Subject: [PATCH 08/10] perf(agentx): raise DSV4 DSpark prefill concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: The DEP16 AgentX ladder assigned only 320 requests per prefill worker. Profiling showed frequent empty pending queues and partially filled prefill batches, so the measured points underused the added prefill capacity.\n\nFix: Raise the one- through four-worker DEP16 ladder to concurrency 480, 960, 1440, and 1920, and rename the corresponding recipes consistently. Keep the independently tuned DEP12 efficiency point at concurrency 1152.\n\nValidation: YAML parsing and matrix generation pass; all four updated recipes pass srtctl dry-run; 114 changelog tests pass; the concurrency-1920 four-worker topology completed a full-duration GB300 AgentX run with saturated prefill batches.\n\n中文：原 DEP16 阶梯每个 prefill worker 只有约 320 个请求，profiling 显示 pending queue 经常为空且 prefill batch 未填满。将一到四个 prefill worker 的并发调整为 480、960、1440 和 1920，并同步重命名 recipe；独立调优的 DEP12 c1152 效率点保持不变。YAML、矩阵生成、四份 recipe dry-run、114 项 changelog 测试以及 c1920 完整时长 GB300 AgentX 测试均通过。 --- ...b300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml} | 4 ++-- ...b300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml} | 4 ++-- ...300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml} | 4 ++-- ...300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml} | 4 ++-- configs/nvidia-master.yaml | 16 ++++++++-------- perf-changelog.yaml | 9 +++++++++ 6 files changed, 25 insertions(+), 16 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml => disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml => disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml => disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml} (98%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/{disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml => disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml} (98%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml index 07775d80a3..96c49f4ec3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload" +name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 320. +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml index 64d6819677..67570f735c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload" +name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" # Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 640. +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. # # DEP8-prefill variant aligned with the measured Pareto point: prefill uses # tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml index 22d80a4cf6..95d1e9205f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload" +name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml index 47e09c2a3c..3408902a6a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml @@ -1,7 +1,7 @@ -name: "disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload" +name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1280. +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. # # Uses the flat single-variant srtctl schema the agentic CI flow expects; # resources + backend (prefill/decode env + sglang_config) are normalized diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index dea7d0940a..6c94488af1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8713,7 +8713,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - dram-utilization: 0.80 search-space: - spec-decoding: draft_model - conc-list: [320] + conc-list: [480] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8724,14 +8724,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c320-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: draft_model - conc-list: [640] + conc-list: [960] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8742,14 +8742,14 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c640-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - spec-decoding: draft_model - conc-list: [960] + conc-list: [1440] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8760,7 +8760,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c960-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -8785,7 +8785,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 12 dp-attn: true - spec-decoding: draft_model - conc-list: [1280] + conc-list: [1920] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -8796,7 +8796,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1280-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be68242f75..349c55c8b3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6842,3 +6842,12 @@ - "Use the DeepSeek-V4-Pro-0813 checkpoint, SGLang 20260829 CUDA 13 nightly, public Dynamo 1.5.0.dev20260902 wheels containing the upstream SGLang config-parser compatibility fix, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." - "Retune the Pareto ladder to concurrency 1, 4, 8, 320, 640, 960, and 1280, while retaining the concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Increase the DEP16 concurrency ladder from 320, 640, 960, and 1280 to 480, 960, 1440, and 1920 after profiling showed request-starved prefill workers at the lower values." + - "Retain the independently tuned concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From 322307a973fd7d565d020559565341c1d6bc3a79 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:40:26 +0800 Subject: [PATCH 09/10] perf(agentx): drop unstable DSV4 c1152 topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: The standalone 2P DEP8 prefill plus DEP12 decode point failed repeated formal runs and did not produce a reliable benchmark result.\n\nFix: Remove the c1152 master-matrix arm and its dedicated recipe, and append a changelog entry recording the removal.\n\nValidation: Generated the filtered GB300 DSV4 Dynamo-SGLang AgentX matrix (7 entries, no c1152) and ran all 246 matrix-logic tests.\n\n中文：删除不稳定的 DSV4 c1152 拓扑。该独立的 2P DEP8 预填充加 DEP12 解码点在正式运行中连续失败，无法产生可靠结果。本提交删除对应矩阵条目和专用 recipe，并追加变更记录。验证后的矩阵包含 7 个条目且不再包含 c1152，矩阵逻辑测试 246 项全部通过。 --- ...0-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml | 230 ------------------ configs/nvidia-master.yaml | 18 -- perf-changelog.yaml | 8 + 3 files changed, 8 insertions(+), 248 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml deleted file mode 100644 index 1138d8f390..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml +++ /dev/null @@ -1,230 +0,0 @@ -name: "disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 1D x DEP12, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1152. -# -# This 28-GPU efficiency point is normalized from the validated local topology; -# formal CI retains the repository model, image, workload, and warmup contracts. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 12 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - nginx_keepalive_timeout: "900s" - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # AgentX warmup can legitimately keep the single wide decode worker busy - # for longer than Dynamo's 10-second TCP request-plane default. - DYN_TCP_REQUEST_TIMEOUT: "60" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 2.1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 12 - dp-size: 12 - ep-size: 12 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "140" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c1cb08cbe9..33ec1208a5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8804,24 +8804,6 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 16 ep: 16 dp-attn: true - - spec-decoding: draft_model - conc-list: [1152] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-kvoffload.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - spec-decoding: draft_model conc-list: [1920] kv-offloading: dram diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f6f1d7e29b..9c8a831c15 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6922,3 +6922,11 @@ - "Increase the DEP16 concurrency ladder from 320, 640, 960, and 1280 to 480, 960, 1440, and 1920 after profiling showed request-starved prefill workers at the lower values." - "Retain the independently tuned concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Remove the unstable concurrency-1152 2P DEP8 prefill plus DEP12 decode point after repeated benchmark failures." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 From 6d97f7c17665487bb8d5bb20aed3f90d8c123421 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Wed, 9 Sep 2026 17:23:37 -0700 Subject: [PATCH 10/10] test(runners): remove stale synthetic acceptance test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除未被引用且包含过时配方数量断言的合成接受长度测试。 --- runners/test_synthetic_acceptance.py | 140 --------------------------- 1 file changed, 140 deletions(-) delete mode 100644 runners/test_synthetic_acceptance.py diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py deleted file mode 100644 index c6d28049ea..0000000000 --- a/runners/test_synthetic_acceptance.py +++ /dev/null @@ -1,140 +0,0 @@ -import os -import subprocess -import sys -from pathlib import Path - -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[1] -INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" -LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" -MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" -RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" -GOLDEN_AL = REPO_ROOT / "golden_al_distribution" / "dsv4-pro-0813-dspark.yaml" -CONFIG_KEYS = ( - "dsv4-fp4-gb300-dynamo-sglang-agentic-agg", - "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", -) - - -def run_injector( - recipe: Path, *, eval_only: bool, acceptance_length: str = "3.77" -) -> subprocess.CompletedProcess[str]: - env = { - **os.environ, - "EVAL_ONLY": str(eval_only).lower(), - "SYNTHETIC_ACCEPTANCE": "true", - "SYNTHETIC_ACCEPTANCE_LENGTH": acceptance_length, - } - return subprocess.run( - [sys.executable, str(INJECTOR), str(recipe), "dynamo-sglang"], - check=False, - capture_output=True, - text=True, - env=env, - ) - - -def _configured_recipe_paths(master: dict) -> set[Path]: - recipe_paths = set() - for config_key in CONFIG_KEYS: - assert master[config_key]["model"] == "deepseek-ai/DeepSeek-V4-Pro-0813" - for group in master[config_key]["scenarios"]["agentic-coding"]: - for point in group["search-space"]: - assert point["spec-decoding"] == "draft_model" - worker = point.get("prefill", point.get("worker")) - assert worker is not None - settings = worker["additional-settings"] - assert "SYNTHETIC_ACCEPTANCE=true" in settings - assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings - config = next( - item for item in settings if item.startswith("CONFIG_FILE=") - ) - recipe_paths.add(RECIPES / config.removeprefix("CONFIG_FILE=recipes/")) - return recipe_paths - - -def test_sglang_injection_is_throughput_only(tmp_path: Path) -> None: - original = " prefill_environment:\n A: B\n decode_environment:\n C: D\n" - - throughput_recipe = tmp_path / "throughput.yaml" - throughput_recipe.write_text(original) - result = run_injector(throughput_recipe, eval_only=False) - assert result.returncode == 0, result.stderr - assert throughput_recipe.read_text().count("SGLANG_SIMULATE_ACC_LEN") == 2 - - eval_recipe = tmp_path / "eval.yaml" - eval_recipe.write_text(original) - result = run_injector(eval_recipe, eval_only=True) - assert result.returncode == 0, result.stderr - assert eval_recipe.read_text() == original - - -def test_gb300_dsv4_configs_inject_only_at_launch() -> None: - master = yaml.safe_load(MASTER.read_text()) - recipe_paths = _configured_recipe_paths(master) - - assert len(recipe_paths) == 7 - for recipe in recipe_paths: - text = recipe.read_text() - parsed = yaml.safe_load(text) - worker_count = 1 if recipe.name.startswith("agg-") else 2 - assert "SGLANG_SIMULATE_ACC_" not in text - assert text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count - assert text.count("speculative-algorithm: DSPARK") == worker_count - assert text.count("speculative-dspark-block-size: 6") == worker_count - assert text.count("speculative-num-draft-tokens: 7") == worker_count - assert "speculative-algorithm: EAGLE" not in text - assert 'path: "deepseek-v4-pro-0813"' in text - assert "AIPERF_TOKENIZER" not in parsed["benchmark"]["env"] - - launcher = LAUNCHER.read_text() - assert 'MODEL == "deepseek-ai/DeepSeek-V4-Pro-0813"' in launcher - assert 'MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-0813"' in launcher - assert 'SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro-0813"' in launcher - source = 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' - inject = 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' - apply = "SRTCTL_OUTPUT=$(srtctl apply" - assert launcher.index(source) < launcher.index(inject) < launcher.index(apply) - - -def test_gb300_dsv4_dspark6_matches_committed_golden_curve() -> None: - golden = yaml.safe_load(GOLDEN_AL.read_text()) - assert golden["deepseek-v4-pro-0813"]["thinking_on"][6] == 3.77 - - master = yaml.safe_load(MASTER.read_text()) - for config_key in CONFIG_KEYS: - for group in master[config_key]["scenarios"]["agentic-coding"]: - for point in group["search-space"]: - worker = point.get("prefill", point.get("worker")) - assert worker is not None - settings = worker["additional-settings"] - assert "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" in settings - - -def test_gb300_dsv4_dspark6_uses_golden_acceptance_only_for_throughput( - tmp_path: Path, -) -> None: - master = yaml.safe_load(MASTER.read_text()) - for recipe in _configured_recipe_paths(master): - original = recipe.read_text() - worker_count = 1 if recipe.name.startswith("agg-") else 2 - - throughput_recipe = tmp_path / recipe.name - throughput_recipe.write_text(original) - result = run_injector(throughput_recipe, eval_only=False) - assert result.returncode == 0, result.stderr - throughput_text = throughput_recipe.read_text() - assert throughput_text.count('SGLANG_SIMULATE_ACC_LEN: "3.77"') == worker_count - assert ( - throughput_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count - ) - - eval_recipe = tmp_path / f"eval-{recipe.name}" - eval_recipe.write_text(original) - result = run_injector(eval_recipe, eval_only=True) - assert result.returncode == 0, result.stderr - eval_text = eval_recipe.read_text() - assert eval_text == original - assert "SGLANG_SIMULATE_ACC_" not in eval_text - assert eval_text.count('SGLANG_RAGGED_VERIFY_MODE: "static"') == worker_count