perf(glm5.2): refresh the B200 FP4 Dynamo SGLang AgentX aggregate curve / perf(glm5.2)：刷新 B200 FP4 Dynamo SGLang AgentX 聚合曲线

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml

@@ -1,4 +1,3 @@
-
 schema: 2
 
 base:
@@ -7,122 +6,168 @@ base:
       revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa
     container: {}
     frameworks:
-      dynamo: 1.5.0.dev20260909
-      sglang: 0.0.0.dev1+g008403017
+      dynamo: 1.5.0.dev20260903
   resources:
     gpu_type: b200
     gpus_per_node: 8
+  slurm:
+    time_limit: '4:00:00'
   dynamo:
-    install: false
+    install: true
+    source:
+      wheel: 1.5.0.dev20260903
+  frontend:
+    type: dynamo
+    env:
+      PIP_BREAK_SYSTEM_PACKAGES: '1'
+    args:
+      router-mode: kv
+      active-decode-blocks-threshold: None
+      active-prefill-tokens-threshold: None
+      active-prefill-tokens-threshold-frac: None
+      router-session-affinity-ttl-secs: 3600
+  observability:
+    enabled: false
+    tachometer:
+      enabled: false
   engine: sglang
   roles:
     agg:
       nodes: 1
       workers: 1
-      gpus: 8
+      gpus: 4
       env:
         TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
         PYTHONUNBUFFERED: '1'
+        PYTHONNOUSERSITE: '1'
+        PIP_BREAK_SYSTEM_PACKAGES: '1'
         DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
-        MC_TE_METRIC: 'true'
-        NCCL_CUMEM_ENABLE: '1'
-        SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
-        SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0'
-        SGLANG_DEFAULT_THINKING: '1'
-        SGLANG_REASONING_EFFORT: max
-        SGLANG_HICACHE_DEBUG_LOG: '1'
-        SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384'
-        SGLANG_MOE_NVFP4_DISPATCH: '1'
+        TORCH_CUDA_ARCH_LIST: '10.0'
+        # Outlast AIPerf's pooled connections past Uvicorn's keep-alive.
+        SGLANG_TIMEOUT_KEEP_ALIVE: '900'
         SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0'
-        SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
-        PIP_BREAK_SYSTEM_PACKAGES: '1'
       args:
         served-model-name: nvidia/GLM-5.2-NVFP4
         trust-remote-code: true
         quantization: modelopt_fp4
         kv-cache-dtype: fp8_e4m3
-        tensor-parallel-size: 8
-        data-parallel-size: 1
+        tensor-parallel-size: 4
         expert-parallel-size: 1
-        enable-dp-attention: false
-        enable-dp-lm-head: false
+        bf16-gemm-backend: cutedsl
+        # GLM-5.2 emits the GLM-4.7 tool-call format.
+        tool-call-parser: glm47
+        reasoning-parser: glm45
         chunked-prefill-size: 8192
         max-prefill-tokens: 8192
-        context-length: 1048576
+        # TP4 weights take ~111 GiB per rank: ~0.7M-token KV pool, ~27 GiB activation headroom.
+        mem-fraction-static: 0.85
         speculative-algorithm: EAGLE
-        speculative-num-steps: 3
+        speculative-num-steps: 5
         speculative-eagle-topk: 1
-        speculative-num-draft-tokens: 4
-        dsa-prefill-backend: trtllm
-        dsa-decode-backend: trtllm
-        moe-runner-backend: flashinfer_trtllm
-        fp4-gemm-backend: flashinfer_trtllm
-        disable-shared-experts-fusion: true
-        enable-flashinfer-allreduce-fusion: true
+        speculative-num-draft-tokens: 6
         weight-loader-prefetch-checkpoints: true
-        model-loader-extra-config: '{"enable_multithread_load": true}'
-        mem-fraction-static: 0.8
+        model-loader-extra-config: '{"enable_multithread_load":true}'
+        # SGLang membinds each rank to its GPU's NUMA node and GPUs 0-3 share node 0
+        # (~1,080 GB): 4 x (170 GB KV + 39 GB DSA indexer) pins ~836 GB of it.
         enable-hierarchical-cache: true
+        hicache-size: 170
         hicache-write-policy: write_back
-        hicache-size: 110
         hicache-io-backend: direct
+        hicache-mem-layout: page_first_direct
         watchdog-timeout: 1800
         enable-metrics: true
-        enable-cache-report: true
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 10
   benchmark:
     env:
       AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
-  health_check:
-    max_attempts: 1440
-    interval_seconds: 10
+      # Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
+      AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
+      AGENTIC_WARMUP_GRACE_PERIOD: '1800'
   services:
     - name: etcd
       type: etcd
       placement:
         node: infra
+      readiness:
+        port: 2379
+        timeout_seconds: 900
     - name: nats
       type: nats
       placement:
         node: infra
       options:
         max_payload_mb: 64
-  frontend:
-    type: dynamo
-    env:
-      PIP_BREAK_SYSTEM_PACKAGES: '1'
-    args:
-      router-mode: kv
-      active-decode-blocks-threshold: None
-      active-prefill-tokens-threshold: None
-      active-prefill-tokens-threshold-frac: None
-      router-session-affinity-ttl-secs: 3600
+      readiness:
+        port: 4222
+        timeout_seconds: 900
   sbatch_directives:
     mem: '0'
   srun_options:
     mem: '0'
     container-remap-root: ''
 
-override_c1:
-  name: agg-b200-tp8-c1-mtp
+override_tp8_c1:
+  name: glm52-b200-agg-tp8-c1-mtp
   roles:
     agg:
+      gpus: 8
       args:
+        tensor-parallel-size: 8
+        # Fan-out subagents queue instead of widening the decode batch past 2.
         max-running-requests: 2
-        cuda-graph-max-bs-decode: 2
+        cuda-graph-max-bs: 2
+
+override_c1:
+  name: glm52-b200-agg-tp4-c1-mtp
+  roles:
+    agg:
+      args:
+        # Fan-out subagents queue behind the running request instead of sharing its decode steps.
+        max-running-requests: 1
+        cuda-graph-max-bs: 1
 
 override_c4:
-  name: agg-b200-tp8-c4-mtp
+  name: glm52-b200-agg-tp4-c4-mtp
   roles:
     agg:
       args:
-        max-running-requests: 8
-        cuda-graph-max-bs-decode: 8
+        max-running-requests: 16
+        cuda-graph-max-bs: 16
 
 override_c8:
-  name: agg-b200-tp8-c8-mtp
+  name: glm52-b200-agg-tp4-c8-mtp
   roles:
     agg:
       args:
-        max-running-requests: 16
-        cuda-graph-max-bs-decode: 16
+        max-running-requests: 32
+        cuda-graph-max-bs: 32
+
+override_c12:
+  name: glm52-b200-agg-tp4-c12-mtp
+  roles:
+    agg:
+      args:
+        max-running-requests: 32
+        cuda-graph-max-bs: 32
+
+override_tp8_dp8attn_c32:
+  name: glm52-b200-agg-tp8-dp8attn-c32-mtp
+  roles:
+    agg:
+      gpus: 8
+      args:
+        tensor-parallel-size: 8
+        data-parallel-size: 8
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        # Aligns the DP ranks' prefills to cut idle time in the shared MoE passes.
+        enable-prefill-delayer: true
+        # Node totals under DP attention: 8192-token chunks and 16 requests per rank.
+        chunked-prefill-size: 65536
+        max-running-requests: 128
+        cuda-graph-max-bs: 16
+        # Unsplit attention weights per rank; leaves room for the DSA indexer's fp32 prefill logits.
+        mem-fraction-static: 0.82

inferencex-e2e/configs/nvidia-master.yaml

@@ -7061,31 +7061,42 @@ glm5.2-fp8-b200-sglang-agentic-mtp:
 # infx/golden_al_distribution/glm5.2_mtp.yaml. Both arms use HiCache write-back
 # DRAM offload for the agentic working set.
 glm5.2-fp4-b200-dynamo-sglang-agentic-agg:
-  image: lmsysorg/sglang:nightly-dev-20260910-00840301
+  image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729
   model: nvidia/GLM-5.2-NVFP4
   model-prefix: glm5.2
   runner: cluster:b200-nscale
   precision: fp4
   framework: dynamo-sglang
-  router: { name: dynamo-router, version: "1.5.0.dev20260909" }
+  router: { name: dynamo-router, version: "1.5.0.dev20260903" }
   multinode: true
   disagg: false
   srt-recipe-dir: glm5.2/sglang/b200-fp4/agentx
   scenarios:
     agentic-coding:
-    - dram-utilization: 0.80
+    - dram-utilization: 1.0
       search-space:
       - spec-decoding: mtp
         conc-list: [1]
         kv-offloading: dram
         kv-offload-backend: { name: hicache }
         num-nodes: 1
-        srt-recipe: agg-variants.yaml:override_c1
+        srt-recipe: agg-variants.yaml:override_tp8_c1
         worker:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: false
+      - spec-decoding: mtp
+        conc-list: [1]
+        kv-offloading: dram
+        kv-offload-backend: { name: hicache }
+        num-nodes: 1
+        srt-recipe: agg-variants.yaml:override_c1
+        worker:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
       - spec-decoding: mtp
         conc-list: [4]
         kv-offloading: dram
@@ -7094,7 +7105,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg:
         srt-recipe: agg-variants.yaml:override_c4
         worker:
           num-worker: 1
-          tp: 8
+          tp: 4
           ep: 1
           dp-attn: false
       - spec-decoding: mtp
@@ -7105,9 +7116,31 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg:
         srt-recipe: agg-variants.yaml:override_c8
         worker:
           num-worker: 1
-          tp: 8
+          tp: 4
+          ep: 1
+          dp-attn: false
+      - spec-decoding: mtp
+        conc-list: [12]
+        kv-offloading: dram
+        kv-offload-backend: { name: hicache }
+        num-nodes: 1
+        srt-recipe: agg-variants.yaml:override_c12
+        worker:
+          num-worker: 1
+          tp: 4
           ep: 1
           dp-attn: false
+      - spec-decoding: mtp
+        conc-list: [32]
+        kv-offloading: dram
+        kv-offload-backend: { name: hicache }
+        num-nodes: 1
+        srt-recipe: agg-variants.yaml:override_tp8_dp8attn_c32
+        worker:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: true
 
 glm5.2-fp4-b200-dynamo-sglang-agentic-disagg:
   image: lmsysorg/sglang:nightly-dev-20260910-00840301

inferencex-e2e/perf-changelog.yaml

@@ -10111,6 +10111,18 @@
     - "Pin Dynamo ffbbaaa6 (dynamo-parsers 8.1.0), use Kimi-K3 parsers only in evaluation recipes, and set DYN_TCP_REQUEST_TIMEOUT=120 for the frontend and workers."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4208
 
+- config-keys:
+    - glm5.2-fp4-b200-dynamo-sglang-agentic-agg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Refresh the GLM-5.2 NVFP4 B200 AgentX aggregate key on Dynamo (KV router) + SGLang: move it to the image and Dynamo build of glm5.2-fp4-b300-dynamo-sglang-agentic-agg-tp4 (lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729, Dynamo 1.5.0.dev20260903) with one aggregated engine per point, EAGLE speculative decoding (5 steps, 6 draft tokens) and HiCache write-back host-DRAM offload."
+    - "Sweep six points in place of the three TP8 points: TP8 and TP4 at conc 1, TP4 at conc 4, 8 and 12, and TP8 MoE with DP8 attention at conc 32."
+    - "Cap admission and decode CUDA graphs at 2 for TP8 conc 1 and at 1 for TP4 conc 1, so fan-out subagents queue instead of widening the decode batch."
+    - "Use mem-fraction-static 0.85 and 170 GB of HiCache per rank, so the KV and DSA indexer host pools of the four ranks on each NUMA node (~836 GB) fit that node; dram-utilization 1.0 records each engine's full GPU share of host DRAM (1,073 GB at TP4, 2,146 GB at TP8) as its offload budget."
+    - "The DP8-attention point runs attention TP1 per rank with the DP LM head and the prefill delayer, mem-fraction-static 0.82, node totals of 65536 chunked-prefill tokens and 128 running requests (8192 and 16 per rank), and decode CUDA graphs up to batch 16."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4286
+
 - config-keys:
     - musespark-fp4-mi355x-sglang-agentic-dflash
     - musespark-fp4-gb200-sglang-agentic-dflash