diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml index a346b23929..a2f9f7ad03 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml @@ -1,4 +1,3 @@ - schema: 2 base: @@ -7,122 +6,168 @@ base: revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa container: {} frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 + dynamo: 1.5.0.dev20260903 resources: gpu_type: b200 gpus_per_node: 8 + slurm: + time_limit: '4:00:00' dynamo: - install: false + install: true + source: + wheel: 1.5.0.dev20260903 + frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + observability: + enabled: false + tachometer: + enabled: false engine: sglang roles: agg: nodes: 1 workers: 1 - gpus: 8 + gpus: 4 env: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' PYTHONUNBUFFERED: '1' + PYTHONNOUSERSITE: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - PIP_BREAK_SYSTEM_PACKAGES: '1' args: served-model-name: nvidia/GLM-5.2-NVFP4 trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 + tensor-parallel-size: 4 expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false + bf16-gemm-backend: cutedsl + # GLM-5.2 emits the GLM-4.7 tool-call format. + tool-call-parser: glm47 + reasoning-parser: glm45 chunked-prefill-size: 8192 max-prefill-tokens: 8192 - context-length: 1048576 + # TP4 weights take ~111 GiB per rank: ~0.7M-token KV pool, ~27 GiB activation headroom. + mem-fraction-static: 0.85 speculative-algorithm: EAGLE - speculative-num-steps: 3 + speculative-num-steps: 5 speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true + speculative-num-draft-tokens: 6 weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 + model-loader-extra-config: '{"enable_multithread_load":true}' + # SGLang membinds each rank to its GPU's NUMA node and GPUs 0-3 share node 0 + # (~1,080 GB): 4 x (170 GB KV + 39 GB DSA indexer) pins ~836 GB of it. enable-hierarchical-cache: true + hicache-size: 170 hicache-write-policy: write_back - hicache-size: 110 hicache-io-backend: direct + hicache-mem-layout: page_first_direct watchdog-timeout: 1800 enable-metrics: true - enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 benchmark: env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - health_check: - max_attempts: 1440 - interval_seconds: 10 + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' services: - name: etcd type: etcd placement: node: infra + readiness: + port: 2379 + timeout_seconds: 900 - name: nats type: nats placement: node: infra options: max_payload_mb: 64 - frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 + readiness: + port: 4222 + timeout_seconds: 900 sbatch_directives: mem: '0' srun_options: mem: '0' container-remap-root: '' -override_c1: - name: agg-b200-tp8-c1-mtp +override_tp8_c1: + name: glm52-b200-agg-tp8-c1-mtp roles: agg: + gpus: 8 args: + tensor-parallel-size: 8 + # Fan-out subagents queue instead of widening the decode batch past 2. max-running-requests: 2 - cuda-graph-max-bs-decode: 2 + cuda-graph-max-bs: 2 + +override_c1: + name: glm52-b200-agg-tp4-c1-mtp + roles: + agg: + args: + # Fan-out subagents queue behind the running request instead of sharing its decode steps. + max-running-requests: 1 + cuda-graph-max-bs: 1 override_c4: - name: agg-b200-tp8-c4-mtp + name: glm52-b200-agg-tp4-c4-mtp roles: agg: args: - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 + max-running-requests: 16 + cuda-graph-max-bs: 16 override_c8: - name: agg-b200-tp8-c8-mtp + name: glm52-b200-agg-tp4-c8-mtp roles: agg: args: - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 + max-running-requests: 32 + cuda-graph-max-bs: 32 + +override_c12: + name: glm52-b200-agg-tp4-c12-mtp + roles: + agg: + args: + max-running-requests: 32 + cuda-graph-max-bs: 32 + +override_tp8_dp8attn_c32: + name: glm52-b200-agg-tp8-dp8attn-c32-mtp + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + # Aligns the DP ranks' prefills to cut idle time in the shared MoE passes. + enable-prefill-delayer: true + # Node totals under DP attention: 8192-token chunks and 16 requests per rank. + chunked-prefill-size: 65536 + max-running-requests: 128 + cuda-graph-max-bs: 16 + # Unsplit attention weights per rank; leaves room for the DSA indexer's fp32 prefill logits. + mem-fraction-static: 0.82 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 68c2763fdf..274fb79696 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7061,31 +7061,42 @@ glm5.2-fp8-b200-sglang-agentic-mtp: # infx/golden_al_distribution/glm5.2_mtp.yaml. Both arms use HiCache write-back # DRAM offload for the agentic working set. glm5.2-fp4-b200-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 + image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 runner: cluster:b200-nscale precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.5.0.dev20260909" } + router: { name: dynamo-router, version: "1.5.0.dev20260903" } multinode: true disagg: false srt-recipe-dir: glm5.2/sglang/b200-fp4/agentx scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 1.0 search-space: - spec-decoding: mtp conc-list: [1] kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 1 - srt-recipe: agg-variants.yaml:override_c1 + srt-recipe: agg-variants.yaml:override_tp8_c1 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false + - spec-decoding: mtp + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c1 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -7094,7 +7105,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: srt-recipe: agg-variants.yaml:override_c4 worker: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false - spec-decoding: mtp @@ -7105,9 +7116,31 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: srt-recipe: agg-variants.yaml:override_c8 worker: num-worker: 1 - tp: 8 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [12] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_c12 + worker: + num-worker: 1 + tp: 4 ep: 1 dp-attn: false + - spec-decoding: mtp + conc-list: [32] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp8_dp8attn_c32 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260910-00840301 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 189bbd0164..18e00c9a43 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10111,6 +10111,18 @@ - "Pin Dynamo ffbbaaa6 (dynamo-parsers 8.1.0), use Kimi-K3 parsers only in evaluation recipes, and set DYN_TCP_REQUEST_TIMEOUT=120 for the frontend and workers." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4208 +- config-keys: + - glm5.2-fp4-b200-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Refresh the GLM-5.2 NVFP4 B200 AgentX aggregate key on Dynamo (KV router) + SGLang: move it to the image and Dynamo build of glm5.2-fp4-b300-dynamo-sglang-agentic-agg-tp4 (lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729, Dynamo 1.5.0.dev20260903) with one aggregated engine per point, EAGLE speculative decoding (5 steps, 6 draft tokens) and HiCache write-back host-DRAM offload." + - "Sweep six points in place of the three TP8 points: TP8 and TP4 at conc 1, TP4 at conc 4, 8 and 12, and TP8 MoE with DP8 attention at conc 32." + - "Cap admission and decode CUDA graphs at 2 for TP8 conc 1 and at 1 for TP4 conc 1, so fan-out subagents queue instead of widening the decode batch." + - "Use mem-fraction-static 0.85 and 170 GB of HiCache per rank, so the KV and DSA indexer host pools of the four ranks on each NUMA node (~836 GB) fit that node; dram-utilization 1.0 records each engine's full GPU share of host DRAM (1,073 GB at TP4, 2,146 GB at TP8) as its offload budget." + - "The DP8-attention point runs attention TP1 per rank with the DP LM head and the prefill delayer, mem-fraction-static 0.82, node totals of 65536 chunked-prefill tokens and 128 running requests (8192 and 16 per rank), and decode CUDA graphs up to batch 16." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4286 + - config-keys: - musespark-fp4-mi355x-sglang-agentic-dflash - musespark-fp4-gb200-sglang-agentic-dflash