[AMD][MI35X] 1012 DSV41 sglang mtp agentic

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml

@@ -5,7 +5,7 @@ base:
   name: dsv41flash-fp4-mi355x-sglang-agentic
   model:
     path: hf:deepseek-ai/DeepSeek-V4.1-Flash
-    container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261006
+    container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012
     precision: fp4
   resources:
     gpu_type: mi355x
@@ -53,7 +53,8 @@ base:
         SGLANG_USE_AITER: '1'
         SGLANG_USE_ROCM700A: '0'
         TRITON_HIP_USE_ASYNC_COPY: '0'
-        ROCM_QUICK_REDUCE_QUANTIZATION: NONE
+        ROCM_QUICK_REDUCE_QUANTIZATION: INT8
+        SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL: 'true'
         AITER_BF16_FP8_MOE_BOUND: '0'
         TORCH_BLAS_PREFER_HIPBLASLT: '1'
         HSA_NO_SCRATCH_RECLAIM: '0'
@@ -69,7 +70,9 @@ base:
       AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
 
-# One variant per point; the saturation point gets a longer warmup drain.
+# One variant per point; the saturation point gets a longer warmup drain. A longer
+# prefill-decode interval trades TTFT for P90 ITL: 192 with lpm up to c8, 96 at c16
+# and 32 at c32. amd-master.yaml runs c32 and c64 on the DP2 variants only.
 override_tp2_c1:
   roles:
     agg:
@@ -78,7 +81,8 @@ override_tp2_c1:
         tensor-parallel-size: 2
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 128
         max-running-requests: 2
   benchmark:
@@ -93,7 +97,8 @@ override_tp2_c2:
         tensor-parallel-size: 2
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 128
         max-running-requests: 4
   benchmark:
@@ -108,7 +113,8 @@ override_tp2_c4:
         tensor-parallel-size: 2
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 256
         max-running-requests: 8
   benchmark:
@@ -123,7 +129,8 @@ override_tp2_c8:
         tensor-parallel-size: 2
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 512
         max-running-requests: 16
   benchmark:
@@ -138,7 +145,7 @@ override_tp2_c16:
         tensor-parallel-size: 2
         mem-fraction-static: 0.80
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 96
         swa-prefix-tails: 1024
         max-running-requests: 32
   benchmark:
@@ -153,7 +160,7 @@ override_tp2_c32:
         tensor-parallel-size: 2
         mem-fraction-static: 0.80
         chunked-prefill-size: 16384
-        prefill-decode-interval: 4
+        prefill-decode-interval: 32
         swa-prefix-tails: 2048
         max-running-requests: 64
   benchmark:
@@ -178,6 +185,89 @@ override_tp2_c64:
       CONC: '64'
       AGENTIC_WARMUP_GRACE_PERIOD: '3600'
 
+# DP2 attention with TP MoE.
+override_tp2_c32_dp:
+  frontend:
+    type: sglang-router
+    args:
+      policy: cache_aware
+      request-id-headers: x-correlation-id
+      dp-aware: true
+      connect-timeout-secs: 900
+      request-timeout-secs: 14400
+      disable-health-check: true
+      disable-retries: true
+  roles:
+    agg:
+      gpus: 2
+      args:
+        tensor-parallel-size: 2
+        data-parallel-size: 2
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        enable-prefill-delayer: true
+        enable-dp-attention-local-control-broadcast: true
+        tokenizer-worker-num: 2
+        prefill-delayer-token-usage-low-watermark: 0.7
+        load-balance-method: total_requests
+        mem-fraction-static: 0.80
+        chunked-prefill-size: 32768
+        prefill-decode-interval: 32
+        swa-prefix-tails: 2048
+        max-running-requests: 64
+      env:
+        SGLANG_SHARED_EXPERT_TP1: '1'
+        SGLANG_DP_USE_GATHERV: '1'
+        SGLANG_DP_USE_REDUCE_SCATTER: '1'
+        SGLANG_AITER_CAPTURE_COPY_IN: '1'
+        GPU_MAX_HW_QUEUES: '5'
+  benchmark:
+    env:
+      CONC: '32'
+      AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
+      AGENTIC_WARMUP_GRACE_PERIOD: '3600'
+
+override_tp2_c64_dp:
+  frontend:
+    type: sglang-router
+    args:
+      policy: cache_aware
+      request-id-headers: x-correlation-id
+      dp-aware: true
+      connect-timeout-secs: 900
+      request-timeout-secs: 14400
+      disable-health-check: true
+      disable-retries: true
+  roles:
+    agg:
+      gpus: 2
+      args:
+        tensor-parallel-size: 2
+        data-parallel-size: 2
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        enable-prefill-delayer: true
+        enable-dp-attention-local-control-broadcast: true
+        tokenizer-worker-num: 2
+        prefill-delayer-token-usage-low-watermark: 0.7
+        load-balance-method: total_requests
+        mem-fraction-static: 0.85
+        chunked-prefill-size: 8192
+        prefill-decode-interval: 16
+        swa-prefix-tails: 4096
+        max-running-requests: 64
+      env:
+        SGLANG_SHARED_EXPERT_TP1: '1'
+        SGLANG_DP_USE_GATHERV: '1'
+        SGLANG_DP_USE_REDUCE_SCATTER: '1'
+        SGLANG_AITER_CAPTURE_COPY_IN: '1'
+        GPU_MAX_HW_QUEUES: '5'
+  benchmark:
+    env:
+      CONC: '64'
+      AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
+      AGENTIC_WARMUP_GRACE_PERIOD: '3600'
+
 override_tp4_c1:
   roles:
     agg:
@@ -186,7 +276,8 @@ override_tp4_c1:
         tensor-parallel-size: 4
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 128
         max-running-requests: 2
   benchmark:
@@ -201,7 +292,8 @@ override_tp4_c2:
         tensor-parallel-size: 4
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 128
         max-running-requests: 4
   benchmark:
@@ -216,7 +308,8 @@ override_tp4_c4:
         tensor-parallel-size: 4
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 256
         max-running-requests: 8
   benchmark:
@@ -231,7 +324,8 @@ override_tp4_c8:
         tensor-parallel-size: 4
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 192
+        schedule-policy: lpm
         swa-prefix-tails: 512
         max-running-requests: 16
   benchmark:
@@ -246,7 +340,7 @@ override_tp4_c16:
         tensor-parallel-size: 4
         mem-fraction-static: 0.70
         chunked-prefill-size: 16384
-        prefill-decode-interval: 16
+        prefill-decode-interval: 96
         swa-prefix-tails: 1024
         max-running-requests: 32
   benchmark:

inferencex-e2e/configs/amd-master.yaml

@@ -1566,10 +1566,10 @@ glm5.3-fp8-mi355x-tilert-agentic:
           ep: 1
           dp-attn: false
 
-# SGLang ROCm 7.2 MI35X release image. TP2/EP1 across the curve, plus TP4/EP1
-# on the interactive band. Host Engram tables on both.
+# SGLang ROCm 7.2 MI35X release image. TP2/EP1 and TP4/EP1 up to c16, DP2
+# attention with TP MoE at c32 and c64. Host Engram tables on all of them.
 dsv41flash-fp4-mi355x-sglang-agentic-dspark:
-  image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261006
+  image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:mi355x-amds
@@ -1580,5 +1580,6 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark:
     agentic-coding:
     - dram-utilization: 0.60
       search-space:
-      - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
+      - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
       - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
+      - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }

inferencex-e2e/perf-changelog.yaml

@@ -9621,3 +9621,14 @@
     - "Switch NIXL KV transfer to the C++ transfer-agent bounce buffer (max_average_descriptor_size 4MB, min_descriptor_count 1) and lower kv_cache_bounce_size_mb from 5120 to 1024."
     - "Move Dynamo on all six points to the ai-dynamo 1.6.0.dev20261010 wheel, which includes ai-dynamo/dynamo#15628 (routed node address selection for metadata and transport advertisement)."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3707
+
+- config-keys:
+    - dsv41flash-fp4-mi355x-sglang-agentic-dspark
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Bump to lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012."
+    - "Trade TTFT for P90 ITL with a longer prefill-decode interval: 192 with lpm scheduling at concurrency 1-8 (TP2 and TP4) and 96 at concurrency 16."
+    - "Enable SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL and ROCM_QUICK_REDUCE_QUANTIZATION=INT8, previously NONE."
+    - "Run TP2 concurrency 32 and 64 on DP2 attention with TP MoE instead of TP-only."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3926