[NVIDIA][AgentX] GB300 DeepSeek-V4.1-Flash Dynamo+SGLang: TP4/TEP2 aggregated and DEP2/DEP16 disaggregated points / [NVIDIA][AgentX] GB300 DeepSeek-V4.1-Flash Dynamo+SGLang：TP4/TEP2 聚合与 DEP2/DEP16 分离式配置点

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml

@@ -1,14 +1,23 @@
-
 schema: 2
 
 base:
   name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg
   identity:
     model: {}
+    frameworks:
+      dynamo: "1.6.0.dev20261014"
+      sglang: "0.0.0.dev1+ga6a8c5601"
+  # The image ships ai-dynamo 1.6.0.dev20261014 (includes ai-dynamo/dynamo#16005).
   dynamo:
-    install: true
-    source:
-      wheel: "1.5.0.dev20260914"
+    install: false
+    request_plane: tcp
+  environment:
+    HF_HOME: /hf_hub_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    HF_DATASETS_CACHE: /hf_hub_cache/datasets
+  observability:
+    tachometer:
+      enabled: false
   slurm:
     time_limit: "4:00:00"
   # Cold weight loads from the shared HF cache plus graph capture take 15-20 min.
@@ -18,7 +27,6 @@ base:
   resources:
     gpu_type: gb300
     gpus_per_node: 4
-  # Dynamo 1.5 discovers workers through etcd and also needs NATS.
   services:
     - name: etcd
       type: etcd
@@ -28,15 +36,24 @@ base:
       type: nats
       placement:
         node: infra
+    - name: otel-collector
+      type: otel-collector
+      enabled: false
   frontend:
     type: dynamo
     enable_multiple_frontends: false
     env:
-      PIP_BREAK_SYSTEM_PACKAGES: "1"
-      DYN_NATS_REQUEST_TIMEOUT_SECS: "1800"
+      DYN_ROUTER_TEMPERATURE: "0"
+      DYN_ROUTER_QUEUE_THRESHOLD: None
+      DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
+      DYN_TOKENIZER: "fastokens"
+      DYN_TOKENIZER_CACHE: "1"
+      DYN_TOKENIZER_CACHE_BYTES: "8000000000"
+      DYN_TCP_CHANNEL_BUFFER: "128"
+      DYN_TCP_REQUEST_TIMEOUT: "60"
     args:
       router-mode: kv
-      router-session-affinity-ttl-secs: "3600"
+      router-session-affinity-ttl-secs: "14400"
       active-decode-blocks-threshold: "None"
       active-prefill-tokens-threshold: "None"
       active-prefill-tokens-threshold-frac: "None"
@@ -45,81 +62,113 @@ base:
     agg:
       nodes: 1
       workers: 1
-      gpus: 2
       env:
-        PIP_BREAK_SYSTEM_PACKAGES: "1"
         PYTHONNOUSERSITE: "1"
         PYTHONUNBUFFERED: "1"
-        HF_HUB_CACHE: /hf_hub_cache
-        # Outlast AIPerf's pooled connections past Uvicorn's keep-alive.
         SGLANG_TIMEOUT_KEEP_ALIVE: "900"
-        # AgentX measures thinking on, the regime of the golden acceptance curve.
         SGLANG_DEFAULT_THINKING: "1"
         SGLANG_DSV41_REASONING_EFFORT: high
         SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True"
-        # TP2 keeps the row-sharded Engram tables in host DRAM.
-        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
-        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
+        # AUTO leaves DSpark draft sampling eager for V4.1 TP shards, which leaves c1
+        # decode launch-bound; fold it into the draft CUDA graph.
+        SGLANG_DSPARK_FOLDED_SAMPLING: "2"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
-        tensor-parallel-size: 2
-        expert-parallel-size: 2
         mem-fraction-static: 0.8
-        # Keep active decode requests progressing while long prefixes are queued.
-        prefill-decode-interval: 16
+        chunked-prefill-size: 16384
         # Admission never exceeds the captured decode graph batch.
         cuda-graph-max-bs-decode: 64
-        # DSpark is the checkpoint's bundled draft; the block size is its only knob.
         speculative-algorithm: DSPARK
         speculative-dspark-block-size: 5
         reasoning-parser: auto
         tool-call-parser: auto
-        # Draft passes under long-context load outlast the 1800 s default watchdog.
         watchdog-timeout: 3600
         enable-metrics: true
-        weight-loader-prefetch-checkpoints: true
-        weight-loader-drop-cache-after-load: false
-        model-loader-extra-config: '{"enable_multithread_load":true}'
   sbatch_directives:
     mem: "0"
     exclusive: ""
   srun_options:
     container-remap-root: ""
+    # Engram host tables need the node's full memory in every worker step.
+    mem: "0"
   benchmark:
     env:
       AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
       AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"
-      # Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
       AIPERF_HTTP_TCP_USER_TIMEOUT: "900000"
+      # The client splits this value on whitespace, so it has none.
+      AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}'
+      AGENTIC_WARMUP_GRACE_PERIOD: "1800"
+
+override_tp2_c16:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16
+  roles:
+    agg:
+      gpus: 2
+      args:
+        tensor-parallel-size: 2
+        expert-parallel-size: 2
+        enable-decoder-swa-bounded-replay: true
+        cuda-graph-backend-prefill: disabled
+        prefill-decode-interval: 96
+        swa-prefix-tails: 1024
+        max-running-requests: 32
+      env:
+        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
+        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
+        PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+        SGLANG_NUMA_BIND_V2: "0"
 
 override_tp4_c1:
-  name: agg-gb300-tp4ep1-c1
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c1
   roles:
     agg:
       gpus: 4
       args:
         tensor-parallel-size: 4
         expert-parallel-size: 1
-        chunked-prefill-size: 4096
-        max-running-requests: 2
+        prefill-decode-interval: 0
+        # A subagent prefill waits instead of stalling the running decode.
+        max-running-requests: 1
+        enable-decoder-swa-bounded-replay: true
+        cuda-graph-backend-prefill: disabled
       env:
         SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "0"
         SGLANG_RAGGED_VERIFY_MODE: static
-  benchmark:
-    env:
-      AGENTIC_WARMUP_GRACE_PERIOD: "1800"
 
-override_tp2_c1:
-  name: agg-gb300-tp2ep2-c1
+override_tp4_c4:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c4
   roles:
     agg:
+      gpus: 4
       args:
-        chunked-prefill-size: 4096
-        max-running-requests: 2
+        tensor-parallel-size: 4
+        expert-parallel-size: 1
+        prefill-decode-interval: 0
+        swa-prefix-tails: 4096
+        max-running-requests: 128
+        cuda-graph-max-bs-decode: 128
       env:
+        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
+        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
+        SGLANG_RAGGED_VERIFY_MODE: static
+
+override_tp4_c16:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c16
+  roles:
+    agg:
+      gpus: 4
+      args:
+        tensor-parallel-size: 4
+        expert-parallel-size: 1
+        prefill-decode-interval: 192
+        swa-prefix-tails: 4096
+        max-running-requests: 128
+        cuda-graph-max-bs-decode: 128
+      env:
+        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
+        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
+        SGLANG_NUMA_BIND_V2: "0"
         SGLANG_RAGGED_VERIFY_MODE: static
-  benchmark:
-    env:
-      AGENTIC_WARMUP_GRACE_PERIOD: "1800"