[NV] Add H200 DeepSeek-V4-Pro AgentX recipes / [NV] 添加 H200 DeepSeek-V4-Pro AgentX 配方

benchmarks/benchmark_lib.sh

@@ -1801,7 +1801,11 @@ build_replay_cmd() {
     # aiperf's conv-aware routing emits nvext.session_control, a removed POC field
     # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400
     # (they moved to router/routing_constraints/agent_context). Default stays on.
-    if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then
+    # New recipes instead set AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true
+    # to route by X-Dynamo-Session-ID header, which needs no routing CLI flag.
+    if [[ "${FRAMEWORK:-}" == dynamo-* \
+          && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" \
+          && "${AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID:-false}" != "true" ]]; then
         REPLAY_CMD+=" --use-dynamo-conv-aware-routing"
         # The upstream 300s affinity TTL is shorter than an overloaded
         # high-concurrency agentic request. Keep bindings alive across long

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml

@@ -0,0 +1,115 @@
+name: "agg-h200-tp8-mtp-kvoffload"
+
+# DeepSeek-V4-Pro AgentX aggregated TP8 serving on one 8xH200 node. The recipe is
+# topology-invariant; configs/nvidia-master.yaml varies only concurrency to
+# produce the latency/throughput Pareto curve.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "dynamo-sglang"
+  precision: "fp8"
+
+dynamo:
+  install: true
+  wheel: "1.3.0.dev20260718"
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: true
+  # AgentX prompts can exceed NATS's 1 MiB default after JSON serialization.
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  nginx_session_affinity: true
+  nginx_session_affinity_header: X-Dynamo-Session-ID
+  enable_multiple_frontends: true
+  num_additional_frontends: 4
+  env:
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+  args:
+    router-mode: "kv"
+    router-session-affinity-ttl-secs: "3600"
+    active-decode-blocks-threshold: "None"
+    active-prefill-tokens-threshold: "None"
+    active-prefill-tokens-threshold-frac: "None"
+
+backend:
+  type: sglang
+
+  aggregated_environment:
+    SGLANG_DEFAULT_THINKING: "1"
+    SGLANG_DSV4_REASONING_EFFORT: high
+    SGLANG_SIMULATE_ACC_LEN: "2.49"
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+    HF_HUB_CACHE: /hf_hub_cache
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
+    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
+    SGLANG_OPT_USE_ONLINE_COMPRESS: "0"
+    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
+    SGLANG_OPT_USE_JIT_NORM: "1"
+    SGLANG_OPT_USE_TOPK_V2: "True"
+    SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
+
+  sglang_config:
+    aggregated:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      trust-remote-code: true
+      stream-interval: 50
+      watchdog-timeout: 1000000
+      mem-fraction-static: 0.88
+      chunked-prefill-size: 4096
+      moe-runner-backend: marlin
+      disable-flashinfer-autotune: true
+      swa-full-tokens-ratio: 0.1
+      max-running-requests: 32
+      cuda-graph-max-bs: 32
+
+      scheduler-recv-interval: 30
+      tp-size: 8
+      dp-size: 1
+      ep-size: 1
+
+      enable-hierarchical-cache: true
+      hicache-ratio: 6
+      hicache-write-policy: write_through
+      hicache-io-backend: kernel
+
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+
+srun_options:
+  # The custom benchmark installs its isolated AIPerf environment at runtime.
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    # Aggregated serving uses one TP8 worker for both prefill and decode.
+    IS_MULTINODE: "false"
+    TP: "8"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache

configs/nvidia-master.yaml

@@ -1695,6 +1695,41 @@ dsv4-fp8-h200-vllm-agentic:
     - search-space:
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] }
 
+# DeepSeek-V4-Pro AgentX on one aggregated TP8 H200 worker. Keep the serving
+# topology fixed and sweep only concurrency to produce the Pareto curve.
+dsv4-fp8-h200-dynamo-sglang-agentic-agg:
+  image: lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:h200-dgxc
+  precision: fp8
+  framework: dynamo-sglang
+  router: { name: dynamo-router, version: "1.3.0.dev20260718" }
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - spec-decoding: mtp
+        conc-list: [1, 2, 4, 8, 16]
+        kv-offloading: dram
+        kv-offload-backend: { name: hicache }
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml"
+        # The aggregated worker serves prefill and decode on the same GPUs.
+        # Keep decode at zero to avoid double-counting the TP8 allocation.
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 1
+          dp-attn: false
+
 # MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image,
 # runner pool, search space) and adds EAGLE speculative decoding via
 # --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp.

perf-changelog.yaml

@@ -5116,3 +5116,11 @@
   description:
     - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349
+
+- config-keys:
+    - dsv4-fp8-h200-dynamo-sglang-agentic-agg
+  description:
+    - "Add one H200 TP8 aggregated DeepSeek-V4-Pro AgentX recipe with MTP and HiCache."
+    - "Generate Pareto points by sweeping concurrency [1, 2, 4, 8, 16] while keeping the serving topology fixed."
+    - "Use Dynamo header-based session affinity and pin AIPerf to the merged PR #17 implementation."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2364

runners/launch_h200-dgxc-slurm.sh

@@ -4,6 +4,8 @@ set -eo pipefail
 # System-specific configuration for H200 DGXC Slurm cluster
 SLURM_PARTITION="main"
 SLURM_ACCOUNT="sa-shared"
+HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/models/gharunners/hf-hub-cache}"
+AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-/home/sa-shared/gharunners/ai-perf-cache}"
 
 set -x
 
@@ -13,7 +15,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
     # The yaml files specify HuggingFace model IDs for portability, but we use
     # local paths to avoid repeated downloading on the shared H200 cluster.
     if [[ $FRAMEWORK == "dynamo-sglang" ]]; then
-        if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then
+        if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp8" ]]; then
+            # The shared HF cache already contains the H200 FP8 checkpoint;
+            # default to that local path (overridable via DSV4_MODEL_PATH) so
+            # srtctl preflight finds the directory instead of trying to pull the
+            # hf: model ID, which fails on the compute node ("path is
+            # unavailable. Pull or register the model yourself").
+            export MODEL_PATH="${DSV4_MODEL_PATH:-${HF_HUB_CACHE_MOUNT}/DeepSeek-V4-Pro}"
+            export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro"
+        elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then
             export MODEL_PATH="/models/DeepSeek-R1-0528"
             export SRT_SLURM_MODEL_PREFIX="dsr1-fp8"
         else
@@ -42,7 +52,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
     fi
 
     # TODO(CJQ): make first class upon srt-slurm upstream refactor
-    if [[ "$IS_AGENTIC" == "1" ]]; then
+    if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then
+        # Overlay the single H200 aggregated recipe on the upstream release
+        # that provides custom benchmarks, Dynamo wheels, and affinity config.
+        git clone --branch v1.0.10 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+        mkdir -p recipes/sglang/deepseek-v4/agentic
+        cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml" \
+            recipes/sglang/deepseek-v4/agentic/
+    elif [[ "$IS_AGENTIC" == "1" ]]; then
         git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR"
         cd "$SRT_REPO_DIR"
     else
@@ -67,6 +85,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
     echo "Configs available at: $SRT_REPO_DIR/"
 
     # Map container images to local squash files based on framework
+    NGINX_IMAGE="nginx:1.27.4"
     NGINX_SQUASH_FILE="/data/containers/nginx+1.27.4.sqsh"
 
     if [[ $FRAMEWORK == "dynamo-sglang" ]]; then
@@ -79,6 +98,31 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
         SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh"
     fi
 
+    if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then
+        # This path intentionally tracks a recent public SGLang image instead
+        # of assuming the H200 cluster has already cached it. Import on a
+        # compute node and serialize concurrent jobs that share the cache.
+        import_squash() {
+            local squash="$1" image="$2"
+            local lock="${squash}.lock"
+            srun -N 1 --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" \
+                --exclusive --time=180 bash -c "
+                exec 9>\"$lock\"
+                flock -w 600 9 || { echo 'Failed to acquire lock for $squash' >&2; exit 1; }
+                if unsquashfs -l \"$squash\" > /dev/null 2>&1; then
+                    echo 'Squash file already exists and is valid, skipping import: $squash'
+                else
+                    rm -f \"$squash\"
+                    enroot import -o \"$squash\" docker://$image
+                    unsquashfs -l \"$squash\" > /dev/null 2>&1
+                fi
+            "
+        }
+
+        import_squash "$SQUASH_FILE" "$IMAGE"
+        import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE"
+    fi
+
     export ISL="$ISL"
     export OSL="$OSL"
     export EVAL_ONLY="${EVAL_ONLY:-false}"
@@ -98,6 +142,11 @@ gpus_per_node: 8
 network_interface: ""
 # Path to srtctl repo root (where the configs live)
 srtctl_root: "${SRTCTL_ROOT}"
+# Persistent AgentX dataset and Hugging Face caches mounted into every
+# server and benchmark container.
+default_mounts:
+  "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache"
+  "${HF_HUB_CACHE_MOUNT}": "/hf_hub_cache"
 # Model path aliases
 model_paths:
   "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
@@ -271,9 +320,6 @@ EOF
     find . -name '.nfs*' -delete 2>/dev/null || true
 
 else
-
-    HF_HUB_CACHE_MOUNT="/models/gharunners/hf-hub-cache"
-    AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache"
     SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
 
     # Convert pyxis image format (nvcr.io#path) to docker format (nvcr.io/path) for enroot import