Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX

benchmarks/benchmark_lib.sh

@@ -2000,7 +2000,17 @@ build_replay_cmd() {
     REPLAY_CMD+=" --endpoint /v1/chat/completions"
     REPLAY_CMD+=" --endpoint-type chat"
     REPLAY_CMD+=" --streaming"
-    REPLAY_CMD+=" --model $MODEL"
+    # SERVED_MODEL_NAME overrides $MODEL when the frontend registers the
+    # model under a different name than the recipe's model.path alias (e.g.
+    # dynamo-trt srt-slurm recipes serve "DeepSeek-V4-Pro" while $MODEL is
+    # the HF id "deepseek-ai/DeepSeek-V4-Pro"). Mismatches 404 at warmup.
+    REPLAY_CMD+=" --model ${SERVED_MODEL_NAME:-$MODEL}"
+    # aiperf's dataset manager resolves the tokenizer from --model by
+    # default, but a SERVED_MODEL_NAME override (above) is a wire name, not
+    # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs
+    # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer
+    # loading. Always pass the real HF id explicitly.
+    REPLAY_CMD+=" --tokenizer $MODEL"
     REPLAY_CMD+=" --concurrency $CONC"
     REPLAY_CMD+=" --benchmark-duration $duration"
     REPLAY_CMD+=" --stats-interval 30"

benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml

@@ -0,0 +1,217 @@
+name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload
+model:
+  path: qwen3.5-fp4
+  container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24
+  precision: fp4
+
+dynamo:
+  install: true
+  wheel: 1.4.0.dev20260807
+
+identity:
+  model:
+    repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+  container:
+    image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24
+  frameworks:
+    tensorrt_llm: 1.3.0rc24
+
+slurm:
+  time_limit: 04:00:00
+health_check:
+  max_attempts: 540
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 4
+  prefill_nodes: 1
+  prefill_workers: 1
+  gpus_per_prefill: 1
+  decode_nodes: 1
+  decode_workers: 1
+  gpus_per_decode: 2
+backend:
+  type: trtllm
+  prefill_environment:
+    CUDA_SCALE_LAUNCH_QUEUES: 4x
+    MIMALLOC_ARENA_RESERVE: '0'
+    MIMALLOC_PURGE_DELAY: ''
+    NCCL_GRAPH_MIXING_SUPPORT: '0'
+    PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+    TLLM_LOG_LEVEL: INFO
+    TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80'
+    TRTLLM_ENABLE_PDL: '1'
+    TRTLLM_PINNED_WEIGHT_STAGING: '1'
+    TRTLLM_SERVER_DISABLE_GC: '1'
+    TRTLLM_WORKER_DISABLE_GC: '1'
+    TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1'
+    UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
+    DYN_ENGINE_CONV_AFFINITY: "1"
+    DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+  decode_environment:
+    MIMALLOC_ARENA_RESERVE: '0'
+    MIMALLOC_PURGE_DELAY: ''
+    NCCL_GRAPH_MIXING_SUPPORT: '0'
+    PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+    TLLM_LOG_LEVEL: INFO
+    TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80'
+    TRTLLM_ENABLE_PDL: '1'
+    TRTLLM_PINNED_WEIGHT_STAGING: '1'
+    TRTLLM_SERVER_DISABLE_GC: '1'
+    TRTLLM_WORKER_DISABLE_GC: '1'
+    TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1'
+    UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
+    DYN_ENGINE_CONV_AFFINITY: "1"
+    DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+  trtllm_config:
+    prefill:
+      attention_dp_config:
+        kv_cache_routing_conversation_affinity: true
+      cache_transceiver_config:
+        backend: NIXL
+        kv_transfer_sender_future_timeout_ms: 20
+        kv_transfer_timeout_ms: 600000
+        transceiver_runtime: PYTHON
+      cuda_graph_config: null
+      disable_overlap_scheduler: false
+      enable_attention_dp: true
+      enable_chunked_prefill: true
+      enable_iter_perf_stats: false
+      enable_iter_req_stats: false
+      kv_cache_config:
+        block_reuse_config:
+          max_num_turns: 3
+          policy: per_conversation
+        dtype: fp8
+        enable_block_reuse: true
+        event_buffer_max_size: 0
+        free_gpu_memory_fraction: 0.8
+        host_cache_size: 137438953472
+        mamba_state_config:
+          additional_snapshot_offsets_from_end:
+          - 2
+          periodic_snapshot_interval: 0
+        pool_ratio:
+        - 0.8
+        - 0.2
+        tokens_per_block: 64
+        use_kv_cache_manager_v2: true
+      max_batch_size: 64
+      max_num_tokens: 8192
+      max_seq_len: 262144
+      moe_config:
+        backend: CUTEDSL
+      moe_expert_parallel_size: 1
+      num_serve_frontends: 8
+      nvfp4_gemm_config:
+        allowed_backends:
+        - cutlass
+        - cublaslt
+        - cutedsl
+        - cuda_core
+      pipeline_parallel_size: 1
+      print_iter_log: true
+      return_perf_metrics: false
+      scheduler_config:
+        capacity_scheduler_policy: MAX_UTILIZATION
+      tensor_parallel_size: 1
+      torch_compile_config:
+        capture_num_tokens:
+        - 1
+        - 2
+        - 4
+        - 8
+        - 16
+        - 32
+        - 64
+        - 128
+        - 256
+        - 512
+        - 1024
+        - 2048
+        - 4096
+        - 8192
+        enable_fullgraph: true
+        enable_piecewise_cuda_graph: true
+      trust_remote_code: true
+    decode:
+      cache_transceiver_config:
+        backend: NIXL
+        kv_transfer_sender_future_timeout_ms: 20
+        kv_transfer_timeout_ms: 600000
+        transceiver_runtime: PYTHON
+      cuda_graph_config:
+        batch_sizes:
+        - 1
+        - 2
+        - 4
+        - 8
+        enable_padding: true
+      enable_attention_dp: false
+      enable_iter_perf_stats: false
+      enable_iter_req_stats: false
+      enable_lm_head_tp_in_adp: false
+      kv_cache_config:
+        avg_seq_len: 102150
+        dtype: fp8
+        enable_block_reuse: false
+        event_buffer_max_size: 0
+        free_gpu_memory_fraction: 0.85
+        host_cache_size: 137438953472
+        tokens_per_block: 64
+        use_kv_cache_manager_v2: true
+      max_batch_size: 8
+      max_num_tokens: 56
+      max_seq_len: 262148
+      moe_config:
+        backend: CUTEDSL
+        use_low_precision_moe_combine: true
+      moe_expert_parallel_size: 2
+      num_postprocess_workers: 4
+      num_serve_frontends: 4
+      nvfp4_gemm_config:
+        allowed_backends:
+        - cutlass
+        - cublaslt
+        - cutedsl
+        - cuda_core
+      pipeline_parallel_size: 1
+      print_iter_log: true
+      return_perf_metrics: false
+      scheduler_config:
+        capacity_scheduler_policy: GUARANTEED_NO_EVICT
+      speculative_config:
+        decoding_type: MTP
+        max_draft_len: 6
+      stream_interval: 20
+      tensor_parallel_size: 2
+      trust_remote_code: true
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  env:
+    ETCD_LEASE_TTL: "120"
+    DYN_TOKENIZER_CACHE: "1"
+    DYN_TOKENIZER_CACHE_BYTES: "8000000000"
+    DYN_TOKENIZER: "fastokens"
+  args:
+    router-mode: kv
+    router-session-affinity-ttl-secs: '14400'
+    active-decode-blocks-threshold: None
+    active-prefill-tokens-threshold: None
+    active-prefill-tokens-threshold-frac: None
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'true'
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2