diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index e7bdf27542..68cd412217 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2000,7 +2000,17 @@ build_replay_cmd() { REPLAY_CMD+=" --endpoint /v1/chat/completions" REPLAY_CMD+=" --endpoint-type chat" REPLAY_CMD+=" --streaming" - REPLAY_CMD+=" --model $MODEL" + # SERVED_MODEL_NAME overrides $MODEL when the frontend registers the + # model under a different name than the recipe's model.path alias (e.g. + # dynamo-trt srt-slurm recipes serve "DeepSeek-V4-Pro" while $MODEL is + # the HF id "deepseek-ai/DeepSeek-V4-Pro"). Mismatches 404 at warmup. + REPLAY_CMD+=" --model ${SERVED_MODEL_NAME:-$MODEL}" + # aiperf's dataset manager resolves the tokenizer from --model by + # default, but a SERVED_MODEL_NAME override (above) is a wire name, not + # necessarily a valid HF repo id (e.g. "Qwen3.5-397B-A17B-NVFP4-V2" vs + # the real "nvidia/Qwen3.5-397B-A17B-NVFP4-V2"), which 404s tokenizer + # loading. Always pass the real HF id explicitly. + REPLAY_CMD+=" --tokenizer $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --stats-interval 30" diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..d7f1aeca78 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml @@ -0,0 +1,217 @@ +name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 1 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 2 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 8 + max_num_tokens: 56 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 2 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml new file mode 100644 index 0000000000..88ed3c1672 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml @@ -0,0 +1,214 @@ +name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 14 + decode_workers: 7 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 1 + max_num_tokens: 8 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 7 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml new file mode 100644 index 0000000000..06410e107c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml @@ -0,0 +1,216 @@ +name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 1 + decode_nodes: 1 + decode_workers: 2 + gpus_per_decode: 2 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 4 + max_num_tokens: 28 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 2 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..726d25ba6c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml @@ -0,0 +1,231 @@ +name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..442a9dec41 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml @@ -0,0 +1,215 @@ +name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 8 + max_num_tokens: 56 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml new file mode 100644 index 0000000000..797ceccdaf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml @@ -0,0 +1,218 @@ +name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 32 + max_num_tokens: 224 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9d55b30631..42f146ac89 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6690,6 +6690,111 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true +qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # 1P1D: 1 prefill (TP1/EP1/dp-attn), 1 decode (TP2/EP2), conc=44 + - spec-decoding: "mtp" + conc-list: [44] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + # 1P7D: 1 prefill (TP4/EP4/dp-attn), 7 decode (TP8/EP8), conc=7 + - spec-decoding: "mtp" + conc-list: [7] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml" + decode: + num-worker: 7 + tp: 8 + ep: 8 + dp-attn: false + # 2P2D: 2 prefill (TP1/EP1/dp-attn), 2 decode (TP2/EP2), conc=52 + - spec-decoding: "mtp" + conc-list: [52] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml" + decode: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP8/EP8), conc=96 + - spec-decoding: "mtp" + conc-list: [96] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + # 3P1D: 3 prefill (TP4/EP4/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=565 + - spec-decoding: "mtp" + conc-list: [565] + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 3P2D: 3 prefill (TP4/EP4/dp-attn), 2 decode (TP4/EP4/dp-attn), conc=704 + - spec-decoding: "mtp" + conc-list: [704] + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57e53d9acb..acd2b5a7ce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6028,3 +6028,15 @@ - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 40, 48, 56 and 70, drafting with DSpark level 7 at concurrency 8 and below, DSpark level 3 at concurrency 16" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2616 +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX with Dynamo router 1.4.0.dev20260807 and NIXL KV transfer." + - "Cover 6 disaggregated topologies (1P1D through 3P2D) with MTP speculative decoding across conc 7-704 using TRT-LLM release:1.3.0rc24." + - "Mixed attention-DP (dep) and non-DP (tep) decode shapes; recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/." + - "EVAL_ONLY runs switch to real block verification by removing synthetic acceptance (TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS stripped from all agentx recipes before srtctl)." + - "Add SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 to all 6 agentx recipe env blocks and make build_replay_cmd prefer SERVED_MODEL_NAME over $MODEL for the aiperf --model flag." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2612 + diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 83252461d0..88b964a8ad 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -190,7 +190,19 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.50 + TRTLLM_RECIPES_DIR="recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" + mkdir -p "$TRTLLM_RECIPES_DIR" + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" \ + "$TRTLLM_RECIPES_DIR" + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ + -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + fi +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check):