From f9f8f156369d0ff0fae47aacd8b93f05a58139f7 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 9 Jul 2026 15:06:28 -0700 Subject: [PATCH 1/6] Add GLM-5 NVFP4 GB200 disaggregated TRT-LLM MTP benchmarks via Dynamo - Add glm5-fp4-gb200-dynamo-trt-mtp config entry (13 ISL1K + 10 ISL8K fixed-seq-len sweep points) using tensorrtllm-runtime:1.3.0-dev.1-cuda13. - Check in the corresponding gb200_nvfp4 MTP recipes locally so CONFIG_FILE=recipes/GLM5/... paths resolve at runtime via the overlay. - Wire runners/launch_gb200-nv.sh to route glm5+fp4 dynamo-trt to the GLM-5-NVFP4 model path and to clone srt-slurm + copy the local GLM5 recipes over the upstream recipes/GLM5 tree. - Append perf-changelog entry. --- ...ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml | 117 ++++++ ...ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml | 121 ++++++ ...ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml | 115 ++++++ .../ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml | 113 ++++++ .../ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml | 114 ++++++ .../ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml | 116 ++++++ .../ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml | 114 ++++++ .../ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml | 114 ++++++ .../ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml | 115 ++++++ .../ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml | 113 ++++++ ...tx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml | 129 +++++++ ...2dep4_gen1dep16_batch256_eplb256_mtp1.yaml | 148 +++++++ ...ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml | 177 +++++++++ ...x12dep4_gen1dep16_batch128_eplb0_mtp1.yaml | 129 +++++++ .../ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml | 113 ++++++ .../ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml | 113 ++++++ .../ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml | 113 ++++++ .../ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml | 113 ++++++ .../ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml | 113 ++++++ .../ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml | 114 ++++++ ...ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml | 117 ++++++ ...ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml | 115 ++++++ ...ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml | 121 ++++++ configs/nvidia-master.yaml | 365 ++++++++++++++++++ perf-changelog.yaml | 9 + runners/launch_gb200-nv.sh | 14 +- 26 files changed, 3154 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml new file mode 100644 index 0000000000..13da898bc4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml @@ -0,0 +1,117 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch32_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '615' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml new file mode 100644 index 0000000000..73b2d10adc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml @@ -0,0 +1,121 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch64_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '1229' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml new file mode 100644 index 0000000000..37912a792f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml @@ -0,0 +1,115 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '666' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml new file mode 100644 index 0000000000..9d09244308 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '180' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml new file mode 100644 index 0000000000..5af56e0452 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml @@ -0,0 +1,114 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '308' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml new file mode 100644 index 0000000000..6a40ed810c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml @@ -0,0 +1,116 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 4 + decode_nodes: 8 + gpus_per_decode: 8 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '92' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml new file mode 100644 index 0000000000..e2919b07c9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml @@ -0,0 +1,114 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 4 + decode_nodes: 8 + gpus_per_decode: 8 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '8' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml new file mode 100644 index 0000000000..cb043e80a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml @@ -0,0 +1,114 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 4 + decode_nodes: 8 + gpus_per_decode: 8 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '24' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml new file mode 100644 index 0000000000..87fc3c9899 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml @@ -0,0 +1,115 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 4 + decode_nodes: 8 + gpus_per_decode: 8 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '40' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml new file mode 100644 index 0000000000..3b76d83207 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 5 + decode_nodes: 5 + gpus_per_decode: 4 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '10' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml new file mode 100644 index 0000000000..ae75aea604 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml @@ -0,0 +1,129 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch128_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '2253' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml new file mode 100644 index 0000000000..ed5867951f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml @@ -0,0 +1,148 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch256_eplb256_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + moe_config: + backend: CUTEDSL + load_balancer: + layer_updates_per_iter: 1 + num_slots: 256 + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml new file mode 100644 index 0000000000..d21c50f2f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml @@ -0,0 +1,177 @@ +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep8_batch512_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 2 + gpus_per_decode: 8 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml new file mode 100644 index 0000000000..1d555a2504 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml @@ -0,0 +1,129 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx12dep4_gen1dep16_batch128_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 12 + prefill_workers: 12 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2253' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml new file mode 100644 index 0000000000..9662511d2d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen1dep32_batch2_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '90' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml new file mode 100644 index 0000000000..f82c0c2c6f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 5 + decode_nodes: 5 + gpus_per_decode: 4 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '5' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml new file mode 100644 index 0000000000..99ff55044f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 5 + decode_nodes: 5 + gpus_per_decode: 4 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '15' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml new file mode 100644 index 0000000000..d72a5d0f2e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_workers: 5 + decode_nodes: 5 + gpus_per_decode: 4 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '30' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml new file mode 100644 index 0000000000..5488068262 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml @@ -0,0 +1,113 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '180' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml new file mode 100644 index 0000000000..13905b0957 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml @@ -0,0 +1,114 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 4 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '333' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml new file mode 100644 index 0000000000..0d28018a20 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml @@ -0,0 +1,117 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx6dep4_gen1dep16_batch32_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 6 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '615' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml new file mode 100644 index 0000000000..3b1c6affcd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml @@ -0,0 +1,115 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx7dep4_gen1dep32_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 7 + prefill_workers: 7 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 8 + gpus_per_decode: 32 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '666' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml new file mode 100644 index 0000000000..48a2401410 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml @@ -0,0 +1,121 @@ +name: glm5_nvfp4_ISL8K_OSL1K_ctx8dep4_gen1dep16_batch64_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + prefill_nodes: 8 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_workers: 1 + decode_nodes: 4 + gpus_per_decode: 16 + gpus_per_node: 4 +backend: + type: trtllm + prefill_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + decode_environment: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + trtllm_config: + prefill: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: null + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1127' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..28f1b28f07 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2630,6 +2630,371 @@ glm5-fp4-gb300-dynamo-trt: ep: 16 dp-attn: true +glm5-fp4-gb200-dynamo-trt-mtp: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb200 + precision: fp4 + framework: dynamo-trt + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 8 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 10 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 24 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 40 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 92 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 308 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 666 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1229 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 4301 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 4301 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 15 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 90 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 333 ] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1127 ] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 666 ] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 12 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp8-b200-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee8..7fbeb7389d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4699,3 +4699,12 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - glm5-fp4-gb200-dynamo-trt-mtp + description: + - "Add GLM-5 NVFP4 GB200 disaggregated TRT-LLM MTP (spec-decoding) benchmarks via Dynamo" + - "New multinode model: glm5 with dynamo-trt framework on GB200, MTP variant" + - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb200_nvfp4 MTP recipes); prefill tp=4/ep=4 (dep4)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 7408df9fd1..7751da26c9 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -141,8 +141,14 @@ elif [[ $FRAMEWORK == "dynamo-trt" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SERVED_MODEL_NAME="kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="nvidia/Kimi-K2.5-NVFP4" + elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then + # SRT_SLURM_MODEL_PREFIX matches the model.path alias + # ("nvidia/GLM-5-NVFP4") in the upstream GLM5 trtllm_dynamo recipes. + export MODEL_PATH="/home/sa-shared/glm-model/GLM-5-NVFP4" + export SERVED_MODEL_NAME="glm-5-nvfp4" + export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" else - echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: gptoss, dsr1, or kimik2.5" + echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: gptoss, dsr1, kimik2.5, or glm5" exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then @@ -359,6 +365,12 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 +elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout sa-submission-q2-2026 + mkdir -p recipes/GLM5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/GLM5" recipes/GLM5 else git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From bfc844122a1497af7530ff8b9dc6f70942eda837 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 9 Jul 2026 15:06:44 -0700 Subject: [PATCH 2/6] Update perf-changelog pr-link for #1800 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7fbeb7389d..4f278aea2a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4707,4 +4707,4 @@ - "New multinode model: glm5 with dynamo-trt framework on GB200, MTP variant" - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb200_nvfp4 MTP recipes); prefill tp=4/ep=4 (dep4)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1800 From c9da83008586895bf74180828a776f6ac85f1979 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 9 Jul 2026 15:51:07 -0700 Subject: [PATCH 3/6] Point glm5-fp4-gb200-dynamo-trt-mtp MODEL_PATH at Lustre Prior /home/sa-shared path was not present on the watchtower login node (the launcher's model-path pre-check surfaced FileNotFoundError immediately). Move MODEL_PATH to /mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4, which is on Lustre and cross-mounted to every watchtower compute node. Mirrors the same fix applied to the non-MTP config. --- runners/launch_gb200-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 7751da26c9..7668ba538e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -144,7 +144,7 @@ elif [[ $FRAMEWORK == "dynamo-trt" ]]; then elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX matches the model.path alias # ("nvidia/GLM-5-NVFP4") in the upstream GLM5 trtllm_dynamo recipes. - export MODEL_PATH="/home/sa-shared/glm-model/GLM-5-NVFP4" + export MODEL_PATH="/mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4" export SERVED_MODEL_NAME="glm-5-nvfp4" export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" else From 5f0faec5c10df5428de834d95486d3ac83eb0352 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 9 Jul 2026 18:06:36 -0700 Subject: [PATCH 4/6] Pin srt-slurm to v1.0.17 + adopt recipes/trtllm/glm5 overlay layout - Rename benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/ gb200_nvfp4/**/MTP/** -> benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/... . - launch_gb200-nv.sh dynamo-trt+glm5 branch: git checkout v1.0.17, mkdir -p recipes/trtllm/glm5, cp -rT src/dst updated. - configs/nvidia-master.yaml: 23 CONFIG_FILE= paths under glm5-fp4-gb200-dynamo-trt-mtp retargeted to recipes/trtllm/glm5/... ; matching doc-URL comments repointed at v1.0.17. - perf-changelog: PR 1800 description updated to v1.0.17 + new overlay path. Mirrors the same treatment applied to the non-MTP config. gb300 sibling entries (recipes/GLM5/.../gb300_nvfp4/**) intentionally untouched - separate PR. --- ...ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml | 0 ...ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml | 0 ...ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml | 0 ...tx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml | 0 ...2dep4_gen1dep16_batch256_eplb256_mtp1.yaml | 0 ...ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml | 0 ...x12dep4_gen1dep16_batch128_eplb0_mtp1.yaml | 0 .../ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml | 0 .../ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml | 0 .../ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml | 0 .../ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml | 0 ...ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml | 0 ...ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml | 0 ...ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml | 0 configs/nvidia-master.yaml | 92 +++++++++---------- perf-changelog.yaml | 2 +- runners/launch_gb200-nv.sh | 6 +- 26 files changed, 50 insertions(+), 50 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml (100%) rename benchmarks/multi_node/srt-slurm-recipes/{GLM5 => trtllm/glm5}/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml (100%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml similarity index 100% rename from benchmarks/multi_node/srt-slurm-recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 28f1b28f07..185643a53e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2653,8 +2653,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2668,8 +2668,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -2683,8 +2683,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2698,8 +2698,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2713,8 +2713,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2728,8 +2728,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2743,8 +2743,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2758,8 +2758,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2773,8 +2773,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2788,8 +2788,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2803,8 +2803,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2818,8 +2818,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2833,8 +2833,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -2852,8 +2852,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -2867,8 +2867,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -2882,8 +2882,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -2897,8 +2897,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2912,8 +2912,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2927,8 +2927,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2942,8 +2942,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2957,8 +2957,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2972,8 +2972,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2987,8 +2987,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4f278aea2a..bf8e9f7d18 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4706,5 +4706,5 @@ - "Add GLM-5 NVFP4 GB200 disaggregated TRT-LLM MTP (spec-decoding) benchmarks via Dynamo" - "New multinode model: glm5 with dynamo-trt framework on GB200, MTP variant" - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" - - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb200_nvfp4 MTP recipes); prefill tp=4/ep=4 (dep4)" + - "Recipes sourced from NVIDIA/srt-slurm tag v1.0.17 (gb200_nvfp4 MTP recipes) and staged under the recipes/trtllm/glm5 overlay; prefill tp=4/ep=4 (dep4)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1800 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 7668ba538e..77cee3bf6e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -368,9 +368,9 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" ]]; then elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 - mkdir -p recipes/GLM5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/GLM5" recipes/GLM5 + git checkout v1.0.17 + mkdir -p recipes/trtllm/glm5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5" recipes/trtllm/glm5 else git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From 8ebe7238deee22e9757d7644eeb6c3ae4a04cc8d Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 14 Jul 2026 16:56:57 -0700 Subject: [PATCH 5/6] update srt-slurm checkpoint --- configs/nvidia-master.yaml | 46 +++++++++++++++++++------------------- runners/launch_gb200-nv.sh | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 185643a53e..2cc6292ea7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2653,7 +2653,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" decode: num-worker: 4 @@ -2668,7 +2668,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" decode: num-worker: 5 @@ -2683,7 +2683,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml" decode: num-worker: 4 @@ -2698,7 +2698,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml" decode: num-worker: 4 @@ -2713,7 +2713,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml" decode: num-worker: 4 @@ -2728,7 +2728,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2743,7 +2743,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2758,7 +2758,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2773,7 +2773,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2788,7 +2788,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml" decode: num-worker: 1 @@ -2803,7 +2803,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml" decode: num-worker: 1 @@ -2818,7 +2818,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml" decode: num-worker: 1 @@ -2833,7 +2833,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml" decode: num-worker: 1 @@ -2852,7 +2852,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" decode: num-worker: 5 @@ -2867,7 +2867,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" decode: num-worker: 5 @@ -2882,7 +2882,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" decode: num-worker: 5 @@ -2897,7 +2897,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2912,7 +2912,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2927,7 +2927,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2942,7 +2942,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2957,7 +2957,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" decode: num-worker: 1 @@ -2972,7 +2972,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" decode: num-worker: 1 @@ -2987,7 +2987,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.17/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" decode: num-worker: 1 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 77cee3bf6e..a5f917da09 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -368,7 +368,7 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" ]]; then elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.17 + git checkout v1.0.26 mkdir -p recipes/trtllm/glm5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5" recipes/trtllm/glm5 else From 851bd20c5e7bd3fbc51c67be7680eaba6569f59c Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Wed, 15 Jul 2026 12:13:40 -0700 Subject: [PATCH 6/6] update kv p2p --- configs/nvidia-master.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a83bff8a44..18d70129b6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2648,6 +2648,8 @@ glm5-fp4-gb200-dynamo-trt-mtp: runner: gb200 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl multinode: true disagg: true scenarios: