From b82c6c6a719612f78c5579bc10529da3a6cf8981 Mon Sep 17 00:00:00 2001 From: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> Date: Thu, 9 Apr 2026 01:31:53 -0700 Subject: [PATCH 1/2] [https://nvbugs/5844149][fix] Fix OOM in ctx server and IMA in gen server Signed-off-by: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> --- .../disaggregated/slurm/benchmark/config.yaml | 128 ++++++++---------- .../config_nvbug5844149_con1_dep32_mtp3.yaml | 123 +++++++++++++++++ ..._nvbug5844149_con2048_dep32_mtp1_eplb.yaml | 126 +++++++++++++++++ ...config_nvbug5844149_con256_dep32_mtp3.yaml | 123 +++++++++++++++++ .../slurm/benchmark/disaggr_torch.slurm | 2 +- jenkins/scripts/perf/local/submit.py | 9 +- ...tx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml | 17 ++- ..._dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml | 12 +- ...x1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml | 14 +- 9 files changed, 467 insertions(+), 87 deletions(-) create mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml create mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml create mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml diff --git a/examples/disaggregated/slurm/benchmark/config.yaml b/examples/disaggregated/slurm/benchmark/config.yaml index e4bda5707d07..c570ac850b3c 100644 --- a/examples/disaggregated/slurm/benchmark/config.yaml +++ b/examples/disaggregated/slurm/benchmark/config.yaml @@ -1,25 +1,25 @@ # SLURM Configuration slurm: script_file: "disaggr_torch.slurm" - partition: "" - account: "" + partition: "gb200" + account: "coreai_comparch_trtllm" job_time: "02:00:00" - job_name: "" + job_name: "nvbug5844149-con1-tep8-mtp3" extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" set_segment: true # Optional: whether to set the segment for the job numa_bind: true # Only enable for GB200/GB300 NVL72 # Benchmark Mode benchmark: - mode: "e2e" # Options: e2e, gen_only, gen_only_no_context + mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script - multi_round: 8 # Number of benchmark rounds - benchmark_ratio: 0.8 # Benchmark ratio + multi_round: 1 # Number of benchmark rounds + benchmark_ratio: 1.0 # Benchmark ratio streaming: true # Enable streaming mode - concurrency_list: "16" - input_length: 1024 # Input sequence length - output_length: 1024 # Output sequence length - dataset_file: "" + concurrency_list: "1" + input_length: 32768 # Input sequence length + output_length: 4096 # Output sequence length + dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" # Hardware Configuration hardware: @@ -29,14 +29,13 @@ hardware: # Environment Configuration environment: - container_mount: "" # Format: path1:path1,path2:path2 - container_image: "" - model_path: "" - trtllm_repo: "" + container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" + container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" + model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" + trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" build_wheel: false # Don't build the wheel when launching multiple jobs - cuda_architectures: "" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures - trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - work_dir: "" + cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures + trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" @@ -51,83 +50,74 @@ accuracy: enable_accuracy_test: false # Set to true to enable accuracy evaluation tasks: gsm8k: - model: "local-completions" # Model type for lm_eval + model: "local-completions" model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" extra_kwargs: trust_remote_code: true worker_config: gen: + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 8 + max_input_len: 32784 + max_seq_len: 40960 tensor_parallel_size: 8 moe_expert_parallel_size: 8 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false pipeline_parallel_size: 1 context_parallel_size: 1 - # Uncomment this section to enable context parallelism. - # cp_config: - # cp_type: "HELIX" - # tokens_per_block: 32 # must match kv_config.tokens_per_block. - max_batch_size: 256 - max_num_tokens: 512 - max_seq_len: 2251 cuda_graph_config: enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 768 - - 1024 - - 2048 - - 256 - print_iter_log: true - trust_remote_code: true + max_batch_size: 1 kv_cache_config: enable_block_reuse: false - tokens_per_block: 32 - free_gpu_memory_fraction: 0.8 + free_gpu_memory_fraction: 0.25 dtype: fp8 + tokens_per_block: 64 moe_config: - backend: CUTLASS - use_low_precision_moe_combine: true - load_balancer: - num_slots: 0 + backend: TRTLLM cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: DEFAULT - stream_interval: 20 - num_postprocess_workers: 4 - speculative_config: # when mtp_size is 0, remove this section. + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL ctx: - max_batch_size: 4 - max_num_tokens: 4608 - max_seq_len: 1227 + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 tensor_parallel_size: 4 - context_parallel_size: 1 moe_expert_parallel_size: 4 - enable_attention_dp: true pipeline_parallel_size: 1 - print_iter_log: true - trust_remote_code: true + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true cuda_graph_config: null - disable_overlap_scheduler: true kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.85 + free_gpu_memory_fraction: 0.4 dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL cache_transceiver_config: - max_tokens_in_buffer: 4608 - backend: DEFAULT - speculative_config: # when mtp_size is 0, remove this section. + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + num_nextn_predict_layers: 3 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml new file mode 100644 index 000000000000..c570ac850b3c --- /dev/null +++ b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml @@ -0,0 +1,123 @@ +# SLURM Configuration +slurm: + script_file: "disaggr_torch.slurm" + partition: "gb200" + account: "coreai_comparch_trtllm" + job_time: "02:00:00" + job_name: "nvbug5844149-con1-tep8-mtp3" + extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" + set_segment: true # Optional: whether to set the segment for the job + numa_bind: true # Only enable for GB200/GB300 NVL72 + +# Benchmark Mode +benchmark: + mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context + use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script + multi_round: 1 # Number of benchmark rounds + benchmark_ratio: 1.0 # Benchmark ratio + streaming: true # Enable streaming mode + concurrency_list: "1" + input_length: 32768 # Input sequence length + output_length: 4096 # Output sequence length + dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" + +# Hardware Configuration +hardware: + gpus_per_node: 4 # Modify this with your hardware configuration + num_ctx_servers: 1 # Number of context servers + num_gen_servers: 1 # Number of generation servers + +# Environment Configuration +environment: + container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" + container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" + model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" + trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" + build_wheel: false # Don't build the wheel when launching multiple jobs + cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures + trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" + server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" + +# Profiling Configuration +profiling: + nsys_on: false # Set to true to enable profiling + ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers + gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers + +# Accuracy Configuration +accuracy: + enable_accuracy_test: false # Set to true to enable accuracy evaluation + tasks: + gsm8k: + model: "local-completions" + model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" + extra_kwargs: + trust_remote_code: true + +worker_config: + gen: + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 8 + max_input_len: 32784 + max_seq_len: 40960 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + context_parallel_size: 1 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.25 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml new file mode 100644 index 000000000000..69962ecf8fef --- /dev/null +++ b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml @@ -0,0 +1,126 @@ +# SLURM Configuration +slurm: + script_file: "disaggr_torch.slurm" + partition: "gb200" + account: "coreai_comparch_trtllm" + job_time: "02:00:00" + job_name: "nvbug5844149-con2048-dep32-mtp1" + extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" + set_segment: true # Optional: whether to set the segment for the job + numa_bind: true # Only enable for GB200/GB300 NVL72 + +# Benchmark Mode +benchmark: + mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context + use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script + multi_round: 1 # Number of benchmark rounds + benchmark_ratio: 1.0 # Benchmark ratio + streaming: true # Enable streaming mode + concurrency_list: "2048" + input_length: 32768 # Input sequence length + output_length: 4096 # Output sequence length + dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" + +# Hardware Configuration +hardware: + gpus_per_node: 4 # Modify this with your hardware configuration + num_ctx_servers: 1 # Number of context servers + num_gen_servers: 1 # Number of generation servers + +# Environment Configuration +environment: + container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" + container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" + model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" + trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" + build_wheel: true # Don't build the wheel when launching multiple jobs + cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures + # trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" + server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" + +# Profiling Configuration +profiling: + nsys_on: false # Set to true to enable profiling + ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers + gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers + +# Accuracy Configuration +accuracy: + enable_accuracy_test: false # Set to true to enable accuracy evaluation + tasks: + gsm8k: + model: "local-completions" + model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" + extra_kwargs: + trust_remote_code: true + +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 128 + max_input_len: 32784 + max_seq_len: 40960 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + context_parallel_size: 1 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL + load_balancer: + num_slots: 288 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + ctx: + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml new file mode 100644 index 000000000000..c228950afe2d --- /dev/null +++ b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml @@ -0,0 +1,123 @@ +# SLURM Configuration +slurm: + script_file: "disaggr_torch.slurm" + partition: "gb200" + account: "coreai_comparch_trtllm" + job_time: "02:00:00" + job_name: "nvbug5844149-con256-dep32-mtp3" + extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" + set_segment: true # Optional: whether to set the segment for the job + numa_bind: true # Only enable for GB200/GB300 NVL72 + +# Benchmark Mode +benchmark: + mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context + use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script + multi_round: 1 # Number of benchmark rounds + benchmark_ratio: 1.0 # Benchmark ratio + streaming: true # Enable streaming mode + concurrency_list: "256" + input_length: 32768 # Input sequence length + output_length: 4096 # Output sequence length + dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" + +# Hardware Configuration +hardware: + gpus_per_node: 4 # Modify this with your hardware configuration + num_ctx_servers: 1 # Number of context servers + num_gen_servers: 1 # Number of generation servers + +# Environment Configuration +environment: + container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" + container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" + model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" + trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" + build_wheel: true # Don't build the wheel when launching multiple jobs + cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures + # trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" + server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" + +# Profiling Configuration +profiling: + nsys_on: false # Set to true to enable profiling + ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers + gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers + +# Accuracy Configuration +accuracy: + enable_accuracy_test: false # Set to true to enable accuracy evaluation + tasks: + gsm8k: + model: "local-completions" + model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" + extra_kwargs: + trust_remote_code: true + +worker_config: + gen: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 64 + max_input_len: 32784 + max_seq_len: 40960 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + context_parallel_size: 1 + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + ctx: + print_iter_log: true + max_batch_size: 1 + max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + dtype: fp8 + tokens_per_block: 64 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + max_tokens_in_buffer: 32832 + backend: UCX + disable_overlap_scheduler: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm index 9e0d66144478..52408db31130 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py index 7d2954429f71..9af04626da86 100755 --- a/jenkins/scripts/perf/local/submit.py +++ b/jenkins/scripts/perf/local/submit.py @@ -286,8 +286,6 @@ def generate_sbatch_params(args, hardware_config, work_dir): f"#SBATCH --segment={total_nodes}", f"#SBATCH --ntasks={total_gpus}", f"#SBATCH --ntasks-per-node={gpus_per_node}", - f"#SBATCH --gpus-per-node={gpus_per_node}", - f"#SBATCH --gres=gpu:{gpus_per_node}", f"#SBATCH --partition={args.partition}", f"#SBATCH --time={args.time}", f"#SBATCH --account={args.account}", @@ -550,6 +548,13 @@ def main(): test_name=select_pattern, ) + if benchmark_mode in ("gen_only",): + config.setdefault("benchmark", {})["multi_round"] = 1 + modified_config_path = os.path.join(work_dir, os.path.basename(config_yaml)) + with open(modified_config_path, "w") as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + config_yaml = modified_config_path + # Generate sbatch params sbatch_lines = generate_sbatch_params(args, hardware_config, work_dir) diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml index 861b1fab55d1..dc6288dfa58b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml @@ -18,7 +18,7 @@ benchmark: mode: e2e use_nv_sa_benchmark: false multi_round: 10 - benchmark_ratio: 0.0 + benchmark_ratio: 1.0 streaming: true concurrency_list: '1' input_length: 32768 @@ -48,24 +48,27 @@ worker_config: gen: print_iter_log: true max_batch_size: 1 - max_num_tokens: 256 + max_num_tokens: 8 + max_input_len: 32784 + max_seq_len: 40960 tensor_parallel_size: 8 moe_expert_parallel_size: 8 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: false + enable_lm_head_tp_in_adp: false cuda_graph_config: enable_padding: true max_batch_size: 1 kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.9 + free_gpu_memory_fraction: 0.25 dtype: fp8 tokens_per_block: 64 moe_config: backend: TRTLLM cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: &id001 @@ -84,6 +87,8 @@ worker_config: print_iter_log: true max_batch_size: 1 max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 @@ -93,14 +98,14 @@ worker_config: cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.4 dtype: fp8 tokens_per_block: 64 moe_config: backend: CUTEDSL use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 502f14dc896d..a2b8f7b1a8f4 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -49,6 +49,8 @@ worker_config: print_iter_log: true max_batch_size: 64 max_num_tokens: 128 + max_input_len: 32784 + max_seq_len: 40960 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 @@ -60,7 +62,7 @@ worker_config: max_batch_size: 64 kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.85 + free_gpu_memory_fraction: 0.5 dtype: fp8 tokens_per_block: 64 moe_config: @@ -70,7 +72,7 @@ worker_config: num_slots: 288 layer_updates_per_iter: 1 cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: &id001 @@ -88,6 +90,8 @@ worker_config: print_iter_log: true max_batch_size: 1 max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 @@ -97,14 +101,14 @@ worker_config: cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.4 dtype: fp8 tokens_per_block: 64 moe_config: backend: CUTEDSL use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml index 24eb58b3b976..bdb9d1ded5ef 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml @@ -48,7 +48,9 @@ worker_config: gen: print_iter_log: true max_batch_size: 8 - max_num_tokens: 256 + max_num_tokens: 64 + max_input_len: 32784 + max_seq_len: 40960 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 @@ -60,14 +62,14 @@ worker_config: max_batch_size: 8 kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.85 + free_gpu_memory_fraction: 0.5 dtype: fp8 tokens_per_block: 64 moe_config: backend: CUTEDSL use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: &id001 @@ -85,6 +87,8 @@ worker_config: print_iter_log: true max_batch_size: 1 max_num_tokens: 32784 + max_input_len: 32784 + max_seq_len: 32832 tensor_parallel_size: 4 moe_expert_parallel_size: 4 pipeline_parallel_size: 1 @@ -94,14 +98,14 @@ worker_config: cuda_graph_config: null kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.6 + free_gpu_memory_fraction: 0.4 dtype: fp8 tokens_per_block: 64 moe_config: backend: CUTEDSL use_low_precision_moe_combine: true cache_transceiver_config: - max_tokens_in_buffer: 32768 + max_tokens_in_buffer: 32832 backend: UCX disable_overlap_scheduler: true speculative_config: *id001 From e0838e92d62ba1405a9ffa0bc05ccfd9f069be5b Mon Sep 17 00:00:00 2001 From: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> Date: Tue, 14 Apr 2026 23:10:37 +0000 Subject: [PATCH 2/2] cleanup Signed-off-by: Balaram Buddharaju <169953907+brb-nv@users.noreply.github.com> --- .../disaggregated/slurm/benchmark/config.yaml | 128 ++++++++++-------- .../config_nvbug5844149_con1_dep32_mtp3.yaml | 123 ----------------- ..._nvbug5844149_con2048_dep32_mtp1_eplb.yaml | 126 ----------------- ...config_nvbug5844149_con256_dep32_mtp3.yaml | 123 ----------------- .../slurm/benchmark/disaggr_torch.slurm | 2 +- 5 files changed, 70 insertions(+), 432 deletions(-) delete mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml delete mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml delete mode 100644 examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml diff --git a/examples/disaggregated/slurm/benchmark/config.yaml b/examples/disaggregated/slurm/benchmark/config.yaml index c570ac850b3c..e4bda5707d07 100644 --- a/examples/disaggregated/slurm/benchmark/config.yaml +++ b/examples/disaggregated/slurm/benchmark/config.yaml @@ -1,25 +1,25 @@ # SLURM Configuration slurm: script_file: "disaggr_torch.slurm" - partition: "gb200" - account: "coreai_comparch_trtllm" + partition: "" + account: "" job_time: "02:00:00" - job_name: "nvbug5844149-con1-tep8-mtp3" + job_name: "" extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" set_segment: true # Optional: whether to set the segment for the job numa_bind: true # Only enable for GB200/GB300 NVL72 # Benchmark Mode benchmark: - mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context + mode: "e2e" # Options: e2e, gen_only, gen_only_no_context use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script - multi_round: 1 # Number of benchmark rounds - benchmark_ratio: 1.0 # Benchmark ratio + multi_round: 8 # Number of benchmark rounds + benchmark_ratio: 0.8 # Benchmark ratio streaming: true # Enable streaming mode - concurrency_list: "1" - input_length: 32768 # Input sequence length - output_length: 4096 # Output sequence length - dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" + concurrency_list: "16" + input_length: 1024 # Input sequence length + output_length: 1024 # Output sequence length + dataset_file: "" # Hardware Configuration hardware: @@ -29,13 +29,14 @@ hardware: # Environment Configuration environment: - container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" - container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" - model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" - trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" + container_mount: "" # Format: path1:path1,path2:path2 + container_image: "" + model_path: "" + trtllm_repo: "" build_wheel: false # Don't build the wheel when launching multiple jobs - cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures - trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + cuda_architectures: "" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures + trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead + work_dir: "" worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" @@ -50,74 +51,83 @@ accuracy: enable_accuracy_test: false # Set to true to enable accuracy evaluation tasks: gsm8k: - model: "local-completions" + model: "local-completions" # Model type for lm_eval model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" extra_kwargs: trust_remote_code: true worker_config: gen: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 8 - max_input_len: 32784 - max_seq_len: 40960 tensor_parallel_size: 8 moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false + enable_attention_dp: true + enable_lm_head_tp_in_adp: true pipeline_parallel_size: 1 context_parallel_size: 1 + # Uncomment this section to enable context parallelism. + # cp_config: + # cp_type: "HELIX" + # tokens_per_block: 32 # must match kv_config.tokens_per_block. + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 2251 cuda_graph_config: enable_padding: true - max_batch_size: 1 + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + - 2048 + - 256 + print_iter_log: true + trust_remote_code: true kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.25 + tokens_per_block: 32 + free_gpu_memory_fraction: 0.8 dtype: fp8 - tokens_per_block: 64 moe_config: - backend: TRTLLM + backend: CUTLASS + use_low_precision_moe_combine: true + load_balancer: + num_slots: 0 cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 + max_tokens_in_buffer: 4608 + backend: DEFAULT stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL + num_postprocess_workers: 4 + speculative_config: # when mtp_size is 0, remove this section. + decoding_type: MTP + num_nextn_predict_layers: 1 ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 32784 - max_input_len: 32784 - max_seq_len: 32832 + max_batch_size: 4 + max_num_tokens: 4608 + max_seq_len: 1227 tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 context_parallel_size: 1 + moe_expert_parallel_size: 4 enable_attention_dp: true - enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true cuda_graph_config: null + disable_overlap_scheduler: true kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.4 + free_gpu_memory_fraction: 0.85 dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: + max_tokens_in_buffer: 4608 + backend: DEFAULT + speculative_config: # when mtp_size is 0, remove this section. decoding_type: MTP - num_nextn_predict_layers: 3 + num_nextn_predict_layers: 1 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml deleted file mode 100644 index c570ac850b3c..000000000000 --- a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con1_dep32_mtp3.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# SLURM Configuration -slurm: - script_file: "disaggr_torch.slurm" - partition: "gb200" - account: "coreai_comparch_trtllm" - job_time: "02:00:00" - job_name: "nvbug5844149-con1-tep8-mtp3" - extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" - set_segment: true # Optional: whether to set the segment for the job - numa_bind: true # Only enable for GB200/GB300 NVL72 - -# Benchmark Mode -benchmark: - mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context - use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script - multi_round: 1 # Number of benchmark rounds - benchmark_ratio: 1.0 # Benchmark ratio - streaming: true # Enable streaming mode - concurrency_list: "1" - input_length: 32768 # Input sequence length - output_length: 4096 # Output sequence length - dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" - -# Hardware Configuration -hardware: - gpus_per_node: 4 # Modify this with your hardware configuration - num_ctx_servers: 1 # Number of context servers - num_gen_servers: 1 # Number of generation servers - -# Environment Configuration -environment: - container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" - container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" - model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" - trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" - build_wheel: false # Don't build the wheel when launching multiple jobs - cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures - trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" - server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" - -# Profiling Configuration -profiling: - nsys_on: false # Set to true to enable profiling - ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers - gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers - -# Accuracy Configuration -accuracy: - enable_accuracy_test: false # Set to true to enable accuracy evaluation - tasks: - gsm8k: - model: "local-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - trust_remote_code: true - -worker_config: - gen: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 8 - max_input_len: 32784 - max_seq_len: 40960 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - context_parallel_size: 1 - cuda_graph_config: - enable_padding: true - max_batch_size: 1 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.25 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: TRTLLM - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL - ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 32784 - max_input_len: 32784 - max_seq_len: 32832 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.4 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml deleted file mode 100644 index 69962ecf8fef..000000000000 --- a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con2048_dep32_mtp1_eplb.yaml +++ /dev/null @@ -1,126 +0,0 @@ -# SLURM Configuration -slurm: - script_file: "disaggr_torch.slurm" - partition: "gb200" - account: "coreai_comparch_trtllm" - job_time: "02:00:00" - job_name: "nvbug5844149-con2048-dep32-mtp1" - extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" - set_segment: true # Optional: whether to set the segment for the job - numa_bind: true # Only enable for GB200/GB300 NVL72 - -# Benchmark Mode -benchmark: - mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context - use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script - multi_round: 1 # Number of benchmark rounds - benchmark_ratio: 1.0 # Benchmark ratio - streaming: true # Enable streaming mode - concurrency_list: "2048" - input_length: 32768 # Input sequence length - output_length: 4096 # Output sequence length - dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" - -# Hardware Configuration -hardware: - gpus_per_node: 4 # Modify this with your hardware configuration - num_ctx_servers: 1 # Number of context servers - num_gen_servers: 1 # Number of generation servers - -# Environment Configuration -environment: - container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" - container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" - model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" - trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" - build_wheel: true # Don't build the wheel when launching multiple jobs - cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures - # trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" - server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" - -# Profiling Configuration -profiling: - nsys_on: false # Set to true to enable profiling - ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers - gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers - -# Accuracy Configuration -accuracy: - enable_accuracy_test: false # Set to true to enable accuracy evaluation - tasks: - gsm8k: - model: "local-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - trust_remote_code: true - -worker_config: - gen: - print_iter_log: true - max_batch_size: 64 - max_num_tokens: 128 - max_input_len: 32784 - max_seq_len: 40960 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - context_parallel_size: 1 - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.5 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - load_balancer: - num_slots: 288 - layer_updates_per_iter: 1 - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 32784 - max_input_len: 32784 - max_seq_len: 32832 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.4 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 diff --git a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml b/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml deleted file mode 100644 index c228950afe2d..000000000000 --- a/examples/disaggregated/slurm/benchmark/config_nvbug5844149_con256_dep32_mtp3.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# SLURM Configuration -slurm: - script_file: "disaggr_torch.slurm" - partition: "gb200" - account: "coreai_comparch_trtllm" - job_time: "02:00:00" - job_name: "nvbug5844149-con256-dep32-mtp3" - extra_args: "" # Cluster specific arguments, e.g. "--gres=gpu:4 --exclude=node1,node2" - set_segment: true # Optional: whether to set the segment for the job - numa_bind: true # Only enable for GB200/GB300 NVL72 - -# Benchmark Mode -benchmark: - mode: "gen_only" # Options: e2e, gen_only, gen_only_no_context - use_nv_sa_benchmark: false # Whether to use NVIDIA SA benchmark script - multi_round: 1 # Number of benchmark rounds - benchmark_ratio: 1.0 # Benchmark ratio - streaming: true # Enable streaming mode - concurrency_list: "256" - input_length: 32768 # Input sequence length - output_length: 4096 # Output sequence length - dataset_file: "/lustre/fsw/coreai_comparch_trtllm/common/disagg_datasets/deepseek-v32-32768-4096-50000-ratio-1_for_serve.json" - -# Hardware Configuration -hardware: - gpus_per_node: 4 # Modify this with your hardware configuration - num_ctx_servers: 1 # Number of context servers - num_gen_servers: 1 # Number of generation servers - -# Environment Configuration -environment: - container_mount: "/lustre/fsw/coreai_comparch_trtllm:/lustre/fsw/coreai_comparch_trtllm" - container_image: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/sqsh/trtllm_pytorch_26.02_py3_sbsa_ubuntu24.04_trt10.15.1.29_202604011104_12600.sqsh" - model_path: "/lustre/fsw/coreai_comparch_trtllm/common/DeepSeek-V3.2-FP4-v2" - trtllm_repo: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM" - build_wheel: true # Don't build the wheel when launching multiple jobs - cuda_architectures: "100-real;103-real" # Optional CUDA architectures to build for (e.g. "90-real;100-real"). If empty, builds for all architectures - # trtllm_wheel_path: "/lustre/fsw/coreai_comparch_trtllm/bbuddharaju/TensorRT-LLM/build/tensorrt_llm-1.3.0rc11-cp312-cp312-linux_aarch64.whl" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - trtllm_wheel_path: "" # Path to pre-built TensorRT-LLM wheel. If provided, install from this wheel instead - worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0" - server_env_var: "TRTLLM_SERVER_DISABLE_GC=1" - -# Profiling Configuration -profiling: - nsys_on: false # Set to true to enable profiling - ctx_profile_range: "10-30" # Set TLLM_PROFILE_START_STOP for ctx workers - gen_profile_range: "200-250" # Set TLLM_PROFILE_START_STOP for gen workers - -# Accuracy Configuration -accuracy: - enable_accuracy_test: false # Set to true to enable accuracy evaluation - tasks: - gsm8k: - model: "local-completions" - model_args_extra: "num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384" - extra_kwargs: - trust_remote_code: true - -worker_config: - gen: - print_iter_log: true - max_batch_size: 256 - max_num_tokens: 64 - max_input_len: 32784 - max_seq_len: 40960 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - context_parallel_size: 1 - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.5 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - num_postprocess_workers: 4 - stream_interval: 20 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 32784 - max_input_len: 32784 - max_seq_len: 32832 - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.4 - dtype: fp8 - tokens_per_block: 64 - moe_config: - backend: CUTEDSL - cache_transceiver_config: - max_tokens_in_buffer: 32832 - backend: UCX - disable_overlap_scheduler: true - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm index 52408db31130..9e0d66144478 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi