diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index a5545060f630..15ebbe1d0380 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -44,6 +44,7 @@ "gemma_3_12b_it": "gemma/gemma-3-12b-it", "gemma_3_12b_it_fp8": "gemma/gemma-3-12b-it-fp8", "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", + "gemma_3_1b_it": "gemma/gemma-3-1b-it", "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml new file mode 100644 index 000000000000..0517cb17b0e5 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gemma-3-1b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metadata: + model_name: gemma_3_1b_it + precision: bf16 + model_dir_name: gemma-3-1b-it + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '256' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 512 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + kv_cache_config: + use_kv_cache_manager_v2: false + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: auto + max_attention_window: [512, 512, 512, 512, 512, 32768] + cache_transceiver_config: + max_tokens_in_buffer: 4096 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: false + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 8192 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: null + kv_cache_config: + use_kv_cache_manager_v2: false + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: auto + max_attention_window: [512, 512, 512, 512, 512, 32768] + cache_transceiver_config: + max_tokens_in_buffer: 4096 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml new file mode 100644 index 000000000000..43bc0a1b8196 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb200_llama-3.1-8b-bf16_1k1k_con256_ctx1_tp1_gen1_tp1_eplb0_mtp0_ccb-NIXL.yaml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metadata: + model_name: llama_v3.1_8b_instruct + precision: bf16 + model_dir_name: Llama-3.1-8B-Instruct + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '256' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 512 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + kv_cache_config: + use_kv_cache_manager_v2: false + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: auto + cache_transceiver_config: + max_tokens_in_buffer: 4096 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: false + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 8192 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: null + kv_cache_config: + use_kv_cache_manager_v2: false + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: auto + cache_transceiver_config: + max_tokens_in_buffer: 4096 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: true