From d25aa3167dbb2b5eb911a35b80980869b178f488 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Mon, 13 Oct 2025 00:11:33 -0700 Subject: [PATCH 1/2] Add perf sanity test Signed-off-by: Chenfei Zhang --- jenkins/L0_Test.groovy | 14 + tests/integration/defs/perf/test_perf.py | 516 +++++- tests/integration/defs/perf/utils.py | 58 + .../test-db/perf_sanity_l0_dgx_b200.yml | 37 + .../test-db/perf_sanity_l0_dgx_b300.yml | 37 + tests/scripts/perf-sanity/README.md | 156 +- tests/scripts/perf-sanity/benchmark-serve.sh | 44 +- .../scripts/perf-sanity/benchmark_config.yaml | 468 ------ tests/scripts/perf-sanity/l0_dgx_b200.yaml | 56 + tests/scripts/perf-sanity/l0_dgx_b300.yaml | 58 + .../perf-sanity/parse_benchmark_results.py | 507 +++--- .../perf-sanity/run_benchmark_serve.py | 1452 +++++++++-------- 12 files changed, 1797 insertions(+), 1606 deletions(-) create mode 100644 tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b200.yml create mode 100644 tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b300.yml delete mode 100644 tests/scripts/perf-sanity/benchmark_config.yaml create mode 100644 tests/scripts/perf-sanity/l0_dgx_b200.yaml create mode 100644 tests/scripts/perf-sanity/l0_dgx_b300.yaml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5e72e3bc9993..927e38fc944f 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2674,6 +2674,20 @@ def launchTestJobs(pipeline, testFilter) parallelJobs += parallelSlurmJobs + // Add Perf Sanity Test Slurm jobs + perfSanityTestConfigs = [ + "DGX_B200-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b200-x4", "perf_sanity_l0_dgx_b200", 1, 1, 4], + "DGX_B300-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b300-x4", "perf_sanity_l0_dgx_b300", 1, 1, 4], + ] + fullSet += perfSanityTestConfigs.keySet() + + parallelPerfSanityJobs = perfSanityTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { + def config = VANILLA_CONFIG + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("Perf"), key, values[2], values[3], values[4] ?: 1) + }]]} + + parallelJobs += parallelPerfSanityJobs + // Try to match what are being tested on x86 H100_PCIe. // The total machine time is scaled proportionally according to the number of each GPU. SBSATestConfigs = [ diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 4df17c3fe6c6..c0ec5402c6c2 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -31,7 +31,8 @@ from .pytorch_model_config import get_model_yaml_config from .utils import (AbstractPerfScriptTestClass, PerfBenchScriptTestCmds, PerfDisaggScriptTestCmds, PerfMetricType, - PerfScriptTestCmds, generate_test_nodes) + PerfScriptTestCmds, PerfServerClientBenchmarkCmds, + generate_test_nodes) if not hasattr(re, "Pattern"): re.Pattern = type(re.compile("")) @@ -205,6 +206,16 @@ } +def get_model_dir(model_name: str): + model_dir = "" + if model_name in MODEL_PATH_DICT.keys(): + model_dir = os.path.join(llm_models_root(), MODEL_PATH_DICT[model_name]) + elif model_name in HF_MODEL_PATH.keys(): + model_dir = os.path.join(llm_models_root(), + MODEL_PATH_DICT[model_name.split('_hf')[0]]) + return model_dir + + def cpu_socket_count_gt_1(): global MAP_BY_SOCKET if MAP_BY_SOCKET is not None: @@ -276,6 +287,7 @@ def import_allowed_perf_config(): PerfMetricType.DISAGG_SERVER_TTFT: re.compile(r"Median TTFT \(ms\):\s*(\d+\.?\d*)"), } + BENCH_PERF_METRIC_LOG_QUERIES = { PerfMetricType.BUILD_TIME: re.compile(r"Engine generation completed in ([\d\.]+) seconds"), @@ -293,25 +305,82 @@ def import_allowed_perf_config(): re.compile(r".*(?:Allocated ([\d\.]+) GiB for max tokens in paged KV cache|" r"Final KV cache size after resize: ([\d\.]+) GiB).*"), } + +SERVER_BENCHMARK_PERF_METRIC_LOG_QUERIES = { + PerfMetricType.SEQ_THROUGHPUT: + re.compile(r"Request throughput \(req\/s\):\s+([\d\.]+)"), + PerfMetricType.TOKEN_THROUGHPUT: + re.compile(r"Output token throughput \(tok\/s\):\s+([\d\.]+)"), + PerfMetricType.TOTAL_TOKEN_THROUGHPUT: + re.compile(r"Total Token throughput \(tok\/s\):\s+([\d\.]+)"), + PerfMetricType.USER_THROUGHPUT: + re.compile(r"User throughput \(tok\/s\):\s+([\d\.]+)"), + PerfMetricType.FIRST_TOKEN_TIME: + re.compile(r"Mean TTFT \(ms\):\s+([\d\.]+)"), + PerfMetricType.MEDIAN_FIRST_TOKEN_TIME: + re.compile(r"Median TTFT \(ms\):\s+([\d\.]+)"), + PerfMetricType.P99_FIRST_TOKEN_TIME: + re.compile(r"P99 TTFT \(ms\):\s+([\d\.]+)"), + PerfMetricType.INTER_TOKEN_TIME: + re.compile(r"Mean ITL \(ms\):\s+([\d\.]+)"), + PerfMetricType.MEDIAN_INTER_TOKEN_TIME: + re.compile(r"Median ITL \(ms\):\s+([\d\.]+)"), + PerfMetricType.P99_INTER_TOKEN_TIME: + re.compile(r"P99 ITL \(ms\):\s+([\d\.]+)"), + PerfMetricType.OUTPUT_TOKEN_TIME: + re.compile(r"Mean TPOT \(ms\):\s+([\d\.]+)"), + PerfMetricType.MEDIAN_OUTPUT_TOKEN_TIME: + re.compile(r"Median TPOT \(ms\):\s+([\d\.]+)"), + PerfMetricType.P99_OUTPUT_TOKEN_TIME: + re.compile(r"P99 TPOT \(ms\):\s+([\d\.]+)"), + PerfMetricType.INFERENCE_TIME: + re.compile(r"Mean E2EL \(ms\):\s+([\d\.]+)"), + PerfMetricType.MEDIAN_INFERENCE_TIME: + re.compile(r"Median E2EL \(ms\):\s+([\d\.]+)"), + PerfMetricType.P99_INFERENCE_TIME: + re.compile(r"P99 E2EL \(ms\):\s+([\d\.]+)"), +} + DISAGG_SERVER_METRICS_LOG_QUERIES = { PerfMetricType.DISAGG_SERVER_E2EL: re.compile(r"Median E2EL \(ms\):\s*(\d+\.?\d*)"), PerfMetricType.DISAGG_SERVER_TTFT: re.compile(r"Median TTFT \(ms\):\s*(\d+\.?\d*)"), } + # (Relative threshold, Absolute threshold) for all metric types PERF_METRIC_THRESHOLD = { PerfMetricType.BUILD_TIME: (0.1, 30), # Ignore build time regression < 30ms PerfMetricType.INFERENCE_TIME: (0.1, 50), # Ignore inference time regression < 50ms + PerfMetricType.MEDIAN_INFERENCE_TIME: + (0.1, 50), # Ignore median inference time regression < 50ms + PerfMetricType.P99_INFERENCE_TIME: + (0.1, 50), # Ignore p99 inference time regression < 50ms PerfMetricType.FIRST_TOKEN_TIME: (0.1, 50), # Ignore first token time regression < 50ms + PerfMetricType.MEDIAN_FIRST_TOKEN_TIME: + (0.1, 50), # Ignore median first token time regression < 50ms + PerfMetricType.P99_FIRST_TOKEN_TIME: + (0.1, 50), # Ignore p99 first token time regression < 50ms PerfMetricType.OUTPUT_TOKEN_TIME: (0.1, 50), # Ignore per output token time regression < 50ms + PerfMetricType.MEDIAN_OUTPUT_TOKEN_TIME: + (0.1, 50), # Ignore median output token time regression < 50ms + PerfMetricType.P99_OUTPUT_TOKEN_TIME: + (0.1, 50), # Ignore p99 output token time regression < 50ms + PerfMetricType.INTER_TOKEN_TIME: + (0.1, 50), # Ignore inter token time regression < 50ms + PerfMetricType.MEDIAN_INTER_TOKEN_TIME: + (0.1, 50), # Ignore median inter token time regression < 50ms + PerfMetricType.P99_INTER_TOKEN_TIME: + (0.1, 50), # Ignore p99 inter token time regression < 50ms PerfMetricType.SEQ_LATENCY: (0.1, 50), # Ignore latency regression < 50ms PerfMetricType.TOKEN_THROUGHPUT: ( -0.1, 10 ), # Ignore throughput regression < 10 tokens/s. Negative rel threshold is to indicate that larger is better. + PerfMetricType.TOTAL_TOKEN_THROUGHPUT: (0.1, 10), + PerfMetricType.USER_THROUGHPUT: (0.1, 10), PerfMetricType.SEQ_THROUGHPUT: ( -0.1, 10 ), # Ignore throughput regression < 10 tokens/s. Negative rel threshold is to indicate that larger is better. @@ -357,6 +426,25 @@ def import_allowed_perf_config(): PerfMetricType.KV_CACHE_SIZE, ] +SERVER_BENCHMARK_METRICS = [ + PerfMetricType.SEQ_THROUGHPUT, + PerfMetricType.TOKEN_THROUGHPUT, + PerfMetricType.TOTAL_TOKEN_THROUGHPUT, + PerfMetricType.USER_THROUGHPUT, + PerfMetricType.FIRST_TOKEN_TIME, + PerfMetricType.MEDIAN_FIRST_TOKEN_TIME, + PerfMetricType.P99_FIRST_TOKEN_TIME, + PerfMetricType.OUTPUT_TOKEN_TIME, + PerfMetricType.MEDIAN_OUTPUT_TOKEN_TIME, + PerfMetricType.P99_OUTPUT_TOKEN_TIME, + PerfMetricType.INTER_TOKEN_TIME, + PerfMetricType.MEDIAN_INTER_TOKEN_TIME, + PerfMetricType.P99_INTER_TOKEN_TIME, + PerfMetricType.INFERENCE_TIME, + PerfMetricType.MEDIAN_INFERENCE_TIME, + PerfMetricType.P99_INFERENCE_TIME, +] + BENCH_INFERENCE_METRICS = [ PerfMetricType.INFERENCE_TIME, PerfMetricType.TOKEN_THROUGHPUT, @@ -391,6 +479,287 @@ class PerfTestMetric(NamedTuple): cmd_idx: int +class ServerConfig: + """ + Configurations of trtllm-server. + """ + + def __init__( + self, + name: str, + model_name: str, + tp: int, + ep: int, + max_num_tokens: int, + attention_backend: str, + max_batch_size: int, + pp: int = 1, + enable_chunked_prefill: bool = False, + disable_overlap_scheduler: bool = False, + moe_backend: str = "", + moe_max_num_tokens: str = "", + stream_interval: int = 10, + enable_attention_dp: bool = False, + attention_dp_balance: bool = False, + batching_wait_iters: int = 10, + timeout_iters: int = 50, + kv_cache_dtype: str = "fp8", + enable_block_reuse: bool = False, + free_gpu_memory_fraction: float = 0.8, + enable_padding: bool = True, + ): + self.name = name + self.model_name = model_name + self.tp = tp + self.ep = ep + self.pp = pp + self.max_num_tokens = max_num_tokens + self.enable_chunked_prefill = enable_chunked_prefill + self.disable_overlap_scheduler = disable_overlap_scheduler + self.attention_backend = attention_backend + self.moe_backend = moe_backend + self.moe_max_num_tokens = moe_max_num_tokens + self.stream_interval = stream_interval + self.enable_attention_dp = enable_attention_dp + self.attention_dp_balance = attention_dp_balance + self.batching_wait_iters = batching_wait_iters + self.timeout_iters = timeout_iters + self.kv_cache_dtype = kv_cache_dtype + self.enable_block_reuse = enable_block_reuse + self.free_gpu_memory_fraction = free_gpu_memory_fraction + self.max_batch_size = max_batch_size + self.enable_padding = enable_padding + + self.model_path = "" + + def to_cmd(self, working_dir: str) -> List[str]: + model_dir = get_model_dir(self.model_name) + self.model_path = model_dir if os.path.exists( + model_dir) else self.model_name + config_path = os.path.join(working_dir, + f"extra-llm-api-config.{self.name}.yml") + return [ + "trtllm-serve", self.model_path, "--host", "localhost", "--port", + "8000", "--backend", "pytorch", "--extra_llm_api_options", + config_path + ] + + def generate_extra_llm_api_config(self) -> str: + """Generate extra-llm-api-config.yml content""" + config_lines = [ + f"tensor_parallel_size: {self.tp}", + f"moe_expert_parallel_size: {self.ep}", + f"pipeline_parallel_size: {self.pp}", + f"max_num_tokens: {self.max_num_tokens}", + f"enable_attention_dp: {str(self.enable_attention_dp).lower()}", + f"disable_overlap_scheduler: {str(self.disable_overlap_scheduler).lower()}", + f"stream_interval: {self.stream_interval}", + f"attn_backend: {self.attention_backend}", + f"enable_chunked_prefill: {str(self.enable_chunked_prefill).lower()}", + "cuda_graph_config:", + f" enable_padding: {str(self.enable_padding).lower()}", + f" max_batch_size: {self.max_batch_size}", + "kv_cache_config:", + f" dtype: {self.kv_cache_dtype}", + f" free_gpu_memory_fraction: {self.free_gpu_memory_fraction}", + f" enable_block_reuse: {str(self.enable_block_reuse).lower()}", + "print_iter_log: false", + ] + + # Add moe_config if moe_backend is specified + if self.moe_backend: + config_lines.append("moe_config:") + config_lines.append(f" backend: {self.moe_backend}") + if self.moe_max_num_tokens: + config_lines.append( + f" max_num_tokens: {self.moe_max_num_tokens}") + + if self.attention_dp_balance: + config_lines.append("attention_dp_balance:") + config_lines.append(" enable_balance: true") + config_lines.append( + f" batching_wait_iters: {self.batching_wait_iters}") + config_lines.append(f" timeout_iters: {self.timeout_iters}") + + return "\n".join(config_lines) + + +class ClientConfig: + """ + Configurations of benchmark client. + """ + + def __init__(self, + name: str, + model_name: str, + concurrency: int, + iterations: int, + isl: int, + osl: int, + random_range_ratio: float = 0.0): + self.name = name + self.model_name = model_name + self.concurrency = concurrency + self.iterations = iterations + self.isl = isl + self.osl = osl + self.random_range_ratio = random_range_ratio + + self.model_path = "" + + def to_cmd(self, working_dir: str) -> List[str]: + model_dir = get_model_dir(self.model_name) + self.model_path = model_dir if os.path.exists( + model_dir) else self.model_name + return [ + "python", "-m", "tensorrt_llm.serve.scripts.benchmark_serving", + "--model", self.model_path, "--dataset-name", "random", + "--random-ids", "--num-prompts", + str(self.concurrency * self.iterations), "--random-input-len", + str(self.isl), "--random-output-len", + str(self.osl), "--random-range-ratio", + str(self.random_range_ratio), "--ignore-eos", + "--percentile-metrics", "ttft,tpot,itl,e2el", "--max-concurrency", + str(self.concurrency) + ] + + +def parse_select_pattern(select_pattern: str): + """Parse select pattern like 'r1_fp4_dep4,r1_fp4_tep4:con1_iter1_1024_1024,r1_fp4_tep4:con8_iter1_1024_1024' + + Format: + - ',' splits different server configs + - ':' means for this server, we choose specific clients + - If no ':', all clients are chosen for that server + + Returns: + - Dict with server name as key and either None (all clients) or set of client names as value + """ + execution_plan = {} + + parts = select_pattern.split(',') + for part in parts: + part = part.strip() + if not part: # Skip empty parts + continue + + if ':' in part: + # Format: "server_name:client_name" + server_name, client_name = part.split(':', 1) + server_name = server_name.strip() + client_name = client_name.strip() + + # Only add if not already set to None (all clients) + if server_name not in execution_plan: + execution_plan[server_name] = set() + + if execution_plan[server_name] is not None: + execution_plan[server_name].add(client_name) + else: + # Format: "server_name" - select all clients for this server + server_name = part.strip() + execution_plan[server_name] = None + + return execution_plan + + +def parse_config_file(config_file_path: str, select_pattern: str = None): + """Parse YAML configuration file and create ServerConfig and ClientConfig objects + + Args: + config_file_path: Path to YAML configuration file + select_pattern: Selection pattern string (e.g., "r1_fp4_dep4,r1_fp4_tep4:con1_iter1_1024_1024") + + Returns: + execution_plan: None (all servers/clients) or dict with server names as keys + server_configs: List of ServerConfig objects + server_client_configs: Dict with server id as key and list of ClientConfig as value + """ + # Parse selection pattern + if select_pattern: + execution_plan = parse_select_pattern(select_pattern) + else: + execution_plan = None + + # Read YAML config file + with open(config_file_path, 'r') as f: + config = yaml.safe_load(f) + + server_configs = [] + server_client_configs = {} + + for server_config_data in config['server_configs']: + server_name = server_config_data['name'] + + # Check if this server should be included based on execution_plan + if execution_plan is not None and server_name not in execution_plan: + continue + + # Create ServerConfig object + server_config = ServerConfig( + name=server_config_data['name'], + model_name=server_config_data['model_name'], + tp=server_config_data['tp'], + ep=server_config_data['ep'], + pp=server_config_data.get('pp', 1), + attention_backend=server_config_data.get('attention_backend', + 'TRTLLM'), + moe_backend=server_config_data.get('moe_backend', ''), + moe_max_num_tokens=server_config_data.get('moe_max_num_tokens', ''), + stream_interval=server_config_data.get('stream_interval', 10), + enable_attention_dp=server_config_data.get('enable_attention_dp', + False), + attention_dp_balance=server_config_data.get('attention_dp_balance', + False), + batching_wait_iters=server_config_data.get('batching_wait_iters', + 10), + timeout_iters=server_config_data.get('timeout_iters', 50), + enable_chunked_prefill=server_config_data.get( + 'enable_chunked_prefill', False), + max_num_tokens=server_config_data.get('max_num_tokens', 2048), + disable_overlap_scheduler=server_config_data.get( + 'disable_overlap_scheduler', False), + kv_cache_dtype=server_config_data.get('kv_cache_dtype', 'fp8'), + enable_block_reuse=server_config_data.get('enable_block_reuse', + False), + free_gpu_memory_fraction=server_config_data.get( + 'free_gpu_memory_fraction', 0.8), + max_batch_size=server_config_data.get('max_batch_size', 256), + enable_padding=server_config_data.get('enable_padding', True)) + + server_id = len(server_configs) + server_configs.append(server_config) + + # Create ClientConfig objects + client_configs = [] + selected_client_names = execution_plan.get( + server_name) if execution_plan else None + + for client_config_data in server_config_data['client_configs']: + client_name = client_config_data['name'] + + # Check if this client should be included + # Include if: execution_plan is None OR selected_client_names is None OR client_name in selected_client_names + if execution_plan is not None and selected_client_names is not None: + if client_name not in selected_client_names: + continue + + client_config = ClientConfig( + name=client_config_data['name'], + model_name=server_config_data['model_name'], + concurrency=client_config_data['concurrency'], + iterations=client_config_data.get('iterations', 1), + isl=client_config_data.get('isl', 1024), + osl=client_config_data.get('osl', 1024), + random_range_ratio=client_config_data.get( + 'random_range_ratio', 0.0)) + client_configs.append(client_config) + + server_client_configs[server_id] = client_configs + + return execution_plan, server_configs, server_client_configs + + class PerfTestConfig: """ Configurations defining the LLM perf test. @@ -496,6 +865,15 @@ def __init__( self.ctx_server_workers = 0 self.gen_server_workers = 0 + # Used for perf sanity test + # config_file: YAML path, select_pattern: server/client selection string + # server_configs: list[ServerConfig], server_client_configs: dict[server_id -> list[ClientConfig]] + self.config_file = None + self.config_path = None + self.select_pattern = None + self.server_configs = [] + self.server_client_configs = {} + def _to_string_disagg(self, entries: List[str]): entries.append(f"disagg_server") if self.ctx_tp_size > 1: @@ -513,10 +891,21 @@ def _to_string_disagg(self, entries: List[str]): return "-".join(entries) def to_string(self, + custom_server_name: str = None, + custom_client_name: str = None, custom_bs: int = None, custom_input_len: int = None, custom_output_len: int = None) -> str: + # Used for perf sanity test + if self.config_file is not None: + entries = ["perf_sanity", self.config_file] + if custom_server_name is not None: + entries.append(f"server:{custom_server_name}") + if custom_client_name is not None: + entries.append(f"client:{custom_client_name}") + return "-".join(entries) + # First, add the model name. entries = [self.model_name] @@ -678,7 +1067,19 @@ def load_from_str(self, test_param_labels) -> None: # Extract configs from test param labels. labels = test_param_labels.split("-") + # Used for perf sanity test + if labels[0] == "perf_sanity": + assert len(labels) > 1, "perf_sanity test must have a config file!" + self.runtime = "server-benchmark" + self.config_file = labels[1] + self.config_path = os.path.join( + "tests/scripts/perf-sanity", f"{labels[1]}.yaml" + if not labels[1].endswith(".yaml") else labels[1]) + self.select_pattern = labels[2] if len(labels) > 2 else None + return + self.model_name = labels.pop(0) + assert labels[0] in ["cpp", "cppmanager", "bench", "disagg_server"], \ f"Invalid runtime {labels[0]}!" self.runtime = labels.pop(0) @@ -899,6 +1300,15 @@ def validate(self): [b >= 32 for b in self.batch_sizes] ), f"gpt_350m and bloom_560m with small BS are very unstable! Please increase to at least 32." + def set_server_client_configs(self, llm_root: str) -> None: + """ + Set the server and client configs. + """ + if self.runtime == "server-benchmark": + config_file_path = os.path.join(llm_root, self.config_path) + _, self.server_configs, self.server_client_configs = parse_config_file( + config_file_path, self.select_pattern) + def get_model_family(self) -> str: """ Get the model family of the current model. @@ -992,20 +1402,28 @@ def set_runtime_configs(self, llm_root, working_dir, llm_root) elif self._config.runtime == "bench": benchmark_script = "trtllm-bench" + elif self._config.runtime == "server-benchmark": + benchmark_script = None + self._config.set_server_client_configs(llm_root) elif self._config.runtime == "disagg_server": benchmark_script = None else: raise RuntimeError(f"Invalid runtime {self._config.runtime}.") + allowed_configs = import_allowed_perf_config() allowed_models = allowed_configs.get_allowed_models() + if self._config.runtime == "bench": build_script = "trtllm-bench" + elif self._config.runtime == "server-benchmark": + build_script = None elif self._config.pp_size > 1 or self._config.model_name not in allowed_models: build_script = "trtllm-build" else: # build.py is used to build engines for both python and cpp runtime build_script = os.path.join(llm_root, "tests/integration/defs/perf/build.py") + self._build_script = build_script self._benchmark_script = benchmark_script self._working_dir = working_dir @@ -1060,6 +1478,28 @@ def get_convert_lora_weights_command(self, model_dir, engine_dir) -> str: return command, checkpoint_dir + def get_trtllm_server_client_commands(self): + server_cmds = [] + client_cmds = [] + names = [] + for server_idx, client_configs in self._config.server_client_configs.items( + ): + server_config = self._config.server_configs[server_idx] + server_cmd = server_config.to_cmd(self._working_dir) + server_cmd = " ".join(server_cmd) + # Generate extra-llm-api-config.yml + config_content = server_config.generate_extra_llm_api_config() + config_filename = f"extra-llm-api-config.{server_config.name}.yml" + config_path = os.path.join(self._working_dir, config_filename) + with open(config_path, 'w') as f: + f.write(config_content) + for client_config in client_configs: + server_cmds.append(server_cmd) + client_cmd = client_config.to_cmd(self._working_dir) + client_cmds.append(client_cmd) + names.append(f"{server_config.name}-{client_config.name}") + return server_cmds, client_cmds, names + def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list: build_cmd = [ self._build_script, f"--output_dir={engine_dir}", @@ -1111,15 +1551,7 @@ def get_trtllm_build_command(self, engine_dir, checkpoint_dir) -> list: return build_cmd def get_trtllm_bench_model(self): - model_dir = "" - if self._config.model_name in MODEL_PATH_DICT.keys(): - model_dir = os.path.join(llm_models_root(), - MODEL_PATH_DICT[self._config.model_name]) - elif self._config.model_name in HF_MODEL_PATH.keys(): - model_dir = os.path.join( - llm_models_root(), - MODEL_PATH_DICT[self._config.model_name.split('_hf')[0]]) - return model_dir + return get_model_dir(self._config.model_name) def get_trtllm_bench_build_command(self, engine_dir) -> list: model_dir = self.get_trtllm_bench_model() @@ -1511,14 +1943,27 @@ def get_commands(self): # Whether this is python or cpp runtime perf test. is_python = self._config.runtime == "python" num_gpus = self._config.num_gpus + is_server_benchmark = self._config.runtime == "server-benchmark" is_disagg = self._config.runtime == "disagg_server" + if is_server_benchmark: + perf_sanity_working_dir = os.path.join(self._working_dir, + "perf-sanity") + if not os.path.exists(perf_sanity_working_dir): + os.makedirs(perf_sanity_working_dir, exist_ok=True) + server_cmds, client_cmds, names = self.get_trtllm_server_client_commands( + ) + return PerfServerClientBenchmarkCmds( + server_cmds=server_cmds, + client_cmds=client_cmds, + names=names, + working_dir=perf_sanity_working_dir) + if is_disagg: ctx_cmd, gen_cmd = self._get_disagg_worker_deploy_command() server_cmd = self._get_disagg_server_deploy_command() client_cmd = self._get_disagg_client_command() benchmark_cmd = self._get_disagg_benchmark_command() - return PerfDisaggScriptTestCmds(ctx_cmd, gen_cmd, server_cmd, client_cmd, benchmark_cmd) @@ -1818,12 +2263,39 @@ def _get_metrics(self) -> List[PerfTestMetric]: """ metrics = [] + if self._config.runtime == "server-benchmark": + cmd_idx = 0 + for server_idx, client_configs in self._config.server_client_configs.items( + ): + server_name = self._config.server_configs[server_idx].name + for client_config in client_configs: + for metric_type in SERVER_BENCHMARK_METRICS: + metrics.append( + PerfTestMetric( + original_test_name=self._full_test_name, + metric_name=self._get_metric_name( + metric_type=metric_type, + server_name=server_name, + client_name=client_config.name), + metric_type=metric_type, + metric_regex=self._get_metric_regex( + metric_type), + metric_threshold=self._get_metric_threshold( + metric_type), + metric_abs_threshold=self. + _get_metric_abs_threshold(metric_type), + cmd_idx=cmd_idx, + )) + cmd_idx += 1 + return metrics + if self._config.runtime == "disagg_server": for metric_type in DISAGG_SERVER_METRICS: metrics.append( PerfTestMetric( original_test_name=self._full_test_name, - metric_name=self._get_metric_name(metric_type), + metric_name=self._get_metric_name( + metric_type=metric_type), metric_type=metric_type, metric_regex=self._get_metric_regex(metric_type), metric_threshold=self._get_metric_threshold( @@ -1852,7 +2324,7 @@ def _get_metrics(self) -> List[PerfTestMetric]: metrics.append( PerfTestMetric( original_test_name=self._full_test_name, - metric_name=self._get_metric_name(metric_type), + metric_name=self._get_metric_name(metric_type=metric_type), metric_type=metric_type, metric_regex=self._get_metric_regex(metric_type), metric_threshold=self._get_metric_threshold(metric_type), @@ -1899,7 +2371,10 @@ def _get_metrics(self) -> List[PerfTestMetric]: PerfTestMetric( original_test_name=self._full_test_name, metric_name=self._get_metric_name( - metric_type, bs, input_len, output_len), + metric_type=metric_type, + bs=bs, + input_len=input_len, + output_len=output_len), metric_type=metric_type, metric_regex=self._get_metric_regex(metric_type), metric_threshold=self._get_metric_threshold( @@ -1915,7 +2390,9 @@ def _get_metric_name(self, metric_type: PerfMetricType, bs: int = None, input_len: int = None, - output_len: int = None) -> str: + output_len: int = None, + server_name: str = None, + client_name: str = None) -> str: """ Construct the metric name for given metric_type, bs, input_len, and output_len. """ @@ -1923,6 +2400,11 @@ def _get_metric_name(self, if metric_type in BUILDER_METRICS: # We build one engine for all benchmark runs, so add all bs and seq lens to the metric name. metric_label = self._config.to_string() + elif self._config.runtime == "server-benchmark": + metric_label = self._config.to_string( + custom_server_name=server_name, + custom_client_name=client_name, + ) else: # Otherwise, generate per-bs and per-seqlen label. metric_label = self._config.to_string( @@ -1942,6 +2424,10 @@ def _get_metric_regex(self, metric_type: PerfMetricType) -> re.Pattern: if metric_type not in BENCH_PERF_METRIC_LOG_QUERIES: raise ValueError(f"Unexpected metric_type: {metric_type}") return BENCH_PERF_METRIC_LOG_QUERIES[metric_type] + elif self._config.runtime == "server-benchmark": + if metric_type not in SERVER_BENCHMARK_PERF_METRIC_LOG_QUERIES: + raise ValueError(f"Unexpected metric_type: {metric_type}") + return SERVER_BENCHMARK_PERF_METRIC_LOG_QUERIES[metric_type] else: if metric_type not in PERF_METRIC_LOG_QUERIES: raise ValueError(f"Unexpected metric_type: {metric_type}") diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py index f5cfb391e37f..c5b1e9039a7a 100644 --- a/tests/integration/defs/perf/utils.py +++ b/tests/integration/defs/perf/utils.py @@ -90,9 +90,20 @@ class PerfMetricType(str, Enum): set up special threshold criteria for each type of metrics (like >50MB for engine size increase, etc.). """ INFERENCE_TIME = "INFERENCE_TIME" + MEDIAN_INFERENCE_TIME = "MEDIAN_INFERENCE_TIME" + P99_INFERENCE_TIME = "P99_INFERENCE_TIME" + INTER_TOKEN_TIME = "INTER_TOKEN_TIME" + MEDIAN_INTER_TOKEN_TIME = "MEDIAN_INTER_TOKEN_TIME" + P99_INTER_TOKEN_TIME = "P99_INTER_TOKEN_TIME" FIRST_TOKEN_TIME = "FIRST_TOKEN_TIME" + MEDIAN_FIRST_TOKEN_TIME = "MEDIAN_FIRST_TOKEN_TIME" + P99_FIRST_TOKEN_TIME = "P99_FIRST_TOKEN_TIME" OUTPUT_TOKEN_TIME = "OUTPUT_TOKEN_TIME" + MEDIAN_OUTPUT_TOKEN_TIME = "MEDIAN_OUTPUT_TOKEN_TIME" + P99_OUTPUT_TOKEN_TIME = "P99_OUTPUT_TOKEN_TIME" TOKEN_THROUGHPUT = "TOKEN_THROUGHPUT" + TOTAL_TOKEN_THROUGHPUT = "TOTAL_TOKEN_THROUGHPUT" + USER_THROUGHPUT = "USER_THROUGHPUT" BUILD_TIME = "BUILD_TIME" BUILD_PEAK_CPU_MEMORY = "BUILD_PEAK_CPU_MEMORY" BUILD_PEAK_GPU_MEMORY = "BUILD_PEAK_GPU_MEMORY" @@ -311,6 +322,53 @@ def get_cmd_str(self, cmd_idx) -> List[str]: return cmd_str +class PerfServerClientBenchmarkCmds(NamedTuple): + server_cmds: List[str] + client_cmds: List[List[str]] + names: List[str] + working_dir: str + + def wait_for_endpoint_ready(self, url: str, timeout: int = 5400): + start = time.monotonic() + while time.monotonic() - start < timeout: + try: + time.sleep(10) + if requests.get(url).status_code == 200: + print(f"endpoint {url} is ready") + return + except Exception as err: + print(f"endpoint {url} is not ready, with exception: {err}") + print_error( + f"Endpoint {url} did not become ready within {timeout} seconds") + + def run_cmd(self, cmd_idx: int, venv) -> str: + output = "" + server_file_path = os.path.join( + self.working_dir, f"trtllm-serve.{self.names[cmd_idx]}.log") + client_file_path = os.path.join( + self.working_dir, f"trtllm-benchmark.{self.names[cmd_idx]}.log") + try: + with ( # Start server process + open(server_file_path, 'w') as server_ctx, + popen(self.server_cmds[cmd_idx], + stdout=server_ctx, + stderr=subprocess.STDOUT, + env=venv._new_env, + shell=True) as server_proc): + self.wait_for_endpoint_ready( + "http://localhost:8000/v1/models", + timeout=5400) # 90 minutes for large models + output += subprocess.check_output(self.client_cmds[cmd_idx], + env=venv._new_env).decode() + finally: + server_proc.terminate() + server_proc.wait() + return output + + def get_cmd_str(self, cmd_idx) -> List[str]: + return ["server-benchmark tests, please check config files"] + + class PerfDisaggScriptTestCmds(NamedTuple): ctx_cmd: str gen_cmd: str diff --git a/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b200.yml b/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b200.yml new file mode 100644 index 000000000000..e6622e610903 --- /dev/null +++ b/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b200.yml @@ -0,0 +1,37 @@ +version: 0.0.1 +perf_sanity_l0_dgx_b200: +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*b200*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: pre_merge + backend: pytorch + orchestrator: mpi + tests: + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b200-r1_fp4_dep4:con1_iter1_1024_1024] + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b300-r1_fp4_dep4:con1_iter1_1024_1024] + +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*b200*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: post_merge + backend: pytorch + orchestrator: mpi + tests: + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b200-r1_fp4_dep4:con1_iter1_1024_1024] + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b300-r1_fp4_dep4:con1_iter1_1024_1024] diff --git a/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b300.yml b/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b300.yml new file mode 100644 index 000000000000..d798eef94fdf --- /dev/null +++ b/tests/integration/test_lists/test-db/perf_sanity_l0_dgx_b300.yml @@ -0,0 +1,37 @@ +version: 0.0.1 +perf_sanity_l0_dgx_b300: +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*gb110*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: pre_merge + backend: pytorch + orchestrator: mpi + tests: + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b200-r1_fp4_dep4:con1_iter1_1024_1024] + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b300-r1_fp4_dep4:con1_iter1_1024_1024] + +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*gb110*' + linux_distribution_name: ubuntu* + cpu: x86_64 + terms: + stage: post_merge + backend: pytorch + orchestrator: mpi + tests: + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b200-r1_fp4_dep4:con1_iter1_1024_1024] + - perf/test_perf.py::test_perf[perf_sanity-l0_dgx_b300-r1_fp4_dep4:con1_iter1_1024_1024] diff --git a/tests/scripts/perf-sanity/README.md b/tests/scripts/perf-sanity/README.md index cd8f5639e89b..ee928939c659 100644 --- a/tests/scripts/perf-sanity/README.md +++ b/tests/scripts/perf-sanity/README.md @@ -6,7 +6,6 @@ Benchmarking scripts for TensorRT-LLM serving performance tests with configurati - Run performance benchmarks across multiple model configurations - Manage test cases through YAML configuration files -- Generate comprehensive CSV reports with complete test case coverage - Support selective execution of specific test cases ## Scripts Overview @@ -16,123 +15,120 @@ Benchmarking scripts for TensorRT-LLM serving performance tests with configurati **Structure**: ```yaml -test_cases: - - id: 1 - model: "70B-FP8" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 16384 +server_configs: + - name: "r1_fp4_dep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" + moe_max_num_tokens: "" + enable_attention_dp: true + enable_chunked_prefill: false + max_num_tokens: 2176 + disable_overlap_scheduler: false + kv_cache_dtype: "fp8" + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + + - name: "r1_fp4_tep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] + enable_attention_dp: false + enable_chunked_prefill: false + max_num_tokens: 2176 + disable_overlap_scheduler: false + kv_cache_dtype: "fp8" + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 ``` -**Configuration Fields**: -- `id`: Unique identifier for the test case -- `model`: Model name (e.g., "70B-FP8", "Scout-FP4") -- `gpus`: Number of GPUs to use -- `tp`: Tensor parallelism size -- `ep`: Expert parallelism size -- `attn_backend`: Attention backend ("TRTLLM", "FLASHINFER") -- `moe_backend`: MoE backend ("DEEPGEMM", "TRTLLM", "CUTLASS", "") -- `enable_attention_dp`: Enable attention data parallelism -- `free_gpu_mem_fraction`: GPU memory fraction to reserve -- `max_batch_size`: Maximum batch size -- `isl`: Input sequence length -- `osl`: Output sequence length -- `max_num_tokens`: Maximum number of tokens -- `moe_max_num_tokens`: Maximum number of tokens for MoE -- `concurrency_iterations`: List of [concurrency, iteration] pairs - - ### 2. `run_benchmark_serve.py` - Main Benchmark Runner **Purpose**: Executes performance benchmarks based on YAML configuration files. **Usage**: ```bash -python run_benchmark_serve.py --output_folder --config_file [--skip ] [--select ] +python run_benchmark_serve.py --log_folder --config_file [--select ] [--timeout 5400] ``` **Arguments**: -- `--output_folder`: Directory to store benchmark results (required) +- `--log_folder`: Directory to store benchmark logs (required) - `--config_file`: Path to YAML configuration file (required) -- `--skip`: Skip pattern for specific test cases/concurrencies (optional, default: no skipping) -- `--select`: Select pattern for specific test cases/concurrencies (optional, default: all test cases) +- `--select`: Select pattern for specific Server and Client Config. (optional, default: all test cases) +- `--timeout`: Timeout for server setup. (optional, default: 3600 seconds) **Examples**: ```bash -# Run all test cases -python run_benchmark_serve.py --output_folder results --config_file benchmark_config.yaml --skip default --select default - -# Skip specific test cases -python run_benchmark_serve.py --output_folder results --config_file benchmark_config.yaml --skip "2-1,4" - -# Run specific concurrencies from specific test cases -python run_benchmark_serve.py --output_folder results --config_file benchmark_config.yaml --select "1,2-3" +# Select +python run_benchmark_serve.py --log_folder ./results --config_file benchmark_config.yaml --select "r1_fp4_dep4:con8_iter1_1024_1024,r1_fp4_tep4:con1_iter1_1024_1024" ``` -**Skip Pattern**: -Format: `"test_case1,test_case2,test_case3"` or `"test_case1-concurrency1,test_case2-concurrency3"` -- `"2,4"`: Skip test cases 2 and 4 entirely -- `"2-1,4-2"`: Skip test case 2's 1st concurrency and test case 4's 2nd concurrency -- `"default"` or empty: No skipping (default) - -**Select Pattern**: -Format: `"test_case1,test_case2,test_case3"` or `"test_case1-concurrency1,test_case2-concurrency3"` -- `"1,3,5"`: Run only test cases 1, 3, and 5 (all concurrencies) -- `"1-1,2-3"`: Run test case 1's 1st concurrency and test case 2's 3rd concurrency -- `"default"` or empty: Run all test cases (default) - - ### 3. `parse_benchmark_results.py` - Results Parser -**Purpose**: Parses benchmark log files and generates comprehensive CSV reports with all test cases from the configuration file. - -**Usage**: -```bash -python parse_benchmark_results.py --input_folder --output_csv --config_file -``` +**Purpose**: Print log's perf. **Arguments**: -- `input_folder`: Folder containing benchmark log files (serve.*.log) (required) -- `output_csv`: Output CSV filename for the results table (required) -- `config_file`: Path to benchmark_config.yaml file (required) +- `--log_folder`: Directory to store benchmark logs (required) -**Examples**: +**Usage**: ```bash -python parse_benchmark_results.py --config_file ./benchmark_logs --output_csv results.csv --input_folder ./benchmark_config.yaml - +python parse_benchmark_results.py --log_folder ``` + ### 4. `benchmark-serve.sh` - SLURM Job Script **Usage**: ```bash -sbatch benchmark-serve.sh [IMAGE] [bench_dir] [output_dir] [select_pattern] [skip_pattern] +sbatch benchmark-serve.sh [IMAGE] [bench_dir] [log_folder] [select_pattern] ``` **Parameters**: - `IMAGE`: Docker image (default: tensorrt-llm-staging/release:main-x86_64) - `bench_dir`: Directory containing config file and benchmark scripts (default: current directory) -- `output_dir`: Directory containing output logs and csv. (default: current directory) +- `log_folder`: Directory containing output logs and csv. (default: current directory) - `select_pattern`: Select pattern (default: default - all test cases) -- `skip_pattern`: Skip pattern (default: default - no skipping) **Examples**: ```bash bench_dir="/path/to/benchmark/scripts" -output_dir="/path/to/store/output/files" -sbatch --reservation=RES--COM-3970 --qos=reservation -D ${output_dir} ${bench_dir}/benchmark-serve.sh urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release:main-x86_64 ${bench_dir} ${output_dir} "1-1" "" +log_folder="/path/to/store/output/files" +sbatch --reservation=RES--COM-3970 --qos=reservation -D ${log_folder} ${bench_dir}/benchmark-serve.sh urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release:main-x86_64 ${bench_dir} ${log_folder} "r1_fp4_dep4:con8_iter1_1024_1024,r1_fp4_tep4:con1_iter1_1024_1024" ``` diff --git a/tests/scripts/perf-sanity/benchmark-serve.sh b/tests/scripts/perf-sanity/benchmark-serve.sh index a3dd58cf723e..4621f571e804 100755 --- a/tests/scripts/perf-sanity/benchmark-serve.sh +++ b/tests/scripts/perf-sanity/benchmark-serve.sh @@ -10,13 +10,15 @@ env && hostname && nvidia-smi DEFAULT_IMAGE="urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release:main-x86_64" IMAGE=${1:-$DEFAULT_IMAGE} -bench_dir=${2:-$(pwd)} -output_dir=${3:-$(pwd)} -select_pattern=${4:-default} -skip_pattern=${5:-default} +config_file=${2:-$(pwd)} +select_pattern=${3:-default} +bench_dir=${4:-$(pwd)} +output_dir=${5:-$(pwd)} +trtllm_dir=${6:-""} +extra_options=${7:-""} start_time=$(date '+%Y-%m-%d-%H:%M:%S') -output_folder=${output_dir}/benchmark.run.${SLURM_JOB_ID}.${start_time}.${select_pattern}.${skip_pattern} +output_folder=${output_dir}/benchmark.run.${SLURM_JOB_ID}.${start_time}.${select_pattern} # Validate bench_dir exists if [[ ! -d "$bench_dir" ]]; then @@ -42,22 +44,38 @@ report_head() { run_benchmark_and_parse() { # Run benchmark and parse results in a single Docker container - docker run --rm --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ + mount=" -v /home/scratch.trt_llm_data:/home/scratch.trt_llm_data:ro -v $output_dir:$output_dir:rw -v $bench_dir:$bench_dir:ro" + if [[ -n "$trtllm_dir" && -d "$trtllm_dir" ]]; then + mount="$mount -v $trtllm_dir:$trtllm_dir:ro" + fi + docker run --rm --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 $extra_options \ --gpus all \ - -v /home/scratch.trt_llm_data:/home/scratch.trt_llm_data:ro \ - -v $output_dir:$output_dir:rw \ - -v $bench_dir:$bench_dir:ro \ + $mount \ -w `pwd` \ --pull always \ ${IMAGE} \ bash -c " echo 'Running benchmarks...' - python3 ${bench_dir}/run_benchmark_serve.py --output_folder ${output_folder} --config_file ${bench_dir}/benchmark_config.yaml --select ${select_pattern} --skip ${skip_pattern} + export LLM_MODELS_ROOT=/home/scratch.trt_llm_data/llm-models - echo 'Benchmarks completed. Generating CSV report...' + # Handle trtllm_dir parameter + if [[ -n \"$trtllm_dir\" && -d \"$trtllm_dir\" ]]; then + echo 'Installing TensorRT-LLM packages from $trtllm_dir...' + pip uninstall tensorrt_llm -y + pip install $trtllm_dir/build/tensorrt_llm*.whl --pre + export PATH=\$HOME/.local/bin:\$PATH + export PYTHONPATH=$trtllm_dir + echo 'TensorRT-LLM packages installed successfully' + else + echo 'No trtllm_dir specified or directory does not exist, running with default packages' + fi + + python3 ${bench_dir}/run_benchmark_serve.py --output_folder ${output_folder} --config_file ${bench_dir}/${config_file} --select ${select_pattern} + + echo 'Benchmarks completed. Parsing results...' if [[ -f '${bench_dir}/parse_benchmark_results.py' ]]; then - python3 ${bench_dir}/parse_benchmark_results.py --config_file ${bench_dir}/benchmark_config.yaml --input_folder ${output_folder} --output_csv ${output_folder}.csv - echo 'CSV report generated successfully' + python3 ${bench_dir}/parse_benchmark_results.py --log_folder ${output_folder} + echo 'Results parsed successfully' else echo 'Warning: parse_benchmark_results.py not found' fi diff --git a/tests/scripts/perf-sanity/benchmark_config.yaml b/tests/scripts/perf-sanity/benchmark_config.yaml deleted file mode 100644 index 6b5d25e69880..000000000000 --- a/tests/scripts/perf-sanity/benchmark_config.yaml +++ /dev/null @@ -1,468 +0,0 @@ -test_cases: - - id: 1 - model: "70B-FP8" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 2 - model: "70B-FP8" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 3 - model: "70B-FP8" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 4 - model: "70B-FP8" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 5 - model: "70B-FP4" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 6 - model: "70B-FP4" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 7 - model: "70B-FP4" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 8 - model: "70B-FP4" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.9 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 16384 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 9 - model: "Scout-FP8" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 10 - model: "Scout-FP8" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9334 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 11 - model: "Scout-FP8" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 12 - model: "Scout-FP8" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9334 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 13 - model: "Scout-FP4" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 14 - model: "Scout-FP4" - gpus: 1 - tp: 1 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9334 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 15 - model: "Scout-FP4" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 16 - model: "Scout-FP4" - gpus: 4 - tp: 4 - ep: 1 - attn_backend: "TRTLLM" - moe_backend: "" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9334 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - [64, 5] - - [512, 2] - - - id: 17 - model: "R1-FP8" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "DEEPGEMM" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - - id: 18 - model: "R1-FP8" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "DEEPGEMM" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 9344 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - - id: 19 - model: "R1-FP8" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "DEEPGEMM" - enable_attention_dp: true - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: 37376 - concurrency_iterations: - - [64, 5] - - [512, 2] - - [4096, 2] - - - id: 20 - model: "R1-FP8" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "DEEPGEMM" - enable_attention_dp: true - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9344 - moe_max_num_tokens: 9344 - concurrency_iterations: - - [64, 5] - - [512, 2] - - [4096, 2] - - - id: 21 - model: "R1-FP4" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "TRTLLM" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 1024 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - - id: 22 - model: "R1-FP4" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "TRTLLM" - enable_attention_dp: false - free_gpu_mem_fraction: 0.8 - max_batch_size: 1024 - isl: 8192 - osl: 1024 - max_num_tokens: 9344 - moe_max_num_tokens: "" - concurrency_iterations: - - [1, 10] - - [8, 10] - - - id: 23 - model: "R1-FP4" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "CUTLASS" - enable_attention_dp: true - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 1024 - osl: 1024 - max_num_tokens: 2176 - moe_max_num_tokens: 37376 - concurrency_iterations: - - [64, 5] - - [512, 2] - - [4096, 2] - - - id: 24 - model: "R1-FP4" - gpus: 8 - tp: 8 - ep: 8 - attn_backend: "TRTLLM" - moe_backend: "CUTLASS" - enable_attention_dp: true - free_gpu_mem_fraction: 0.8 - max_batch_size: 512 - isl: 8192 - osl: 1024 - max_num_tokens: 9344 - moe_max_num_tokens: 9344 - concurrency_iterations: - - [64, 5] - - [512, 2] - - [4096, 2] diff --git a/tests/scripts/perf-sanity/l0_dgx_b200.yaml b/tests/scripts/perf-sanity/l0_dgx_b200.yaml new file mode 100644 index 000000000000..b36c36afc698 --- /dev/null +++ b/tests/scripts/perf-sanity/l0_dgx_b200.yaml @@ -0,0 +1,56 @@ +server_configs: + - name: "r1_fp4_dep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" + enable_attention_dp: true + enable_chunked_prefill: false + max_num_tokens: 2176 + kv_cache_dtype: "fp8" + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + + - name: "r1_fp4_tep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" + enable_attention_dp: false + enable_chunked_prefill: false + max_num_tokens: 2176 + kv_cache_dtype: "fp8" + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 diff --git a/tests/scripts/perf-sanity/l0_dgx_b300.yaml b/tests/scripts/perf-sanity/l0_dgx_b300.yaml new file mode 100644 index 000000000000..27a30bec50f6 --- /dev/null +++ b/tests/scripts/perf-sanity/l0_dgx_b300.yaml @@ -0,0 +1,58 @@ +server_configs: + - name: "r1_fp4_dep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" + moe_max_num_tokens: "" + enable_attention_dp: true + enable_chunked_prefill: false + max_num_tokens: 2176 + kv_cache_dtype: "fp8" + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + + - name: "r1_fp4_tep4" + model_name: "deepseek_r1_0528_fp4" + tp: 4 + ep: 4 + pp: 1 + attention_backend: "TRTLLM" + moe_backend: "CUTLASS" + moe_max_num_tokens: "" + enable_attention_dp: false + enable_chunked_prefill: false + max_num_tokens: 2176 + kv_cache_dtype: "fp8" + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + enable_padding: true + client_configs: + - name: "con1_iter1_1024_1024" + concurrency: 1 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 + - name: "con8_iter1_1024_1024" + concurrency: 8 + iterations: 1 + isl: 1024 + osl: 1024 + random_range_ratio: 0.0 diff --git a/tests/scripts/perf-sanity/parse_benchmark_results.py b/tests/scripts/perf-sanity/parse_benchmark_results.py index ae3c87d3b4a7..65f8e24a00d8 100644 --- a/tests/scripts/perf-sanity/parse_benchmark_results.py +++ b/tests/scripts/perf-sanity/parse_benchmark_results.py @@ -4,377 +4,240 @@ import sys from pathlib import Path -import pandas as pd -import yaml +class PerfMetrics: + """Class to store and parse performance metrics from benchmark logs""" + + def __init__(self): + # Basic metrics + self.total_requests = 0 + self.successful_requests = 0 + self.failed_requests = 0 + self.benchmark_duration = 0.0 + self.total_input_tokens = 0 + self.total_generated_tokens = 0 + self.request_throughput = 0.0 + self.output_token_throughput = 0.0 + self.total_token_throughput = 0.0 + self.user_throughput = 0.0 + self.avg_decoded_tokens_per_iter = 0.0 + + # Time to First Token (TTFT) + self.mean_ttft_ms = 0.0 + self.median_ttft_ms = 0.0 + self.p99_ttft_ms = 0.0 + + # Time per Output Token (TPOT) + self.mean_tpot_ms = 0.0 + self.median_tpot_ms = 0.0 + self.p99_tpot_ms = 0.0 + + # Inter-token Latency (ITL) + self.mean_itl_ms = 0.0 + self.median_itl_ms = 0.0 + self.p99_itl_ms = 0.0 + + # End-to-end Latency (E2EL) + self.mean_e2el_ms = 0.0 + self.median_e2el_ms = 0.0 + self.p99_e2el_ms = 0.0 + + def to_str(self) -> str: + return f"Total Requests: {self.total_requests}, Successful Requests: {self.successful_requests}, Failed Requests: {self.failed_requests}, Benchmark Duration (s): {self.benchmark_duration}, Total Input Tokens: {self.total_input_tokens}, Total Generated Tokens: {self.total_generated_tokens}, Request Throughput (req/s): {self.request_throughput}, Output Token Throughput (tok/s): {self.output_token_throughput}, Total Token Throughput (tok/s): {self.total_token_throughput}, User Throughput (tok/s): {self.user_throughput}, Avg Decoded Tokens per Iter: {self.avg_decoded_tokens_per_iter}, Mean TTFT (ms): {self.mean_ttft_ms}, Median TTFT (ms): {self.median_ttft_ms}, P99 TTFT (ms): {self.p99_ttft_ms}, Mean TPOT (ms): {self.mean_tpot_ms}, Median TPOT (ms): {self.median_tpot_ms}, P99 TPOT (ms): {self.p99_tpot_ms}, Mean ITL (ms): {self.mean_itl_ms}, Median ITL (ms): {self.median_itl_ms}, P99 ITL (ms): {self.p99_itl_ms}, Mean E2EL (ms): {self.mean_e2el_ms}, Median E2EL (ms): {self.median_e2el_ms}, P99 E2EL (ms): {self.p99_e2el_ms}" + + @classmethod + def from_log_content(cls, log_content: str) -> 'PerfMetrics': + """Parse performance metrics from log content""" + metrics = cls() + + # Define patterns for each metric + patterns = { + 'total_requests': r'Total requests:\s+(\d+)', + 'successful_requests': r'Successful requests:\s+(\d+)', + 'failed_requests': r'Failed requests:\s+(\d+)', + 'benchmark_duration': r'Benchmark duration \(s\):\s+([\d.]+)', + 'total_input_tokens': r'Total input tokens:\s+(\d+)', + 'total_generated_tokens': r'Total generated tokens:\s+(\d+)', + 'request_throughput': r'Request throughput \(req/s\):\s+([\d.]+)', + 'output_token_throughput': + r'Output token throughput \(tok/s\):\s+([\d.]+)', + 'total_token_throughput': + r'Total Token throughput \(tok/s\):\s+([\d.]+)', + 'user_throughput': r'User throughput \(tok/s\):\s+([\d.]+)', + 'avg_decoded_tokens_per_iter': + r'Avg Decoded Tokens per Iter:\s+([\d.]+)', + 'mean_ttft_ms': r'Mean TTFT \(ms\):\s+([\d.]+)', + 'median_ttft_ms': r'Median TTFT \(ms\):\s+([\d.]+)', + 'p99_ttft_ms': r'P99 TTFT \(ms\):\s+([\d.]+)', + 'mean_tpot_ms': r'Mean TPOT \(ms\):\s+([\d.]+)', + 'median_tpot_ms': r'Median TPOT \(ms\):\s+([\d.]+)', + 'p99_tpot_ms': r'P99 TPOT \(ms\):\s+([\d.]+)', + 'mean_itl_ms': r'Mean ITL \(ms\):\s+([\d.]+)', + 'median_itl_ms': r'Median ITL \(ms\):\s+([\d.]+)', + 'p99_itl_ms': r'P99 ITL \(ms\):\s+([\d.]+)', + 'mean_e2el_ms': r'Mean E2EL \(ms\):\s+([\d.]+)', + 'median_e2el_ms': r'Median E2EL \(ms\):\s+([\d.]+)', + 'p99_e2el_ms': r'P99 E2EL \(ms\):\s+([\d.]+)', + } -def extract_config_from_log_content(log_file): - """ - Extract configuration from log file content using "Completed benchmark with Configuration:" pattern - """ - try: - with open(log_file, 'r') as f: - for line in f: - if "Completed benchmark with Configuration:" in line: - # Extract values using regex patterns - model_label_match = re.search(r'model_label=([^,]+)', line) - gpus_match = re.search(r'GPUs=(\d+)', line) - tp_match = re.search(r'TP=(\d+)', line) - ep_match = re.search(r'EP=(\d+)', line) - attn_backend_match = re.search(r'attn_backend=([^,]+)', - line) - moe_backend_match = re.search(r'moe_backend=([^,]+)', line) - enable_attention_dp_match = re.search( - r'enable_attention_dp=([^,]+)', line) - free_gpu_mem_fraction_match = re.search( - r'free_gpu_mem_fraction=([^,]+)', line) - max_batch_size_match = re.search(r'max_batch_size=(\d+)', - line) - isl_match = re.search(r'ISL=(\d+)', line) - osl_match = re.search(r'OSL=(\d+)', line) - max_num_tokens_match = re.search(r'max_num_tokens=(\d+)', - line) - moe_max_num_tokens_match = re.search( - r'moe_max_num_tokens=([^,]+)', line) - concurrency_match = re.search(r'Concurrency=(\d+)', line) - - # Extract values, use empty string if not found - model_label = model_label_match.group( - 1) if model_label_match else "" - gpus = int(gpus_match.group(1)) if gpus_match else "" - tp = int(tp_match.group(1)) if tp_match else "" - ep = int(ep_match.group(1)) if ep_match else "" - attn_backend = attn_backend_match.group( - 1) if attn_backend_match else "" - moe_backend = moe_backend_match.group( - 1) if moe_backend_match else "" - enable_attention_dp = enable_attention_dp_match.group( - 1) if enable_attention_dp_match else "" - free_gpu_mem_fraction = float( - free_gpu_mem_fraction_match.group( - 1)) if free_gpu_mem_fraction_match else "" - max_batch_size = int(max_batch_size_match.group( - 1)) if max_batch_size_match else "" - isl = int(isl_match.group(1)) if isl_match else "" - osl = int(osl_match.group(1)) if osl_match else "" - max_num_tokens = int(max_num_tokens_match.group( - 1)) if max_num_tokens_match else "" - moe_max_num_tokens_str = moe_max_num_tokens_match.group( - 1) if moe_max_num_tokens_match else "" - concurrency = int( - concurrency_match.group(1)) if concurrency_match else "" - - # Handle moe_max_num_tokens (could be "N/A", empty, or a number) - moe_max_num_tokens = "" - if moe_max_num_tokens_str and moe_max_num_tokens_str != "N/A": - try: - moe_max_num_tokens = int(moe_max_num_tokens_str) - except ValueError: - moe_max_num_tokens = "" - elif not moe_max_num_tokens_str: - moe_max_num_tokens = "" - - # Handle enable_attention_dp (convert string to boolean) - enable_attention_dp_bool = "" - if enable_attention_dp: - enable_attention_dp_bool = enable_attention_dp.lower( - ) == "true" - - # Check if all required fields are present (not empty strings) - if (model_label and gpus != "" and tp != "" and ep != "" - and attn_backend and free_gpu_mem_fraction != "" - and max_batch_size != "" and isl != "" and osl != "" - and max_num_tokens != "" and concurrency != ""): - return { - 'model_name': model_label, - 'gpus': gpus, - 'tp': tp, - 'ep': ep, - 'attn_backend': attn_backend, - 'moe_backend': moe_backend, - 'enable_attention_dp': enable_attention_dp_bool, - 'free_gpu_mem_fraction': free_gpu_mem_fraction, - 'max_batch_size': max_batch_size, - 'isl': isl, - 'osl': osl, - 'max_num_tokens': max_num_tokens, - 'moe_max_num_tokens': moe_max_num_tokens, - 'concurrency': concurrency, - 'found_in_log': True - } + # Parse each metric + for attr_name, pattern in patterns.items(): + match = re.search(pattern, log_content) + if match: + value = match.group(1) + try: + if '.' in value: + setattr(metrics, attr_name, float(value)) else: - print( - f"Warning: Incomplete configuration in {log_file} - missing required fields" - ) - return None - except Exception as e: - print(f"Warning: Could not read {log_file}: {e}") + setattr(metrics, attr_name, int(value)) + except ValueError: + # Keep default value if parsing fails + pass - return None + return metrics -def extract_metrics_from_log(log_file): +def extract_server_and_client_name_from_log(log_file): """ - Extract Total Token throughput and User throughput from log file + Extract server name, client name, and performance metrics from log file. + Looks for pattern: Server-Config: - """ - total_throughput = "" - user_throughput = "" - try: with open(log_file, 'r') as f: - for line in f: - if "Total Token throughput (tok/s):" in line: - parts = line.strip().split() - if len(parts) >= 5: - total_throughput = parts[4] - elif "User throughput (tok/s):" in line: - parts = line.strip().split() - if len(parts) >= 4: - user_throughput = parts[3] - except Exception as e: - print(f"Warning: Could not read {log_file}: {e}") + content = f.read() - return total_throughput, user_throughput + # Look for Server-Config pattern + server_config_match = re.search(r'Server-Config:\s*(\S+)', content) + if not server_config_match: + print( + f"Warning: Could not find 'Server-Config:' pattern in {log_file}" + ) + return None, None, None + # Extract the full config name + config_name = server_config_match.group(1) -def generate_all_test_cases(benchmark_config): - """ - Generate all test cases from benchmark_config.yaml including all concurrency iterations - """ - all_test_cases = [] - - for test_case in benchmark_config['test_cases']: - base_config = { - 'model_name': test_case['model'], - 'gpus': test_case['gpus'], - 'tp': test_case['tp'], - 'ep': test_case['ep'], - 'attn_backend': test_case['attn_backend'], - 'moe_backend': test_case['moe_backend'], - 'enable_attention_dp': test_case['enable_attention_dp'], - 'free_gpu_mem_fraction': test_case['free_gpu_mem_fraction'], - 'max_batch_size': test_case['max_batch_size'], - 'isl': test_case['isl'], - 'osl': test_case['osl'], - 'max_num_tokens': test_case['max_num_tokens'], - 'moe_max_num_tokens': test_case['moe_max_num_tokens'], - } + # Split on the last '-' to separate server and client names + # Format: - + parts = config_name.rsplit('-', 1) + if len(parts) != 2: + print( + f"Warning: Invalid Server-Config format in {log_file}: {config_name}" + ) + return None, None, None - # Generate a test case for each concurrency iteration - for concurrency, iterations in test_case['concurrency_iterations']: - test_case_config = base_config.copy() - test_case_config['concurrency'] = concurrency - test_case_config['iterations'] = iterations - test_case_config['TPS/System'] = "" - test_case_config['TPS/User'] = "" - all_test_cases.append(test_case_config) + server_name = parts[0] + client_name = parts[1] - return all_test_cases + # Extract PerfMetrics + perf_metrics = PerfMetrics.from_log_content(content) + return server_name, client_name, perf_metrics -def match_log_to_test_case(log_config, test_case): - """ - Check if a log configuration matches a test case configuration - Returns True if all parameters match exactly - """ - if not log_config: - return False - - # Check if all key parameters match exactly - return (log_config['model_name'] == test_case['model_name'] - and log_config['gpus'] == test_case['gpus'] - and log_config['tp'] == test_case['tp'] - and log_config['ep'] == test_case['ep'] - and log_config['attn_backend'] == test_case['attn_backend'] - and log_config['moe_backend'] == test_case['moe_backend'] - and log_config['enable_attention_dp'] - == test_case['enable_attention_dp'] - and log_config['free_gpu_mem_fraction'] - == test_case['free_gpu_mem_fraction'] - and log_config['max_batch_size'] == test_case['max_batch_size'] - and log_config['isl'] == test_case['isl'] - and log_config['osl'] == test_case['osl'] - and log_config['max_num_tokens'] == test_case['max_num_tokens'] and - (log_config['moe_max_num_tokens'] == test_case['moe_max_num_tokens'] - or (not log_config['moe_max_num_tokens'] - and not test_case['moe_max_num_tokens'])) - and log_config['concurrency'] == test_case['concurrency']) - - -def create_test_case_row(test_case): - """ - Create a row for a test case with empty performance data - """ - return { - 'model_name': test_case['model_name'], - 'GPUs': test_case['gpus'], - 'TP': test_case['tp'], - 'EP': test_case['ep'], - 'attn_backend': test_case['attn_backend'], - 'moe_backend': test_case['moe_backend'], - 'enable_attention_dp': test_case['enable_attention_dp'], - 'free_gpu_mem_fraction': test_case['free_gpu_mem_fraction'], - 'max_batch_size': test_case['max_batch_size'], - 'ISL': test_case['isl'], - 'OSL': test_case['osl'], - 'max_num_tokens': test_case['max_num_tokens'], - 'moe_max_num_tokens': test_case['moe_max_num_tokens'], - 'Concurrency': test_case['concurrency'], - 'Iterations': test_case['iterations'], - 'TPS/System': test_case['TPS/System'], - 'TPS/User': test_case['TPS/User'], - } - - -def parse_benchmark_results(input_folder, output_csv, config_file): + except Exception as e: + print(f"Warning: Could not read {log_file}: {e}") + return None, None, None + + +def parse_benchmark_results(log_folder): """ - Parse benchmark results and generate CSV table + Parse benchmark results from log files and print grouped by server and client names """ - input_folder = Path(input_folder) - config_file = Path(config_file) + log_folder = Path(log_folder) # Validate inputs - if not input_folder.exists(): - print(f"Error: Input folder '{input_folder}' does not exist") - return - - if not input_folder.is_dir(): - print(f"Error: '{input_folder}' is not a directory") - return - - if not config_file.exists(): - print(f"Error: Config file '{config_file}' does not exist") + if not log_folder.exists(): + print(f"Error: Input folder '{log_folder}' does not exist") return - # Load benchmark configuration - try: - with open(config_file, 'r') as f: - benchmark_config = yaml.safe_load(f) - print(f"Loaded benchmark configuration from: {config_file}") - except Exception as e: - print(f"Error: Could not load {config_file}: {e}") + if not log_folder.is_dir(): + print(f"Error: '{log_folder}' is not a directory") return - # Generate all test cases from config - all_test_cases = generate_all_test_cases(benchmark_config) - print(f"Generated {len(all_test_cases)} test cases from configuration") - - # Find all serve.*.log files - log_files = list(input_folder.glob("serve.*.log")) + # Find all trtllm-benchmark.*.log files + log_files = list(log_folder.glob("trtllm-benchmark.*.log")) print(f"Found {len(log_files)} log files to process") + # Dictionary to group results by server name and client name + # Structure: {server_name: {client_name: perf_metrics}} + results_by_server = {} + # Process each log file - matched_count = 0 + parsed_count = 0 for log_file in log_files: - print(f"Processing: {log_file.name}") - - # Extract configuration from log - log_config = extract_config_from_log_content(log_file) - if not log_config: - print(f" Skipped - could not parse configuration") + # Extract server name, client name, and PerfMetrics from log + server_name, client_name, perf_metrics = extract_server_and_client_name_from_log( + log_file) + if not server_name or not client_name or not perf_metrics: continue - # Extract performance metrics - total_throughput, user_throughput = extract_metrics_from_log(log_file) - - # Find matching test case in table - matched = False - for test_case in all_test_cases: - if match_log_to_test_case(log_config, test_case): - # Update performance data - test_case['TPS/System'] = total_throughput - test_case['TPS/User'] = user_throughput - matched = True - matched_count += 1 - break - - if not matched: + parsed_count += 1 + + # Group results by server name and client name + if server_name not in results_by_server: + results_by_server[server_name] = {} + + results_by_server[server_name][client_name] = perf_metrics + + print(f"Successfully parsed {parsed_count} log files\n") + + # Print grouped results + print_grouped_results(results_by_server) + + +def print_grouped_results(results_by_server): + """ + Print benchmark results grouped by server name and client name + """ + print("=" * 100) + + # Sort server names for consistent output + for server_name in sorted(results_by_server.keys()): + print(f"Server Name: {server_name}") + + # Sort client names for consistent output + for client_name in sorted(results_by_server[server_name].keys()): + perf_metrics = results_by_server[server_name][client_name] + + print(f"Client Name: {client_name}") print( - f" Skipped - no matching test case found for test case {test_case}" - ) - - print(f"Successfully matched {matched_count} log files to test cases") - - table_rows = [] - for test_case in all_test_cases: - row = create_test_case_row(test_case) - table_rows.append(row) - - # Add empty rows between different test configurations - final_table = [] - for i, row in enumerate(table_rows): - if i > 0: - prev_row = table_rows[i - 1] - # Check if any key parameters changed - if (row['model_name'] != prev_row['model_name'] - or row['TP'] != prev_row['TP'] - or row['EP'] != prev_row['EP'] - or row['moe_backend'] != prev_row['moe_backend'] - or row['ISL'] != prev_row['ISL'] - or row['OSL'] != prev_row['OSL']): - # Add empty row - empty_row = {key: '' for key in row.keys()} - final_table.append(empty_row) - - final_table.append(row) - - # Create DataFrame and save to CSV - df = pd.DataFrame(final_table) - - # Ensure output directory exists - output_path = Path(output_csv) - output_path.parent.mkdir(parents=True, exist_ok=True) - - # Save to CSV - df.to_csv(output_path, index=False) - - # Print summary - print(f"\nCSV table saved to: {output_path}") - print( - f"Total rows: {len(final_table)} (including {len(final_table) - len(table_rows)} empty separator rows)" - ) + f"Benchmark duration (s): {perf_metrics.benchmark_duration:.2f} " + f"Request throughput (req/s): {perf_metrics.request_throughput:.2f} " + f"Output token throughput (tok/s): {perf_metrics.output_token_throughput:.2f} " + f"Total Token throughput (tok/s): {perf_metrics.total_token_throughput:.2f} " + f"User throughput (tok/s): {perf_metrics.user_throughput:.2f} " + f"Mean TTFT (ms): {perf_metrics.mean_ttft_ms:.2f} " + f"Median TTFT (ms): {perf_metrics.median_ttft_ms:.2f} " + f"P99 TTFT (ms): {perf_metrics.p99_ttft_ms:.2f}") - return df + print("=" * 100) def main(): parser = argparse.ArgumentParser( description= - "Script to parse benchmark metrics from a specified folder and generate CSV table", + "Script to parse benchmark metrics from log files and print grouped by server and client names", epilog= - "Example: python parse_benchmark_results.py ./benchmark_logs results.csv ./benchmark_config.yaml" + "Example: python parse_benchmark_results.py --log_folder ./benchmark_logs" ) parser.add_argument( - "--input_folder", - help="Folder containing benchmark log files (serve.*.log)") - parser.add_argument("--output_csv", - help="Output CSV filename for the results table") - parser.add_argument("--config_file", - help="Path to benchmark_config.yaml file") + "--log_folder", + required=True, + help="Folder containing benchmark log files (trtllm-benchmark.*.log)") args = parser.parse_args() # Validate inputs - input_folder_path = Path(args.input_folder) - config_file_path = Path(args.config_file) + log_folder_path = Path(args.log_folder) - if not input_folder_path.exists(): - print(f"Error: Input folder '{args.input_folder}' not found.") + if not log_folder_path.exists(): + print(f"Error: Input folder '{args.log_folder}' not found.") sys.exit(1) - if not input_folder_path.is_dir(): - print(f"Error: '{args.input_folder}' is not a directory.") + if not log_folder_path.is_dir(): + print(f"Error: '{args.log_folder}' is not a directory.") sys.exit(1) - if not config_file_path.exists(): - print(f"Error: Config file '{args.config_file}' not found.") - sys.exit(1) - - print(f"Using input folder: {input_folder_path}") - print(f"Using config file: {config_file_path}") - print(f"Output will be saved to: {args.output_csv}") - print() - parse_benchmark_results(args.input_folder, args.output_csv, - args.config_file) + parse_benchmark_results(args.log_folder) if __name__ == "__main__": diff --git a/tests/scripts/perf-sanity/run_benchmark_serve.py b/tests/scripts/perf-sanity/run_benchmark_serve.py index 2d4928ae325d..a0cf4ebd3619 100644 --- a/tests/scripts/perf-sanity/run_benchmark_serve.py +++ b/tests/scripts/perf-sanity/run_benchmark_serve.py @@ -1,753 +1,771 @@ #!/usr/bin/env python3 import argparse +import ast import os import subprocess import sys import time from pathlib import Path -from typing import Any, Dict, List, Set +from typing import Dict, List, NamedTuple import requests import yaml -class BenchmarkRunner: +def get_node_name() -> str: + """Get the current node name""" + try: + result = subprocess.run("hostname", + shell=True, + capture_output=True, + text=True, + check=True) + return result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return "unknown" + + +def get_gpu_info() -> str: + """Get GPU information from nvidia-smi""" + try: + result = subprocess.run("nvidia-smi", + shell=True, + capture_output=True, + text=True, + check=True) + return result.stdout + except subprocess.CalledProcessError as e: + return f"nvidia-smi failed with error code {e.returncode}\nError output: {e.stderr}" + except FileNotFoundError: + return "nvidia-smi not found" + except Exception as e: + return f"Failed to get GPU information: {e}" + + +# Model PATH of local dir synced from internal LLM models repo +MODEL_PATH_DICT = { + "llama_v2_7b": "llama-models-v2/llama-v2-7b-hf", # not safetensors repo + "llama_v2_13b": "llama-models-v2/llama-v2-13b-hf", # not safetensors repo + "llama_v2_70b": "llama-models-v2/llama-v2-70b-hf", # not safetensors repo + "llama_v3.1_8b": "llama-3.1-model/Meta-Llama-3.1-8B", + "llama_v3.1_8b_instruct": "llama-3.1-model/Llama-3.1-8B-Instruct", + "llama_v3.1_8b_instruct_fp8": "llama-3.1-model/Llama-3.1-8B-Instruct-FP8", + "llama_v3.1_8b_instruct_fp4": + "modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4", + "llama_v3.1_70b": "llama-3.1-model/Meta-Llama-3.1-70B", + "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", + "llama_v3.1_70b_instruct_fp8": "llama-3.1-model/Llama-3.1-70B-Instruct-FP8", + "llama_v3.3_70b_instruct_fp8": + "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8", + "llama_v3.3_70b_instruct_fp4": + "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4", + "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", + "llama_v3.1_405b_instruct_fp8": + "llama-3.1-model/Llama-3.1-405B-Instruct-FP8", + "llama_v3.1_405b_instruct_fp4": + "modelopt-hf-model-hub/Llama-3.1-405B-Instruct-fp4", + "llama_v3.1_70b_instruct": "llama-3.1-model/Meta-Llama-3.1-70B-Instruct", + "llama_v3.2_1b": "llama-3.2-models/Llama-3.2-1B", + "llama_v3.1_nemotron_nano_8b": "Llama-3.1-Nemotron-Nano-8B-v1", + "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", + "llama_v3.3_nemotron_super_49b": + "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1", + "llama_v3.3_nemotron_super_49b_fp8": + "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8", + "llama_v3.1_nemotron_ultra_253b": + "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1", + "llama_v3.1_nemotron_ultra_253b_fp8": + "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", + "llama_v4_scout_17b_16e_instruct": + "llama4-models/Llama-4-Scout-17B-16E-Instruct", + "llama_v4_scout_17b_16e_instruct_fp8": + "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", + "llama_v4_scout_17b_16e_instruct_fp4": + "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", + "llama_v4_maverick_17b_128e_instruct": + "llama4-models/Llama-4-Maverick-17B-128E-Instruct", + "llama_v4_maverick_17b_128e_instruct_fp8": + "llama4-models/nvidia/Llama-4-Maverick-17B-128E-Instruct-FP8", + "mixtral_8x7b_v0.1": "Mixtral-8x7B-v0.1", + "mixtral_8x7b_v0.1_instruct": "Mixtral-8x7B-Instruct-v0.1", + "mixtral_8x7b_v0.1_instruct_fp8": "Mixtral-8x7B-Instruct-v0.1-fp8", + "mixtral_8x7b_v0.1_instruct_fp4": + "modelopt-hf-model-hub/Mixtral-8x7B-Instruct-v0.1-fp4", + "mistral_nemo_12b_base": "Mistral-Nemo-Base-2407", + "deepseek_r1_distill_qwen_32b": "DeepSeek-R1/DeepSeek-R1-Distill-Qwen-32B", + "mixtral_8x22b_v0.1": "Mixtral-8x22B-v0.1", + "mistral_7b_v0.1": "mistral-7b-v0.1", + "ministral_8b": "Ministral-8B-Instruct-2410", + "ministral_8b_fp8": "Ministral-8B-Instruct-2410-FP8", + "gemma_3_1b_it": "gemma/gemma-3-1b-it", + "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", + "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", + "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", + "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", + "deepseek_v3_lite_fp8": "DeepSeek-V3-Lite/fp8", + "deepseek_v3_lite_nvfp4": "DeepSeek-V3-Lite/nvfp4_moe_only", + "qwen2_7b_instruct": "Qwen2-7B-Instruct", + "qwen_14b_chat": "Qwen-14B-Chat", + "qwen3_235b_a22b_fp8": "Qwen3/saved_models_Qwen3-235B-A22B_fp8_hf", + "qwen3_235b_a22b_fp4": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", + "starcoder2_3b": "starcoder2-3b", + "starcoder_15b": "starcoder2-15b", + "t5": "t5-small", # not supported for trtllm-bench build config + "flan_t5_base": + "flan-t5-small", # not supported for trtllm-bench build config + "flan_t5_large": + "flan-t5-xl", # not supported for trtllm-bench build config + "whisper_large_v3": + "whisper-models/large-v3", # not supported for trtllm-bench tokenizer + "bart_large_cnn": "bart-large-cnn", # not safetensors repo + "mbart_large_50_many_to_one_mmt": "mbart-large-50-many-to-one-mmt", + "mamba_130m": "mamba/mamba-130m-hf", + "mamba_370m": "mamba/mamba-370m-hf", + "mamba_2.8b": "mamba/mamba-2.8b-hf", + "gpt_20b": "gpt-neox-20b", + "gpt_350m_moe": "gpt2-medium", + "phi_3_mini_4k_instruct": "Phi-3/Phi-3-mini-4k-instruct", + "phi_3_mini_128k_instruct": "Phi-3/Phi-3-mini-128k-instruct", + "phi_4_mini_instruct": "Phi-4-mini-instruct", + "phi_4_multimodal_instruct": "multimodals/Phi-4-multimodal-instruct", + "phi_4_multimodal_instruct_image": "multimodals/Phi-4-multimodal-instruct", + "phi_4_multimodal_instruct_audio": "multimodals/Phi-4-multimodal-instruct", + "bielik_11b_v2.2_instruct": "Bielik-11B-v2.2-Instruct", + "bielik_11b_v2.2_instruct_fp8": "Bielik-11B-v2.2-Instruct-FP8", + "mistral_small_v3.1_24b": "Mistral-Small-3.1-24B-Instruct-2503", + "gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b", +} +# Model PATH of HuggingFace +HF_MODEL_PATH = { + "llama_v2_7b_hf": "meta-llama/Llama-2-7b-hf", + "llama_v2_70b_hf": "meta-llama/Llama-2-70b-hf", + "falcon_180b_hf": "tiiuae/falcon-180B", + "gptj_6b_hf": "EleutherAI/gpt-j-6b", + "llama_v3_8b_hf": "meta-llama/Meta-Llama-3-8B", + "llama_v3.1_8b_hf": "meta-llama/Llama-3.1-8B", + "llama_v3.1_8b_instruct_hf": "nvidia/Llama-3.1-8B-Instruct-FP8", + "llama_v3.1_70b_instruct_hf": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "llama_v3_70b_hf": "meta-llama/Meta-Llama-3-70B", + "llama_v3.1_70b_hf": "meta-llama/Llama-3.1-70B", + "llama_v3.1_405b_hf": "meta-llama/Llama-3.1-405B", + "llama_v3.1_nemotron_nano_8b_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1", + "llama_v3.1_nemotron_nano_8b_fp8_hf": + "nvidia/Llama-3.1-Nemotron-Nano-8B-v1-FP8", + "llama_v3.3_nemotron_super_49b_hf": + "nvidia/Llama-3_3-Nemotron-Super-49B-v1", + "llama_v3.3_nemotron_super_49b_fp8_hf": + "nvidia/Llama-3_3-Nemotron-Super-49B-v1-FP8", + "llama_v3.1_nemotron_ultra_253b_fp8_hf": + "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", + "mixtral_8x7b_v0.1_hf": "mistralai/Mixtral-8x7B-v0.1", + "mixtral_8x7b_v0.1_instruct_hf": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "mistral_7b_v0.1_hf": "mistralai/Mistral-7B-v0.1", + "ministral_8b_hf": "mistralai/Ministral-8B-Instruct-2410", + "flan_t5_base_hf": "google/flan-t5-small", + "phi_4_mini_instruct_hf": "microsoft/Phi-4-mini-instruct", + "gemma_3_1b_it_hf": "google/gemma-3-1b-it", +} + +LLM_MODELS_ROOT = os.environ.get('LLM_MODELS_ROOT', + '/home/scratch.trt_llm_data/llm-models') + + +# Model path mapping +def llm_models_root(): + return LLM_MODELS_ROOT + + +def get_model_dir(model_name: str) -> str: + model_dir = "" + if model_name in MODEL_PATH_DICT.keys(): + model_dir = os.path.join(llm_models_root(), MODEL_PATH_DICT[model_name]) + elif model_name in HF_MODEL_PATH.keys(): + model_dir = os.path.join(llm_models_root(), + MODEL_PATH_DICT[model_name.split('_hf')[0]]) + return model_dir + + +def str_to_bool(value: str) -> bool: + return ast.literal_eval(value) + + +# {metric_name: (is_optional, type)} +SERVER_CONFIG_METRICS = { + "model_name": (False, str), + "tp": (False, int), + "ep": (False, int), + "pp": (True, int), + "isl": (False, int), + "osl": (False, int), + "max_num_tokens": (False, int), + "enable_chunked_prefill": (True, str_to_bool), + "disable_overlap_scheduler": (True, str_to_bool), + "attention_backend": (False, str), + "moe_backend": (True, str), + "moe_max_num_tokens": (True, str), + "stream_interval": (True, int), + "enable_attention_dp": (True, str_to_bool), + "attention_dp_balance": (True, str_to_bool), + "batching_wait_iters": (True, int), + "timeout_iters": (True, int), + "kv_cache_dtype": (True, str), + "enable_block_reuse": (True, str_to_bool), + "free_gpu_memory_fraction": (True, float), + "max_batch_size": (False, int), + "enable_padding": (True, str_to_bool), +} + +CLIENT_CONFIG_METRICS = { + "concurrency": (False, int), + "iterations": (False, int), + "random_range_ratio": (True, float), +} + + +class ServerConfig: + """ + Configurations of trtllm-server. + """ + + def __init__( + self, + name: str, + model_name: str, + tp: int, + ep: int, + max_num_tokens: int, + attention_backend: str, + max_batch_size: int, + pp: int = 1, + enable_chunked_prefill: bool = False, + disable_overlap_scheduler: bool = False, + moe_backend: str = "", + moe_max_num_tokens: str = "", + stream_interval: int = 10, + enable_attention_dp: bool = False, + attention_dp_balance: bool = False, + batching_wait_iters: int = 10, + timeout_iters: int = 50, + kv_cache_dtype: str = "fp8", + enable_block_reuse: bool = False, + free_gpu_memory_fraction: float = 0.8, + enable_padding: bool = True, + ): + self.name = name + self.model_name = model_name + self.tp = tp + self.ep = ep + self.pp = pp + self.max_num_tokens = max_num_tokens + self.enable_chunked_prefill = enable_chunked_prefill + self.disable_overlap_scheduler = disable_overlap_scheduler + self.attention_backend = attention_backend + self.moe_backend = moe_backend + self.moe_max_num_tokens = moe_max_num_tokens + self.stream_interval = stream_interval + self.enable_attention_dp = enable_attention_dp + self.attention_dp_balance = attention_dp_balance + self.batching_wait_iters = batching_wait_iters + self.timeout_iters = timeout_iters + self.kv_cache_dtype = kv_cache_dtype + self.enable_block_reuse = enable_block_reuse + self.free_gpu_memory_fraction = free_gpu_memory_fraction + self.max_batch_size = max_batch_size + self.enable_padding = enable_padding + + self.model_path = "" + + def to_cmd(self, working_dir: str) -> List[str]: + model_dir = get_model_dir(self.model_name) + self.model_path = model_dir if os.path.exists( + model_dir) else self.model_name + config_path = os.path.join(working_dir, + f"extra-llm-api-config.{self.name}.yml") + return [ + "trtllm-serve", self.model_path, "--host", "localhost", "--port", + "8000", "--backend", "pytorch", "--extra_llm_api_options", + config_path + ] - def __init__(self, - output_folder: str, - config_file: str, - skip_pattern: str = None, - select_pattern: str = None): - self.output_folder = Path(output_folder) - self.config_file = Path(config_file) - - # Treat empty or "default" values as None (default behavior) - self.skip_pattern = None if not skip_pattern or skip_pattern.lower( - ) == "default" else skip_pattern - self.select_pattern = None if not select_pattern or select_pattern.lower( - ) == "default" else select_pattern - - self.skip_test_cases: Set[int] = set() - self.skip_concurrencies: Dict[int, Set[int]] = {} - self.select_test_cases: Set[int] = set() - self.select_concurrencies: Dict[int, Set[int]] = {} - - if self.skip_pattern: - self.parse_skip_pattern(self.skip_pattern) - - if self.select_pattern: - self.parse_select_pattern(self.select_pattern) - - # Execution plan: {test_case_id: [concurrency_indices]} - self.execution_plan: Dict[int, List[int]] = {} - - # Model path mapping - self.model_paths = { - "70B-FP4": - "/home/scratch.trt_llm_data/llm-models/llama-3.3-models/Llama-3.3-70B-Instruct-FP4", - "70B-FP8": - "/home/scratch.trt_llm_data/llm-models/llama-3.3-models/Llama-3.3-70B-Instruct-FP8", - "Scout-FP4": - "/home/scratch.trt_llm_data/llm-models/llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", - "Scout-FP8": - "/home/scratch.trt_llm_data/llm-models/llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", - "R1-FP8": - "/home/scratch.trt_llm_data/llm-models/DeepSeek-R1/DeepSeek-R1/", - "R1-FP4": - "/home/scratch.trt_llm_data/llm-models/DeepSeek-R1/DeepSeek-R1-0528-FP4" - } - - # Set environment variables - os.environ['TQDM_MININTERVAL'] = '1000' - os.environ['PRINT_ITER_LOG'] = 'false' - - # Capture system information - self.node_name = self.get_node_name() - self.gpu_info = self.get_gpu_info() - - # Change to output directory - os.chdir(self.output_folder) - - def get_node_name(self) -> str: - """Get the current node name""" - try: - result = subprocess.run("hostname", - shell=True, - capture_output=True, - text=True, - check=True) - return result.stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return "unknown" - - def get_gpu_info(self) -> str: - """Get GPU information from nvidia-smi""" - try: - result = subprocess.run("nvidia-smi", - shell=True, - capture_output=True, - text=True, - check=True) - return result.stdout - except subprocess.CalledProcessError as e: - return f"nvidia-smi failed with error code {e.returncode}\nError output: {e.stderr}" - except FileNotFoundError: - return "nvidia-smi not found" - - def parse_skip_pattern(self, skip_pattern: str) -> None: - """Parse skip pattern like '2,4-1' to determine what to skip""" - if not skip_pattern: - return - - parts = skip_pattern.split(',') - for part in parts: - part = part.strip() - if not part: # Skip empty parts - continue - - if '-' in part: - # Format: "test_case-concurrency_index" (1-based) - try: - test_case_str, concurrency_str = part.split('-') - test_case_id = int(test_case_str) - concurrency_index = int( - concurrency_str) - 1 # Convert to 0-based - - if test_case_id not in self.skip_concurrencies: - self.skip_concurrencies[test_case_id] = set() - self.skip_concurrencies[test_case_id].add(concurrency_index) - except ValueError: - raise ValueError( - f"Invalid skip pattern '{part}'. Expected format: 'test_case-concurrency_index' (e.g., '2-1')" - ) - else: - # Format: "test_case" - skip entire test case - try: - test_case_id = int(part) - self.skip_test_cases.add(test_case_id) - except ValueError: - raise ValueError( - f"Invalid test case ID '{part}' in skip pattern. Must be a valid integer." - ) - - print(f"Skipping test cases: {sorted(self.skip_test_cases)}") - print(f"Skipping concurrencies: {self.skip_concurrencies}") - - def parse_select_pattern(self, select_pattern: str) -> None: - """Parse select pattern like '1,3,5' or '1-1,2-3' to determine which test cases/concurrencies to run""" - if not select_pattern: - return - - self.select_concurrencies: Dict[int, Set[int]] = {} - - parts = select_pattern.split(',') - for part in parts: - part = part.strip() - if not part: # Skip empty parts - continue - - if '-' in part: - # Format: "test_case-concurrency_index" (1-based) - try: - test_case_str, concurrency_str = part.split('-') - test_case_id = int(test_case_str) - concurrency_index = int( - concurrency_str) - 1 # Convert to 0-based - - if test_case_id not in self.select_concurrencies: - self.select_concurrencies[test_case_id] = set() - self.select_concurrencies[test_case_id].add( - concurrency_index) - except ValueError: - raise ValueError( - f"Invalid select pattern '{part}'. Expected format: 'test_case-concurrency_index' (e.g., '2-1')" - ) - else: - # Format: "test_case" - select entire test case - try: - test_case_id = int(part) - self.select_test_cases.add(test_case_id) - except ValueError: - raise ValueError( - f"Invalid test case ID '{part}' in select pattern. Must be a valid integer." - ) - - print(f"Selected test cases: {sorted(self.select_test_cases)}") - print(f"Selected concurrencies: {self.select_concurrencies}") - - def build_execution_plan(self, test_cases: List[Dict[str, Any]]) -> None: - """Build execution plan by analyzing config file, skip_pattern, and select_pattern""" - self.execution_plan.clear() - - # Step 1: Initialize execution plan based on select_pattern - if not self.select_pattern: - # If select_pattern is empty or default, include all test cases with all concurrencies - for test_case in test_cases: - test_case_id = test_case['id'] - all_concurrencies = list( - range(len(test_case['concurrency_iterations']))) - self.execution_plan[test_case_id] = all_concurrencies - else: - # If select_pattern is specified, only include selected test cases and concurrencies - for test_case in test_cases: - test_case_id = test_case['id'] - - # Check if this test case is selected - if test_case_id in self.select_test_cases: - # Test case is selected - include all concurrencies - all_concurrencies = list( - range(len(test_case['concurrency_iterations']))) - self.execution_plan[test_case_id] = all_concurrencies - elif test_case_id in self.select_concurrencies: - # Specific concurrencies are selected for this test case - selected_concurrencies = list( - self.select_concurrencies[test_case_id]) - # Validate that selected concurrencies exist in config - max_concurrency_index = len( - test_case['concurrency_iterations']) - 1 - valid_concurrencies = [ - c for c in selected_concurrencies - if 0 <= c <= max_concurrency_index - ] - if valid_concurrencies: - self.execution_plan[test_case_id] = valid_concurrencies - - # Step 2: Apply skip_pattern to remove test cases and concurrencies - # Remove entire test cases that are in skip_test_cases - for test_case_id in self.skip_test_cases: - if test_case_id in self.execution_plan: - del self.execution_plan[test_case_id] - - # Remove specific concurrencies that are in skip_concurrencies - for test_case_id, skip_concurrency_indices in self.skip_concurrencies.items( - ): - if test_case_id in self.execution_plan: - # Remove skipped concurrencies from the list - remaining_concurrencies = [ - c for c in self.execution_plan[test_case_id] - if c not in skip_concurrency_indices - ] - if remaining_concurrencies: - self.execution_plan[test_case_id] = remaining_concurrencies - else: - # If no concurrencies remain, remove the entire test case - del self.execution_plan[test_case_id] - - # Step 3: Clean up - remove test cases with empty concurrency lists - # (This should not happen with the above logic, but just to be safe) - test_cases_to_remove = [] - for test_case_id, concurrencies in self.execution_plan.items(): - if not concurrencies: - test_cases_to_remove.append(test_case_id) - - for test_case_id in test_cases_to_remove: - del self.execution_plan[test_case_id] - - def print_execution_plan(self, test_cases: List[Dict[str, Any]]) -> None: - """Print which test cases and concurrencies will be executed""" - print("\n" + "=" * 80) - print("EXECUTION PLAN") - print("=" * 80) - - total_test_cases = 0 - total_concurrencies = 0 - - for test_case in test_cases: - test_case_id = test_case['id'] - model_label = test_case['model'] - - # Check if this test case is in execution plan - if test_case_id not in self.execution_plan: - print(f"Test Case {test_case_id}: {model_label} - SKIPPED") - continue - - total_test_cases += 1 - print(f"\nTest Case {test_case_id}: {model_label}") - print( - f" Config: GPUs={test_case['gpus']}, TP={test_case['tp']}, EP={test_case['ep']}, attn_backend={test_case['attn_backend']}, moe_backend={test_case['moe_backend']}" - ) - - # Get concurrencies from execution plan - concurrencies_to_run = [] - for concurrency_index in self.execution_plan[test_case_id]: - concurrency, iteration = test_case['concurrency_iterations'][ - concurrency_index] - concurrencies_to_run.append( - (concurrency_index + 1, concurrency, - iteration)) # +1 for 1-based display - total_concurrencies += 1 - - print( - f" Concurrencies to run ({len(concurrencies_to_run)}/{len(test_case['concurrency_iterations'])}):" - ) - for concurrency_num, concurrency, iteration in concurrencies_to_run: - print( - f" {concurrency_num}. Concurrency={concurrency}, Iteration={iteration}" - ) - - print("\n" + "=" * 80) - print( - f"SUMMARY: {total_test_cases} test cases, {total_concurrencies} concurrencies will be executed" - ) - print("=" * 80 + "\n") - - def generate_extra_llm_api_config(self, test_case: Dict[str, Any]) -> str: + def generate_extra_llm_api_config(self) -> str: """Generate extra-llm-api-config.yml content""" config_lines = [ - "print_iter_log: true", - f"enable_attention_dp: {str(test_case['enable_attention_dp']).lower()}", - "disable_overlap_scheduler: false", - "stream_interval: 10", - f"attn_backend: {test_case['attn_backend']}", + f"tensor_parallel_size: {self.tp}", + f"moe_expert_parallel_size: {self.ep}", + f"pipeline_parallel_size: {self.pp}", + f"max_num_tokens: {self.max_num_tokens}", + f"enable_attention_dp: {str(self.enable_attention_dp).lower()}", + f"disable_overlap_scheduler: {str(self.disable_overlap_scheduler).lower()}", + f"stream_interval: {self.stream_interval}", + f"attn_backend: {self.attention_backend}", + f"enable_chunked_prefill: {str(self.enable_chunked_prefill).lower()}", "cuda_graph_config:", - " enable_padding: true", - f" max_batch_size: {test_case['max_batch_size']}", + f" enable_padding: {str(self.enable_padding).lower()}", + f" max_batch_size: {self.max_batch_size}", "kv_cache_config:", - " dtype: fp8", - f" free_gpu_memory_fraction: {test_case['free_gpu_mem_fraction']}", - " enable_block_reuse: false", + f" dtype: {self.kv_cache_dtype}", + f" free_gpu_memory_fraction: {self.free_gpu_memory_fraction}", + f" enable_block_reuse: {str(self.enable_block_reuse).lower()}", + "print_iter_log: false", ] # Add moe_config if moe_backend is specified - if test_case['moe_backend']: + if self.moe_backend: config_lines.append("moe_config:") - config_lines.append(f" backend: {test_case['moe_backend']}") - - if test_case['moe_max_num_tokens']: + config_lines.append(f" backend: {self.moe_backend}") + if self.moe_max_num_tokens: config_lines.append( - f" max_num_tokens: {test_case['moe_max_num_tokens']}") + f" max_num_tokens: {self.moe_max_num_tokens}") + + if self.attention_dp_balance: + config_lines.append("attention_dp_balance:") + config_lines.append(" enable_balance: true") + config_lines.append( + f" batching_wait_iters: {self.batching_wait_iters}") + config_lines.append(f" timeout_iters: {self.timeout_iters}") return "\n".join(config_lines) - def wait_for_server(self, - server_pid: int, - server_log_filename: str, - max_attempts: int = 360) -> bool: - """Wait for server to be ready""" - print("Waiting for trtllm-serve to be ready...") - for attempt in range(1, max_attempts + 1): - # Check if server is still running - try: - os.kill(server_pid, 0) # Check if process exists - except OSError: - print("Error: Server process has died") - return False - - # Check server log for runtime errors - if self.check_for_runtime_error(server_log_filename): - print( - f"RuntimeError detected in server log: {server_log_filename}" - ) - print("Killing server process due to runtime error") - try: - subprocess.run(f"kill -9 {server_pid}", - shell=True, - check=False) - subprocess.run(f"wait {server_pid} 2>/dev/null || true", - shell=True, - check=False) - except Exception as e: - print(f"Warning: Error killing server process: {e}") - return False - - # Try to connect to server - try: - response = requests.get("http://localhost:8000/v1/models", - timeout=5) - if response.status_code == 200: - print( - f"Server is ready! HTTP status: {response.status_code}") - return True - except requests.RequestException: - pass - - print( - f"Attempt {attempt}/{max_attempts}: Server not ready yet, waiting..." - ) - time.sleep(10) - - print( - f"Error: Server did not become ready after {max_attempts} attempts") - return False - - def check_for_runtime_error(self, log_file_path: str) -> bool: - """Check if RuntimeError exists in log file""" - try: - if os.path.exists(log_file_path): - with open(log_file_path, 'r') as f: - content = f.read() - if "RuntimeError" in content or "runtime error" in content or "illegal memory access" in content or "terminate called" in content: - return True - except Exception as e: - print(f"Warning: Could not read log file {log_file_path}: {e}") - return False - - def run_benchmark(self, test_case: Dict[str, Any], concurrency: int, - iteration: int, model_path: str, - server_log_filename: str) -> bool: - """Run a single benchmark with monitoring. Returns True if successful, False if should skip test case""" - num_prompts = concurrency * iteration - - print( - f'Running benchmark with concurrency: {concurrency}, iteration: {iteration}, num-prompts: {num_prompts}' - ) - - # Build benchmark command - benchmark_cmd = [ +class ClientConfig: + """ + Configurations of benchmark client. + """ + + def __init__(self, + name: str, + model_name: str, + concurrency: int, + iterations: int, + isl: int, + osl: int, + random_range_ratio: float = 0.0): + self.name = name + self.model_name = model_name + self.concurrency = concurrency + self.iterations = iterations + self.isl = isl + self.osl = osl + self.random_range_ratio = random_range_ratio + + self.model_path = "" + + def to_cmd(self, working_dir: str) -> List[str]: + model_dir = get_model_dir(self.model_name) + self.model_path = model_dir if os.path.exists( + model_dir) else self.model_name + return [ "python", "-m", "tensorrt_llm.serve.scripts.benchmark_serving", - "--model", model_path, "--dataset-name", "random", "--random-ids", - "--num-prompts", - str(num_prompts), "--random-input-len", - str(test_case['isl']), "--random-output-len", - str(test_case['osl']), "--random-range-ratio", "0.0", - "--ignore-eos", "--percentile-metrics", "ttft,tpot,itl,e2el", - "--max-concurrency", - str(concurrency) + "--model", self.model_path, "--dataset-name", "random", + "--random-ids", "--num-prompts", + str(self.concurrency * self.iterations), "--random-input-len", + str(self.isl), "--random-output-len", + str(self.osl), "--random-range-ratio", + str(self.random_range_ratio), "--ignore-eos", + "--percentile-metrics", "ttft,tpot,itl,e2el", "--max-concurrency", + str(self.concurrency) ] - print(f'Running benchmark with command:') - print(' '.join(benchmark_cmd)) - print() - # Prepare log filename - benchmark_log_filename = ( - f"serve.{test_case['model']}.tp{test_case['tp']}.ep{test_case['ep']}." - f"attn{test_case['attn_backend']}.moe{test_case['moe_backend']}." - f"gpu{test_case['free_gpu_mem_fraction']}.batch{test_case['max_batch_size']}." - f"isl{test_case['isl']}.osl{test_case['osl']}." - f"tokens{test_case['max_num_tokens']}.moetokens{test_case['moe_max_num_tokens']}." - f"concurrency{concurrency}.iter{iteration}.log") +def parse_select_pattern(select_pattern: str): + """Parse select pattern like 'r1_fp4_dep4,r1_fp4_tep4:con1_iter1_1024_1024,r1_fp4_tep4:con8_iter1_1024_1024' - try: - with open(benchmark_log_filename, 'w') as f: - f.write(f"GPU Info: {self.gpu_info}\n") - - # Start benchmark as subprocess - with open(benchmark_log_filename, 'a') as log_file: - benchmark_process = subprocess.Popen(benchmark_cmd, - stdout=log_file, - stderr=subprocess.STDOUT) - - # Monitor logs every 60 seconds with timeout - print( - f"Starting log monitoring for benchmark process (PID: {benchmark_process.pid})" - ) - - start_time = time.time() - timeout_seconds = 3600 # 1 hour timeout - - while benchmark_process.poll() is None: # Process is still running - time.sleep(60) # Wait 60 seconds - - # Check if benchmark has been running for more than 1 hour - elapsed_time = time.time() - start_time - if elapsed_time > timeout_seconds: - print( - f"Benchmark timeout after {elapsed_time:.0f} seconds (>{timeout_seconds} seconds)" - ) - print("Killing benchmark process due to timeout") - try: - subprocess.run(f"kill -9 {benchmark_process.pid}", - shell=True, - check=False) - benchmark_process.wait(timeout=10) - except Exception as e: - print(f"Warning: Error killing benchmark process: {e}") - return False # Signal to skip test case - - print( - f"Checking logs for RuntimeError... (benchmark PID: {benchmark_process.pid}, elapsed: {elapsed_time:.0f}s)" - ) - - # Check server log for RuntimeError - if self.check_for_runtime_error(server_log_filename): - print( - f"RuntimeError found in server log: {server_log_filename}" - ) - print( - "Killing benchmark process and skipping this test case") - try: - subprocess.run(f"kill -9 {benchmark_process.pid}", - shell=True, - check=False) - benchmark_process.wait(timeout=10) - except Exception as e: - print(f"Warning: Error killing benchmark process: {e}") - return False # Signal to skip test case - - # Check benchmark log for RuntimeError - if self.check_for_runtime_error(benchmark_log_filename): - print( - f"RuntimeError found in benchmark log: {benchmark_log_filename}" - ) - print( - "Killing benchmark process and skipping this test case") - try: - subprocess.run(f"kill -9 {benchmark_process.pid}", - shell=True, - check=False) - benchmark_process.wait(timeout=10) - except Exception as e: - print(f"Warning: Error killing benchmark process: {e}") - return False # Signal to skip test case - - # Process completed, check final return code - return_code = benchmark_process.returncode - if return_code != 0: - print( - f"Benchmark process completed with error code: {return_code}" - ) - - # Read and display error output - try: - with open(benchmark_log_filename, 'r') as f: - error_content = f.read() - print( - f"Benchmark error output:\n{error_content[-1000:]}" - ) # Last 1000 chars - except Exception as e: - print(f"Could not read benchmark log: {e}") - - print( - f"Skipping this concurrency level and continuing with next one..." - ) - print("-----------------------------------------") - return True # Continue with next concurrency, don't skip test case - - # Success case - print( - f"Benchmark completed successfully (PID: {benchmark_process.pid})" - ) - - # Add configuration summary to log file - config_summary = ( - f"Completed benchmark with Configuration: " - f"model_label={test_case['model']}, GPUs={test_case['gpus']}, " - f"TP={test_case['tp']}, EP={test_case['ep']}, " - f"attn_backend={test_case['attn_backend']}, " - f"moe_backend={test_case['moe_backend']}, " - f"enable_attention_dp={test_case['enable_attention_dp']}, " - f"free_gpu_mem_fraction={test_case['free_gpu_mem_fraction']}, " - f"max_batch_size={test_case['max_batch_size']}, " - f"ISL={test_case['isl']}, OSL={test_case['osl']}, " - f"max_num_tokens={test_case['max_num_tokens']}, " - f"moe_max_num_tokens={test_case['moe_max_num_tokens']}, " - f"Concurrency={concurrency}") - with open(benchmark_log_filename, 'a') as f: - f.write(f"\n{config_summary}\n") - - print("-----------------------------------------") - return True # Continue with next concurrency - - except Exception as e: - print( - f"Error running benchmark with concurrency {concurrency}: {e}") - print( - f"Skipping this concurrency level and continuing with next one..." - ) - print("-----------------------------------------") - return True # Continue with next concurrency, don't skip test case - - def run_test_case(self, test_case: Dict[str, Any]) -> None: - """Run a test case using the execution plan""" - model_label = test_case['model'] - test_case_id = test_case['id'] - - # Get model path - model_path = self.model_paths.get(model_label) - if not model_path: - print(f"Error: No model path found for {model_label}") - return - - # Use local path if it exists, otherwise use model name - if os.path.exists(model_path): - MODEL = model_path - else: - MODEL = model_label + Format: + - ',' splits different server configs + - ':' means for this server, we choose specific clients + - If no ':', all clients are chosen for that server - # Generate extra-llm-api-config.yml - config_content = self.generate_extra_llm_api_config(test_case) - config_path = "/tmp/extra-llm-api-config.yml" + Returns: + - Dict with server name as key and either None (all clients) or set of client names as value + """ + execution_plan = {} - with open(config_path, 'w') as f: - f.write(config_content) + parts = select_pattern.split(',') + for part in parts: + part = part.strip() + if not part: # Skip empty parts + continue - print("extra-llm-api-config.yml:") - print(config_content) - - # Build trtllm-serve command - serve_cmd = [ - "trtllm-serve", MODEL, "--backend", "pytorch", "--tp_size", - str(test_case['tp']), "--ep_size", - str(test_case['ep']), "--max_batch_size", - str(test_case['max_batch_size']), "--max_num_tokens", - str(test_case['max_num_tokens']), - "--kv_cache_free_gpu_memory_fraction", - str(test_case['free_gpu_mem_fraction']), "--extra_llm_api_options", - config_path - ] + if ':' in part: + # Format: "server_name:client_name" + server_name, client_name = part.split(':', 1) + server_name = server_name.strip() + client_name = client_name.strip() - print("Starting trtllm-serve with command:") - print(' '.join(serve_cmd)) - print() - - # Start server - server_log_filename = ( - f"trtllm-serve.{model_label}.tp{test_case['tp']}.ep{test_case['ep']}." - f"attn{test_case['attn_backend']}.moe{test_case['moe_backend']}." - f"gpu{test_case['free_gpu_mem_fraction']}.batch{test_case['max_batch_size']}." - f"isl{test_case['isl']}.osl{test_case['osl']}." - f"tokens{test_case['max_num_tokens']}.moetokens{test_case['moe_max_num_tokens']}.log" - ) + # Only add if not already set to None (all clients) + if server_name not in execution_plan: + execution_plan[server_name] = set() + if execution_plan[server_name] is not None: + execution_plan[server_name].add(client_name) + else: + # Format: "server_name" - select all clients for this server + server_name = part.strip() + execution_plan[server_name] = None + + return execution_plan + + +def parse_config_file(config_file_path: str, select_pattern: str = None): + """Parse YAML configuration file and create ServerConfig and ClientConfig objects + + Args: + config_file_path: Path to YAML configuration file + select_pattern: Selection pattern string (e.g., "r1_fp4_dep4,r1_fp4_tep4:con1_iter1_1024_1024") + + Returns: + execution_plan: None (all servers/clients) or dict with server names as keys + server_configs: List of ServerConfig objects + server_client_configs: Dict with server id as key and list of ClientConfig as value + """ + # Parse selection pattern + if select_pattern: + execution_plan = parse_select_pattern(select_pattern) + else: + execution_plan = None + + # Read YAML config file + with open(config_file_path, 'r') as f: + config = yaml.safe_load(f) + + server_configs = [] + server_client_configs = {} + + for server_config_data in config['server_configs']: + server_name = server_config_data['name'] + + # Check if this server should be included based on execution_plan + if execution_plan is not None and server_name not in execution_plan: + continue + + # Create ServerConfig object + server_config = ServerConfig( + name=server_config_data['name'], + model_name=server_config_data['model_name'], + tp=server_config_data['tp'], + ep=server_config_data['ep'], + pp=server_config_data.get('pp', 1), + attention_backend=server_config_data.get('attention_backend', + 'TRTLLM'), + moe_backend=server_config_data.get('moe_backend', ''), + moe_max_num_tokens=server_config_data.get('moe_max_num_tokens', ''), + enable_attention_dp=server_config_data.get('enable_attention_dp', + False), + attention_dp_balance=server_config_data.get('attention_dp_balance', + False), + batching_wait_iters=server_config_data.get('batching_wait_iters', + 10), + timeout_iters=server_config_data.get('timeout_iters', 50), + enable_chunked_prefill=server_config_data.get( + 'enable_chunked_prefill', False), + max_num_tokens=server_config_data.get('max_num_tokens', 2048), + disable_overlap_scheduler=server_config_data.get( + 'disable_overlap_scheduler', False), + kv_cache_dtype=server_config_data.get('kv_cache_dtype', 'fp8'), + enable_block_reuse=server_config_data.get('enable_block_reuse', + False), + free_gpu_memory_fraction=server_config_data.get( + 'free_gpu_memory_fraction', 0.8), + max_batch_size=server_config_data.get('max_batch_size', 256), + enable_padding=server_config_data.get('enable_padding', True)) + + server_id = len(server_configs) + server_configs.append(server_config) + + # Create ClientConfig objects + client_configs = [] + selected_client_names = execution_plan.get( + server_name) if execution_plan else None + + for client_config_data in server_config_data['client_configs']: + client_name = client_config_data['name'] + + # Check if this client should be included + # Include if: execution_plan is None OR selected_client_names is None OR client_name in selected_client_names + if execution_plan is not None and selected_client_names is not None: + if client_name not in selected_client_names: + continue + + client_config = ClientConfig( + name=client_config_data['name'], + model_name=server_config_data['model_name'], + concurrency=client_config_data['concurrency'], + iterations=client_config_data.get('iterations', 1), + isl=client_config_data.get('isl', 1024), + osl=client_config_data.get('osl', 1024), + random_range_ratio=client_config_data.get( + 'random_range_ratio', 0.0)) + client_configs.append(client_config) + + server_client_configs[server_id] = client_configs + + return execution_plan, server_configs, server_client_configs + + +def get_trtllm_server_client_commands( + server_configs: List[ServerConfig], + server_client_configs: Dict[int, List[ClientConfig]], working_dir: str): + server_cmds = [] + client_cmds = [] + names = [] + for server_idx, client_configs in server_client_configs.items(): + server_config = server_configs[server_idx] + server_cmd = server_config.to_cmd(working_dir) + # Generate extra-llm-api-config.yml + config_content = server_config.generate_extra_llm_api_config() + config_filename = f"extra-llm-api-config.{server_config.name}.yml" + config_path = os.path.join(working_dir, config_filename) + with open(config_path, 'w') as f: + f.write(config_content) + for client_config in client_configs: + server_cmds.append(server_cmd) + client_cmd = client_config.to_cmd(working_dir) + client_cmds.append(client_cmd) + names.append(f"{server_config.name}-{client_config.name}") + return server_cmds, client_cmds, names + + +class PerfServerBenchmarkCmds(NamedTuple): + server_cmds: List[List[str]] + client_cmds: List[List[str]] + names: List[str] + working_dir: str + + def wait_for_endpoint_ready(self, url: str, timeout: int = 5400): + start = time.monotonic() + while time.monotonic() - start < timeout: + try: + time.sleep(10) + if requests.get(url, timeout=5).status_code == 200: + print(f"endpoint {url} is ready") + return + except Exception as err: + print(f"endpoint {url} is not ready, with exception: {err}") + print_error( + f"Endpoint {url} did not become ready within {timeout} seconds") + + def run_cmd(self, + cmd_idx: int, + node_name: str, + gpu_info: str, + max_timeout: int = 5400) -> str: + output = "" + server_file_path = os.path.join( + self.working_dir, f"trtllm-serve.{self.names[cmd_idx]}.log") + client_file_path = os.path.join( + self.working_dir, f"trtllm-benchmark.{self.names[cmd_idx]}.log") + + server_proc = None try: - with open(server_log_filename, 'w') as log_file: - log_file.write(f"extra-llm-api-config.yml:\n") - log_file.write(config_content) - log_file.write("\n") - - with open(server_log_filename, 'a') as log_file: - server_process = subprocess.Popen(serve_cmd, - stdout=log_file, - stderr=subprocess.STDOUT) + # Run server command + with open(server_file_path, 'w') as server_ctx: + server_proc = subprocess.Popen(self.server_cmds[cmd_idx], + stdout=server_ctx, + stderr=subprocess.STDOUT) # Wait for server to be ready - if not self.wait_for_server(server_process.pid, - server_log_filename): - print( - "Failed to start server, killing process and skipping this test case" - ) - try: - subprocess.run(f"kill -9 {server_process.pid}", - shell=True, - check=False) - subprocess.run( - f"wait {server_process.pid} 2>/dev/null || true", - shell=True, - check=False) - except Exception as e: - print(f"Warning: Error during server cleanup: {e}") - return - - # Run benchmarks based on execution plan - for concurrency_index in self.execution_plan[test_case_id]: - concurrency, iteration = test_case['concurrency_iterations'][ - concurrency_index] - should_continue = self.run_benchmark(test_case, concurrency, - iteration, MODEL, - server_log_filename) - - # If run_benchmark returns False, skip the entire test case - if not should_continue: - print( - f"RuntimeError detected - skipping remaining concurrencies for test case {test_case_id}" - ) - break - + self.wait_for_endpoint_ready("http://localhost:8000/v1/models", + timeout=max_timeout) + + # Save node name, gpu info, server config, client config output to server file path + with open(client_file_path, 'w') as client_ctx: + client_ctx.write(f"Node: {node_name}\n") + client_ctx.write(f"GPU Info: {gpu_info}\n") + client_ctx.write(f"Server-Config: {self.names[cmd_idx]}\n") + # Run client command + subprocess.run(self.client_cmds[cmd_idx], + stdout=client_ctx, + stderr=subprocess.STDOUT, + check=True) finally: - # Cleanup: Kill server process using shell commands like in the original bash script - print(f"Stopping server for {model_label}") - try: - # Use shell commands for more reliable process killing - subprocess.run(f"kill -9 {server_process.pid}", - shell=True, - check=False) - subprocess.run(f"wait {server_process.pid} 2>/dev/null || true", - shell=True, - check=False) - except Exception as e: - print(f"Warning: Error during server cleanup: {e}") - - time.sleep(5) # Give it time to clean up resources - print(f"Benchmark completed for {model_label}") - print() - - def run_benchmarks(self) -> None: - """Main function to run all benchmarks from config file""" - script_start_time = time.time() - - print(f"Using config file: {self.config_file}") - if self.select_pattern: - print(f"Select pattern: {self.select_pattern}") - else: - print("Select pattern: default (all test cases)") - if self.skip_pattern: - print(f"Skip pattern: {self.skip_pattern}") - else: - print("Skip pattern: default (no skipping)") - - # Load configuration - with open(self.config_file, 'r') as f: - config = yaml.safe_load(f) - - test_cases = config['test_cases'] - - # Build execution plan - self.build_execution_plan(test_cases) - - # Print execution plan before starting benchmarks - self.print_execution_plan(test_cases) - - # Run each test case based on execution plan - for i, test_case in enumerate(test_cases, 1): - test_case_id = test_case['id'] - - if test_case_id not in self.execution_plan: - print("=" * 57) - print( - f"Test case {i}/{len(test_cases)} (ID: {test_case_id}): {test_case['model']} - SKIPPED" - ) - print("=" * 57) - continue - - print("=" * 57) - print( - f"Test case {i}/{len(test_cases)} (ID: {test_case_id}): {test_case['model']}" - ) - print( - f"Config: GPUs={test_case['gpus']}, TP={test_case['tp']}, EP={test_case['ep']}, attn_backend={test_case['attn_backend']}, moe_backend={test_case['moe_backend']}" - ) - print("=" * 57) - - self.run_test_case(test_case) - - # Calculate and display total script runtime - script_total_time = time.time() - script_start_time - hours = int(script_total_time // 3600) - minutes = int((script_total_time % 3600) // 60) - seconds = int(script_total_time % 60) - - print("=" * 80) - print("SCRIPT COMPLETION SUMMARY") - print("=" * 80) - print( - f"Total script runtime: {hours:02d}:{minutes:02d}:{seconds:02d} (HH:MM:SS)" - ) - print(f"Total runtime in seconds: {script_total_time:.2f}") - print("=" * 80) - print("All benchmarks completed!") + server_proc.terminate() + server_proc.wait() + + return output + + def get_cmd_str(self, cmd_idx) -> List[str]: + return ["server-benchmark tests, please check config files"] + + +def run_perf_tests(server_configs: List[ServerConfig], + server_client_configs: Dict[int, List[ClientConfig]], + max_timeout: int, working_dir: str, node_name: str, + gpu_info: str) -> None: + """Main function to run all benchmarks from config file""" + + server_cmds, client_cmds, names = get_trtllm_server_client_commands( + server_configs, server_client_configs, working_dir) + commands = PerfServerBenchmarkCmds(server_cmds=server_cmds, + client_cmds=client_cmds, + names=names, + working_dir=working_dir) + + # Run each server config based on execution plan + for cmd_idx in range(len(client_cmds)): + print(f"Server cmd: {server_cmds[cmd_idx]}") + print(f"Client cmd: {client_cmds[cmd_idx]}") + commands.run_cmd(cmd_idx, node_name, gpu_info, max_timeout) + + +def generate_repro_scripts(server_configs: List[ServerConfig], + server_client_configs: Dict[int, List[ClientConfig]], + node_name: str, max_timeout: int): + """Generate reproduction scripts for all server configs""" + for server_id, server_config in enumerate(server_configs): + script_content = generate_repro_script(server_config, + server_client_configs[server_id], + node_name, max_timeout) + script_filename = f"reproduce.server-{server_config.name}.sh" + + with open(script_filename, 'w') as f: + f.write(script_content) + + # Make script executable + os.chmod(script_filename, 0o755) + + +def generate_repro_script(server_config: ServerConfig, + client_configs: List[ClientConfig], node_name: str, + max_timeout: int) -> str: + """Generate a shell script to reproduce a server config""" + model_path = server_config.model_path + script_content = f"""#!/bin/bash +# Reproduction script for server: {server_config.name}) +# Node: {node_name} + +set -e + +# Function to wait for server to be ready +wait_for_server() {{ + local timeout={max_timeout} + local attempt=0 + + echo "Waiting for trtllm-serve to be ready..." + + while [ $((attempt * 60)) -le $timeout ]; do + if ! kill -0 $SERVER_PID 2>/dev/null; then + echo "Error: Server process has died" + return 1 + fi + + # Check for runtime errors in server log + if grep -q "RuntimeError\\|runtime error\\|CUDA error\\|illegal memory access\\|terminate called" "$SERVER_LOG" 2>/dev/null; then + echo "RuntimeError detected in server log: $SERVER_LOG" + echo "Killing server process due to runtime error" + kill -9 $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true + return 1 + fi + + # Try to connect to server + if curl -s "http://localhost:8000/v1/models" > /dev/null 2>&1; then + echo "Server is ready! HTTP status: 200" + return 0 + fi + + echo "Elapsed time: $((attempt * 60)) / $timeout seconds: Server not ready yet, waiting..." + sleep 60 + attempt=$((attempt + 1)) + done + + echo "Error: Server did not become ready after $timeout seconds" + return 1 +}} + +# Function to cleanup server process +cleanup_server() {{ + if [ -n "$SERVER_PID" ]; then + echo "Stopping server" + kill -9 $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true + sleep 5 # Give it time to clean up resources + echo "Server cleanup completed" + fi +}} + +# Set trap to cleanup server on script exit +trap cleanup_server EXIT + +# Generate extra-llm-api-config.yml +CONFIG_FILENAME="extra-llm-api-config.{server_config.name}.yml" + +cat > "$CONFIG_FILENAME" << 'EOF' +{server_config.generate_extra_llm_api_config()} +EOF + +# Start trtllm-serve in background +SERVER_LOG="trtllm-serve.{server_config.name}.log" + +echo "Starting trtllm-serve with command:" +echo "trtllm-serve {model_path} --host localhost --port 8000 --backend pytorch --extra_llm_api_options $CONFIG_FILENAME" + +trtllm-serve {model_path} --host localhost --port 8000 --backend pytorch --extra_llm_api_options "$CONFIG_FILENAME" > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! +echo "Server started with PID: $SERVER_PID" + +# Wait for server to be ready +if ! wait_for_server; then + echo "Failed to start server, exiting" + exit 1 +fi + +echo "Server is ready, starting benchmarks..." + +# Run benchmarks for each concurrency level +""" + # Add benchmark commands for each client config + for client_config in client_configs: + num_prompts = client_config.concurrency * client_config.iterations + script_content += f""" +echo "Running benchmark with concurrency: {client_config.concurrency}, iterations: {client_config.iterations}, num-prompts: {num_prompts}" + +BENCHMARK_LOG="trtllm-benchmark.{server_config.name}.{client_config.name}.log" + +echo "Running benchmark with command:" +echo "python -m tensorrt_llm.serve.scripts.benchmark_serving --model {model_path} --dataset-name random --random-ids --num-prompts {num_prompts} --random-input-len {client_config.isl} --random-output-len {client_config.osl} --random-range-ratio {client_config.random_range_ratio} --ignore-eos --percentile-metrics ttft,tpot,itl,e2el --max-concurrency {client_config.concurrency}" + +python -m tensorrt_llm.serve.scripts.benchmark_serving \\ + --model {model_path} \\ + --dataset-name random \\ + --random-ids \\ + --num-prompts {num_prompts} \\ + --random-input-len {client_config.isl} \\ + --random-output-len {client_config.osl} \\ + --random-range-ratio {client_config.random_range_ratio} \\ + --ignore-eos \\ + --percentile-metrics ttft,tpot,itl,e2el \\ + --max-concurrency {client_config.concurrency} > "$BENCHMARK_LOG" 2>&1 + +if [ $? -eq 0 ]; then + echo "Benchmark completed successfully" +else + echo "Benchmark failed with error code $?" +fi + +echo "-----------------------------------------" +""" + + script_content += f""" + +echo "All benchmarks completed successfully!" +echo "Server will be automatically cleaned up on script exit" +""" + return script_content def main(): parser = argparse.ArgumentParser( description='Run benchmarks from YAML configuration file') - parser.add_argument('--output_folder', + parser.add_argument('--log_folder', required=True, help='Output folder for benchmark results') parser.add_argument('--config_file', required=True, help='Path to YAML configuration file') parser.add_argument( - '--skip', - help= - 'Skip pattern: "2,4-1" means skip test case 2 and test case 4\'s 1st concurrency' - ) - parser.add_argument( - '--select', - help= - 'Select pattern: "1,3,5" means only run test cases 1, 3, and 5; "1-1,2-3" means only run test case 1\'s 1st concurrency and test case 2\'s 3rd concurrency' - ) + '--select', help='Select pattern: "r1_fp4_dep4:con1_iter1_1024_1024"') + parser.add_argument('--timeout', help='Timeout in seconds', default=5400) args = parser.parse_args() @@ -762,17 +780,35 @@ def main(): print(f"Error: Config file '{args.config_file}' does not exist") sys.exit(1) - if not os.path.exists(args.output_folder): - print(f"Error: Output folder '{args.output_folder}' does not exist") + if not os.path.exists(args.log_folder): + print(f"Error: Output folder '{args.log_folder}' does not exist") sys.exit(1) - try: - runner = BenchmarkRunner(args.output_folder, args.config_file, - args.skip, args.select) - runner.run_benchmarks() - except Exception as e: - print(f"Error: {e}") - sys.exit(1) + # Capture system information + node_name = get_node_name() + gpu_info = get_gpu_info() + + log_folder = Path(args.log_folder) + config_file = Path(args.config_file) + + default_max_timeout = 5400 + max_timeout = int(args.timeout) if args.timeout else default_max_timeout + + # Change to output directory + os.chdir(log_folder) + + # Treat empty or "default" values as None (default behavior) + select_pattern = None if not args.select or args.select.lower( + ) == "default" else args.select + + execution_plan, server_configs, server_client_configs = parse_config_file( + config_file, select_pattern) + + generate_repro_scripts(server_configs, server_client_configs, node_name, + max_timeout) + + run_perf_tests(server_configs, server_client_configs, max_timeout, + log_folder, node_name, gpu_info) if __name__ == "__main__": From cf15a7a90a705f26045f79663ce7fc874264cc29 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Mon, 20 Oct 2025 19:28:58 -0700 Subject: [PATCH 2/2] Add perf-sanity tests Signed-off-by: Chenfei Zhang --- jenkins/L0_Test.groovy | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 927e38fc944f..e0361d907e3e 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2655,6 +2655,9 @@ def launchTestJobs(pipeline, testFilter) "DGX_B200-8_GPUs-PyTorch-1": ["b200-x8", "l0_dgx_b200", 1, 1, 8], "DGX_B200-4_GPUs-PyTorch-Post-Merge-1": ["b200-trtllm", "l0_dgx_b200", 1, 1, 4, 1, true], "DGX_B300-4_GPUs-PyTorch-Post-Merge-1": ["b300-x4", "l0_dgx_b300", 1, 1, 4], + // Perf sanity post merge test + "DGX_B200-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b200-x4", "perf_sanity_l0_dgx_b200", 1, 1, 4], + "DGX_B300-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b300-x4", "perf_sanity_l0_dgx_b300", 1, 1, 4], ] fullSet += x86SlurmTestConfigs.keySet() @@ -2674,20 +2677,6 @@ def launchTestJobs(pipeline, testFilter) parallelJobs += parallelSlurmJobs - // Add Perf Sanity Test Slurm jobs - perfSanityTestConfigs = [ - "DGX_B200-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b200-x4", "perf_sanity_l0_dgx_b200", 1, 1, 4], - "DGX_B300-4_GPUs-PyTorch-Perf-Sanity-Post-Merge-1": ["b300-x4", "perf_sanity_l0_dgx_b300", 1, 1, 4], - ] - fullSet += perfSanityTestConfigs.keySet() - - parallelPerfSanityJobs = perfSanityTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { - def config = VANILLA_CONFIG - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("Perf"), key, values[2], values[3], values[4] ?: 1) - }]]} - - parallelJobs += parallelPerfSanityJobs - // Try to match what are being tested on x86 H100_PCIe. // The total machine time is scaled proportionally according to the number of each GPU. SBSATestConfigs = [