value
+ # Using regex to find operator right after field name, avoiding false matches in values
+ m = re.match(r"^\s*(\w+)\s*(>=|<=|!=|>|<|=)\s*(.*)", clause)
+ if not m:
+ return None, f"Invalid clause (missing operator): {clause}"
+
+ key = m.group(1).strip()
+ found_op = m.group(2)
+ value = _parse_value(m.group(3))
+
+ if not key:
+ return None, f"Invalid clause (empty field name): {clause}"
+
+ # Check if comparison operator is allowed for this field
+ # != is allowed for all fields, but >, <, >=, <= are restricted
+ if found_op not in ("=", "!=") and not _can_use_comparison_operator(key):
+ return None, (
+ f"Comparison operator '{found_op}' not allowed for field '{key}'. "
+ f"Only fields starting with 'd_', 'l_', or field 'ts_created' can use >, <, >=, <= operators."
+ )
+
+ # Convert date string to timestamp for ts_created
+ if key == "ts_created" and isinstance(value, str):
+ try:
+ value = _parse_date_string(value)
+ except ValueError as e:
+ return None, str(e)
+
+ result.append((key, found_op, value))
+
+ return result, None
+
+
+def _build_opensearch_clause(field, operator, value):
+ """Build OpenSearch query clause from field, operator, and value.
+
+ Returns a tuple (clause_type, clause) where clause_type is "must" or "must_not".
+ """
+ if operator == "=":
+ return ("must", {"term": {field: value}})
+
+ if operator == "!=":
+ return ("must_not", {"term": {field: value}})
+
+ op_map = {
+ ">": "gt",
+ "<": "lt",
+ ">=": "gte",
+ "<=": "lte",
}
- json_data = json.dumps(json_data)
+ return ("must", {"range": {field: {op_map[operator]: value}}})
+
+
+def parse_update_operation(operation):
+ match = re.match(
+ r"^\s*UPDATE\s+SET\s+(.+?)(?:\s+WHERE\s+(.+))?\s*$", operation, flags=re.IGNORECASE
+ )
+ if not match:
+ return None, None, "Invalid UPDATE operation format"
+ set_text = match.group(1).strip()
+ where_text = match.group(2).strip() if match.group(2) else ""
+ set_values, error = _parse_assignments(set_text)
+ if error:
+ return None, None, f"Invalid SET clause: {error}"
+ where_clauses = []
+ if match.group(2) is not None:
+ if not where_text:
+ return None, None, "Invalid WHERE clause: empty scope"
+ where_clauses, error = _parse_where_clauses(where_text)
+ if error:
+ return None, None, f"Invalid WHERE clause: {error}"
+ return set_values, where_clauses, None
+
+
+def update_perf_data_fields(data_list, set_values):
+ updated_list = []
+ for data in data_list:
+ updated_data = data.copy()
+ for key, value in set_values.items():
+ updated_data[key] = value
+ updated_list.append(updated_data)
+ return updated_list
- data_list = []
+
+def post_perf_data(data_list, project_name):
+ if not data_list:
+ print(f"No data to post to {project_name}")
+ return False
try:
- res = OpenSearchDB.queryFromOpenSearchDB(json_data, project_name)
- if res is None:
- print(f"Failed to query from {project_name}, returned no response")
- return None
- payload = res.json().get("hits", {}).get("hits", [])
- if len(payload) == 0:
- print(f"No regression data found in {project_name}, returned empty list")
- return []
- for hit in payload:
- data_dict = hit.get("_source", {})
- data_dict["_id"] = hit.get("_id", "")
- if data_dict["_id"] == "":
- print(f"Failed to query from {project_name}, returned data with no _id")
- return None
- data_list.append(data_dict)
- print(f"Successfully queried from {project_name}, queried {len(data_list)} entries")
- return data_list
+ print(f"Ready to post {len(data_list)} data to {project_name}")
+ return OpenSearchDB.postToOpenSearchDB(data_list, project_name)
except Exception as e:
- print(f"Failed to query from {project_name}, returned error: {e}")
- return None
+ print(f"Failed to post data to {project_name}, error: {e}")
+ return False
-def get_regression_data_by_job_id(data_list, query_job_number):
- """Returns a dict with job_id as key and list of regression data as value.
+def get_regression_dict(data_list, query_job_number, lookback_job_number=LOOKBACK_JOBS):
+ """Returns a dict with job_id as key and list of regression tuples as value.
+ Each tuple is (test_case_name, gpu_type, runtime, history_regression_job_ids, data).
Only returns the latest query_job_number jobs.
"""
if data_list is None or len(data_list) == 0:
return {}
# Group data by job_id
- job_data_dict = {}
+ job_test_dict = {}
for data in data_list:
- job_id = data.get("s_job_id", "")
- if job_id == "":
+ raw_job_id = data.get("s_job_id", "")
+ if raw_job_id == "":
continue
- if job_id not in job_data_dict:
- job_data_dict[job_id] = []
- job_data_dict[job_id].append(data)
-
- # Sort job_ids by the latest ts_created in each group (descending)
- def get_latest_timestamp(job_id):
- timestamps = [d.get("ts_created", 0) for d in job_data_dict[job_id]]
- return max(timestamps) if timestamps else 0
-
- sorted_job_ids = sorted(job_data_dict.keys(), key=get_latest_timestamp, reverse=True)
+ try:
+ job_id = int(raw_job_id)
+ except (TypeError, ValueError):
+ continue
+ job_test_dict.setdefault(job_id, []).append(data)
- # Only keep the latest query_job_number jobs
- latest_job_ids = sorted_job_ids[:query_job_number]
+ if not job_test_dict:
+ return {}
- result = {}
+ # Sort job_ids (descending: latest -> oldest)
+ sorted_job_id_list = sorted(job_test_dict.keys(), reverse=True)
+
+ # Build (test_case_name, gpu_type, runtime) -> job_ids dict
+ test_job_dict = {}
+ for job_id, data_list in job_test_dict.items():
+ for data in data_list:
+ test_case_name = data.get("s_test_case_name") or ""
+ gpu_type = data.get("s_gpu_type") or ""
+ runtime = data.get("s_runtime") or ""
+ if not test_case_name or not gpu_type or not runtime:
+ continue
+ key = (test_case_name, gpu_type, runtime)
+ test_job_dict.setdefault(key, set()).add(job_id)
+
+ # Sort job ids for each test case (descending: latest -> oldest)
+ for key, job_id_set in list(test_job_dict.items()):
+ test_job_dict[key] = sorted(job_id_set, reverse=True)
+
+ # Only keep the latest query_job_number jobs in the result
+ latest_job_ids = sorted_job_id_list[:query_job_number]
+
+ regression_dict = {}
for job_id in latest_job_ids:
- result[job_id] = job_data_dict[job_id]
-
- return result
-
-
-def process_regression_message(regression_dict):
+ entries = []
+ for data in job_test_dict.get(job_id, []):
+ test_case_name = data.get("s_test_case_name") or ""
+ gpu_type = data.get("s_gpu_type") or ""
+ runtime = data.get("s_runtime") or ""
+ if not test_case_name or not gpu_type or not runtime:
+ continue
+ key = (test_case_name, gpu_type, runtime)
+ history_ids = test_job_dict.get(key, [])
+ lower_bound = job_id - lookback_job_number + 1
+ history_regression_job_ids = [
+ jid for jid in history_ids if lower_bound <= jid <= job_id
+ ]
+ entries.append((test_case_name, gpu_type, runtime, history_regression_job_ids, data))
+ regression_dict[job_id] = entries
+
+ return regression_dict
+
+
+def split_regression_message(regression_dict):
"""Process regression data into message chunks.
Returns a list of messages, each containing at most MAX_TEST_CASES_PER_MSG test cases.
@@ -114,12 +299,17 @@ def process_regression_message(regression_dict):
if not regression_dict:
return []
- # Flatten all test cases into a list with (job_id, idx, data) tuples
+ # Flatten all test cases into a list with
+ # (job_id, idx, test_case_name, gpu_type, runtime, history_regression_job_ids, data) tuples
all_test_cases = []
for job_id, data_list in regression_dict.items():
- sorted_data_list = sorted(data_list, key=lambda x: x.get("s_test_case_name", ""))
- for idx, data in enumerate(sorted_data_list, start=1):
- all_test_cases.append((job_id, idx, data))
+ sorted_data_list = sorted(data_list, key=lambda x: x[0])
+ for idx, (test_case_name, gpu_type, runtime, history_regression_job_ids, data) in enumerate(
+ sorted_data_list, start=1
+ ):
+ all_test_cases.append(
+ (job_id, idx, test_case_name, gpu_type, runtime, history_regression_job_ids, data)
+ )
# Split into chunks of MAX_TEST_CASES_PER_MSG
chunks = []
@@ -131,7 +321,15 @@ def process_regression_message(regression_dict):
for chunk in chunks:
msg_parts = []
current_job_id = None
- for job_id, idx, data in chunk:
+ for (
+ job_id,
+ idx,
+ test_case_name,
+ gpu_type,
+ runtime,
+ history_regression_job_ids,
+ data,
+ ) in chunk:
# Add job header when switching to a new job_id
if job_id != current_job_id:
if msg_parts:
@@ -140,12 +338,46 @@ def process_regression_message(regression_dict):
msg_parts.append(job_header)
current_job_id = job_id
- test_case_name = data.get("s_test_case_name", "N/A")
regression_info = data.get("s_regression_info", "N/A")
+ history_text = (
+ ", ".join(str(jid) for jid in history_regression_job_ids)
+ if history_regression_job_ids
+ else "N/A"
+ )
msg_parts.append(f"*REGRESSION TEST CASE {idx}: {test_case_name}*\n")
+ msg_parts.append(f"*GPU: {gpu_type} Mode: {runtime}*\n")
+ msg_parts.append(f"*History Regression Post-Merge Job IDs: {history_text}*\n")
+
+ # Parse regression_info to extract baseline info and metrics
+ baseline_date = "N/A"
+ baseline_branch = "N/A"
+ baseline_commit = "N/A"
for part in regression_info.split(","):
part = part.strip()
- if part and "baseline_id" not in part:
+ if "baseline_date:" in part:
+ baseline_date = _timestamp_to_date(part.split(":", 1)[-1].strip())
+ elif "baseline_branch:" in part:
+ baseline_branch = part.split(":", 1)[-1].strip()
+ elif "baseline_commit:" in part:
+ baseline_commit = part.split(":", 1)[-1].strip()
+
+ # Get regression branch and commit from data
+ regression_date = _timestamp_to_date(data.get("ts_created", "N/A"))
+ regression_branch = data.get("s_branch", "N/A")
+ regression_commit = data.get("s_commit", "N/A")
+
+ msg_parts.append(
+ f"*Baseline date, branch and commit: "
+ f"{baseline_date} {baseline_branch} {baseline_commit}*\n"
+ )
+ msg_parts.append(
+ f"*Regression date, branch and commit: "
+ f"{regression_date} {regression_branch} {regression_commit}*\n"
+ )
+
+ for part in regression_info.split(","):
+ part = part.strip()
+ if part and "baseline_" not in part:
msg_parts.append(f" {part}\n")
msg = "".join(msg_parts).strip()
@@ -235,14 +467,64 @@ def main():
print(f"Query Job Number: {args.query_job_number}")
if args.operation == "SLACK BOT SENDS MESSAGE":
- data_list = query_regression_data(args.project_name)
+ last_days = QUERY_LOOKBACK_DAYS
+ must_clauses = [
+ {"term": {"b_is_valid": True}},
+ {"term": {"b_is_post_merge": True}},
+ {"term": {"b_is_regression": True}},
+ {"term": {"b_is_baseline": False}},
+ {
+ "range": {
+ "ts_created": {
+ "gte": int(time.time() - 24 * 3600 * last_days)
+ // (24 * 3600)
+ * 24
+ * 3600
+ * 1000,
+ }
+ }
+ },
+ ]
+ data_list = OpenSearchDB.queryPerfDataFromOpenSearchDB(
+ args.project_name, must_clauses, size=MAX_QUERY_SIZE
+ )
if data_list is None:
print("Failed to query regression data")
return
- regression_dict = get_regression_data_by_job_id(data_list, args.query_job_number)
- messages = process_regression_message(regression_dict)
+ regression_dict = get_regression_dict(data_list, args.query_job_number)
+ messages = split_regression_message(regression_dict)
send_regression_message(messages, args.channel_id, args.bot_token)
+ elif args.operation.strip().upper().startswith("UPDATE"):
+ set_values, where_clauses, error = parse_update_operation(args.operation)
+ if error:
+ print(error)
+ return
+
+ must_clauses = []
+ must_not_clauses = []
+ for field, operator, value in where_clauses:
+ clause_type, clause = _build_opensearch_clause(field, operator, value)
+ if clause_type == "must":
+ must_clauses.append(clause)
+ else:
+ must_not_clauses.append(clause)
+
+ data_list = OpenSearchDB.queryPerfDataFromOpenSearchDB(
+ args.project_name, must_clauses, size=MAX_QUERY_SIZE, must_not_clauses=must_not_clauses
+ )
+ if data_list is None:
+ print("Failed to query data for update")
+ return
+ if len(data_list) == 0:
+ print("No data matched the update scope")
+ return
+
+ updated_data_list = update_perf_data_fields(data_list, set_values)
+ if not post_perf_data(updated_data_list, args.project_name):
+ print("Failed to post updated data")
+ return
+ print(f"Updated {len(updated_data_list)} entries successfully")
else:
print(f"Unknown operation: {args.operation}")
diff --git a/jenkins/scripts/perf/perf_utils.py b/jenkins/scripts/perf/perf_utils.py
new file mode 100644
index 000000000000..aae00a35d86a
--- /dev/null
+++ b/jenkins/scripts/perf/perf_utils.py
@@ -0,0 +1,1620 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared utilities for perf sanity scripts.
+
+Contains constants, regression detection algorithms, OpenSearch query helpers,
+and HTML/SVG report generation functions used by test.py, get_pre_merge_html.py,
+and perf_sanity_triage.py.
+"""
+
+import json as _json
+import math
+import os
+import sys
+import time
+from collections import defaultdict
+from datetime import datetime
+from html import escape as escape_html
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from open_search_db import OpenSearchDB
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+PERF_SANITY_PROJECT_NAME = "swdl-trtllm-infra-ci-prod-perf_sanity_info"
+QUERY_LOOKBACK_DAYS = 90
+MAX_QUERY_SIZE = 9999
+DEFAULT_THRESHOLD = 0.05
+
+CHART_METRICS = [
+ "d_seq_throughput",
+ "d_token_throughput",
+ "d_total_token_throughput",
+ "d_user_throughput",
+]
+
+# Only these 2 metrics determine the overall test-case classification.
+CLASSIFICATION_METRICS = [
+ "d_token_throughput",
+ "d_total_token_throughput",
+]
+
+METRIC_LABELS = {
+ "d_seq_throughput": "Request Throughput (req/s)",
+ "d_token_throughput": "Output Token Throughput (tok/s)",
+ "d_total_token_throughput": "Total Token Throughput (tok/s)",
+ "d_user_throughput": "User Throughput (tok/s)",
+}
+
+# Algorithm parameters
+_STABILITY_CV_THRESHOLD = 0.03 # 3%
+_REGRESSION_THRESHOLD = 0.05 # 5%
+_ROLLING_WINDOW = 7
+_MIN_STABLE_SEGMENT = 7
+_MIN_CONFIRMATION_DAYS = 3
+_DIRECTION_CHANGE_THRESHOLD = 6 # per 30 days
+_OUTLIER_ZSCORE = 2.0
+
+# Curve type display
+_CURVE_TYPE_COLORS = {
+ "no_regression": "#0d904f",
+ "sudden_drop": "#d93025",
+ "gradual_decline": "#e8710a",
+ "significant_fluctuation": "#7b1fa2",
+ "occasional_spike": "#c5a600",
+ "other_reasons": "#607d8b",
+}
+
+_CURVE_TYPE_LABELS = {
+ "no_regression": "No Regression",
+ "sudden_drop": "Sudden Drop",
+ "gradual_decline": "Gradual Decline",
+ "significant_fluctuation": "Significant Fluctuation",
+ "occasional_spike": "Occasional Spike",
+ "other_reasons": "Other Reasons",
+}
+
+# ---------------------------------------------------------------------------
+# Timestamp / data utilities
+# ---------------------------------------------------------------------------
+
+_TIME_FORMATS = [
+ "%Y-%m-%dT%H:%M:%S.%fZ",
+ "%Y-%m-%dT%H:%M:%SZ",
+ "%Y-%m-%dT%H:%M:%S.%f",
+ "%Y-%m-%dT%H:%M:%S",
+ "%b %d, %Y @ %H:%M:%S.%f",
+]
+
+
+def _parse_timestamp(timestamp):
+ """Parse a timestamp value into a datetime object."""
+ if isinstance(timestamp, (int, float)):
+ if timestamp > 1e12:
+ timestamp = timestamp / 1000
+ return datetime.fromtimestamp(timestamp)
+ if isinstance(timestamp, datetime):
+ return timestamp
+ timestamp_str = str(timestamp)
+ for fmt in _TIME_FORMATS:
+ try:
+ return datetime.strptime(timestamp_str, fmt)
+ except ValueError:
+ continue
+ return datetime.fromtimestamp(0)
+
+
+def _ts_to_date(ts):
+ """Convert a millisecond timestamp to a datetime."""
+ try:
+ return datetime.fromtimestamp(int(ts) / 1000)
+ except (ValueError, TypeError, OSError):
+ return datetime.fromtimestamp(0)
+
+
+def _extract_points(data_list, metric):
+ """Extract (datetime, float_value, data_dict) triples from data dicts."""
+ points = []
+ for d in data_list:
+ ts = d.get("ts_created") or d.get("@timestamp")
+ val = d.get(metric)
+ if ts is not None and val is not None:
+ try:
+ points.append((_ts_to_date(ts), float(val), d))
+ except (ValueError, TypeError):
+ pass
+ points.sort(key=lambda p: p[0])
+ return points
+
+
+def _data_dict_to_json_attr(data_dict):
+ """Serialize a data dict to an HTML-safe JSON string for embedding in attributes."""
+ return escape_html(_json.dumps(data_dict, default=str, ensure_ascii=True))
+
+
+# ---------------------------------------------------------------------------
+# Baseline computation
+# ---------------------------------------------------------------------------
+
+
+def _daily_aggregate(points):
+ """Aggregate multiple data points on the same day to a single mean value.
+
+ Args:
+ points: list of (datetime, float) or (datetime, float, data_dict)
+ tuples.
+
+ Returns:
+ list of (date_str, float, [data_dicts]) triples sorted by date.
+ The third element is a list of original data dicts for that day
+ (empty list when input items have no third element).
+ """
+ by_day = defaultdict(list)
+ entries = defaultdict(list)
+ for item in points:
+ dt, val = item[0], item[1]
+ day_key = dt.strftime("%Y-%m-%d")
+ by_day[day_key].append(val)
+ if len(item) > 2 and item[2] is not None:
+ entries[day_key].append(item[2])
+ result = []
+ for day in sorted(by_day):
+ vals = by_day[day]
+ result.append((day, sum(vals) / len(vals), entries[day]))
+ return result
+
+
+def _rolling_smooth(values, window=3):
+ """Trailing rolling mean with same-length output.
+
+ Early elements use fewer samples (i.e. the first element is itself,
+ the second is the mean of the first two, etc.).
+ """
+ if not values:
+ return []
+ smoothed = []
+ for i in range(len(values)):
+ start = max(0, i - window + 1)
+ w = values[start : i + 1]
+ smoothed.append(sum(w) / len(w))
+ return smoothed
+
+
+def _percentile(values, p):
+ """Compute the p-th percentile with linear interpolation.
+
+ Args:
+ values: non-empty list of floats.
+ p: percentile in [0, 100].
+ """
+ if not values:
+ return 0.0
+ s = sorted(values)
+ k = (p / 100.0) * (len(s) - 1)
+ lo = int(k)
+ hi = min(lo + 1, len(s) - 1)
+ frac = k - lo
+ return s[lo] + frac * (s[hi] - s[lo])
+
+
+def get_baseline(grouped_data):
+ """Compute rolling-smooth + P95 baselines and daily data for all entries.
+
+ For each (test_case, gpu_type) key and each metric, this function:
+ 1. Extracts data points as 3-tuples (datetime, float, data_dict).
+ 2. Aggregates to daily values preserving original data entries.
+ 3. Applies rolling smooth (window=3) to daily values.
+ 4. Computes P95 of the smoothed values as the baseline.
+
+ Mutates ``grouped_data[key]`` to add:
+ "daily_data": {metric: {"dates": [...], "values": [...],
+ "entries": [[data_dicts], ...]}},
+ "baselines": {metric: float},
+ """
+ for key, bucket in grouped_data.items():
+ history_data = bucket["history_data"]
+ daily_data = {}
+ baselines = {}
+ for metric in CHART_METRICS:
+ points = _extract_points(history_data, metric)
+ daily = _daily_aggregate(points)
+ daily_dates = [d for d, _, _ in daily]
+ daily_vals = [v for _, v, _ in daily]
+ daily_entries = [e for _, _, e in daily]
+
+ smoothed = _rolling_smooth(daily_vals, window=3)
+ baseline = _percentile(smoothed, 95) if smoothed else 0.0
+
+ daily_data[metric] = {
+ "dates": daily_dates,
+ "values": daily_vals,
+ "entries": daily_entries,
+ }
+ baselines[metric] = baseline
+ bucket["daily_data"] = daily_data
+ bucket["baselines"] = baselines
+
+
+# ---------------------------------------------------------------------------
+# Regression classification
+# ---------------------------------------------------------------------------
+
+
+def _extract_jump_commits(daily_entries, daily_dates, js_idx, je_idx):
+ """Extract commit and timestamp info at jump interval endpoints.
+
+ Args:
+ daily_entries: list of lists of data_dicts (one list per day).
+ daily_dates: list of date strings corresponding to daily_entries.
+ js_idx: jump-start day index (left endpoint).
+ je_idx: jump-end day index (right endpoint).
+
+ Returns:
+ {"left": {"s_commit": str, "timestamp": str},
+ "right": {"s_commit": str, "timestamp": str}}
+ or None if data is unavailable.
+ """
+ if not daily_entries or not daily_dates:
+ return None
+ js_idx = max(0, min(js_idx, len(daily_entries) - 1))
+ je_idx = max(0, min(je_idx, len(daily_entries) - 1))
+
+ def _pick_last(entries_list):
+ """Pick the last chronological entry from a day's entries."""
+ if not entries_list:
+ return None
+ best = entries_list[-1]
+ for e in entries_list:
+ ts_e = e.get("ts_created") or e.get("@timestamp", 0)
+ ts_b = best.get("ts_created") or best.get("@timestamp", 0)
+ if ts_e is not None and ts_b is not None and ts_e > ts_b:
+ best = e
+ commit = best.get("s_commit", "")
+ ts_raw = best.get("ts_created") or best.get("@timestamp", "")
+ if isinstance(ts_raw, (int, float)):
+ if ts_raw > 1e12:
+ ts_raw = ts_raw / 1000
+ ts_str = datetime.fromtimestamp(ts_raw).strftime("%Y-%m-%d %H:%M")
+ else:
+ ts_str = str(ts_raw)
+ return {"s_commit": str(commit), "timestamp": ts_str}
+
+ left = _pick_last(daily_entries[js_idx])
+ right = _pick_last(daily_entries[je_idx])
+ if left is None and right is None:
+ return None
+ return {"left": left, "right": right}
+
+
+def _cv(values):
+ """Coefficient of variation (std / mean). Returns 0 if mean is 0."""
+ if len(values) < 2:
+ return 0.0
+ mean = sum(values) / len(values)
+ if mean == 0:
+ return 0.0
+ variance = sum((v - mean) ** 2 for v in values) / len(values)
+ return math.sqrt(variance) / abs(mean)
+
+
+def _is_stable(values, threshold=_STABILITY_CV_THRESHOLD):
+ """Check if CV < threshold."""
+ return _cv(values) < threshold
+
+
+def _rolling_stats(values, window=_ROLLING_WINDOW):
+ """Compute rolling means, rolling CVs, and direction change count.
+
+ Returns:
+ (rolling_means, rolling_cvs, direction_changes)
+ """
+ if len(values) < window:
+ return [], [], 0
+
+ rolling_means = []
+ rolling_cvs = []
+ for i in range(len(values) - window + 1):
+ w = values[i : i + window]
+ m = sum(w) / len(w)
+ rolling_means.append(m)
+ rolling_cvs.append(_cv(w))
+
+ direction_changes = 0
+ for i in range(2, len(rolling_means)):
+ d_prev = rolling_means[i - 1] - rolling_means[i - 2]
+ d_curr = rolling_means[i] - rolling_means[i - 1]
+ if d_prev * d_curr < 0:
+ direction_changes += 1
+
+ return rolling_means, rolling_cvs, direction_changes
+
+
+def _find_change_point(values, window=_ROLLING_WINDOW):
+ """Find the optimal split point using segmented approach (Phase 4).
+
+ Returns:
+ (split_index, jump_start_index, jump_end_index) or None.
+ """
+ n = len(values)
+ if n < 2 * window:
+ return None
+
+ best_score = -1
+ best_idx = -1
+ eps = 1e-12
+
+ for i in range(window, n - window + 1):
+ left = values[:i]
+ right = values[i:]
+ left_mean = sum(left) / len(left)
+ right_mean = sum(right) / len(right)
+ left_var = sum((v - left_mean) ** 2 for v in left) / len(left)
+ right_var = sum((v - right_mean) ** 2 for v in right) / len(right)
+ score = (left_mean - right_mean) ** 2 / (left_var + right_var + eps)
+ if score > best_score:
+ best_score = score
+ best_idx = i
+
+ if best_idx < 0:
+ return None
+
+ pre_level = sum(values[:best_idx]) / best_idx
+ post_level = sum(values[best_idx:]) / (n - best_idx)
+
+ if pre_level == post_level:
+ return best_idx, best_idx, best_idx
+
+ threshold_start = pre_level - 0.2 * (pre_level - post_level)
+ threshold_end = pre_level - 0.8 * (pre_level - post_level)
+
+ jump_start = best_idx
+ jump_end = best_idx
+
+ if pre_level > post_level:
+ for j in range(n):
+ if values[j] < threshold_start:
+ jump_start = j
+ break
+ for j in range(n):
+ if values[j] < threshold_end:
+ jump_end = j
+ break
+ else:
+ for j in range(n):
+ if values[j] > threshold_start:
+ jump_start = j
+ break
+ for j in range(n):
+ if values[j] > threshold_end:
+ jump_end = j
+ break
+
+ return best_idx, jump_start, jump_end
+
+
+def _is_regression(daily_values, baseline, threshold=_REGRESSION_THRESHOLD):
+ """Step 1: Determine whether the metric shows a regression.
+
+ A regression exists when the recent average drops more than
+ ``threshold`` compared to the baseline.
+
+ Returns True if regression is detected, False otherwise.
+ """
+ if not daily_values or baseline == 0:
+ return False
+ recent_count = min(5, max(3, len(daily_values)))
+ recent_avg = sum(daily_values[-recent_count:]) / recent_count
+ drop_ratio = (baseline - recent_avg) / baseline
+ return drop_ratio > threshold
+
+
+def _classify_regression_type(daily_values):
+ """Step 2: Given that a regression exists, determine its subtype.
+
+ Checks in priority order:
+ 1. Significant Fluctuation
+ 2. Occasional Spike
+ 3. Sudden Drop
+ 4. Gradual Decline
+
+ If none of the four patterns match, falls back to ``"other_reasons"``.
+
+ Returns (regression_type, jump_interval) where regression_type is one of
+ ``"significant_fluctuation"``, ``"occasional_spike"``,
+ ``"sudden_drop"``, ``"gradual_decline"``, ``"other_reasons"``.
+ """
+ n_days = len(daily_values)
+ rolling_means, rolling_cvs, direction_changes = _rolling_stats(daily_values)
+
+ # --- Significant Fluctuation ---
+ normalized_dir_changes = direction_changes * 30 / n_days if n_days > 0 else 0
+ oscillation_windows = 0
+ if rolling_means:
+ for i in range(len(rolling_means)):
+ w = daily_values[i : i + _ROLLING_WINDOW]
+ if w and max(w) > 0:
+ amp = (max(w) - min(w)) / max(w)
+ if amp > _REGRESSION_THRESHOLD:
+ oscillation_windows += 1
+ has_long_stable = False
+ stable_run = 0
+ for cv_val in rolling_cvs:
+ if cv_val < _STABILITY_CV_THRESHOLD:
+ stable_run += 1
+ if stable_run >= 2 * _ROLLING_WINDOW:
+ has_long_stable = True
+ break
+ else:
+ stable_run = 0
+
+ if (
+ normalized_dir_changes > _DIRECTION_CHANGE_THRESHOLD
+ and oscillation_windows > len(rolling_means) * 0.3
+ and not has_long_stable
+ ):
+ return "significant_fluctuation", None
+
+ # --- Occasional Spike ---
+ if n_days >= 3:
+ mean_val = sum(daily_values) / n_days
+ std_val = math.sqrt(sum((v - mean_val) ** 2 for v in daily_values) / n_days)
+ if std_val > 0:
+ outlier_indices = [
+ i
+ for i, v in enumerate(daily_values)
+ if abs(v - mean_val) / std_val > _OUTLIER_ZSCORE
+ ]
+ else:
+ outlier_indices = []
+ non_outlier_vals = [v for i, v in enumerate(daily_values) if i not in outlier_indices]
+ if len(outlier_indices) < 3 and non_outlier_vals and _is_stable(non_outlier_vals):
+ max_consecutive_low = 0
+ consecutive = 0
+ low_threshold = mean_val - _REGRESSION_THRESHOLD * mean_val
+ for v in daily_values:
+ if v < low_threshold:
+ consecutive += 1
+ max_consecutive_low = max(max_consecutive_low, consecutive)
+ else:
+ consecutive = 0
+ if max_consecutive_low < _MIN_CONFIRMATION_DAYS:
+ return "occasional_spike", None
+
+ # --- Sudden Drop / Gradual Decline (via change-point analysis) ---
+ cp = _find_change_point(daily_values)
+ if cp is not None:
+ split_idx, jump_start, jump_end = cp
+ pre_segment = daily_values[:split_idx]
+ post_segment = daily_values[split_idx:]
+
+ adj_left = max(0, jump_start - 1)
+ adj_right = jump_end
+ if adj_left >= adj_right:
+ adj_left = max(0, adj_right - 1)
+ if adj_left == adj_right:
+ adj_right = min(n_days - 1, adj_right + 1)
+
+ if len(pre_segment) >= _MIN_STABLE_SEGMENT and len(post_segment) >= _MIN_CONFIRMATION_DAYS:
+ pre_stable = _is_stable(pre_segment)
+ post_stable = _is_stable(post_segment)
+ pre_mean = sum(pre_segment) / len(pre_segment)
+ post_mean = sum(post_segment) / len(post_segment)
+
+ transition_width = abs(jump_end - jump_start) + 1
+ shift = (pre_mean - post_mean) / pre_mean if pre_mean > 0 else 0
+
+ if (
+ pre_stable
+ and post_stable
+ and shift > _REGRESSION_THRESHOLD
+ and transition_width <= 2
+ ):
+ return "sudden_drop", (adj_left, adj_right)
+
+ if (
+ pre_stable
+ and post_stable
+ and shift > _REGRESSION_THRESHOLD
+ and transition_width > 2
+ ):
+ decline_vals = daily_values[jump_start : jump_end + 1]
+ if len(decline_vals) >= 3:
+ x_vals = list(range(len(decline_vals)))
+ x_mean = sum(x_vals) / len(x_vals)
+ y_mean = sum(decline_vals) / len(decline_vals)
+ ss_xy = sum((x - x_mean) * (y - y_mean) for x, y in zip(x_vals, decline_vals))
+ ss_xx = sum((x - x_mean) ** 2 for x in x_vals)
+ ss_yy = sum((y - y_mean) ** 2 for y in decline_vals)
+ if ss_xx > 0 and ss_yy > 0:
+ slope = ss_xy / ss_xx
+ r_squared = (ss_xy**2) / (ss_xx * ss_yy)
+ if slope < 0 and r_squared > 0.7:
+ return "gradual_decline", (adj_left, adj_right)
+
+ return "other_reasons", (jump_start, jump_end)
+
+ return "other_reasons", None
+
+
+def classify_single_metric(daily_values, baseline, threshold=_REGRESSION_THRESHOLD):
+ """Two-step classification for one metric's time series.
+
+ Step 1 -- Regression check:
+ Is the recent average more than ``threshold`` below the baseline?
+ If **no** -> ``"no_regression"``.
+
+ Step 2 -- Regression subtype (only when Step 1 says *yes*):
+ Classify into one of ``"significant_fluctuation"``,
+ ``"occasional_spike"``, ``"sudden_drop"``,
+ ``"gradual_decline"``, or ``"other_reasons"``.
+
+ Returns:
+ (curve_type, jump_interval) where jump_interval is
+ (start_index, end_index) or None.
+ """
+ if not daily_values:
+ return "no_regression", None
+
+ if not _is_regression(daily_values, baseline, threshold):
+ return "no_regression", None
+
+ regression_type, jump_interval = _classify_regression_type(daily_values)
+ return regression_type, jump_interval
+
+
+def _get_threshold_for_metric(baseline_data_list, metric):
+ """Get the pre-merge threshold for a metric from the latest baseline data.
+
+ Looks for d_threshold_pre_merge_{metric_suffix} in the latest baseline
+ entry. Returns DEFAULT_THRESHOLD (5%) if not found.
+ """
+ if not baseline_data_list:
+ return DEFAULT_THRESHOLD
+ latest_baseline = baseline_data_list[-1]
+ metric_suffix = metric[2:] # Remove "d_" prefix
+ threshold_key = f"d_threshold_pre_merge_{metric_suffix}"
+ if threshold_key in latest_baseline:
+ return latest_baseline[threshold_key]
+ return DEFAULT_THRESHOLD
+
+
+def classify_test_case(grouped_data):
+ """Run classification on all metrics and aggregate results.
+
+ Uses threshold from baseline data for each metric. Reads pre-computed
+ ``daily_data`` and ``baselines`` from each entry (populated by
+ :func:`get_baseline`) and stores classification results back into
+ ``grouped_data[key]``:
+ "curve_type": str (overall)
+ "per_metric_info": {metric: {"curve_type": str,
+ "jump_interval": (date_str, date_str) or None,
+ "jump_commits": {...} or None}}
+ """
+ for key, bucket in grouped_data.items():
+ daily_data = bucket.get("daily_data", {})
+ baselines = bucket.get("baselines", {})
+ baseline_data_list = bucket.get("baseline_data", [])
+ per_metric_results = {}
+ per_metric_info = {}
+
+ for metric in CHART_METRICS:
+ md = daily_data.get(metric, {})
+ daily_vals = md.get("values", [])
+ daily_dates = md.get("dates", [])
+ daily_entries = md.get("entries", [])
+ baseline = baselines.get(metric, 0.0)
+
+ threshold = _get_threshold_for_metric(baseline_data_list, metric)
+ curve_type, jump = classify_single_metric(daily_vals, baseline, threshold)
+ per_metric_results[metric] = curve_type
+
+ jump_dates = None
+ jump_commits = None
+ if jump is not None and daily_dates:
+ js, je = jump
+ js = max(0, min(js, len(daily_dates) - 1))
+ je = max(0, min(je, len(daily_dates) - 1))
+ jump_dates = (daily_dates[js], daily_dates[je])
+ if curve_type in ("sudden_drop", "gradual_decline", "other_reasons"):
+ jump_commits = _extract_jump_commits(daily_entries, daily_dates, js, je)
+
+ per_metric_info[metric] = {
+ "curve_type": curve_type,
+ "jump_interval": jump_dates,
+ "jump_commits": jump_commits,
+ }
+
+ # Aggregate overall type using only CLASSIFICATION_METRICS.
+ # Both NR and OS are "transparent" (defer to the other metric).
+ # Priority: SF > OR > GD > SD > OS > NR
+ classification_types = [
+ per_metric_results[m] for m in CLASSIFICATION_METRICS if m in per_metric_results
+ ]
+
+ if not classification_types:
+ overall = "no_regression"
+ elif len(classification_types) == 1:
+ overall = classification_types[0]
+ else:
+ # 6x6 aggregation: merge two types via priority, where NR and
+ # OS are transparent (defer to the other curve's type).
+ _PRIORITY = {
+ "significant_fluctuation": 5,
+ "other_reasons": 4,
+ "gradual_decline": 3,
+ "sudden_drop": 2,
+ "occasional_spike": 1,
+ "no_regression": 0,
+ }
+ a, b = classification_types[0], classification_types[1]
+ pa, pb = _PRIORITY.get(a, 0), _PRIORITY.get(b, 0)
+ overall = a if pa >= pb else b
+
+ bucket["curve_type"] = overall
+ bucket["per_metric_info"] = per_metric_info
+
+
+# ---------------------------------------------------------------------------
+# OpenSearch query + grouping
+# ---------------------------------------------------------------------------
+
+
+def get_history_data(extra_must_clauses=None):
+ """Query perf data from OpenSearch and group by (s_test_case_name, s_gpu_type).
+
+ Queries both baseline and non-baseline data from the last
+ QUERY_LOOKBACK_DAYS days. Additional filters can be passed via
+ *extra_must_clauses*.
+
+ Returns:
+ dict mapping (test_case, gpu_type) -> {
+ "history_data": [non-baseline entries sorted by time],
+ "baseline_data": [baseline entries sorted by time],
+ }
+ or None on query failure.
+ """
+ must_clauses = [
+ {"term": {"b_is_valid": True}},
+ {
+ "range": {
+ "ts_created": {
+ "gte": int(time.time() - 24 * 3600 * QUERY_LOOKBACK_DAYS)
+ // (24 * 3600)
+ * 24
+ * 3600
+ * 1000,
+ }
+ }
+ },
+ ]
+ if extra_must_clauses:
+ must_clauses.extend(extra_must_clauses)
+
+ data_list = OpenSearchDB.queryPerfDataFromOpenSearchDB(
+ PERF_SANITY_PROJECT_NAME, must_clauses, size=MAX_QUERY_SIZE
+ )
+
+ if data_list is None:
+ return None
+
+ groups = {}
+ for data in data_list:
+ key = (
+ data.get("s_test_case_name", ""),
+ data.get("s_gpu_type", ""),
+ )
+ groups.setdefault(key, {"history_data": [], "baseline_data": []})
+ if data.get("b_is_baseline"):
+ groups[key]["baseline_data"].append(data)
+ else:
+ groups[key]["history_data"].append(data)
+
+ for key, bucket in groups.items():
+ bucket["history_data"] = sorted(
+ bucket["history_data"],
+ key=lambda d: _parse_timestamp(d.get("ts_created") or d.get("@timestamp", 0)),
+ )
+ bucket["baseline_data"] = sorted(
+ bucket["baseline_data"],
+ key=lambda d: _parse_timestamp(d.get("ts_created") or d.get("@timestamp", 0)),
+ )
+
+ return groups
+
+
+# ---------------------------------------------------------------------------
+# SVG chart generation
+# ---------------------------------------------------------------------------
+
+_SVG_WIDTH = 620
+_SVG_HEIGHT = 280
+_MARGIN = {"top": 30, "right": 20, "bottom": 55, "left": 75}
+_PLOT_W = _SVG_WIDTH - _MARGIN["left"] - _MARGIN["right"]
+_PLOT_H = _SVG_HEIGHT - _MARGIN["top"] - _MARGIN["bottom"]
+
+
+def _generate_svg_chart(
+ history_points,
+ metric,
+ label,
+ new_points=None,
+ baseline_value=None,
+ threshold_line_value=None,
+ curve_type=None,
+ jump_interval=None,
+):
+ """Return an SVG string for a single metric chart.
+
+ Args:
+ history_points: list of (datetime, value) or (datetime, value, data_dict)
+ sorted by date.
+ metric: metric key string.
+ label: display label for the chart title.
+ new_points: optional list of (datetime, value) for new data (red dots).
+ baseline_value: optional float drawn as a horizontal dashed red line.
+ threshold_line_value: optional float drawn as a horizontal dashed
+ orange line (regression threshold).
+ curve_type: optional str -- the regression classification for this
+ metric (used for badge display).
+ jump_interval: optional (start_date_str, end_date_str) -- regression
+ window shading.
+ """
+ all_values = [v for _, v, *_ in history_points if v is not None]
+ if new_points:
+ all_values.extend(v for _, v in new_points if v is not None)
+ if baseline_value is not None:
+ all_values.append(baseline_value)
+ if threshold_line_value is not None:
+ all_values.append(threshold_line_value)
+
+ if not history_points and not new_points and baseline_value is None:
+ return f'No data for {escape_html(label)}
'
+ if not all_values:
+ return (
+ f'No numeric data for {escape_html(label)}
'
+ )
+
+ min_val = min(all_values)
+ max_val = max(all_values)
+ val_range = max_val - min_val if max_val != min_val else 1.0
+ min_val -= val_range * 0.05
+ max_val += val_range * 0.05
+ val_range = max_val - min_val
+
+ dates = [d for d, *_ in history_points]
+ if new_points:
+ dates.extend(d for d, _ in new_points)
+ if not dates:
+ return (
+ f'No data points for {escape_html(label)}
'
+ )
+
+ min_ts = min(dates).timestamp()
+ max_ts = max(dates).timestamp()
+ ts_range = max_ts - min_ts if max_ts != min_ts else 1.0
+
+ def _x(dt):
+ return _MARGIN["left"] + (dt.timestamp() - min_ts) / ts_range * _PLOT_W
+
+ def _x_date_str(date_str):
+ dt = datetime.strptime(date_str, "%Y-%m-%d")
+ ts = dt.timestamp()
+ ts = max(min_ts, min(ts, max_ts))
+ return _MARGIN["left"] + (ts - min_ts) / ts_range * _PLOT_W
+
+ def _y(v):
+ return _MARGIN["top"] + _PLOT_H - (v - min_val) / val_range * _PLOT_H
+
+ svg = [
+ f'")
+ return "\n".join(svg)
+
+
+# ---------------------------------------------------------------------------
+# HTML report generation (post-merge dashboard)
+# ---------------------------------------------------------------------------
+
+
+def generate_post_merge_html(grouped_data, output_file):
+ """Generate a post-merge HTML dashboard from grouped perf data.
+
+ This produces a full interactive report with three-way cascading filters
+ (GPU Type, Test Case, Curve Type), summary tables, and click-to-inspect
+ data-point popups.
+ """
+ all_gpu_types = sorted(set(gpu for _, gpu in grouped_data.keys()))
+ all_test_cases = sorted(set(tc for tc, _ in grouped_data.keys()))
+ all_curve_types_set = set()
+
+ sections = []
+ section_tuples = []
+
+ for (test_case, gpu_type), bucket in sorted(grouped_data.items()):
+ history_data = bucket["history_data"]
+ curve_type = bucket.get("curve_type", "no_regression")
+ baselines = bucket.get("baselines", {})
+ per_metric_info = bucket.get("per_metric_info", {})
+
+ all_curve_types_set.add(curve_type)
+ section_tuples.append((gpu_type, test_case, curve_type))
+
+ charts = []
+ for metric in CHART_METRICS:
+ label = METRIC_LABELS.get(metric, metric)
+ hist_pts = _extract_points(history_data, metric)
+ baseline_val = baselines.get(metric)
+ m_info = per_metric_info.get(metric, {})
+ charts.append(
+ _generate_svg_chart(
+ hist_pts,
+ metric,
+ label,
+ baseline_value=baseline_val,
+ curve_type=m_info.get("curve_type"),
+ jump_interval=m_info.get("jump_interval"),
+ )
+ )
+
+ # Summary table
+ summary_rows = ""
+ if history_data:
+ latest = history_data[-1]
+ for metric in CHART_METRICS:
+ val = latest.get(metric)
+ bl_val = baselines.get(metric)
+ diff_str = ""
+ if val is not None and bl_val is not None and bl_val != 0:
+ diff_pct = (val - bl_val) / bl_val * 100
+ color = "#0d904f" if diff_pct >= 0 else "#d93025"
+ diff_str = f' ({diff_pct:+.2f}%)'
+ val_str = f"{val:.2f}" if val is not None else "N/A"
+ bl_str = f"{bl_val:.2f}" if bl_val is not None else "N/A"
+ m_info = per_metric_info.get(metric, {})
+ m_ct = m_info.get("curve_type", "no_regression")
+ m_ct_color = _CURVE_TYPE_COLORS.get(m_ct, "#888")
+ m_ct_label = _CURVE_TYPE_LABELS.get(m_ct, m_ct)
+ m_jump = m_info.get("jump_interval")
+ jump_str = ""
+ if m_jump:
+ jump_str = (
+ f' '
+ f"[{m_jump[0]} ~ {m_jump[1]}]"
+ )
+ ct_cell = (
+ f'{m_ct_label}'
+ f"{jump_str}"
+ )
+ jc = m_info.get("jump_commits")
+ jl_cell = ""
+ jr_cell = ""
+ if jc:
+ left = jc.get("left")
+ right = jc.get("right")
+ if left and left.get("s_commit"):
+ short = left["s_commit"][:8]
+ ts = left.get("timestamp", "")
+ jl_cell = (
+ f"{escape_html(short)}"
+ f'
'
+ f"{escape_html(ts)}"
+ )
+ if right and right.get("s_commit"):
+ short = right["s_commit"][:8]
+ ts = right.get("timestamp", "")
+ jr_cell = (
+ f"{escape_html(short)}"
+ f'
'
+ f"{escape_html(ts)}"
+ )
+ summary_rows += (
+ f"| {METRIC_LABELS.get(metric, metric)} | "
+ f"{val_str}{diff_str} | "
+ f"{bl_str} | "
+ f"{ct_cell} | "
+ f"{jl_cell} | "
+ f"{jr_cell} |
"
+ )
+
+ n_points = len(history_data)
+ ct_color = _CURVE_TYPE_COLORS.get(curve_type, "#888")
+ ct_label = _CURVE_TYPE_LABELS.get(curve_type, curve_type)
+
+ header = escape_html(f"{test_case} [{gpu_type}]")
+ data_gpu = escape_html(gpu_type)
+ data_test = escape_html(test_case)
+ data_curve = escape_html(curve_type)
+ table_header = (
+ "| Metric | Latest Value | "
+ "Baseline (P95) | Curve Type | "
+ "Jump Left | Jump Right |
|---|
"
+ )
+ section = f"""
+
+ {header}
+ {n_points} runs
+ {ct_label}
+
+
+ {"".join(charts)}
+
+ {
+ ""
+ if not summary_rows
+ else f'''
+
+ {table_header}
+ {summary_rows}
+
+ '''
+ }
+
+ """
+ sections.append(section)
+
+ all_curve_types = sorted(all_curve_types_set)
+
+ gpu_to_tests = {}
+ test_to_gpus = {}
+ for tc, gpu in grouped_data.keys():
+ gpu_to_tests.setdefault(gpu, [])
+ if tc not in gpu_to_tests[gpu]:
+ gpu_to_tests[gpu].append(tc)
+ test_to_gpus.setdefault(tc, [])
+ if gpu not in test_to_gpus[tc]:
+ test_to_gpus[tc].append(gpu)
+ for k in gpu_to_tests:
+ gpu_to_tests[k].sort()
+ for k in test_to_gpus:
+ test_to_gpus[k].sort()
+
+ triples_json = _json.dumps(section_tuples)
+
+ gpu_chips = [
+ ''
+ ]
+ for gpu in all_gpu_types:
+ gpu_chips.append(
+ f''
+ )
+
+ test_chips = [
+ ''
+ ]
+ for tc in all_test_cases:
+ test_chips.append(
+ f''
+ )
+
+ curve_chips = [
+ ''
+ ]
+ for ct in all_curve_types:
+ ct_color = _CURVE_TYPE_COLORS.get(ct, "#888")
+ ct_label = _CURVE_TYPE_LABELS.get(ct, ct)
+ curve_chips.append(
+ f'"
+ )
+
+ html = f"""
+
+
+
+ Perf Sanity History Dashboard
+
+
+
+ Perf Sanity History Dashboard
+
+ {len(grouped_data)} test case(s) ·
+ Lookback: {QUERY_LOOKBACK_DAYS} days ·
+ Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
+
+
+
+
GPU Type
+
+ {"".join(gpu_chips)}
+
+
Test Case
+
+ {"".join(test_chips)}
+
+
Curve Type
+
+ {"".join(curve_chips)}
+
+
+
+ {"".join(sections)}
+
+
+
+
+
+"""
+ with open(output_file, "w", encoding="utf-8") as f:
+ f.write(html)
+ print(f"Generated perf history report with {len(grouped_data)} test cases: {output_file}")
diff --git a/tests/integration/defs/perf/open_search_db_utils.py b/tests/integration/defs/perf/open_search_db_utils.py
index cfc7cd62d3a7..9b7f7895ace8 100644
--- a/tests/integration/defs/perf/open_search_db_utils.py
+++ b/tests/integration/defs/perf/open_search_db_utils.py
@@ -22,6 +22,7 @@
import time
from datetime import datetime
+import yaml
from defs.trt_test_alternative import print_info, print_warning
_project_root = os.path.abspath(
@@ -224,93 +225,6 @@ def get_common_values(new_data_dict, match_keys):
return common_values_dict
-def query_history_data(common_values_dict):
- """
- Query post-merge data with common values to narrow down scope.
- """
- # Query data from the last 90 days
- last_days = QUERY_LOOKBACK_DAYS
-
- # Build must clauses with base filters
- must_clauses = [
- {
- "term": {
- "b_is_valid": True
- }
- },
- {
- "term": {
- "b_is_post_merge": True
- }
- },
- {
- "term": {
- "b_is_regression": False
- }
- },
- {
- "range": {
- "ts_created": {
- "gte":
- int(time.time() - 24 * 3600 * last_days) // (24 * 3600) *
- 24 * 3600 * 1000,
- }
- }
- },
- ]
-
- # Add common values as term filters to narrow down the query
- for key, value in common_values_dict.items():
- must_clauses.append({"term": {key: value}})
-
- json_data = {
- "query": {
- "bool": {
- "must": must_clauses
- },
- },
- "size": MAX_QUERY_SIZE,
- }
- json_data = json.dumps(json_data)
-
- data_list = []
- try:
- res = OpenSearchDB.queryFromOpenSearchDB(json_data,
- TEST_INFO_PROJECT_NAME)
- if res is None:
- # No response from database, return None
- print_info(
- f"Failed to query from {TEST_INFO_PROJECT_NAME}, returned no response"
- )
- return None
- payload = res.json().get("hits", {}).get("hits", [])
- if len(payload) == 0:
- # No history data found in database, return empty list
- print_info(
- f"No history data found in {TEST_INFO_PROJECT_NAME}, returned empty list"
- )
- return []
- for hit in payload:
- data_dict = hit.get("_source", {})
- data_dict["_id"] = hit.get("_id", "")
- if data_dict["_id"] == "":
- print_info(
- f"Failed to query from {TEST_INFO_PROJECT_NAME}, returned data with no _id"
- )
- # Invalid data, return None
- return None
- data_list.append(data_dict)
- print_info(
- f"Successfully queried from {TEST_INFO_PROJECT_NAME}, queried {len(data_list)} entries"
- )
- return data_list
- except Exception as e:
- print_info(
- f"Failed to query from {TEST_INFO_PROJECT_NAME}, returned error: {e}"
- )
- return None
-
-
def match(history_data, new_data, match_keys):
"""
Check if the server and client config of history data match the new data
@@ -329,16 +243,76 @@ def is_empty(value):
return True
-def calculate_best_perf_result(history_data_list, new_data):
+def _rolling_smooth(values, window=3):
+ """Trailing rolling mean with same-length output.
+
+ Early elements use fewer samples (i.e. the first element is itself,
+ the second is the mean of the first two, etc.).
"""
- Get the best performance metrics from history data and new data
+ if not values:
+ return []
+ smoothed = []
+ for i in range(len(values)):
+ start = max(0, i - window + 1)
+ w = values[start:i + 1]
+ smoothed.append(sum(w) / len(w))
+ return smoothed
+
+
+def _percentile(values, p):
+ """Compute the p-th percentile with linear interpolation."""
+ if not values:
+ return 0.0
+ s = sorted(values)
+ k = (p / 100.0) * (len(s) - 1)
+ lo = int(k)
+ hi = min(lo + 1, len(s) - 1)
+ frac = k - lo
+ return s[lo] + frac * (s[hi] - s[lo])
+
+
+def _daily_aggregate_values(data_list, metric):
+ """Aggregate multiple data points on the same day to a single mean value.
+
+ Returns a list of daily-aggregated metric values sorted by date.
+ """
+ by_day = {}
+ for data in data_list:
+ if data.get("b_is_baseline"):
+ continue
+ val = data.get(metric)
+ if val is None:
+ continue
+ ts = data.get("ts_created") or data.get("@timestamp")
+ if ts is None:
+ continue
+ if isinstance(ts, (int, float)):
+ if ts > 1e12:
+ ts = ts / 1000
+ day_key = datetime.fromtimestamp(ts).strftime("%Y-%m-%d")
+ elif isinstance(ts, datetime):
+ day_key = ts.strftime("%Y-%m-%d")
+ else:
+ day_key = str(ts)[:10]
+ by_day.setdefault(day_key, []).append(val)
+ result = []
+ for day in sorted(by_day):
+ vals = by_day[day]
+ result.append(sum(vals) / len(vals))
+ return result
+
+
+def calculate_baseline_metrics(history_data_list, new_data):
+ """Calculate baseline metrics using rolling smooth + percentile algorithm.
+
+ For each metric, aggregates data to daily values, applies a trailing
+ rolling mean (window=3), then takes:
+ - P95 for MAXIMIZE_METRICS (larger is better, e.g. throughput)
+ - P5 for MINIMIZE_METRICS (smaller is better, e.g. latency)
"""
- # Combine history data and new data
all_data = []
if history_data_list:
all_data.extend(history_data_list)
-
- # Handle new_data as either a single dict or list
if isinstance(new_data, list):
all_data.extend(new_data)
elif new_data:
@@ -347,35 +321,18 @@ def calculate_best_perf_result(history_data_list, new_data):
if not all_data:
return {}
- best_metrics = {}
-
- # Calculate best values for maximize metrics
- for metric in MAXIMIZE_METRICS:
- values = []
- for data in all_data:
- # Skip baseline data
- if data.get("b_is_baseline") and data.get("b_is_baseline") == True:
- continue
- if metric not in data:
- continue
- values.append(data.get(metric))
- if values:
- best_metrics[metric] = max(values)
-
- # Calculate best values for minimize metrics
- for metric in MINIMIZE_METRICS:
- values = []
- for data in all_data:
- # Skip baseline data
- if data.get("b_is_baseline") and data.get("b_is_baseline") == True:
- continue
- if metric not in data:
- continue
- values.append(data.get(metric))
- if values:
- best_metrics[metric] = min(values)
+ baseline_metrics = {}
+ for metric in MAXIMIZE_METRICS + MINIMIZE_METRICS:
+ daily_vals = _daily_aggregate_values(all_data, metric)
+ if not daily_vals:
+ continue
+ smoothed = _rolling_smooth(daily_vals, window=3)
+ if metric in MAXIMIZE_METRICS:
+ baseline_metrics[metric] = _percentile(smoothed, 95)
+ else:
+ baseline_metrics[metric] = _percentile(smoothed, 5)
- return best_metrics
+ return baseline_metrics
def get_history_data(new_data_dict, match_keys, common_values_dict):
@@ -423,7 +380,32 @@ def parse_timestamp(timestamp):
cmd_idxs = new_data_dict.keys()
history_data_list = None
if cmd_idxs:
- history_data_list = query_history_data(common_values_dict)
+ last_days = QUERY_LOOKBACK_DAYS
+ must_clauses = [
+ {
+ "term": {
+ "b_is_valid": True
+ }
+ },
+ {
+ "term": {
+ "b_is_post_merge": True
+ }
+ },
+ {
+ "range": {
+ "ts_created": {
+ "gte":
+ int(time.time() - 24 * 3600 * last_days) //
+ (24 * 3600) * 24 * 3600 * 1000,
+ }
+ }
+ },
+ ]
+ for key, value in common_values_dict.items():
+ must_clauses.append({"term": {key: value}})
+ history_data_list = OpenSearchDB.queryPerfDataFromOpenSearchDB(
+ TEST_INFO_PROJECT_NAME, must_clauses, size=MAX_QUERY_SIZE)
# If query_history_data returned None, it means network failure
if history_data_list is None:
@@ -504,9 +486,12 @@ def prepare_regressive_test_cases(history_baseline_dict, new_data_dict):
continue
is_post_merge = new_data.get("b_is_post_merge", False)
- baseline_id = history_baseline.get("_id", "")
-
- info_parts = [f"baseline_id: {baseline_id}"]
+ info_parts = [
+ f"baseline_id: {history_baseline.get('_id', '')}",
+ f"baseline_branch: {history_baseline.get('s_branch', '')}",
+ f"baseline_commit: {history_baseline.get('s_commit', '')}",
+ f"baseline_date: {history_baseline.get('ts_created', '')}",
+ ]
regressive_metrics = []
# Check all metrics and build info string
for metric in MAXIMIZE_METRICS + MINIMIZE_METRICS:
@@ -563,8 +548,8 @@ def prepare_baseline_data(history_baseline_dict, history_data_dict,
cmd_idxs = new_data_dict.keys()
# Find the best history post-merge data for each cmd
for cmd_idx in cmd_idxs:
- # Calculate best metrics from history post-merge data and new data
- best_metrics = calculate_best_perf_result(history_data_dict[cmd_idx],
+ # Calculate baseline metrics using rolling smooth + P95 algorithm
+ best_metrics = calculate_baseline_metrics(history_data_dict[cmd_idx],
new_data_dict[cmd_idx])
# Create new_baseline_data from new_data_dict and set b_is_baseline
@@ -643,82 +628,115 @@ def _get_metric_keys():
return metric_keys
-def _print_regression_data(data, print_func=None):
- """
- Print regression info and config.
+def generate_perf_yaml(new_data_dict, output_dir=None):
"""
- if print_func is None:
- print_func = print_info
-
- if "s_regression_info" in data:
- print_func("=== Regression Info ===")
- for item in data["s_regression_info"].split(","):
- print_func(item.strip())
+ Save new perf data entries to perf_data.yaml for post-processing.
- metric_keys = _get_metric_keys()
-
- print_func("\n=== Config ===")
- config_keys = sorted([key for key in data.keys() if key not in metric_keys])
- for key in config_keys:
- if key == "s_regression_info":
- continue
- value = data[key]
- print_func(f'"{key}": {value}')
+ Each entry in the output list is a dict with:
+ - "new_data": the new perf data dict
+ """
+ all_entries = []
+ for cmd_idx, new_data in new_data_dict.items():
+ entry = {"new_data": new_data}
+ all_entries.append(entry)
+
+ if output_dir is not None and len(all_entries) > 0:
+ perf_data_file = os.path.join(output_dir, "perf_data.yaml")
+ with open(perf_data_file, 'w') as f:
+ yaml.dump(all_entries, f, default_flow_style=False)
+ print_info(
+ f"Saved {len(all_entries)} perf data entries to {perf_data_file}")
+ elif len(all_entries) == 0:
+ print_info("No perf data to save.")
-def check_perf_regression(new_data_dict, fail_on_regression=False):
- """
- Check performance regression by printing regression data from new_data_dict.
- If fail_on_regression is True, raises RuntimeError when regressions are found.
- (This is a temporary feature to fail regression tests. We are observing the stability and will fail them by default soon.)
- """
- # Filter regression data from new_data_dict
- regressive_data_list = [
- data for data in new_data_dict.values()
- if data.get("b_is_regression", False)
- ]
- # Split regression data into post-merge and pre-merge
- post_merge_regressions = [
- data for data in regressive_data_list
- if data.get("b_is_post_merge", False)
- ]
- pre_merge_regressions = [
- data for data in regressive_data_list
- if not data.get("b_is_post_merge", False)
- ]
-
- # Print pre-merge regression data with print_warning
- if len(pre_merge_regressions) > 0:
- print_warning(
- f"Found {len(pre_merge_regressions)} pre-merge perf regression data"
- )
- for i, data in enumerate(pre_merge_regressions):
- print_warning(f"\n{'=' * 60}")
- print_warning(f"Pre-merge Regression Data #{i + 1}")
- print_warning("=" * 60)
- _print_regression_data(data, print_func=print_warning)
-
- if fail_on_regression:
- raise RuntimeError(
- f"Found {len(pre_merge_regressions)} pre-merge perf regression data"
- )
-
- # Print post-merge regression data with print_warning
- if len(post_merge_regressions) > 0:
- print_warning(
- f"Found {len(post_merge_regressions)} post-merge perf regression data"
- )
- for i, data in enumerate(post_merge_regressions):
- print_warning(f"\n{'=' * 60}")
- print_warning(f"Post-merge Regression Data #{i + 1}")
- print_warning("=" * 60)
- _print_regression_data(data, print_func=print_warning)
-
- if fail_on_regression:
- raise RuntimeError(
- f"Found {len(post_merge_regressions)} post-merge perf regression data"
- )
-
- # Print summary if no regressions
- if len(regressive_data_list) == 0:
- print_info("No regression data found.")
+# def _print_regression_data(data, print_func=None):
+# """
+# Print regression info and config.
+# """
+# if print_func is None:
+# print_func = print_info
+#
+# if "s_regression_info" in data:
+# print_func("=== Regression Info ===")
+# for item in data["s_regression_info"].split(","):
+# print_func(item.strip())
+#
+# metric_keys = _get_metric_keys()
+#
+# print_func("\n=== Config ===")
+# config_keys = sorted([key for key in data.keys() if key not in metric_keys])
+# for key in config_keys:
+# if key == "s_regression_info":
+# continue
+# value = data[key]
+# print_func(f'"{key}": {value}')
+
+# def check_perf_regression(new_data_dict,
+# fail_on_regression=False,
+# output_dir=None):
+# """
+# Check performance regression by printing regression data from new_data_dict.
+# If fail_on_regression is True, raises RuntimeError when regressions are found.
+# (This is a temporary feature to fail regression tests. We are observing the stability and will fail them by default soon.)
+# If output_dir is provided, saves regression data to regression_data.yaml.
+# """
+# # Filter regression data from new_data_dict
+# regressive_data_list = [
+# data for data in new_data_dict.values()
+# if data.get("b_is_regression", False)
+# ]
+# # Split regression data into post-merge and pre-merge
+# post_merge_regressions = [
+# data for data in regressive_data_list
+# if data.get("b_is_post_merge", False)
+# ]
+# pre_merge_regressions = [
+# data for data in regressive_data_list
+# if not data.get("b_is_post_merge", False)
+# ]
+#
+# # Save regression data to yaml file if output_dir is provided
+# if output_dir is not None and len(regressive_data_list) > 0:
+# regression_data_file = os.path.join(output_dir, "regression_data.yaml")
+# with open(regression_data_file, 'w') as f:
+# yaml.dump(regressive_data_list, f, default_flow_style=False)
+# print_info(
+# f"Saved {len(regressive_data_list)} regression data to {regression_data_file}"
+# )
+#
+# # Print pre-merge regression data with print_warning
+# if len(pre_merge_regressions) > 0:
+# print_warning(
+# f"Found {len(pre_merge_regressions)} pre-merge perf regression data"
+# )
+# for i, data in enumerate(pre_merge_regressions):
+# print_warning(f"\n{'=' * 60}")
+# print_warning(f"Pre-merge Regression Data #{i + 1}")
+# print_warning("=" * 60)
+# _print_regression_data(data, print_func=print_warning)
+#
+# if fail_on_regression:
+# raise RuntimeError(
+# f"Found {len(pre_merge_regressions)} pre-merge perf regression data"
+# )
+#
+# # Print post-merge regression data with print_warning
+# if len(post_merge_regressions) > 0:
+# print_warning(
+# f"Found {len(post_merge_regressions)} post-merge perf regression data"
+# )
+# for i, data in enumerate(post_merge_regressions):
+# print_warning(f"\n{'=' * 60}")
+# print_warning(f"Post-merge Regression Data #{i + 1}")
+# print_warning("=" * 60)
+# _print_regression_data(data, print_func=print_warning)
+#
+# if fail_on_regression:
+# raise RuntimeError(
+# f"Found {len(post_merge_regressions)} post-merge perf regression data"
+# )
+#
+# # Print summary if no regressions
+# if len(regressive_data_list) == 0:
+# print_info("No regression data found.")
diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py
index 6b1b9124e278..161eb4dd5509 100644
--- a/tests/integration/defs/perf/test_perf_sanity.py
+++ b/tests/integration/defs/perf/test_perf_sanity.py
@@ -14,10 +14,8 @@
# limitations under the License.
"""TensorRT LLM perf sanity tests."""
-import contextlib
import copy
import glob
-import io
import os
import re
import socket
@@ -26,18 +24,18 @@
from typing import Dict, List, NamedTuple, Optional, Tuple
import pytest
-import requests
import yaml
+from test_common.error_utils import report_error
from test_common.http_utils import wait_for_endpoint_ready
-from defs.trt_test_alternative import print_error, print_info
+from defs.trt_test_alternative import print_info
from tensorrt_llm._utils import get_free_port
from ..conftest import get_llm_root, llm_models_root
from .open_search_db_utils import (
SCENARIO_MATCH_FIELDS,
add_id,
- check_perf_regression,
+ generate_perf_yaml,
get_common_values,
get_history_data,
get_job_info,
@@ -45,7 +43,6 @@
prepare_baseline_data,
prepare_regressive_test_cases,
)
-from .utils import collect_and_clean_myelin_time
# Model PATH of local dir synced from internal LLM models repo
MODEL_PATH_DICT = {
@@ -54,18 +51,25 @@
"deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/",
"deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/",
"deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/",
+ "deepseek_v32_fp4": "DeepSeek-V3.2-Exp-FP4-v2",
"gpt_oss_120b_fp4": "gpt_oss/gpt-oss-120b",
+ "k2_thinking_fp4": "Kimi-K2-Thinking-NVFP4",
+ "qwen3_235b_a22b_fp4": "Qwen3/saved_models_Qwen3-235B-A22B_nvfp4_hf", # Qwen3-235B-A22B-FP4
}
-SUPPORTED_GPU_TYPE = [
- "H200",
- "B200",
- "B300",
- "GB200",
- "GB300",
-]
+SUPPORTED_GPU_MAPPING = {
+ "GB200": "gb200",
+ "GB300": "gb300",
+ "B200": "b200",
+ "B300": "b300",
+ "H200": "h200",
+}
-DEFAULT_TIMEOUT = 7200
+DEFAULT_TIMEOUT = 5400
+AGG_CONFIG_FOLDER = os.environ.get("AGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/aggregated")
+DISAGG_CONFIG_FOLDER = os.environ.get(
+ "DISAGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/disaggregated"
+)
# Regex patterns for parsing benchmark output metrics
# Key is the metric name used in database (e.g., "mean_e2el", "seq_throughput")
@@ -96,9 +100,20 @@ def get_model_dir(model_name: str) -> str:
return ""
-def get_dataset_path() -> str:
- """Get dataset path for benchmark."""
- return os.path.join(llm_models_root(), "datasets", "ShareGPT_V3_unfiltered_cleaned_split.json")
+def get_dataset_dir(dataset_file: Optional[str]) -> str:
+ """Get dataset directory path from dataset file."""
+ if not dataset_file or dataset_file == "":
+ return ""
+
+ # return os.path.join(llm_models_root(), "datasets", "ShareGPT_V3_unfiltered_cleaned_split.json")
+ llm_models_path = os.path.join(llm_models_root(), dataset_file)
+ if os.path.exists(llm_models_path):
+ return llm_models_path
+ elif os.path.exists(dataset_file):
+ return dataset_file
+ else:
+ print_info(f"Dataset file not found in {llm_models_path} and {dataset_file}")
+ return ""
def to_env_dict(env_vars: str) -> Dict[str, str]:
@@ -140,6 +155,7 @@ def __init__(self, server_config_data: dict, env_vars: str = ""):
self.disable_overlap_scheduler = server_config_data.get("disable_overlap_scheduler", False)
self.num_postprocess_workers = server_config_data.get("num_postprocess_workers", 0)
self.stream_interval = server_config_data.get("stream_interval", 10)
+ self.print_iter_log = server_config_data.get("print_iter_log", False)
self.attn_backend = server_config_data.get("attn_backend", "TRTLLM")
self.enable_chunked_prefill = server_config_data.get("enable_chunked_prefill", False)
self.enable_attention_dp = server_config_data.get("enable_attention_dp", False)
@@ -212,6 +228,7 @@ def __init__(self, server_config_data: dict, env_vars: str = ""):
self.eagle3_layers_to_capture = []
self.max_draft_len = speculative_config.get("max_draft_len", 0)
self.speculative_model = speculative_config.get("speculative_model", "")
+ self.eagle3_one_model = speculative_config.get("eagle3_one_model", False)
# match_mode: "config" (default) or "scenario"
self.match_mode = server_config_data.get("match_mode", "config")
@@ -269,15 +286,11 @@ def to_match_keys(self) -> List[str]:
"l_gpus_per_node",
"l_max_batch_size",
"b_disable_overlap_scheduler",
- "l_num_postprocess_workers",
- "s_attn_backend",
"b_enable_chunked_prefill",
"b_enable_attention_dp",
"b_enable_lm_head_tp_in_adp",
# attention_dp_config
"b_attention_dp_balance",
- # moe_config
- "s_moe_backend",
# cuda_graph_config
"b_enable_cuda_graph",
# kv_cache_config
@@ -339,6 +352,7 @@ def to_db_data(self) -> dict:
"s_eagle3_layers_to_capture": ",".join(map(str, self.eagle3_layers_to_capture)),
"l_max_draft_len": self.max_draft_len,
"s_speculative_model_dir": self.speculative_model,
+ "b_eagle3_one_model": self.eagle3_one_model,
"s_server_log_link": "",
"s_server_env_var": self.env_vars,
}
@@ -365,7 +379,12 @@ def generate_extra_llm_api_config(self) -> str:
class ClientConfig:
"""Configurations of benchmark client."""
- def __init__(self, client_config_data: dict, model_name: str, env_vars: str = ""):
+ def __init__(
+ self,
+ client_config_data: dict,
+ model_name: str,
+ env_vars: str = "",
+ ):
self.model_name = model_name
self.concurrency = client_config_data.get("concurrency", 1)
self.iterations = client_config_data.get("iterations", 1)
@@ -375,7 +394,9 @@ def __init__(self, client_config_data: dict, model_name: str, env_vars: str = ""
self.backend = client_config_data.get("backend", "openai")
self.use_chat_template = client_config_data.get("use_chat_template", False)
self.streaming = client_config_data.get("streaming", True)
+ self.trust_remote_code = client_config_data.get("trust_remote_code", True)
self.model_path = ""
+ self.dataset_file = client_config_data.get("dataset_file", "")
self.env_vars = env_vars
# Generate default name if not provided
@@ -387,7 +408,7 @@ def to_cmd(self) -> List[str]:
"""Generate benchmark command."""
model_dir = get_model_dir(self.model_name)
self.model_path = model_dir if os.path.exists(model_dir) else self.model_name
- dataset_path = get_dataset_path()
+ dataset_path = get_dataset_dir(self.dataset_file)
benchmark_cmd = [
"python",
"-m",
@@ -396,27 +417,36 @@ def to_cmd(self) -> List[str]:
self.model_path,
"--tokenizer",
self.model_path,
- "--dataset-name",
- "random",
- "--random-ids",
"--num-prompts",
str(self.concurrency * self.iterations),
"--max-concurrency",
str(self.concurrency),
- "--random-input-len",
- str(self.isl),
- "--random-output-len",
- str(self.osl),
- "--random-range-ratio",
- str(self.random_range_ratio),
- "--trust-remote-code",
"--ignore-eos",
+ "--no-test-input",
"--percentile-metrics",
"ttft,tpot,itl,e2el",
]
- if dataset_path and os.path.exists(dataset_path):
+ if dataset_path:
+ benchmark_cmd.append("--dataset-name")
+ benchmark_cmd.append("trtllm_custom")
benchmark_cmd.append("--dataset-path")
benchmark_cmd.append(dataset_path)
+ print_info(f"Dataset: {dataset_path} exists. Use trtllm_custom dataset for benchmark.")
+ else:
+ benchmark_cmd.append("--dataset-name")
+ benchmark_cmd.append("random")
+ benchmark_cmd.append("--random-ids")
+ benchmark_cmd.append("--tokenize-on-client")
+ benchmark_cmd.append("--random-input-len")
+ benchmark_cmd.append(str(self.isl))
+ benchmark_cmd.append("--random-output-len")
+ benchmark_cmd.append(str(self.osl))
+ benchmark_cmd.append("--random-range-ratio")
+ benchmark_cmd.append(str(self.random_range_ratio))
+ print_info(
+ f"Dataset: {dataset_path} is not provided or does not exist. "
+ f"Use random dataset (random_range_ratio={self.random_range_ratio}) for benchmark."
+ )
if self.backend:
benchmark_cmd.append("--backend")
benchmark_cmd.append(self.backend)
@@ -424,6 +454,8 @@ def to_cmd(self) -> List[str]:
benchmark_cmd.append("--use-chat-template")
if not self.streaming:
benchmark_cmd.append("--non-streaming")
+ if self.trust_remote_code:
+ benchmark_cmd.append("--trust-remote-code")
return benchmark_cmd
def to_env(self) -> Dict[str, str]:
@@ -450,9 +482,11 @@ def to_db_data(self) -> dict:
"l_isl": self.isl,
"l_osl": self.osl,
"d_random_range_ratio": self.random_range_ratio,
+ "s_dataset_file": self.dataset_file,
"s_backend": self.backend,
"b_use_chat_template": self.use_chat_template,
"b_streaming": self.streaming,
+ "b_trust_remote_code": self.trust_remote_code,
"s_client_log_link": "",
"s_client_env_vars": self.env_vars,
}
@@ -498,6 +532,11 @@ class AggrTestCmds(NamedTuple):
client_cmds: Dict[int, List[List[str]]]
timeout: int
output_dir: str
+ test_output_dir: str
+
+ def get_server_logs(self, server_idx) -> List[str]:
+ server_file_path = os.path.join(self.test_output_dir, f"trtllm-serve.{server_idx}.log")
+ return [server_file_path]
def run_cmd(self, server_idx: int) -> List[str]:
"""Run all clients for a server and return outputs."""
@@ -510,29 +549,28 @@ def run_cmd(self, server_idx: int) -> List[str]:
server_port = get_free_port()
server_cmd_with_port = add_host_port_to_cmd(server_cmd, server_hostname, server_port)
- server_file_path = os.path.join(self.output_dir, f"trtllm-serve.{server_idx}.log")
-
print_info(f"Starting server. cmd is {server_cmd_with_port}")
+ server_file_path = os.path.join(self.test_output_dir, f"trtllm-serve.{server_idx}.log")
with open(server_file_path, "w") as server_ctx:
server_proc = subprocess.Popen(
server_cmd_with_port,
+ env=copy.deepcopy(os.environ),
stdout=server_ctx,
stderr=subprocess.STDOUT,
- env=copy.deepcopy(os.environ),
)
- wait_for_endpoint_ready(
- f"http://{server_hostname}:{server_port}/health",
- timeout=self.timeout,
- server_proc=server_proc,
- )
+ wait_for_endpoint_ready(
+ f"http://{server_hostname}:{server_port}/health",
+ timeout=self.timeout,
+ check_files=[server_file_path],
+ server_proc=server_proc,
+ )
# Run all clients for this server
for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]):
client_file_path = os.path.join(
- self.output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log"
+ self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log"
)
-
client_cmd_with_port = add_host_port_to_cmd(
client_cmd, server_hostname, server_port
)
@@ -546,7 +584,6 @@ def run_cmd(self, server_idx: int) -> List[str]:
with open(client_file_path, "w") as client_ctx:
client_ctx.write(output)
-
outputs.append(output)
finally:
@@ -571,20 +608,21 @@ class DisaggTestCmds(NamedTuple):
num_ctx_servers: int
num_gen_servers: int
output_dir: str
+ test_output_dir: str
def _generate_hostname_file(self, server_idx: int, port: int):
"""Create hostname file for coordination."""
- hostnames_dir = os.path.join(self.output_dir, f"hostnames-{server_idx}")
+ hostnames_dir = os.path.join(self.test_output_dir, f"hostnames-{server_idx}")
if not os.path.exists(hostnames_dir):
os.makedirs(hostnames_dir, exist_ok=True)
hostname_file = os.path.join(hostnames_dir, f"{self.disagg_serving_type}.txt")
with open(hostname_file, "w") as f:
f.write(f"{self.hostname}:{port}")
- def _generate_disagg_server_config(self, server_idx: int, disagg_server_port: int) -> str:
+ def _generate_disagg_server_config(self, server_idx: int) -> str:
"""Generate disagg server config from hostname files."""
print_info(f"Generating disagg server config for server index {server_idx}")
- hostnames_folder = os.path.join(self.output_dir, f"hostnames-{server_idx}")
+ hostnames_folder = os.path.join(self.test_output_dir, f"hostnames-{server_idx}")
expected_count = self.num_ctx_servers + self.num_gen_servers
start_time = time.time()
hostnames = []
@@ -597,8 +635,7 @@ def _generate_disagg_server_config(self, server_idx: int, disagg_server_port: in
f"expected: {expected_count}"
)
if elapsed_time > self.timeout:
- print_error(f"Time out. Hostnames files are not ready after {self.timeout}s")
- break
+ raise RuntimeError(f"Time out. Hostnames files are not ready after {self.timeout}s")
time.sleep(10)
if not os.path.exists(hostnames_folder):
continue
@@ -620,6 +657,11 @@ def _generate_disagg_server_config(self, server_idx: int, disagg_server_port: in
elif hostname_file.startswith("GEN"):
gen_hostnames.append(hostname_port)
+ # Allocate port here (after waiting) to minimize the window between
+ # port allocation and actual use, avoiding TOCTOU race conditions
+ # where another process on the same node grabs the port.
+ disagg_server_port = get_free_port()
+
server_config = {
"hostname": self.hostname,
"port": disagg_server_port,
@@ -633,7 +675,7 @@ def _generate_disagg_server_config(self, server_idx: int, disagg_server_port: in
"urls": gen_hostnames,
},
}
- config_path = os.path.join(self.output_dir, f"server_config.{server_idx}.yaml")
+ config_path = os.path.join(self.test_output_dir, f"server_config.{server_idx}.yaml")
with open(config_path, "w") as f:
yaml.dump(server_config, f)
print_info(f"Server config file {config_path} generated")
@@ -641,7 +683,7 @@ def _generate_disagg_server_config(self, server_idx: int, disagg_server_port: in
def _get_disagg_server_hostname_and_port(self, server_idx: int) -> Tuple[str, int]:
"""Wait for and read disagg server config."""
- config_path = os.path.join(self.output_dir, f"server_config.{server_idx}.yaml")
+ config_path = os.path.join(self.test_output_dir, f"server_config.{server_idx}.yaml")
start_time = time.time()
while True:
if os.path.exists(config_path):
@@ -649,8 +691,9 @@ def _get_disagg_server_hostname_and_port(self, server_idx: int) -> Tuple[str, in
break
elapsed_time = time.time() - start_time
if elapsed_time > self.timeout:
- print_error(f"Server config file {config_path} not found after {self.timeout}s")
- break
+ raise RuntimeError(
+ f"Server config file {config_path} not found after {self.timeout}s"
+ )
print_info(f"Waiting for server config file, elapsed time: {elapsed_time}s")
time.sleep(10)
@@ -670,60 +713,39 @@ def wait_for_benchmark_ready(self, benchmark_status_file: str):
elapsed_time = time.time() - start_time
print_info(f"Waiting for benchmark status file, elapsed time: {elapsed_time}s")
if elapsed_time > self.timeout:
- print_error(f"Timeout waiting for benchmark status file after {self.timeout}s")
- break
- time.sleep(10)
-
- def wait_for_endpoint_ready(self, url: str, server_files: List[str] = None):
- """Wait for endpoint to be ready."""
- start = time.monotonic()
- iteration = 0
- error_keywords = ["RuntimeError", "out of memory", "ValueError"]
- while True:
- iteration += 1
- elapsed_time = time.monotonic() - start
- if elapsed_time > self.timeout:
- print_error(
- f"Timeout waiting for endpoint {url} to be ready after {self.timeout} seconds"
+ raise RuntimeError(
+ f"Timeout waiting for benchmark status file after {self.timeout}s"
)
- break
- print_info(f"Waiting for endpoint {url} to be ready, elapsed time: {elapsed_time}s")
-
- if server_files and iteration % 30 == 0:
- for server_file in server_files:
- if os.path.exists(server_file):
- try:
- with open(server_file, "r") as f:
- content = f.read()
- for line in content.splitlines():
- for keyword in error_keywords:
- if keyword in line:
- print_error(
- f"Found '{keyword}' in server file {server_file}: {line}"
- )
- except Exception as e:
- print_info(f"Failed to read server file {server_file}: {e}")
+ time.sleep(10)
- try:
- time.sleep(10)
- if requests.get(url).status_code == 200:
- print_info(f"endpoint {url} is ready")
- return
- except Exception as err:
- print_info(f"endpoint {url} is not ready, with exception: {err}")
+ def get_server_logs(self, server_idx: int) -> List[str]:
+ server_logs = []
+ for i in range(self.num_ctx_servers):
+ server_logs.append(
+ os.path.join(self.test_output_dir, f"trtllm-serve.CTX_{i}.{server_idx}.log")
+ )
+ server_logs.append(os.path.join(self.output_dir, f"ctx_server_{i}.log"))
+ for i in range(self.num_gen_servers):
+ server_logs.append(
+ os.path.join(self.test_output_dir, f"trtllm-serve.GEN_{i}.{server_idx}.log")
+ )
+ server_logs.append(os.path.join(self.output_dir, f"gen_server_{i}.log"))
+ server_logs.append(
+ os.path.join(self.test_output_dir, f"trtllm-serve.DISAGG_SERVER.{server_idx}.log")
+ )
+ server_logs.append(os.path.join(self.output_dir, "disagg_server.log"))
+ return server_logs
def run_cmd(self, server_idx: int) -> List[str]:
"""Run commands for a server and return outputs."""
outputs = []
- benchmark_status_file = os.path.join(self.output_dir, f"benchmark_status.{server_idx}.txt")
- port = get_free_port()
-
+ benchmark_status_file = os.path.join(
+ self.test_output_dir, f"benchmark_status.{server_idx}.txt"
+ )
ctx_cmd, gen_cmd, disagg_cmd = self.server_cmds[server_idx]
if "CTX" in self.disagg_serving_type or "GEN" in self.disagg_serving_type:
+ port = get_free_port()
self._generate_hostname_file(server_idx, port)
- server_file_path = os.path.join(
- self.output_dir, f"trtllm-serve.{server_idx}.{self.disagg_serving_type}.log"
- )
is_ctx = "CTX" in self.disagg_serving_type
server_cmd = ctx_cmd if is_ctx else gen_cmd
server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, port)
@@ -731,34 +753,39 @@ def run_cmd(self, server_idx: int) -> List[str]:
print_info(
f"Starting server. disagg_serving_type: {self.disagg_serving_type} cmd is {server_cmd}"
)
+ server_file_path = os.path.join(
+ self.test_output_dir,
+ f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log",
+ )
with open(server_file_path, "w") as server_ctx:
server_proc = subprocess.Popen(
server_cmd,
+ env=copy.deepcopy(os.environ),
stdout=server_ctx,
stderr=subprocess.STDOUT,
- env=copy.deepcopy(os.environ),
)
- self.wait_for_benchmark_ready(benchmark_status_file)
+ self.wait_for_benchmark_ready(benchmark_status_file)
finally:
print_info(f"Server {self.disagg_serving_type} stopped")
server_proc.terminate()
server_proc.wait()
elif self.disagg_serving_type == "DISAGG_SERVER":
- disagg_server_file_path = os.path.join(
- self.output_dir, f"trtllm-serve.{server_idx}.{self.disagg_serving_type}.log"
- )
try:
- self._generate_disagg_server_config(server_idx, port)
+ self._generate_disagg_server_config(server_idx)
print_info(f"Starting disagg server. cmd is {disagg_cmd}")
+ disagg_server_file_path = os.path.join(
+ self.test_output_dir,
+ f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log",
+ )
with open(disagg_server_file_path, "w") as disagg_server_ctx:
disagg_server_proc = subprocess.Popen(
disagg_cmd,
+ env=copy.deepcopy(os.environ),
stdout=disagg_server_ctx,
stderr=subprocess.STDOUT,
- env=copy.deepcopy(os.environ),
)
- self.wait_for_benchmark_ready(benchmark_status_file)
+ self.wait_for_benchmark_ready(benchmark_status_file)
finally:
print_info(f"Disagg server {self.disagg_serving_type} stopped")
disagg_server_proc.terminate()
@@ -769,32 +796,18 @@ def run_cmd(self, server_idx: int) -> List[str]:
disagg_server_hostname, disagg_server_port = (
self._get_disagg_server_hostname_and_port(server_idx)
)
- server_files = [
- os.path.join(self.output_dir, f"trtllm-serve.{server_idx}.DISAGG_SERVER.log"),
- ]
- for ctx_idx in range(self.num_ctx_servers):
- server_files.append(
- os.path.join(
- self.output_dir, f"trtllm-serve.{server_idx}.CTX_{ctx_idx}.log"
- )
- )
- for gen_idx in range(self.num_gen_servers):
- server_files.append(
- os.path.join(
- self.output_dir, f"trtllm-serve.{server_idx}.GEN_{gen_idx}.log"
- )
- )
- self.wait_for_endpoint_ready(
+
+ wait_for_endpoint_ready(
f"http://{disagg_server_hostname}:{disagg_server_port}/health",
- server_files=server_files,
+ timeout=self.timeout,
+ check_files=self.get_server_logs(server_idx),
)
# Run all clients for this server
for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]):
benchmark_file_path = os.path.join(
- self.output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log"
+ self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log"
)
-
client_cmd_with_port = add_host_port_to_cmd(
client_cmd, disagg_server_hostname, disagg_server_port
)
@@ -833,6 +846,79 @@ def parse_select_pattern(select_pattern: str) -> list:
return [name.strip() for name in select_pattern.split(",")]
+def parse_test_string(test_case_name: str):
+ """Parse test case name to get config base name, select pattern, runtime, and benchmark_mode.
+
+ Test name formats:
+ - Disagg e2e: disagg_upload-e2e-{config_base}
+ - Disagg gen_only: disagg_upload-gen_only-{config_base}
+ - ctx_only: aggr_upload-ctx_only-{config_base} (runs aggr mode but reads disagg config)
+ - Regular aggr: aggr_upload-{config}-{server_name}
+
+ Returns:
+ tuple: (config_base_name, select_pattern, runtime_mode, benchmark_mode)
+ - runtime_mode: "aggregated" or "disaggregated"
+ - benchmark_mode: "e2e", "gen_only", "ctx_only", or None (for normal aggr)
+ """
+ labels = test_case_name.split("-")
+
+ assert len(labels) > 1, "perf_sanity test must have a config file!"
+
+ prefix = labels[0]
+ is_disagg_prefix = "disagg" in prefix
+ is_aggr_prefix = "aggr" in prefix
+
+ if is_disagg_prefix:
+ # Disagg format: disagg_upload-{e2e|gen_only}-{config_base}
+ assert len(labels) > 2, "Disagg test must have benchmark_mode and config!"
+ benchmark_mode = labels[1] # e2e or gen_only
+ assert benchmark_mode in ("e2e", "gen_only"), (
+ f"Invalid benchmark_mode for disagg: {benchmark_mode}"
+ )
+ runtime_mode = "disaggregated"
+ config_base_name = "-".join(labels[2:])
+ select_pattern = None
+ elif is_aggr_prefix:
+ # Check if this is ctx_only (aggr_upload-ctx_only-{config_base})
+ if len(labels) > 2 and labels[1] == "ctx_only":
+ # ctx_only: aggr_upload-ctx_only-{config_base}
+ # Runs in aggregated mode but reads disagg config
+ benchmark_mode = "ctx_only"
+ runtime_mode = "aggregated"
+ config_base_name = "-".join(labels[2:])
+ select_pattern = None
+ else:
+ # Regular aggr: aggr_upload-config_yml or aggr_upload-config_yml-server_config_name
+ benchmark_mode = None
+ runtime_mode = "aggregated"
+ config_base_name = labels[1]
+ # select_pattern is server config name (e.g., "r1_fp8_dep8_mtp1_1k1k")
+ select_pattern = "-".join(labels[2:]) if len(labels) > 2 else None
+ else:
+ raise ValueError(f"Invalid test name prefix: {prefix}")
+
+ return config_base_name, select_pattern, runtime_mode, benchmark_mode
+
+
+def get_config_dir(benchmark_mode: Optional[str]) -> str:
+ """Get config directory based on benchmark_mode.
+
+ Args:
+ benchmark_mode: "e2e", "gen_only", "ctx_only", or None (for normal aggr)
+
+ Returns:
+ str: Absolute config directory path
+ """
+ if benchmark_mode in ("e2e", "gen_only", "ctx_only"):
+ config_dir = DISAGG_CONFIG_FOLDER
+ else:
+ config_dir = AGG_CONFIG_FOLDER
+ # If relative path, join with llm root
+ if not os.path.isabs(config_dir):
+ config_dir = os.path.join(get_llm_root(), config_dir)
+ return config_dir
+
+
class PerfSanityTestConfig:
"""Configuration for perf sanity tests."""
@@ -840,6 +926,10 @@ def __init__(self, test_case_name: str, output_dir: str):
self._output_dir = output_dir
self._perf_results: Dict[int, List[Dict[str, float]]] = {}
+ # Initialize server configs
+ self.server_configs: List = []
+ self.server_client_configs: Dict[int, List[ClientConfig]] = {}
+
# Parse test case name
self.parse_test_case_name(test_case_name)
@@ -847,68 +937,55 @@ def parse_test_case_name(self, test_case_name: str):
"""Parse test case name into components."""
self._test_param_labels = test_case_name
- # Extract configs from test param labels
- labels = self._test_param_labels.split("-")
-
def get_gpu_type() -> str:
try:
output = subprocess.check_output(
- ["nvidia-smi", "-L"], stderr=subprocess.DEVNULL, text=True
+ "nvidia-smi -q | grep 'Product Name' | head -1",
+ shell=True,
+ stderr=subprocess.DEVNULL,
+ text=True,
)
- first_line = output.strip().split("\n")[0]
- gpu_models = SUPPORTED_GPU_TYPE
- for model in gpu_models:
- if model in first_line:
- if model.startswith("B") and not model.startswith("GB"):
- return f"dgx_{model.lower()}"
- return model.lower()
+ model = output.split()[-1]
+ return SUPPORTED_GPU_MAPPING.get(model, "unsupported")
except (subprocess.CalledProcessError, FileNotFoundError, IndexError):
- print_error("Failed to get GPU type")
- return ""
+ raise RuntimeError("Failed to get GPU type")
- assert len(labels) > 1, "perf_sanity test must have a config file!"
- is_disagg = "disagg" in labels[0]
- self.upload_to_db = "upload" in labels[0]
+ self.upload_to_db = "upload" in test_case_name.split("-")[0]
self.gpu_type = get_gpu_type()
- if is_disagg:
- # For disagg: disagg_upload-deepseek-r1-fp4_8k1k_ctx1_gen1_dep32_bs128_eplb0_mtp0_ccb-UCX
+ # Parse test case name to get config_base_name, select_pattern, runtime, benchmark_mode
+ config_base_name, self.select_pattern, runtime, self.benchmark_mode = parse_test_string(
+ test_case_name
+ )
+
+ # Set runtime based on parsed result
+ if runtime == "disaggregated":
self.runtime = "multi_node_disagg_server"
- self.config_dir = "tests/integration/defs/perf/disagg/test_configs/disagg/perf"
- config_base = "-".join(labels[1:])
- self.config_file = (
- f"{config_base}.yaml" if not config_base.endswith(".yaml") else config_base
- )
- self.select_pattern = None
else:
- # For aggr: aggr_upload-config_yml or aggr_upload-config_yml-server_config_name
self.runtime = "aggr_server"
- self.config_dir = "tests/scripts/perf-sanity"
- config_base = labels[1]
- self.config_file = (
- f"{config_base}.yaml"
- if config_base and not config_base.endswith(".yaml")
- else config_base
- )
- # select_pattern is server config name (e.g., "r1_fp8_dep8_mtp1_1k1k")
- self.select_pattern = "-".join(labels[2:]) if len(labels) > 2 else None
- self.config_dir = os.getenv(
- "TRTLLM_CONFIG_FOLDER", os.path.join(get_llm_root(), self.config_dir)
+ # Set config_file
+ self.config_file = (
+ f"{config_base_name}.yaml"
+ if not config_base_name.endswith(".yaml")
+ else config_base_name
)
- # Initialize server configs
- self.server_configs: List = []
- self.server_client_configs: Dict[int, List[ClientConfig]] = {}
+ # Get config_dir based on benchmark_mode
+ self.config_dir = get_config_dir(self.benchmark_mode)
def parse_config_file(self):
- """Parse config file based on runtime."""
+ """Parse config file based on runtime and benchmark_mode."""
config_file_path = os.path.join(self.config_dir, self.config_file)
- if self.runtime == "aggr_server":
- self._parse_aggr_config_file(config_file_path)
- elif self.runtime == "multi_node_disagg_server":
+ # benchmark_mode determines which parser to use:
+ # - e2e, gen_only, ctx_only: use _parse_disagg_config_file (reads disagg config)
+ # - None (normal aggr): use _parse_aggr_config_file
+ if self.benchmark_mode in ("e2e", "gen_only", "ctx_only"):
self._parse_disagg_config_file(config_file_path, self.config_file)
+ else:
+ # Normal aggregated mode
+ self._parse_aggr_config_file(config_file_path)
def _parse_aggr_config_file(self, config_file_path: str):
"""Parse YAML config file for aggregated server."""
@@ -956,7 +1033,9 @@ def _parse_aggr_config_file(self, config_file_path: str):
client_configs = []
for client_config_data in server_config_data["client_configs"]:
client_config = ClientConfig(
- client_config_data, server_config_data["model_name"], client_env_var
+ client_config_data,
+ server_config_data["model_name"],
+ env_vars=client_env_var,
)
client_configs.append(client_config)
@@ -966,7 +1045,12 @@ def _parse_aggr_config_file(self, config_file_path: str):
self.server_client_configs = server_client_configs
def _parse_disagg_config_file(self, config_file_path: str, config_file: str):
- """Parse YAML config file for disaggregated server."""
+ """Parse YAML config file for disaggregated server.
+
+ This method handles e2e, gen_only, and ctx_only modes.
+ For ctx_only: output is on par with _parse_aggr_config_file (single ServerConfig),
+ OSL is set to 1, and cache_transceiver_config is ignored.
+ """
disagg_serving_type = os.environ.get("DISAGG_SERVING_TYPE", "BENCHMARK")
# Get config file base name (without extension)
@@ -988,9 +1072,13 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str):
model_name = metadata.get("model_name", "")
assert model_name, "model_name is required in metadata section"
- benchmark_mode = benchmark.get("mode", "e2e")
- if "gen_only" in benchmark_mode:
- hardware["num_ctx_servers"] = 0
+ # Use self.benchmark_mode instead of reading from config file
+ benchmark_mode = self.benchmark_mode
+ if benchmark_mode == "gen_only":
+ # Check if it's gen_only_no_context from config
+ config_mode = benchmark.get("mode", "e2e")
+ if "gen_only_no_context" in config_mode:
+ hardware["num_ctx_servers"] = 0
worker_env_var = environment.get("worker_env_var", "")
server_env_var = environment.get("server_env_var", "")
@@ -1005,90 +1093,120 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str):
else:
concurrency_values = [int(concurrency_str)]
- # Gen only mode only runs max concurrency
- if "gen_only" in benchmark_mode:
- concurrency_values = [max(concurrency_values)]
-
- # Create ctx server config
- ctx_server_config_data = {
- "concurrency": max(concurrency_values),
- "name": config_file_base_name,
- "model_name": model_name,
- "gpus_per_node": gpus_per_node,
- "disagg_run_type": "ctx",
- **worker_config.get("ctx", {}),
- }
+ # Gen only mode only runs the first concurrency
+ if benchmark_mode == "gen_only":
+ concurrency_values = [concurrency_values[0]]
+
+ # Handle ctx_only mode specially - output should be on par with _parse_aggr_config_file
+ if benchmark_mode == "ctx_only":
+ # Get ctx worker config and modify it
+ ctx_config = dict(worker_config.get("ctx", {}))
+ # Ignore cache_transceiver_config for ctx_only
+ ctx_config.pop("cache_transceiver_config", None)
+ # Disable overlap scheduler for ctx_only
+ ctx_config["disable_overlap_scheduler"] = True
+
+ # Create server config for ctx_only (single ServerConfig, not tuple)
+ ctx_server_config_data = {
+ "concurrency": -1, # Same as aggr
+ "name": f"{benchmark_mode}-{config_file_base_name}",
+ "model_name": model_name,
+ "gpus_per_node": gpus_per_node,
+ "disagg_run_type": "aggr", # Run as aggr
+ **ctx_config,
+ }
- # Create gen server config
- gen_server_config_data = {
- "concurrency": max(concurrency_values),
- "name": config_file_base_name,
- "model_name": model_name,
- "gpus_per_node": gpus_per_node,
- "disagg_run_type": "gen",
- **worker_config.get("gen", {}),
- }
+ ctx_server_config = ServerConfig(ctx_server_config_data, worker_env_var)
+ self.server_configs = [ctx_server_config]
+ else:
+ # For e2e and gen_only modes - create ctx and gen server configs
+ ctx_server_config_data = {
+ "concurrency": concurrency_values[0],
+ "name": f"{benchmark_mode}-{config_file_base_name}",
+ "model_name": model_name,
+ "gpus_per_node": gpus_per_node,
+ "disagg_run_type": "ctx",
+ **worker_config.get("ctx", {}),
+ }
- ctx_server_config = ServerConfig(ctx_server_config_data, worker_env_var)
- gen_server_config = ServerConfig(gen_server_config_data, worker_env_var)
-
- # Create disagg config
- disagg_config = DisaggConfig(
- name=config_file_base_name,
- disagg_serving_type=disagg_serving_type,
- hostname=socket.gethostname(),
- numa_bind=numa_bind,
- timeout=timeout,
- benchmark_mode=benchmark_mode,
- model_name=model_name,
- hardware=hardware,
- server_env_var=server_env_var,
- )
+ gen_server_config_data = {
+ "concurrency": concurrency_values[0],
+ "name": f"{benchmark_mode}-{config_file_base_name}",
+ "model_name": model_name,
+ "gpus_per_node": gpus_per_node,
+ "disagg_run_type": "gen",
+ **worker_config.get("gen", {}),
+ }
- # server_configs is a list with one element (tuple of ctx, gen, disagg config)
- self.server_configs = [(ctx_server_config, gen_server_config, disagg_config)]
+ ctx_server_config = ServerConfig(ctx_server_config_data, worker_env_var)
+ gen_server_config = ServerConfig(gen_server_config_data, worker_env_var)
+
+ disagg_config = DisaggConfig(
+ name=f"{benchmark_mode}-{config_file_base_name}",
+ disagg_serving_type=disagg_serving_type,
+ hostname=socket.gethostname(),
+ numa_bind=numa_bind,
+ timeout=timeout,
+ benchmark_mode=benchmark_mode,
+ model_name=model_name,
+ hardware=hardware,
+ server_env_var=server_env_var,
+ )
+
+ # server_configs is a list with one element (tuple of ctx, gen, disagg config)
+ self.server_configs = [(ctx_server_config, gen_server_config, disagg_config)]
# Create client configs for each concurrency value
+ # For ctx_only: OSL is set to 1 and dataset_file is empty
+ osl = 1 if benchmark_mode == "ctx_only" else benchmark.get("output_length", 1024)
+ dataset_file = "" if benchmark_mode == "ctx_only" else benchmark.get("dataset_file", "")
+
client_configs = []
for concurrency in concurrency_values:
client_config_data = {
"concurrency": concurrency,
"iterations": benchmark.get("multi_round", 1),
"isl": benchmark.get("input_length", 1024),
- "osl": benchmark.get("output_length", 1024),
+ "osl": osl,
"random_range_ratio": benchmark.get("benchmark_ratio", 0.0),
"backend": "openai",
"use_chat_template": False,
"streaming": benchmark.get("streaming", True),
+ "dataset_file": dataset_file,
}
- client_config = ClientConfig(client_config_data, model_name, client_env_var)
+ client_config = ClientConfig(
+ client_config_data,
+ model_name,
+ env_vars=client_env_var,
+ )
client_configs.append(client_config)
self.server_client_configs = {0: client_configs}
def get_commands(self):
- """Get commands based on runtime."""
- perf_sanity_output_dir = os.path.join(self._output_dir, self._test_param_labels)
- os.makedirs(perf_sanity_output_dir, exist_ok=True)
+ """Get commands based on runtime and benchmark_mode."""
+ self.test_output_dir = os.path.join(self._output_dir, self._test_param_labels)
+ os.makedirs(self.test_output_dir, exist_ok=True)
+ # ctx_only runs in aggregated mode (uses _get_aggr_commands)
if self.runtime == "aggr_server":
- return self._get_aggr_commands(perf_sanity_output_dir)
- elif self.runtime == "multi_node_disagg_server":
- return self._get_disagg_commands(perf_sanity_output_dir)
+ return self._get_aggr_commands(self._output_dir, self.test_output_dir)
+ else:
+ return self._get_disagg_commands(self._output_dir, self.test_output_dir)
- def _get_aggr_commands(self, output_dir: str):
+ def _get_aggr_commands(self, output_dir: str, test_output_dir: str):
"""Get commands for aggregated server."""
server_cmds = []
client_cmds = {}
for server_idx, client_configs in self.server_client_configs.items():
server_config = self.server_configs[server_idx]
- server_cmd = server_config.to_cmd(output_dir)
+ server_cmd = server_config.to_cmd(test_output_dir)
# Generate extra-llm-api-config.yml
config_content = server_config.generate_extra_llm_api_config()
config_filename = f"extra-llm-api-config.aggr.{server_config.name}.yml"
- config_path = os.path.join(output_dir, config_filename)
+ config_path = os.path.join(test_output_dir, config_filename)
with open(config_path, "w") as f:
f.write(config_content)
@@ -1104,9 +1222,10 @@ def _get_aggr_commands(self, output_dir: str):
client_cmds=client_cmds,
timeout=DEFAULT_TIMEOUT,
output_dir=output_dir,
+ test_output_dir=test_output_dir,
)
- def _get_disagg_commands(self, output_dir: str):
+ def _get_disagg_commands(self, output_dir: str, test_output_dir: str):
"""Get commands for disaggregated server."""
server_cmds = []
client_cmds = {}
@@ -1117,21 +1236,21 @@ def _get_disagg_commands(self, output_dir: str):
disagg_serving_type = disagg_config.disagg_serving_type
# Generate ctx server command
- ctx_cmd = ctx_config.to_cmd(output_dir, numa_bind, "CTX")
+ ctx_cmd = ctx_config.to_cmd(test_output_dir, numa_bind, "CTX")
if "CTX" in disagg_serving_type:
config_content = ctx_config.generate_extra_llm_api_config()
config_path = os.path.join(
- output_dir, f"extra-llm-api-config.ctx.{ctx_config.name}.yml"
+ test_output_dir, f"extra-llm-api-config.ctx.{ctx_config.name}.yml"
)
with open(config_path, "w") as f:
f.write(config_content)
# Generate gen server command
- gen_cmd = gen_config.to_cmd(output_dir, numa_bind, "GEN")
+ gen_cmd = gen_config.to_cmd(test_output_dir, numa_bind, "GEN")
if "GEN" in disagg_serving_type:
config_content = gen_config.generate_extra_llm_api_config()
config_path = os.path.join(
- output_dir, f"extra-llm-api-config.gen.{gen_config.name}.yml"
+ test_output_dir, f"extra-llm-api-config.gen.{gen_config.name}.yml"
)
with open(config_path, "w") as f:
f.write(config_content)
@@ -1141,7 +1260,7 @@ def _get_disagg_commands(self, output_dir: str):
"trtllm-serve",
"disaggregated",
"-c",
- f"{output_dir}/server_config.{server_idx}.yaml",
+ f"{test_output_dir}/server_config.{server_idx}.yaml",
"-t",
str(timeout),
"-r",
@@ -1166,40 +1285,10 @@ def _get_disagg_commands(self, output_dir: str):
num_ctx_servers=disagg_config.num_ctx_servers,
num_gen_servers=disagg_config.num_gen_servers,
output_dir=output_dir,
+ test_output_dir=test_output_dir,
)
- def run_ex(self, commands) -> Dict[int, List[str]]:
- """Run commands and collect outputs."""
- outputs = {}
-
- for server_idx in range(len(commands.server_cmds)):
- try:
- with io.StringIO() as buf:
- with contextlib.redirect_stdout(buf):
- server_outputs = commands.run_cmd(server_idx)
- for output in server_outputs:
- print(collect_and_clean_myelin_time(output))
-
- # Check for errors in each output
- for output in server_outputs:
- self._check_benchmark_output_for_errors(output)
-
- print(buf.getvalue())
-
- outputs[server_idx] = server_outputs
-
- except Exception as e:
- print_error(f"Test command failed for server {server_idx}. Error: {e}")
- if isinstance(e, subprocess.CalledProcessError):
- print_error("--- stdout ---")
- if e.stdout:
- print_error(e.stdout.decode() if isinstance(e.stdout, bytes) else e.stdout)
- print_error("--------------")
- outputs[server_idx] = []
-
- return outputs
-
- def _check_benchmark_output_for_errors(self, output: str) -> None:
+ def _check_benchmark_errors(self, output: str) -> None:
"""Check whether the benchmark output contains error messages."""
if not output:
return
@@ -1210,12 +1299,31 @@ def _check_benchmark_output_for_errors(self, output: str) -> None:
failed_count = int(failed_requests_match.group(1))
if failed_count > 0:
error_msg = f"Benchmark output contains {failed_count} failed requests."
- raise Exception(error_msg)
+ raise RuntimeError(error_msg)
# Check for explicit failure markers
if "!FAILED REQUESTS!" in output or "!CHECK LOG FOR ERRORS!" in output:
error_msg = "Benchmark output contains failure markers."
- raise Exception(error_msg)
+ raise RuntimeError(error_msg)
+
+ def run_ex(self, commands) -> Dict[int, List[str]]:
+ """Run commands and collect outputs."""
+ outputs = {}
+ for server_idx in range(len(commands.server_cmds)):
+ try:
+ server_outputs = commands.run_cmd(server_idx)
+ for output in server_outputs:
+ self._check_benchmark_errors(output)
+ outputs[server_idx] = server_outputs
+
+ except Exception as e:
+ outputs[server_idx] = []
+ report_error(
+ error_msg=e,
+ log_files=commands.get_server_logs(server_idx),
+ )
+
+ return outputs
def get_perf_result(self, outputs: Dict[int, List[str]]):
"""Parse performance results from outputs."""
@@ -1257,12 +1365,9 @@ def check_test_failure(self):
f"Some metrics in Server {server_idx} Client {client_idx} are missing. "
f"The broken metrics is {metrics}. "
)
-
if error_msg:
raise Exception(error_msg)
- print_info("All servers passed")
-
def upload_test_results_to_database(self):
"""Upload test results and baseline to database."""
@@ -1327,7 +1432,7 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict:
if not match_keys:
if server_config.match_mode == "scenario":
match_keys = SCENARIO_MATCH_FIELDS.copy()
- is_scenario_mode = True
+ is_scenario_mode = True # noqa: F841
else:
match_keys.extend(["s_gpu_type", "s_runtime"])
match_keys.extend(server_config.to_match_keys())
@@ -1440,16 +1545,18 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict:
# Upload the new perf data and baseline data to database
post_new_perf_data(new_baseline_data_dict, new_data_dict)
- check_perf_regression(new_data_dict, fail_on_regression=is_scenario_mode)
+ generate_perf_yaml(
+ new_data_dict,
+ output_dir=self.test_output_dir,
+ )
+ # TODO: Re-enable regression failure check if needed
+ # check_perf_regression(new_data_dict, fail_on_regression=is_scenario_mode, output_dir=self.test_output_dir)
# Perf sanity test case parameters
AGG_TEST_TYPES = ["aggr_upload", "aggr"]
DISAGG_TEST_TYPES = ["disagg_upload", "disagg"]
-AGGR_CONFIG_FOLDER = "tests/scripts/perf-sanity"
-DISAGG_CONFIG_FOLDER = "tests/integration/defs/perf/disagg/test_configs/disagg/perf"
-
def get_server_config_names(yaml_path: str) -> List[str]:
"""Read a YAML file and return the list of server_config names."""
@@ -1476,8 +1583,10 @@ def get_yaml_files_with_server_names(directory: str) -> Dict[str, List[str]]:
def get_aggr_test_cases() -> List[str]:
"""Generate aggr test cases based on actual server_config names in YAML files."""
- llm_root = get_llm_root()
- aggr_config_dir = os.path.join(llm_root, AGGR_CONFIG_FOLDER)
+ aggr_config_dir = AGG_CONFIG_FOLDER
+ # If relative path, join with llm root
+ if not os.path.isabs(aggr_config_dir):
+ aggr_config_dir = os.path.join(get_llm_root(), aggr_config_dir)
yaml_server_names = get_yaml_files_with_server_names(aggr_config_dir)
test_cases = []
@@ -1494,16 +1603,24 @@ def get_aggr_test_cases() -> List[str]:
def get_disagg_test_cases() -> List[str]:
- """Generate disagg test cases."""
- llm_root = get_llm_root()
- disagg_config_dir = os.path.join(llm_root, DISAGG_CONFIG_FOLDER)
+ """Generate disagg test cases with benchmark modes."""
+ disagg_config_dir = DISAGG_CONFIG_FOLDER
+ # If relative path, join with llm root
+ if not os.path.isabs(disagg_config_dir):
+ disagg_config_dir = os.path.join(get_llm_root(), disagg_config_dir)
yaml_files = glob.glob(os.path.join(disagg_config_dir, "*.yaml"))
basenames = sorted([os.path.splitext(os.path.basename(f))[0] for f in yaml_files])
test_cases = []
for config_yml in basenames:
+ # Disagg e2e and gen_only test cases
for test_type in DISAGG_TEST_TYPES:
- test_cases.append(f"{test_type}-{config_yml}")
+ test_cases.append(f"{test_type}-e2e-{config_yml}")
+ test_cases.append(f"{test_type}-gen_only-{config_yml}")
+
+ # ctx_only test cases (uses aggr prefix)
+ for test_type in AGG_TEST_TYPES:
+ test_cases.append(f"{test_type}-ctx_only-{config_yml}")
return test_cases
diff --git a/tests/scripts/perf-sanity/README.md b/tests/scripts/perf-sanity/README.md
index 1fcd31d84c6b..6dc64b03680c 100644
--- a/tests/scripts/perf-sanity/README.md
+++ b/tests/scripts/perf-sanity/README.md
@@ -2,6 +2,8 @@
Performance sanity testing scripts for TensorRT-LLM with configuration-driven test cases supporting single-node, multi-node aggregated, and multi-node disaggregated architectures.
+This document serves as a reference for both developers and AI agents working with the perf sanity system.
+
## Overview
- Run performance sanity benchmarks across multiple model configs
@@ -9,17 +11,53 @@ Performance sanity testing scripts for TensorRT-LLM with configuration-driven te
- Manage test cases through YAML config files
- Automated resource calculation and job submission via SLURM
-## Configuration File Types
+## System Architecture
+
+### Key Scripts
+
+| Script | Purpose | SLURM Launch Draft |
+|--------|---------|-------------------|
+| `tests/integration/defs/perf/test_perf_sanity.py` | Main pytest entry point for all perf sanity tests | N/A |
+| `jenkins/scripts/perf/disaggregated/submit.py` | CI submission script (disaggregated tests only) | Uses `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` |
+| `jenkins/scripts/perf/local/submit.py` | Local submission script (both aggregated and disaggregated tests) | Uses `jenkins/scripts/perf/aggregated/slurm_launch_draft.sh` or `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` |
+| `jenkins/L0_Test.groovy` | Jenkins CI pipeline for test orchestration | N/A |
+
+### SLURM Launch Script Generation
+
+The submit scripts generate `slurm_launch.sh` from draft templates:
+
+| Submit Script | Mode | Draft Template Used |
+|---------------|------|---------------------|
+| `jenkins/scripts/perf/disaggregated/submit.py` | Disaggregated (CI only) | `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` |
+| `jenkins/scripts/perf/local/submit.py` | Aggregated (local) | `jenkins/scripts/perf/aggregated/slurm_launch_draft.sh` |
+| `jenkins/scripts/perf/local/submit.py` | Disaggregated (local) | `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` |
+
+## Environment Variables
+
+The config folder paths can be overridden via environment variables. Both submit scripts (`local/submit.py` and `disaggregated/submit.py`) propagate these into the pytest execution environment.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `AGG_CONFIG_FOLDER` | `tests/scripts/perf-sanity/aggregated` | Path to aggregated config YAML files |
+| `DISAGG_CONFIG_FOLDER` | `tests/scripts/perf-sanity/disaggregated` | Path to disaggregated config YAML files |
+
+**Example**: Run with custom config folders:
+```bash
+AGG_CONFIG_FOLDER=my/custom/agg DISAGG_CONFIG_FOLDER=my/custom/disagg \
+ python jenkins/scripts/perf/local/submit.py ...
+```
+
+## Configuration Files
There are two modes for perf sanity tests: aggregated (aggr) and disaggregated (disagg).
-### Aggregated Mode (aggr)
+### Aggregated Mode Config Files
-**Config Location**: [`tests/scripts/perf-sanity`](./)
+**Location**: `tests/scripts/perf-sanity/aggregated`
**File Naming**: `xxx.yaml` where words are connected by `_` (underscore), not `-` (hyphen).
-**File Examples**:
+**Examples**:
- `deepseek_r1_fp4_v2_grace_blackwell.yaml` - Single-node aggregated test
- `deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml` - Multi-node aggregated test
@@ -27,53 +65,171 @@ There are two modes for perf sanity tests: aggregated (aggr) and disaggregated (
- Single-node: Performance tests on a single server with multiple GPUs
- Multi-node: Model runs across multiple nodes with unified execution
-**Test Case Names**:
+### Disaggregated Mode Config Files
+
+**Location**: `tests/scripts/perf-sanity/disaggregated`
+
+**File Naming**: `xxx.yaml` (can contain `-` hyphen).
+
+**Example**: `deepseek-r1-fp4_1k1k_ctx1_gen1_dep8_bs768_eplb0_mtp0_ccb-UCX.yaml`
+
+**Use Case**: Disaggregated architecture where model runs across multiple nodes with separate context (prefill) and generation (decode) servers.
+
+## Test Case Formats
+
+In each test db yml file (with keyword `perf_sanity`), there are four test types:
+
+### 1. Normal Aggregated Test
+
+Uses aggregated config files from `tests/scripts/perf-sanity/aggregated`.
+
+**Format**:
```
-perf/test_perf_sanity.py::test_e2e[aggr_upload-{config yaml file base name}]
-perf/test_perf_sanity.py::test_e2e[aggr_upload-{config yaml file base name}-{server_config_name}]
+perf/test_perf_sanity.py::test_e2e[aggr_upload-{agg config file base name}-{test name}]
```
-- Without server config name: runs all server configs in the YAML file
-- With server config name: runs only the specified server config (the `name` field in `server_configs`)
-
-**Examples**:
+**Example**:
```
-perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell]
perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k]
-perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k1k]
```
-### Disaggregated Mode (disagg)
+### 2. Aggregated ctx_only Test
-**Config Location**: [`tests/integration/defs/perf/disagg/test_configs/disagg/perf`](../../integration/defs/perf/disagg/test_configs/disagg/perf)
+Uses disaggregated config files but runs context (prefill) phase only in aggregated mode.
-**File Naming**: `xxx.yaml` (can contain `-` hyphen).
+**Format**:
+```
+perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-{disagg config file base name}]
+```
-**File Example**: `deepseek-r1-fp4_1k1k_ctx1_gen1_dep8_bs768_eplb0_mtp0_ccb-UCX.yaml`
+**Example**:
+```
+perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
+```
-**Use Case**: Disaggregated architecture where model runs across multiple nodes with separate context (prefill) and generation (decode) servers.
+### 3. Disaggregated gen_only Test
+
+Uses disaggregated config files and runs generation (decode) phase only.
-**Test Case Name**:
+**Format**:
```
-perf/test_perf_sanity.py::test_e2e[disagg_upload-{config yaml file base name}]
+perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-{disagg config file base name}]
```
**Example**:
```
-perf/test_perf_sanity.py::test_e2e[disagg_upload-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8_bs768_eplb0_mtp0_ccb-UCX]
+perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
```
+### 4. Disaggregated e2e Test
+
+Uses disaggregated config files and runs full end-to-end disaggregated flow.
+
+**Format**:
+```
+perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-{disagg config file base name}]
+```
+
+**Example**:
+```
+perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
+```
+
+## CI Test Database
+
+Test lists are defined in `tests/integration/test_lists/test-db/`.
+
+### YAML File Naming Convention
+
+| Test Type | File Pattern | Example |
+|-----------|--------------|---------|
+| Single-node aggregated | `l0_{gpu_type}_multi_gpus_perf_sanity.yml` | `l0_b200_multi_gpus_perf_sanity.yml` |
+| Multi-node aggregated | `l0_{gpu_type}_multi_nodes_perf_sanity_node{node count}_gpu{gpu count per test}.yml` | `l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml` |
+| Multi-node disaggregated | `l0_{gpu_type}_multi_gpus_perf_sanity_ctx{ctx worker count}node{node count per ctx worker}_gpu{gpu count per ctx worker}_gen{gen worker count}node{node count per gen worker}_gpu{gen gpus per gen worker}.yml` | `l0_b200_multi_gpus_perf_sanity_ctx1node1_gpu8_gen1node1_gpu8.yml` |
+
+### Jenkins Pipeline Configuration
+
+Tests are defined in `jenkins/L0_Test.groovy` under the `launchTestJobs` function:
+
+| Config Variable | Test Type |
+|-----------------|-----------|
+| `x86SlurmTestConfigs` | Single-node aggregated tests |
+| `SBSASlurmTestConfigs` | Multi-node aggregated tests |
+| `multiNodesSBSAConfigs` | Multi-node disaggregated tests |
+
+## CI Stage Rules
+
+### Test Batching Rules
+
+| Test Type | Nodes per Test | Max Tests per Stage | Notes |
+|-----------|----------------|---------------------|-------|
+| Normal aggregated test | 1 | 6 | Multiple tests can share a stage |
+| Aggregated ctx_only test | 1 | 6 | Multiple tests can share a stage |
+| Normal aggregated test | > 1 | 1 | One test per stage |
+| Aggregated ctx_only test | > 1 | 1 | One test per stage |
+| Disaggregated gen_only test | Any | 1 | Always one test per stage |
+| Disaggregated e2e test | Any | 1 | Always one test per stage |
+
+**Important**: Pre-merge and post-merge tests must be in separate stages.
+
+### GPU Hours Calculation
+
+- Each CI stage runtime is approximately **1 hour**
+- GPU hours = (number of stages) x (GPUs per stage) x 1 hour
+
+**Example**: A test configuration with 12 single-node tests (6 tests x 2 stages) using 8 GPUs each = 2 stages x 8 GPUs x 1 hour = 16 GPU hours
+
## Running Tests
**Important**: Do NOT add `--perf` flag when running pytest. Perf sanity tests are static test cases and do not use perf mode.
-```bash
-# Run all server configs in an aggregated test
-pytest perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell]
+### Local Run Examples
-# Run a specific server config in an aggregated test
+```bash
+# Run a normal aggregated test
pytest perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k1k]
-# Run a specific disaggregated test
-pytest perf/test_perf_sanity.py::test_e2e[disagg_upload-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8_bs768_eplb0_mtp0_ccb-UCX]
+# Run an aggregated ctx_only test
+pytest perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
+
+# Run a disaggregated gen_only test
+pytest perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
+
+# Run a disaggregated e2e test
+pytest perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8]
+```
+
+### Using Local Submit Script
+
+For local SLURM job submission (supports both aggregated and disaggregated tests):
+
+```bash
+python jenkins/scripts/perf/local/submit.py --help
```
+
+## Disaggregated Test SLURM Execution
+
+A disaggregated test runs **four srun steps** within a single multi-node SLURM job allocation. Each step has a different role set via `DISAGG_SERVING_TYPE`:
+
+| Step | `DISAGG_SERVING_TYPE` | Needs MPI | Notes |
+|------|-----------------------|-----------|-------|
+| Context worker(s) | `CTX_0`, `CTX_1`, ... | Yes | Launched via `trtllm-llmapi-launch`, multi-GPU |
+| Generation worker(s) | `GEN_0`, `GEN_1`, ... | Yes | Launched via `trtllm-llmapi-launch`, multi-GPU |
+| Disagg server | `DISAGG_SERVER` | No | Runs `trtllm-serve disaggregated`, single process |
+| Benchmark client | `BENCHMARK` | No | Runs benchmark pytest, single process |
+
+All four srun steps share the same `srunArgs` array, but `--mpi=pmix` is added **only** to the CTX/GEN worker srun commands in `slurm_launch_draft.sh` (not in srunArgs). This prevents unwanted MPI initialization in the disagg server and benchmark processes. See the MPI/PMI section in `jenkins/scripts/perf/README.md` for details.
+
+## Quick Reference for AI Agents
+
+When working with perf sanity tests, use these paths:
+
+| Resource | Path |
+|----------|------|
+| Pytest script | `tests/integration/defs/perf/test_perf_sanity.py` |
+| Aggregated configs | `tests/scripts/perf-sanity/aggregated/*.yaml` |
+| Disaggregated configs | `tests/scripts/perf-sanity/disaggregated/*.yaml` |
+| CI submit (disagg only) | `jenkins/scripts/perf/disaggregated/submit.py` |
+| Local submit (all) | `jenkins/scripts/perf/local/submit.py` |
+| Jenkins pipeline | `jenkins/L0_Test.groovy` |
+| Test database | `tests/integration/test_lists/test-db/` |
diff --git a/tests/scripts/perf-sanity/config_database_b200_nvl.yaml b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml
similarity index 59%
rename from tests/scripts/perf-sanity/config_database_b200_nvl.yaml
rename to tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml
index b579f03170c9..2a3e8b3549ee 100644
--- a/tests/scripts/perf-sanity/config_database_b200_nvl.yaml
+++ b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml
@@ -1,12 +1,12 @@
server_configs:
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc4_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc1_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -15,29 +15,31 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
+ max_num_tokens: 3136
max_seq_len: 2068
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl1024
+ concurrency: 1
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc32_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc32_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 32
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -46,11 +48,13 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
+ max_num_tokens: 3136
max_seq_len: 2068
client_configs:
- name: con32_isl1024_osl1024
@@ -61,10 +65,11 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc256_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
max_batch_size: 512
@@ -76,34 +81,36 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: CUTLASS
+ backend: DEEPGEMM
attention_dp_config:
batching_wait_iters: 0
enable_balance: true
timeout_iters: 60
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1344
+ max_num_tokens: 2112
max_seq_len: 2068
client_configs:
- - name: con256_isl1024_osl1024
- concurrency: 256
+ - name: con2048_isl1024_osl1024
+ concurrency: 2048
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc4_gpu8
- model_name: deepseek_r1_0528_fp4_v2
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc1_gpu8
+ model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -112,29 +119,31 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
- max_seq_len: 2068
+ max_num_tokens: 3136
+ max_seq_len: 9416
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl8192
+ concurrency: 1
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc32_gpu8
- model_name: deepseek_r1_0528_fp4_v2
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc32_gpu8
+ model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 32
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -143,28 +152,30 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
- max_seq_len: 2068
+ max_seq_len: 9416
client_configs:
- - name: con32_isl1024_osl1024
+ - name: con32_isl1024_osl8192
concurrency: 32
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc256_gpu8
- model_name: deepseek_r1_0528_fp4_v2
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc2048_gpu8
+ model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
+ max_batch_size: 256
enable_attention_dp: true
print_iter_log: true
kv_cache_config:
@@ -173,34 +184,36 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: CUTLASS
+ backend: DEEPGEMM
attention_dp_config:
batching_wait_iters: 0
enable_balance: true
timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1344
- max_seq_len: 2068
+ max_num_tokens: 3072
+ max_seq_len: 9416
client_configs:
- - name: con256_isl1024_osl1024
- concurrency: 256
+ - name: con2048_isl1024_osl8192
+ concurrency: 2048
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc4_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc1_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -209,42 +222,51 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 10304
max_seq_len: 9416
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con1_isl8192_osl1024
+ concurrency: 1
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc32_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc32_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 32
+ enable_attention_dp: true
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.8
+ free_gpu_memory_fraction: 0.7
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ backend: DEEPGEMM
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 100
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 9280
max_seq_len: 9416
client_configs:
- name: con32_isl8192_osl1024
@@ -255,10 +277,11 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc256_gpu4
- model_name: deepseek_r1_0528_fp4_v2
- gpus: 4
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc2048_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
max_batch_size: 512
@@ -266,38 +289,40 @@ server_configs:
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.8
+ free_gpu_memory_fraction: 0.7
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: CUTLASS
+ backend: DEEPGEMM
attention_dp_config:
batching_wait_iters: 0
enable_balance: true
- timeout_iters: 60
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ timeout_iters: 100
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8512
+ max_num_tokens: 9280
max_seq_len: 9416
client_configs:
- - name: con256_isl8192_osl1024
- concurrency: 256
+ - name: con2048_isl8192_osl1024
+ concurrency: 2048
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc4_gpu8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc1_gpu8
model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -306,29 +331,30 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
- max_seq_len: 9416
+ max_seq_len: 2068
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl1024
+ concurrency: 1
iterations: 10
- isl: 8192
+ isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc32_gpu8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc32_gpu8
model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 512
- enable_attention_dp: false
+ max_batch_size: 32
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -337,25 +363,28 @@ server_configs:
stream_interval: 10
moe_config:
backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
- max_seq_len: 9416
+ max_num_tokens: 3136
+ max_seq_len: 2068
client_configs:
- - name: con32_isl8192_osl1024
+ - name: con32_isl1024_osl1024
concurrency: 32
iterations: 10
- isl: 8192
+ isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc256_gpu8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_1024_conc2048_gpu8
model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
max_batch_size: 512
@@ -375,26 +404,24 @@ server_configs:
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8512
- max_seq_len: 9416
+ max_seq_len: 2068
client_configs:
- - name: con256_isl8192_osl1024
- concurrency: 256
+ - name: con2048_isl1024_osl1024
+ concurrency: 2048
iterations: 10
- isl: 8192
+ isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc4_gpu8
- model_name: deepseek_r1_0528_fp8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc1_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 1024
cuda_graph_config:
enable_padding: true
- max_batch_size: 256
- enable_attention_dp: false
+ max_batch_size: 1024
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -402,30 +429,32 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
- max_seq_len: 2068
+ max_num_tokens: 5248
+ max_seq_len: 9416
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl8192
+ concurrency: 1
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc16_gpu8
- model_name: deepseek_r1_0528_fp8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc32_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 1024
cuda_graph_config:
enable_padding: true
- max_batch_size: 256
- enable_attention_dp: false
+ max_batch_size: 1024
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -433,30 +462,33 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
- max_seq_len: 2068
+ max_num_tokens: 5248
+ max_seq_len: 9416
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
+ - name: con32_isl1024_osl8192
+ concurrency: 32
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc64_gpu8
- model_name: deepseek_r1_0528_fp8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc1024_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
max_batch_size: 256
- enable_attention_dp: false
+ enable_attention_dp: true
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -464,30 +496,36 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: CUTLASS
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
- max_seq_len: 2068
+ max_seq_len: 9416
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con1024_isl1024_osl8192
+ concurrency: 1024
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc4_gpu8
- model_name: deepseek_r1_0528_fp8
- gpus: 8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_1024_8192_conc2048_gpu4
+ model_name: deepseek_r1_0528_fp4_v2
+ gpus: 4
match_mode: scenario
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
max_batch_size: 256
- enable_attention_dp: false
+ enable_attention_dp: true
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -495,30 +533,68 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: CUTLASS
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ trust_remote_code: true
+ max_seq_len: 9416
+ client_configs:
+ - name: con2048_isl1024_osl8192
+ concurrency: 2048
+ iterations: 10
+ isl: 1024
+ osl: 8192
+ random_range_ratio: 0.0
+ backend: openai
+ streaming: true
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc1_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
+ gpus: 8
+ match_mode: scenario
+ max_batch_size: 512
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ print_iter_log: true
+ kv_cache_config:
+ dtype: fp8
+ free_gpu_memory_fraction: 0.8
+ enable_block_reuse: false
+ stream_interval: 10
+ moe_config:
+ backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 10304
max_seq_len: 9416
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con1_isl8192_osl1024
+ concurrency: 1
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc16_gpu8
- model_name: deepseek_r1_0528_fp8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc16_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 256
- enable_attention_dp: false
+ max_batch_size: 16
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -526,12 +602,14 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: TRTLLM
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 10304
max_seq_len: 9416
client_configs:
- name: con16_isl8192_osl1024
@@ -542,13 +620,12 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc64_gpu8
- model_name: deepseek_r1_0528_fp8
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc512_gpu8
+ model_name: deepseek_r1_0528_fp4_v2
gpus: 8
match_mode: scenario
cuda_graph_config:
enable_padding: true
- max_batch_size: 256
enable_attention_dp: true
print_iter_log: true
kv_cache_config:
@@ -557,1217 +634,753 @@ server_configs:
enable_block_reuse: false
stream_interval: 10
moe_config:
- backend: DEEPGEMM
+ backend: TRTLLM
attention_dp_config:
batching_wait_iters: 0
enable_balance: true
timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 8768
max_seq_len: 9416
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con512_isl8192_osl1024
+ concurrency: 512
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc1024_gpu4
+ model_name: deepseek_r1_0528_fp4_v2
+ gpus: 4
match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 256
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.85
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
+ stream_interval: 10
moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ backend: CUTLASS
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
+ max_num_tokens: 8768
+ max_seq_len: 9416
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1024_isl8192_osl1024
+ concurrency: 1024
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
+- name: nvidia_DeepSeek_R1_0528_FP4_v2_8192_1024_conc2048_gpu4
+ model_name: deepseek_r1_0528_fp4_v2
+ gpus: 4
match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 256
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.85
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
+ stream_interval: 10
moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ backend: CUTLASS
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
+ max_num_tokens: 8768
+ max_seq_len: 9416
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con2048_isl8192_osl1024
+ concurrency: 2048
iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
- iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu8
+- name: openai_gpt_oss_120b_1024_1024_conc1_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
+ max_batch_size: 1
print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
- client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con1_isl1024_osl1024
+ concurrency: 1
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu1
+- name: openai_gpt_oss_120b_1024_1024_conc384_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- NCCL_GRAPH_REGISTER: 0
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: fp8
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- print_iter_log: true
- stream_interval: 20
- num_postprocess_workers: 4
- moe_config:
- backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 384
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 384
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ num_postprocess_workers: 4
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 9236
+ max_seq_len: 2068
client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
+ - name: con384_isl1024_osl1024
+ concurrency: 384
iterations: 10
isl: 1024
- osl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu2
+- name: openai_gpt_oss_120b_1024_1024_conc2048_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 256
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
- dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
- backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ backend: CUTLASS
+ attention_dp_config:
+ enable_balance: true
+ num_postprocess_workers: 4
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 9236
+ max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
+ - name: con2048_isl1024_osl1024
+ concurrency: 2048
iterations: 10
isl: 1024
- osl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc8_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 8
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 8
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 9236
+ max_seq_len: 2068
client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
+ - name: con8_isl1024_osl1024
+ concurrency: 8
iterations: 10
isl: 1024
- osl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc256_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 256
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 9236
+ max_seq_len: 2068
client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
+ - name: con256_isl1024_osl1024
+ concurrency: 256
iterations: 10
isl: 1024
- osl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc1536_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 1536
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 1536
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 9236
+ max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
+ - name: con1536_isl1024_osl1024
+ concurrency: 1536
iterations: 10
isl: 1024
- osl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc1_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 1
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
+ - name: con1_isl1024_osl8192
+ concurrency: 1
iterations: 10
isl: 1024
osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc32_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 32
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 32
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
+ - name: con32_isl1024_osl8192
+ concurrency: 32
iterations: 10
isl: 1024
osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc2048_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 256
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
- dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
- backend: TRTLLM
+ backend: CUTLASS
+ attention_dp_config:
+ enable_balance: true
+ num_postprocess_workers: 4
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
+ - name: con2048_isl1024_osl8192
+ concurrency: 2048
iterations: 10
isl: 1024
osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu1
+- name: openai_gpt_oss_120b_1024_8192_conc4_gpu2
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 2
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 4
cuda_graph_config:
enable_padding: true
max_batch_size: 4
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ num_postprocess_workers: 4
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl8192_osl1024
+ - name: con4_isl1024_osl8192
concurrency: 4
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu1
+- name: openai_gpt_oss_120b_1024_8192_conc256_gpu2
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 2
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 256
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ num_postprocess_workers: 4
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
+ - name: con256_isl1024_osl8192
+ concurrency: 256
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu1
+- name: openai_gpt_oss_120b_1024_8192_conc10_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 10
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 10
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ num_postprocess_workers: 4
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con10_isl1024_osl8192
+ concurrency: 10
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu2
+- name: openai_gpt_oss_120b_1024_8192_conc128_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 128
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ num_postprocess_workers: 4
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con128_isl1024_osl8192
+ concurrency: 128
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu2
+- name: openai_gpt_oss_120b_1024_8192_conc896_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 896
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 896
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ num_postprocess_workers: 4
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
+ - name: con896_isl1024_osl8192
+ concurrency: 896
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu2
+- name: openai_gpt_oss_120b_8192_1024_conc1_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 1
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ num_postprocess_workers: 4
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con1_isl8192_osl1024
+ concurrency: 1
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu4
+- name: openai_gpt_oss_120b_8192_1024_conc32_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 32
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 32
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con32_isl8192_osl1024
+ concurrency: 32
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu4
+- name: openai_gpt_oss_120b_8192_1024_conc1792_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 1792
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 1792
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
+ - name: con1792_isl8192_osl1024
+ concurrency: 1792
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu4
+- name: openai_gpt_oss_120b_8192_1024_conc8_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 4
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 8
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 8
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ num_postprocess_workers: 4
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con8_isl8192_osl1024
+ concurrency: 8
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu8
+- name: openai_gpt_oss_120b_8192_1024_conc128_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 128
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con128_isl8192_osl1024
+ concurrency: 128
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu8
+- name: openai_gpt_oss_120b_8192_1024_conc2048_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
@@ -1776,61 +1389,98 @@ server_configs:
NCCL_GRAPH_REGISTER: 0
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 2048
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
- num_postprocess_workers: 4
moe_config:
backend: TRTLLM
+ num_postprocess_workers: 4
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
+ - name: con2048_isl8192_osl1024
+ concurrency: 2048
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu8
+- name: openai_gpt_oss_120b_8192_1024_conc768_gpu2
model_name: gpt_oss_120b_fp4
- gpus: 8
+ gpus: 2
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 384
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 384
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
dtype: fp8
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- print_iter_log: true
stream_interval: 20
+ moe_config:
+ backend: TRTLLM
+ attention_dp_config:
+ enable_balance: true
num_postprocess_workers: 4
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
+ trust_remote_code: true
+ max_num_tokens: 20000
+ max_seq_len: 9236
+ client_configs:
+ - name: con768_isl8192_osl1024
+ concurrency: 768
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ random_range_ratio: 0.0
+ backend: openai
+ streaming: true
+- name: openai_gpt_oss_120b_8192_1024_conc1280_gpu2
+ model_name: gpt_oss_120b_fp4
+ gpus: 2
+ match_mode: scenario
+ env_overrides:
+ TRTLLM_ENABLE_PDL: 1
+ NCCL_GRAPH_REGISTER: 0
+ max_batch_size: 640
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 640
+ enable_attention_dp: true
+ print_iter_log: true
+ kv_cache_config:
+ dtype: fp8
+ free_gpu_memory_fraction: 0.85
+ enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRTLLM
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
+ attention_dp_config:
+ enable_balance: true
+ num_postprocess_workers: 4
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con1280_isl8192_osl1024
+ concurrency: 1280
iterations: 10
isl: 8192
osl: 1024
diff --git a/tests/scripts/perf-sanity/config_database_h200_sxm.yaml b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml
similarity index 50%
rename from tests/scripts/perf-sanity/config_database_h200_sxm.yaml
rename to tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml
index f8e3d7bd294e..9179ca664585 100644
--- a/tests/scripts/perf-sanity/config_database_h200_sxm.yaml
+++ b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml
@@ -1,74 +1,79 @@
server_configs:
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc4_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc1_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.75
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
stream_interval: 10
moe_config:
backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
+ max_num_tokens: 3136
max_seq_len: 2068
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl1024
+ concurrency: 1
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc16_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc32_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 1024
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
- enable_attention_dp: false
+ max_batch_size: 1024
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.75
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
stream_interval: 10
moe_config:
backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
+ max_num_tokens: 5248
max_seq_len: 2068
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
+ - name: con32_isl1024_osl1024
+ concurrency: 32
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc64_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_1024_conc2048_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
- enable_attention_dp: false
+ max_batch_size: 512
+ enable_attention_dp: true
print_iter_log: true
kv_cache_config:
dtype: fp8
@@ -77,90 +82,100 @@ server_configs:
stream_interval: 10
moe_config:
backend: CUTLASS
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 1152
+ max_num_tokens: 2112
max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con2048_isl1024_osl1024
+ concurrency: 2048
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc4_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc1_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
- enable_attention_dp: false
+ max_batch_size: 1
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.75
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
stream_interval: 10
moe_config:
backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 3136
max_seq_len: 9416
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl8192
+ concurrency: 1
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc16_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc16_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
- enable_attention_dp: false
+ max_batch_size: 16
print_iter_log: true
kv_cache_config:
dtype: fp8
- free_gpu_memory_fraction: 0.75
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
stream_interval: 10
moe_config:
backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 3136
max_seq_len: 9416
client_configs:
- - name: con16_isl8192_osl1024
+ - name: con16_isl1024_osl8192
concurrency: 16
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc64_gpu8
+- name: deepseek_ai_DeepSeek_R1_0528_1024_8192_conc512_gpu8
model_name: deepseek_r1_0528_fp8
gpus: 8
match_mode: scenario
+ max_batch_size: 128
cuda_graph_config:
enable_padding: true
- max_batch_size: 128
enable_attention_dp: true
print_iter_log: true
kv_cache_config:
@@ -174,135 +189,132 @@ server_configs:
batching_wait_iters: 0
enable_balance: true
timeout_iters: 60
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 8320
+ max_num_tokens: 1344
max_seq_len: 9416
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con512_isl1024_osl8192
+ concurrency: 512
iterations: 10
- isl: 8192
- osl: 1024
+ isl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc1_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 1
+ print_iter_log: true
kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
+ dtype: fp8
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
+ stream_interval: 10
moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
+ max_num_tokens: 10304
+ max_seq_len: 9416
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl8192_osl1024
+ concurrency: 1
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc16_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
max_batch_size: 16
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
+ dtype: fp8
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
+ stream_interval: 10
moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
+ max_num_tokens: 10304
+ max_seq_len: 9416
client_configs:
- - name: con16_isl1024_osl1024
+ - name: con16_isl8192_osl1024
concurrency: 16
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
+- name: deepseek_ai_DeepSeek_R1_0528_8192_1024_conc256_gpu8
+ model_name: deepseek_r1_0528_fp8
+ gpus: 8
match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 256
+ print_iter_log: true
kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
+ dtype: fp8
+ free_gpu_memory_fraction: 0.8
enable_block_reuse: false
+ stream_interval: 10
moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ backend: CUTLASS
+ speculative_config:
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 2068
+ max_num_tokens: 10304
+ max_seq_len: 9416
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con256_isl8192_osl1024
+ concurrency: 256
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu2
+- name: openai_gpt_oss_120b_1024_1024_conc4_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
cuda_graph_config:
enable_padding: true
max_batch_size: 4
- enable_attention_dp: false
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
moe_config:
@@ -310,10 +322,9 @@ server_configs:
num_postprocess_workers: 4
print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
@@ -325,18 +336,16 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu2
+- name: openai_gpt_oss_120b_1024_1024_conc64_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 64
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
moe_config:
@@ -344,690 +353,427 @@ server_configs:
num_postprocess_workers: 4
print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
+ - name: con64_isl1024_osl1024
+ concurrency: 64
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu2
+- name: openai_gpt_oss_120b_1024_1024_conc2048_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 256
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
+ attention_dp_config:
+ enable_balance: true
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con2048_isl1024_osl1024
+ concurrency: 2048
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc128_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 128
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con128_isl1024_osl1024
+ concurrency: 128
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc384_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 384
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 384
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
+ - name: con384_isl1024_osl1024
+ concurrency: 384
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu4
+- name: openai_gpt_oss_120b_1024_1024_conc1024_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 1024
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 1024
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 2068
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con1024_isl1024_osl1024
+ concurrency: 1024
iterations: 10
isl: 1024
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc4_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc1_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 1
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 2068
+ max_seq_len: 9236
client_configs:
- - name: con4_isl1024_osl1024
- concurrency: 4
+ - name: con1_isl1024_osl8192
+ concurrency: 1
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc16_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc64_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 64
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 64
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 2068
+ max_seq_len: 9236
client_configs:
- - name: con16_isl1024_osl1024
- concurrency: 16
+ - name: con64_isl1024_osl8192
+ concurrency: 64
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_1024_conc64_gpu8
+- name: openai_gpt_oss_120b_1024_8192_conc1280_gpu8
model_name: gpt_oss_120b_fp4
gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 1280
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
+ max_batch_size: 1280
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
- max_seq_len: 2068
+ max_seq_len: 9236
client_configs:
- - name: con64_isl1024_osl1024
- concurrency: 64
+ - name: con1280_isl1024_osl8192
+ concurrency: 1280
iterations: 10
isl: 1024
- osl: 1024
+ osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu1
+- name: openai_gpt_oss_120b_1024_8192_conc512_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 512
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 512
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
+ - name: con512_isl1024_osl8192
+ concurrency: 512
iterations: 10
isl: 1024
osl: 8192
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu1
+- name: openai_gpt_oss_120b_8192_1024_conc1_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 1
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 1
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
+ - name: con1_isl8192_osl1024
+ concurrency: 1
iterations: 10
- isl: 1024
- osl: 8192
+ isl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu1
+- name: openai_gpt_oss_120b_8192_1024_conc64_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 64
cuda_graph_config:
enable_padding: true
max_batch_size: 64
- enable_attention_dp: false
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl1024_osl8192
+ - name: con64_isl8192_osl1024
concurrency: 64
iterations: 10
- isl: 1024
- osl: 8192
+ isl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu2
+- name: openai_gpt_oss_120b_8192_1024_conc1536_gpu8
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 8
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 1536
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
+ max_batch_size: 1536
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
+ - name: con1536_isl8192_osl1024
+ concurrency: 1536
iterations: 10
- isl: 1024
- osl: 8192
+ isl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu2
+- name: openai_gpt_oss_120b_8192_1024_conc2_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 2
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 2
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 2
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 4
moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
+ - name: con2_isl8192_osl1024
+ concurrency: 2
iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 8192
+ isl: 8192
+ osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu4
+- name: openai_gpt_oss_120b_8192_1024_conc4_gpu4
model_name: gpt_oss_120b_fp4
gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc4_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 4
cuda_graph_config:
enable_padding: true
max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con4_isl1024_osl8192
- concurrency: 4
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc16_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl1024_osl8192
- concurrency: 16
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_1024_8192_conc64_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl1024_osl8192
- concurrency: 64
- iterations: 10
- isl: 1024
- osl: 8192
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
@@ -1039,102 +785,32 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu1
+- name: openai_gpt_oss_120b_8192_1024_conc768_gpu4
model_name: gpt_oss_120b_fp4
- gpus: 1
+ gpus: 4
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 768
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
+ max_batch_size: 768
print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu1
- model_name: gpt_oss_120b_fp4
- gpus: 1
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 1
- moe_expert_parallel_size: 1
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu2
- model_name: gpt_oss_120b_fp4
- gpus: 2
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
+ - name: con768_isl8192_osl1024
+ concurrency: 768
iterations: 10
isl: 8192
osl: 1024
@@ -1147,23 +823,22 @@ server_configs:
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 8
cuda_graph_config:
enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
+ max_batch_size: 8
+ enable_attention_dp: true
+ print_iter_log: true
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
+ stream_interval: 20
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
tensor_parallel_size: 2
moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
@@ -1175,238 +850,64 @@ server_configs:
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu2
+- name: openai_gpt_oss_120b_8192_1024_conc256_gpu2
model_name: gpt_oss_120b_fp4
gpus: 2
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 256
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
+ max_batch_size: 256
print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 2
- moe_expert_parallel_size: 2
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu4
- model_name: gpt_oss_120b_fp4
- gpus: 4
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con256_isl8192_osl1024
+ concurrency: 256
iterations: 10
isl: 8192
osl: 1024
random_range_ratio: 0.0
backend: openai
streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc4_gpu8
+- name: openai_gpt_oss_120b_8192_1024_conc384_gpu2
model_name: gpt_oss_120b_fp4
- gpus: 8
+ gpus: 2
match_mode: scenario
env_overrides:
TRTLLM_ENABLE_PDL: 1
+ max_batch_size: 384
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
+ max_batch_size: 384
print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con4_isl8192_osl1024
- concurrency: 4
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc16_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 16
- enable_attention_dp: false
kv_cache_config:
- dtype: auto
free_gpu_memory_fraction: 0.85
enable_block_reuse: false
- moe_config:
- backend: TRITON
- num_postprocess_workers: 4
- print_iter_log: true
stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- trust_remote_code: true
- backend: pytorch
- max_num_tokens: 20000
- max_seq_len: 9236
- client_configs:
- - name: con16_isl8192_osl1024
- concurrency: 16
- iterations: 10
- isl: 8192
- osl: 1024
- random_range_ratio: 0.0
- backend: openai
- streaming: true
-- name: openai_gpt_oss_120b_8192_1024_conc64_gpu8
- model_name: gpt_oss_120b_fp4
- gpus: 8
- match_mode: scenario
- env_overrides:
- TRTLLM_ENABLE_PDL: 1
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 64
- enable_attention_dp: false
- kv_cache_config:
- dtype: auto
- free_gpu_memory_fraction: 0.85
- enable_block_reuse: false
moe_config:
backend: TRITON
num_postprocess_workers: 4
- print_iter_log: true
- stream_interval: 20
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
trust_remote_code: true
- backend: pytorch
max_num_tokens: 20000
max_seq_len: 9236
client_configs:
- - name: con64_isl8192_osl1024
- concurrency: 64
+ - name: con384_isl8192_osl1024
+ concurrency: 384
iterations: 10
isl: 8192
osl: 1024
diff --git a/tests/scripts/perf-sanity/deepseek_r1_fp8_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
similarity index 51%
rename from tests/scripts/perf-sanity/deepseek_r1_fp8_blackwell.yaml
rename to tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
index 6ff3d94545f9..937356b9ce44 100644
--- a/tests/scripts/perf-sanity/deepseek_r1_fp8_blackwell.yaml
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
@@ -1,11 +1,14 @@
metadata:
- model_name: deepseek_r1_0528_fp8
+ model_name: deepseek_r1_0528_fp4_v2
supported_gpus:
- - B200
- - B300
+ - GB200
+hardware:
+ gpus_per_node: 4
server_configs:
- - name: "r1_fp8_dep8_mtp1_1k1k"
- model_name: "deepseek_r1_0528_fp8"
+ # 1k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_1k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ trust_remote_code: true
tensor_parallel_size: 8
moe_expert_parallel_size: 8
pipeline_parallel_size: 1
@@ -18,7 +21,7 @@ server_configs:
enable_balance: true
timeout_iters: 60
moe_config:
- backend: 'DEEPGEMM'
+ backend: 'CUTLASS'
cuda_graph_config:
enable_padding: true
max_batch_size: 512
@@ -30,70 +33,77 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 1
client_configs:
- - name: "con4096_iter5_1k1k"
- concurrency: 4096
- iterations: 5
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
- - name: "r1_fp8_tep8_mtp3_1k1k"
- model_name: "deepseek_r1_0528_fp8"
+ # 8k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_8k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ trust_remote_code: true
tensor_parallel_size: 8
moe_expert_parallel_size: 8
pipeline_parallel_size: 1
- max_batch_size: 64
- max_num_tokens: 8192
+ max_batch_size: 512
+ max_num_tokens: 12288
attn_backend: "TRTLLM"
- enable_attention_dp: false
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
moe_config:
- backend: 'DEEPGEMM'
+ backend: 'CUTLASS'
cuda_graph_config:
enable_padding: true
- max_batch_size: 64
+ max_batch_size: 512
kv_cache_config:
dtype: 'fp8'
enable_block_reuse: false
free_gpu_memory_fraction: 0.8
speculative_config:
decoding_type: 'MTP'
- num_nextn_predict_layers: 3
+ num_nextn_predict_layers: 1
client_configs:
- - name: "con64_iter10_1k1k"
- concurrency: 64
+ - name: "con1024_iter10_8k1k"
+ concurrency: 1024
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
- - name: "r1_fp8_tp8_mtp3_1k1k"
- model_name: "deepseek_r1_0528_fp8"
+ # 1k1k configs - TEP8 with TRTLLM, MTP3
+ - name: "r1_fp4_v2_tep8_mtp3"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ trust_remote_code: true
tensor_parallel_size: 8
- moe_expert_parallel_size: 1
+ moe_expert_parallel_size: 8
pipeline_parallel_size: 1
- max_batch_size: 8
- max_num_tokens: 8192
+ max_batch_size: 512
+ max_num_tokens: 3136
attn_backend: "TRTLLM"
enable_attention_dp: false
moe_config:
- backend: 'TRTLLM'
+ backend: "TRTLLM"
cuda_graph_config:
enable_padding: true
- max_batch_size: 8
+ max_batch_size: 512
kv_cache_config:
dtype: 'fp8'
- enable_block_reuse: false
- free_gpu_memory_fraction: 0.8
+ free_gpu_memory_fraction: 0.5
speculative_config:
decoding_type: 'MTP'
num_nextn_predict_layers: 3
client_configs:
- - name: "con8_iter10_1k1k"
- concurrency: 8
- iterations: 10
+ - name: "con32_iter12_1k1k"
+ concurrency: 32
+ iterations: 12
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml
new file mode 100644
index 000000000000..588fdf4286e5
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml
@@ -0,0 +1,138 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ supported_gpus:
+ - B200
+hardware:
+ gpus_per_node: 8
+server_configs:
+ # 1k1k configs - TP4 with TRTLLM, MTP3
+ - name: "r1_fp4_v2_tp4_mtp3_1k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con2_iter10_1k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+
+ # 1k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_1k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 128
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - TP4 with TRTLLM, MTP3
+ - name: "r1_fp4_v2_tp4_mtp3_8k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con4_iter10_8k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_8k1k"
+ model_name: "deepseek_r1_0528_fp4_v2"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 32
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml
similarity index 82%
rename from tests/scripts/perf-sanity/deepseek_r1_fp4_v2_grace_blackwell.yaml
rename to tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml
index 46c89fe768fa..8ba596e9b334 100644
--- a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_grace_blackwell.yaml
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml
@@ -3,8 +3,10 @@ metadata:
supported_gpus:
- GB200
- GB300
+hardware:
+ gpus_per_node: 4
server_configs:
- # 1k1k configs
+ # 1k1k configs - DEP4 with CUTLASS, MTP1
- name: "r1_fp4_v2_dep4_mtp1_1k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -31,14 +33,15 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 1
client_configs:
- - name: "con2048_iter5_1k1k"
- concurrency: 2048
- iterations: 5
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+ # 1k1k configs - TEP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tep4_mtp3_1k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -66,9 +69,10 @@ server_configs:
iterations: 10
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+ # 1k1k configs - TP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tp4_mtp3_1k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -91,15 +95,15 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 3
client_configs:
- - name: "con4_iter10_1k1k"
- concurrency: 4
+ - name: "con2_iter10_1k1k"
+ concurrency: 2
iterations: 10
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
- # 8k1k configs
+ # 8k1k configs - DEP4 with CUTLASS, MTP1
- name: "r1_fp4_v2_dep4_mtp1_8k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -126,14 +130,15 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 1
client_configs:
- - name: "con2048_iter5_8k1k"
- concurrency: 2048
- iterations: 5
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
isl: 8192
osl: 1024
- random_range_ratio: 0.2
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+ # 8k1k configs - TEP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tep4_mtp3_8k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -161,9 +166,10 @@ server_configs:
iterations: 10
isl: 8192
osl: 1024
- random_range_ratio: 0.2
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+ # 8k1k configs - TP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tp4_mtp3_8k1k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -186,15 +192,15 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 3
client_configs:
- - name: "con4_iter10_8k1k"
- concurrency: 4
+ - name: "con2_iter10_8k1k"
+ concurrency: 2
iterations: 10
isl: 8192
osl: 1024
- random_range_ratio: 0.2
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
- # 1k8k configs
+ # 1k8k configs - DEP4 with CUTLASS, MTP1
- name: "r1_fp4_v2_dep4_mtp1_1k8k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -226,9 +232,10 @@ server_configs:
iterations: 5
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k8k-20480-ratio-1_for_serve.json
+ # 1k8k configs - TEP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tep4_mtp3_1k8k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -256,9 +263,10 @@ server_configs:
iterations: 10
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k8k-20480-ratio-1_for_serve.json
+ # 1k8k configs - TP4 with TRTLLM, MTP3
- name: "r1_fp4_v2_tp4_mtp3_1k8k"
model_name: "deepseek_r1_0528_fp4_v2"
tensor_parallel_size: 4
@@ -286,5 +294,5 @@ server_configs:
iterations: 10
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k8k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml
new file mode 100644
index 000000000000..8be30076a811
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml
@@ -0,0 +1,166 @@
+metadata:
+ model_name: deepseek_r1_0528_fp8
+ supported_gpus:
+ - B200
+hardware:
+ gpus_per_node: 8
+server_configs:
+ # 1k1k configs - TP8 with TRTLLM, MTP3
+ - name: "r1_fp8_tp8_mtp3_1k1k"
+ model_name: "deepseek_r1_0528_fp8"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 8
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 8
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con4_iter10_1k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+
+ # 1k1k configs - DEP8 with DEEPGEMM, MTP1
+ - name: "r1_fp8_dep8_mtp1_1k1k"
+ model_name: "deepseek_r1_0528_fp8"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 128
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'DEEPGEMM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - TP8 with TRTLLM, MTP3
+ - name: "r1_fp8_tp8_mtp3_8k1k"
+ model_name: "deepseek_r1_0528_fp8"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 8
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 8
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con4_iter10_8k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP8 with DEEPGEMM, MTP1
+ - name: "r1_fp8_dep8_mtp1_8k1k"
+ model_name: "deepseek_r1_0528_fp8"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 32
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'DEEPGEMM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+
+ # 6k1k configs - TP8 with TRTLLM, MTP1
+ - name: "r1_fp8_tp8_6k1k"
+ model_name: "deepseek_r1_0528_fp8"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ client_configs:
+ - name: "con64_iter10_6k1k"
+ concurrency: 64
+ iterations: 10
+ isl: 6144
+ osl: 1024
+ backend: "openai"
+ random_range_ratio: 0.2
diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml
new file mode 100644
index 000000000000..7835cce594eb
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml
@@ -0,0 +1,72 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ supported_gpus:
+ - B200
+hardware:
+ gpus_per_node: 8
+server_configs:
+ # 8k1k configs - TEP8 with TRTLLM, MTP3
+ - name: "v32_fp4_tep8_mtp3_8k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con2_iter10_8k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "v32_fp4_dep8_mtp1_8k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 32
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml
new file mode 100644
index 000000000000..591303abb6c7
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml
@@ -0,0 +1,138 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ supported_gpus:
+ - GB200
+hardware:
+ gpus_per_node: 4
+server_configs:
+ # 1k1k configs - TEP4 with TRTLLM, MTP3
+ - name: "v32_fp4_tep4_mtp3_1k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con2_iter10_1k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json
+
+ # 1k1k configs - DEP4 with CUTLASS, MTP1
+ - name: "v32_fp4_dep4_mtp1_1k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 256
+ max_num_tokens: 8192
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 256
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - TEP4 with TRTLLM, MTP3
+ - name: "v32_fp4_tep4_mtp3_8k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 3
+ client_configs:
+ - name: "con2_iter10_8k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP4 with CUTLASS, MTP1
+ - name: "v32_fp4_dep4_mtp1_8k1k"
+ model_name: "deepseek_v32_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 64
+ max_num_tokens: 12288
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ speculative_config:
+ decoding_type: 'MTP'
+ num_nextn_predict_layers: 1
+ client_configs:
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
similarity index 57%
rename from tests/scripts/perf-sanity/deepseek_r1_fp4_v2_blackwell.yaml
rename to tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
index 8661c2413984..f92bdca6a3d8 100644
--- a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_blackwell.yaml
+++ b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
@@ -1,13 +1,16 @@
metadata:
model_name: deepseek_r1_0528_fp4_v2
supported_gpus:
- - B200
- - B300
+ - GB300
+hardware:
+ gpus_per_node: 4
server_configs:
- - name: "r1_fp4_v2_dep4_mtp1_1k1k"
+ # 1k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_1k1k"
model_name: "deepseek_r1_0528_fp4_v2"
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ trust_remote_code: true
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
pipeline_parallel_size: 1
max_batch_size: 512
max_num_tokens: 8192
@@ -30,69 +33,76 @@ server_configs:
decoding_type: 'MTP'
num_nextn_predict_layers: 1
client_configs:
- - name: "con2048_iter5_1k1k"
- concurrency: 2048
- iterations: 5
+ - name: "con1024_iter10_1k1k"
+ concurrency: 1024
+ iterations: 10
isl: 1024
osl: 1024
- random_range_ratio: 0.8
+ random_range_ratio: 0.2
backend: "openai"
- - name: "r1_fp4_v2_tep4_mtp3_1k1k"
+ # 8k1k configs - DEP8 with CUTLASS, MTP1
+ - name: "r1_fp4_v2_dep8_mtp1_8k1k"
model_name: "deepseek_r1_0528_fp4_v2"
- tensor_parallel_size: 4
- moe_expert_parallel_size: 4
+ trust_remote_code: true
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
pipeline_parallel_size: 1
- max_batch_size: 32
- max_num_tokens: 8192
+ max_batch_size: 512
+ max_num_tokens: 12288
attn_backend: "TRTLLM"
- enable_attention_dp: false
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
moe_config:
- backend: 'TRTLLM'
+ backend: 'CUTLASS'
cuda_graph_config:
enable_padding: true
- max_batch_size: 32
+ max_batch_size: 512
kv_cache_config:
dtype: 'fp8'
enable_block_reuse: false
free_gpu_memory_fraction: 0.8
speculative_config:
decoding_type: 'MTP'
- num_nextn_predict_layers: 3
+ num_nextn_predict_layers: 1
client_configs:
- - name: "con32_iter10_1k1k"
- concurrency: 32
+ - name: "con1024_iter10_8k1k"
+ concurrency: 1024
iterations: 10
- isl: 1024
+ isl: 8192
osl: 1024
- random_range_ratio: 0.8
+ random_range_ratio: 0.2
backend: "openai"
- - name: "r1_fp4_v2_tp4_mtp3_1k1k"
+ # 1k1k configs - TEP8 with TRTLLM, MTP3
+ - name: "r1_fp4_v2_tep8_mtp3"
model_name: "deepseek_r1_0528_fp4_v2"
- tensor_parallel_size: 4
- moe_expert_parallel_size: 1
+ trust_remote_code: true
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
pipeline_parallel_size: 1
- max_batch_size: 4
- max_num_tokens: 8192
+ max_batch_size: 512
+ max_num_tokens: 3136
attn_backend: "TRTLLM"
enable_attention_dp: false
moe_config:
- backend: 'TRTLLM'
+ backend: "TRTLLM"
cuda_graph_config:
enable_padding: true
- max_batch_size: 4
+ max_batch_size: 512
kv_cache_config:
dtype: 'fp8'
- enable_block_reuse: false
- free_gpu_memory_fraction: 0.8
+ free_gpu_memory_fraction: 0.5
speculative_config:
decoding_type: 'MTP'
num_nextn_predict_layers: 3
client_configs:
- - name: "con4_iter10_1k1k"
- concurrency: 4
- iterations: 10
+ - name: "con32_iter12_1k1k"
+ concurrency: 32
+ iterations: 12
isl: 1024
osl: 1024
random_range_ratio: 0.8
diff --git a/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_blackwell.yaml
new file mode 100644
index 000000000000..cde7a71b2a2d
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_blackwell.yaml
@@ -0,0 +1,103 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ supported_gpus:
+ - B200
+hardware:
+ gpus_per_node: 8
+server_configs:
+ # 1k1k configs - TP2 with TRTLLM, MTP0
+ - name: "gpt_oss_fp4_tp2_mtp0_1k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con4_iter10_1k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+
+ # 1k1k configs - TP1 with TRTLLM, MTP0
+ - name: "gpt_oss_fp4_tp1_mtp0_1k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con256_iter10_1k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - TP1 with TRTLLM, MTP0
+ - name: "gpt_oss_fp4_tp1_mtp0_8k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con4_iter10_8k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/gpt_oss_120b_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml
similarity index 54%
rename from tests/scripts/perf-sanity/gpt_oss_120b_fp4_grace_blackwell.yaml
rename to tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml
index d5993c46deb3..cc5dbe906753 100644
--- a/tests/scripts/perf-sanity/gpt_oss_120b_fp4_grace_blackwell.yaml
+++ b/tests/scripts/perf-sanity/aggregated/gpt_oss_120b_fp4_grace_blackwell.yaml
@@ -1,8 +1,9 @@
metadata:
model_name: gpt_oss_120b_fp4
supported_gpus:
- - B200
- - B300
+ - GB200
+hardware:
+ gpus_per_node: 4
server_configs:
- name: "gpt_oss_fp4_dep4_1k8k"
model_name: "gpt_oss_120b_fp4"
@@ -32,8 +33,8 @@ server_configs:
iterations: 5
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k8k-20480-ratio-1_for_serve.json
- name: "gpt_oss_fp4_dep2_1k1k"
model_name: "gpt_oss_120b_fp4"
@@ -63,8 +64,8 @@ server_configs:
iterations: 5
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
- name: "gpt_oss_fp4_tep2_1k8k"
model_name: "gpt_oss_120b_fp4"
@@ -92,8 +93,8 @@ server_configs:
iterations: 10
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k8k-20480-ratio-1_for_serve.json
- name: "gpt_oss_fp4_tp2_1k8k"
model_name: "gpt_oss_120b_fp4"
@@ -121,8 +122,8 @@ server_configs:
iterations: 10
isl: 1024
osl: 8192
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k8k-20480-ratio-1_for_serve.json
- name: "gpt_oss_fp4_tp4_eagle3_1k1k"
model_name: "gpt_oss_120b_fp4"
@@ -155,5 +156,102 @@ server_configs:
iterations: 32
isl: 1024
osl: 1024
- random_range_ratio: 0.8
backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+
+ # GPT-OSS configs from Excel - 1k1k TP2 (TP2EP1) con4
+ - name: "gpt_oss_fp4_tp2_mtp0_1k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con4_iter10_1k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+
+ # GPT-OSS configs from Excel - 1k1k TP1 con256
+ - name: "gpt_oss_fp4_tp1_mtp0_1k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con256_iter10_1k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 1024
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+
+ # GPT-OSS configs from Excel - 8k1k TP1 con4 and con256
+ - name: "gpt_oss_fp4_tp1_mtp0_8k1k"
+ model_name: "gpt_oss_120b_fp4"
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ max_batch_size: 512
+ max_num_tokens: 20000
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ num_postprocess_workers: 4
+ stream_interval: 20
+ client_configs:
+ - name: "con4_iter10_8k1k"
+ concurrency: 4
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_2_nodes_grace_blackwell.yaml
new file mode 100644
index 000000000000..d7f9a5cc728f
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_2_nodes_grace_blackwell.yaml
@@ -0,0 +1,72 @@
+metadata:
+ model_name: k2_thinking_fp4
+ supported_gpus:
+ - GB200
+hardware:
+ gpus_per_node: 4
+server_configs:
+ # 32k8k configs - TEP8 with TRTLLM
+ - name: "k2_thinking_fp4_tep8_32k8k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 8192
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_chunked_prefill: true
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con2_iter10_32k8k"
+ concurrency: 2
+ iterations: 10
+ isl: 32768
+ osl: 8192
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-32k8k-20480-ratio-1_for_serve.json
+
+ # 32k8k configs - DEP8 with CUTLASS
+ - name: "k2_thinking_fp4_dep8_32k8k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 32
+ max_num_tokens: 8192
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_chunked_prefill: true
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con128_iter10_32k8k"
+ concurrency: 128
+ iterations: 10
+ isl: 32768
+ osl: 8192
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-32k8k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_blackwell.yaml
new file mode 100644
index 000000000000..10b5b26a86b3
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_blackwell.yaml
@@ -0,0 +1,136 @@
+metadata:
+ model_name: k2_thinking_fp4
+ supported_gpus:
+ - B200
+hardware:
+ gpus_per_node: 8
+server_configs:
+ # 8k1k configs - TEP8 with TRTLLM
+ - name: "k2_thinking_fp4_tep8_8k1k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 12288
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con2_iter10_8k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP8 with CUTLASS
+ - name: "k2_thinking_fp4_dep8_8k1k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 64
+ max_num_tokens: 12288
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con512_iter10_8k1k"
+ concurrency: 512
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+
+ # 32k8k configs - TEP8 with TRTLLM
+ - name: "k2_thinking_fp4_tep8_32k8k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 8192
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_chunked_prefill: true
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con2_iter10_32k8k"
+ concurrency: 2
+ iterations: 10
+ isl: 32768
+ osl: 8192
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-32k8k-20480-ratio-1_for_serve.json
+
+ # 32k8k configs - DEP8 with CUTLASS
+ - name: "k2_thinking_fp4_dep8_32k8k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ max_batch_size: 32
+ max_num_tokens: 8192
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_chunked_prefill: true
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con128_iter10_32k8k"
+ concurrency: 128
+ iterations: 10
+ isl: 32768
+ osl: 8192
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-32k8k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_grace_blackwell.yaml
new file mode 100644
index 000000000000..dbef10e3cbf1
--- /dev/null
+++ b/tests/scripts/perf-sanity/aggregated/k2_thinking_fp4_grace_blackwell.yaml
@@ -0,0 +1,70 @@
+metadata:
+ model_name: k2_thinking_fp4
+ supported_gpus:
+ - GB200
+hardware:
+ gpus_per_node: 4
+server_configs:
+ # 8k1k configs - TEP4 with TRTLLM
+ - name: "k2_thinking_fp4_tep4_8k1k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 4
+ max_num_tokens: 12288
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_attention_dp: false
+ moe_config:
+ backend: 'TRTLLM'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con2_iter10_8k1k"
+ concurrency: 2
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+
+ # 8k1k configs - DEP4 with CUTLASS
+ - name: "k2_thinking_fp4_dep4_8k1k"
+ model_name: "k2_thinking_fp4"
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ max_batch_size: 64
+ max_num_tokens: 12288
+ trust_remote_code: true
+ attn_backend: "TRTLLM"
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ moe_config:
+ backend: 'CUTLASS'
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ dtype: 'fp8'
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ client_configs:
+ - name: "con256_iter10_8k1k"
+ concurrency: 256
+ iterations: 10
+ isl: 8192
+ osl: 1024
+ backend: "openai"
+ trust_remote_code: true
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
diff --git a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
deleted file mode 100644
index 0d663651ee39..000000000000
--- a/tests/scripts/perf-sanity/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-metadata:
- model_name: deepseek_r1_0528_fp4_v2
- supported_gpus:
- - GB200
- - GB300
-hardware:
- gpus_per_node: 4
-server_configs:
- - name: "r1_fp4_v2_dep8_mtp1"
- model_name: "deepseek_r1_0528_fp4_v2"
- trust_remote_code: true
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- pipeline_parallel_size: 1
- max_batch_size: 512
- max_num_tokens: 3136
- attn_backend: "TRTLLM"
- enable_attention_dp: true
- attention_dp_config:
- batching_wait_iters: 0
- enable_balance: true
- timeout_iters: 60
- moe_config:
- backend: 'CUTLASS'
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 512
- kv_cache_config:
- dtype: 'fp8'
- enable_block_reuse: false
- free_gpu_memory_fraction: 0.5
- client_configs:
- - name: "con32_iter12_1k1k"
- concurrency: 32
- iterations: 12
- isl: 1024
- osl: 1024
- random_range_ratio: 0.8
- backend: "openai"
-
- - name: "r1_fp4_v2_tep8_mtp3"
- model_name: "deepseek_r1_0528_fp4_v2"
- trust_remote_code: true
- tensor_parallel_size: 8
- moe_expert_parallel_size: 8
- pipeline_parallel_size: 1
- max_batch_size: 512
- max_num_tokens: 3136
- attn_backend: "TRTLLM"
- enable_attention_dp: false
- moe_config:
- backend: "TRTLLM"
- cuda_graph_config:
- enable_padding: true
- max_batch_size: 512
- kv_cache_config:
- dtype: 'fp8'
- free_gpu_memory_fraction: 0.5
- speculative_config:
- decoding_type: 'MTP'
- num_nextn_predict_layers: 3
- client_configs:
- - name: "con32_iter12_1k1k"
- concurrency: 32
- iterations: 12
- isl: 1024
- osl: 1024
- random_range_ratio: 0.8
- backend: "openai"
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..f406982f0671
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,96 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 256
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..d92871914265
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '2048'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 256
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 256
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..10346b399f16
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '256'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 256
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..8bf5046d96a3
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1536'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 192
+ max_num_tokens: 384
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 192
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..0545a4f5d793
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,96 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 256
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..77cf540bcac6
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - B200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:8
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '256'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 8
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp2_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp2_ccb-UCX.yaml
new file mode 100644
index 000000000000..288617016b17
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp2_ccb-UCX.yaml
@@ -0,0 +1,96 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 128k8k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '128'
+ input_length: 131072
+ output_length: 8192
+ dataset_file: datasets/perf-ci/deepseek_r1-128k8k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 8
+ max_num_tokens: 32
+ tensor_parallel_size: 16
+ moe_expert_parallel_size: 16
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 8
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 2
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 131104
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 8
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.3
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..8730f9c9ad2e
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,96 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 128k8k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 131072
+ output_length: 8192
+ dataset_file: datasets/perf-ci/deepseek_r1-128k8k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 4
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 131104
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 8
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.3
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..6001cd94297c
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,96 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 128k8k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '64'
+ input_length: 131072
+ output_length: 8192
+ dataset_file: datasets/perf-ci/deepseek_r1-128k8k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 8
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 2
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 131104
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 8
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.3
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 131104
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..023a4bee0c46
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..3dba0456a22c
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,93 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ - GB300
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+ config_index: -1
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: "--gres=gpu:4"
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: true
+ multi_round: 8
+ benchmark_ratio: 0.8
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: "TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes"
+ server_env_var: "TRTLLM_SERVER_DISABLE_GC=1"
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+worker_config:
+ gen:
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ max_batch_size: 768
+ max_num_tokens: 768
+ max_seq_len: 2068
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 768
+ print_iter_log: true
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTLASS
+ use_low_precision_moe_combine: true
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ stream_interval: 100
+ num_postprocess_workers: 4
+ ctx:
+ max_batch_size: 16
+ max_num_tokens: 16896
+ max_seq_len: 2044
+ tensor_parallel_size: 4
+ context_parallel_size: 1
+ moe_expert_parallel_size: 4
+ enable_attention_dp: true
+ pipeline_parallel_size: 1
+ print_iter_log: true
+ cuda_graph_config: null
+ disable_overlap_scheduler: true
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.75
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..343256ea737e
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 128
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..fecfd0c9b60e
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '3072'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 768
+ max_num_tokens: 1536
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 768
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..f195391d47f5
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.75
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..fd74cf5fc5a1
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 128
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..6bc440bc7ba3
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4096'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 256
+ max_num_tokens: 512
+ tensor_parallel_size: 16
+ moe_expert_parallel_size: 16
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 256
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..065a4e3eef5b
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml
@@ -0,0 +1,108 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ load_balancer:
+ num_slots: 256
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..f393fe8c16df
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,105 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..14ee30497a71
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml
@@ -0,0 +1,105 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '2048'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 512
+ max_num_tokens: 1024
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..75e40a71e3c8
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,105 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 32k4k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 32768
+ output_length: 4096
+ dataset_file: datasets/perf-ci/deepseek_v32-32k4k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 256
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 32784
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml
new file mode 100644
index 000000000000..1532c16be662
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml
@@ -0,0 +1,108 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 32k4k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '2048'
+ input_length: 32768
+ output_length: 4096
+ dataset_file: datasets/perf-ci/deepseek_v32-32k4k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 64
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ load_balancer:
+ num_slots: 288
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 1
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 32784
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..93a047c09259
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,105 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 32k4k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '256'
+ input_length: 32768
+ output_length: 4096
+ dataset_file: datasets/perf-ci/deepseek_v32-32k4k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 8
+ max_num_tokens: 256
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 8
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.85
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 32784
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..a724179fc510
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-UCX.yaml
@@ -0,0 +1,108 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.75
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ load_balancer:
+ num_slots: 256
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..ca8db584e74f
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,105 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..ee9948f72846
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX.yaml
@@ -0,0 +1,104 @@
+metadata:
+ model_name: deepseek_v32_fp4
+ precision: fp4
+ model_dir_name: DeepSeek-V3.2-FP4-v2
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4096'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_v32-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 128
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ load_balancer:
+ num_slots: 256
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ nvfp4_gemm_config:
+ allowed_backends:
+ - cutlass
+ - cublaslt
+ - cutedsl
+ - cuda_core
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ tokens_per_block: 64
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..5ae5902c87b7
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,95 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '2048'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1536
+ max_num_tokens: 20000
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1536
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..e0ac186b2033
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,95 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '512'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1536
+ max_num_tokens: 20000
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1536
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..0acc76bb0aec
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,91 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '64'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 256
+ max_num_tokens: 20000
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 256
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 1024
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..196c915b00cd
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,91 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '128'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1024
+ max_num_tokens: 20000
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1024
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..a1c940d944cc
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,91 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 1024
+ max_num_tokens: 20000
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 1024
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..667e345b8954
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,95 @@
+metadata:
+ model_name: gpt_oss_120b_fp4
+ precision: fp4
+ model_dir_name: GPT-OSS-120B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '512'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/gpt_oss_120b-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 512
+ max_num_tokens: 20000
+ tensor_parallel_size: 2
+ moe_expert_parallel_size: 2
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ attention_dp_config:
+ batching_wait_iters: 0
+ enable_balance: true
+ timeout_iters: 60
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 20000
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 8448
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb384_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb384_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..1cc06715d9b2
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb384_mtp0_ccb-UCX.yaml
@@ -0,0 +1,98 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '2048'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 64
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: WIDEEP
+ load_balancer:
+ num_slots: 384
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 8192
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..d049701d6374
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,95 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4096'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 512
+ max_num_tokens: 512
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 512
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: CUTLASS
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 8192
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..77f285c14180
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,94 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 4
+ max_num_tokens: 128
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 8192
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..7f64ef39347e
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-UCX.yaml
@@ -0,0 +1,104 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: WIDEEP
+ load_balancer:
+ num_slots: 416
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: Eagle
+ max_draft_len: 3
+ eagle3_one_model: true
+ speculative_model: Kimi-K2-Thinking-NVFP4-Eagle3
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 8768
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb384_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb384_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..9ba96ccfd86b
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb384_mtp0_ccb-UCX.yaml
@@ -0,0 +1,98 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 5
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4096'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 256
+ max_num_tokens: 256
+ tensor_parallel_size: 16
+ moe_expert_parallel_size: 16
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 256
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: WIDEEP
+ load_balancer:
+ num_slots: 384
+ layer_updates_per_iter: 1
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 8768
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..a133bb9b885e
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k2-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,101 @@
+metadata:
+ model_name: k2_thinking_fp4
+ precision: fp4
+ model_dir_name: Kimi-K2-Thinking-NVFP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '4'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/k2_thinking-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 4
+ max_num_tokens: 128
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 4
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.8
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: Eagle
+ max_draft_len: 3
+ eagle3_one_model: true
+ speculative_model: Kimi-K2-Thinking-NVFP4-Eagle3
+ trust_remote_code: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ allreduce_strategy: MNNVL
+ ctx:
+ print_iter_log: true
+ max_batch_size: 2
+ max_num_tokens: 8768
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
+ trust_remote_code: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..6b79f12fd74a
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,92 @@
+metadata:
+ model_name: qwen3_235b_a22b_fp4
+ precision: fp4
+ model_dir_name: Qwen3-235B-A22B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/qwen3_235b-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 128
+ max_num_tokens: 128
+ tensor_parallel_size: 8
+ moe_expert_parallel_size: 8
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 128
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 4
+ max_num_tokens: 32768
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml
new file mode 100644
index 000000000000..001b3f2b3975
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-UCX.yaml
@@ -0,0 +1,91 @@
+metadata:
+ model_name: qwen3_235b_a22b_fp4
+ precision: fp4
+ model_dir_name: Qwen3-235B-A22B-FP4
+ supported_gpus:
+ - GB200
+ script_file: disaggr_torch.slurm
+ benchmark_type: 8k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '64'
+ input_length: 8192
+ output_length: 1024
+ dataset_file: datasets/perf-ci/qwen3_235b-8k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 64
+ max_num_tokens: 64
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 64
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.9
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 4
+ max_num_tokens: 32768
+ tensor_parallel_size: 1
+ moe_expert_parallel_size: 1
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: false
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 32768
+ backend: UCX
+ disable_overlap_scheduler: true
diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
new file mode 100644
index 000000000000..95ef58484ee0
--- /dev/null
+++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-UCX.yaml
@@ -0,0 +1,97 @@
+metadata:
+ model_name: deepseek_r1_0528_fp4_v2
+ precision: fp4
+ model_dir_name: DeepSeek-R1-0528-FP4-v2
+ supported_gpus:
+ - GB300
+ script_file: disaggr_torch.slurm
+ benchmark_type: 1k1k
+slurm:
+ script_file: disaggr_torch.slurm
+ partition:
+ account:
+ job_time: 02:00:00
+ job_name: unified-benchmark
+ extra_args: --gres=gpu:4
+ numa_bind: true
+benchmark:
+ mode: e2e
+ use_nv_sa_benchmark: false
+ multi_round: 10
+ benchmark_ratio: 0.0
+ streaming: true
+ concurrency_list: '1024'
+ input_length: 1024
+ output_length: 1024
+ dataset_file: datasets/perf-ci/deepseek_r1-1k1k-20480-ratio-1_for_serve.json
+hardware:
+ gpus_per_node: 4
+ num_ctx_servers: 1
+ num_gen_servers: 1
+environment:
+ container_mount:
+ container_image:
+ model_path:
+ trtllm_repo: ''
+ build_wheel: false
+ work_dir:
+ worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes
+ server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+profiling:
+ nsys_on: false
+accuracy:
+ enable_accuracy_test: false
+ model: local-completions
+ tasks: gsm8k
+ model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096
+worker_config:
+ gen:
+ print_iter_log: true
+ max_batch_size: 32
+ max_num_tokens: 128
+ tensor_parallel_size: 32
+ moe_expert_parallel_size: 32
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config:
+ enable_padding: true
+ max_batch_size: 32
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: CUTEDSL
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: &id001
+ decoding_type: MTP
+ num_nextn_predict_layers: 3
+ num_postprocess_workers: 4
+ stream_interval: 20
+ ctx:
+ print_iter_log: true
+ max_batch_size: 16
+ max_num_tokens: 16384
+ tensor_parallel_size: 4
+ moe_expert_parallel_size: 4
+ pipeline_parallel_size: 1
+ context_parallel_size: 1
+ enable_attention_dp: true
+ enable_lm_head_tp_in_adp: true
+ cuda_graph_config: null
+ kv_cache_config:
+ enable_block_reuse: false
+ free_gpu_memory_fraction: 0.6
+ dtype: fp8
+ moe_config:
+ backend: TRTLLM
+ cache_transceiver_config:
+ max_tokens_in_buffer: 16384
+ backend: UCX
+ disable_overlap_scheduler: true
+ speculative_config: *id001
diff --git a/tests/test_common/error_utils.py b/tests/test_common/error_utils.py
new file mode 100644
index 000000000000..9cb57aaf88f7
--- /dev/null
+++ b/tests/test_common/error_utils.py
@@ -0,0 +1,82 @@
+import os
+
+ERROR_KEYWORDS = [
+ "RuntimeError",
+ "out of memory",
+ "ValueError",
+ "FileNotFoundError",
+ "ConnectionRefusedError",
+ "ClientConnectorError",
+ "CancelledError",
+ "TimeoutError",
+ "PMI2_Init failed to initialize",
+ "OSError",
+]
+SLURM_LOG_TAIL_LINES = 200 # Number of lines to print from slurm job logs
+ERROR_CONTEXT_LINES = 100 # Number of lines to print before and after error line
+
+
+def check_error(file_path: str) -> list[tuple[int, str]]:
+ if not os.path.exists(file_path):
+ return []
+
+ error_lines = []
+ with open(file_path, "r", errors="replace") as f:
+ for line_idx, line in enumerate(f, start=1):
+ for keyword in ERROR_KEYWORDS:
+ if keyword in line:
+ error_lines.append((line_idx, line.strip()))
+ break # Only add line once even if multiple keywords match
+
+ return error_lines
+
+
+def report_error(
+ error_msg: str | Exception, log_files: list[str], tail_lines: int = SLURM_LOG_TAIL_LINES
+) -> None:
+ # Convert Exception to string if needed
+ if isinstance(error_msg, Exception):
+ error_msg_str = str(error_msg)
+ else:
+ error_msg_str = error_msg
+
+ messages = [error_msg_str]
+
+ for log_file in log_files:
+ if not os.path.exists(log_file):
+ messages.append(f"Failed to read {log_file}: Path doesn't exist")
+
+ all_lines = None
+ error_lines = []
+ try:
+ with open(log_file, "r", errors="replace") as f:
+ all_lines = f.readlines()
+ for line_idx, line in enumerate(f, start=1):
+ for keyword in ERROR_KEYWORDS:
+ if keyword in line:
+ error_lines.append((line_idx, line.strip()))
+ break
+ except Exception as e:
+ all_lines = None
+ error_lines = []
+ messages.append(f"Failed to read {log_file}: {e}")
+
+ if error_lines:
+ error_lines_str = ", ".join(
+ [f"Error line {line_idx}: {line_str}" for line_idx, line_str in error_lines]
+ )
+ messages.append(error_lines_str)
+ # Find the first error line number for context
+ first_idx = min(line_idx for line_idx, _ in error_lines)
+ if all_lines is not None:
+ # Use all_lines for error context
+ start_idx = max(0, first_idx - ERROR_CONTEXT_LINES - 1)
+ end_idx = min(len(all_lines), first_idx + ERROR_CONTEXT_LINES)
+ context_lines = all_lines[start_idx:end_idx]
+ messages.append("".join(context_lines))
+ else:
+ tail_content = "".join(all_lines[-tail_lines:]) if all_lines else "(empty)"
+ messages.append(f"--- {log_file} [last {tail_lines} lines] ---")
+ messages.append(tail_content)
+
+ raise RuntimeError("\n".join(messages))
diff --git a/tests/test_common/http_utils.py b/tests/test_common/http_utils.py
index 07826751b74f..9628dff1809f 100644
--- a/tests/test_common/http_utils.py
+++ b/tests/test_common/http_utils.py
@@ -3,16 +3,37 @@
import requests
+from test_common.error_utils import check_error
-def wait_for_endpoint_ready(url: str, timeout: int = 300, server_proc: subprocess.Popen = None):
+
+def wait_for_endpoint_ready(
+ url: str,
+ timeout: int = 300,
+ check_files: list[str] | None = None,
+ server_proc: subprocess.Popen | None = None,
+):
start = time.monotonic()
+ iteration = 0
while time.monotonic() - start < timeout:
+ # Check server_proc if provided (singular)
if server_proc is not None:
exit_code = server_proc.poll()
if exit_code is not None:
raise RuntimeError(
f"Server process exited with code {exit_code} before becoming ready."
)
+
+ iteration += 1
+ if check_files and iteration % 300 == 0:
+ for check_file in check_files:
+ error_lines = check_error(check_file)
+ if error_lines:
+ error_lines_str = ", ".join(
+ [f"line {line_idx}: {line_str}" for line_idx, line_str in error_lines]
+ )
+ raise RuntimeError(
+ f"Found error in server file {check_file}: {error_lines_str}"
+ )
try:
time.sleep(1)
if requests.get(url, timeout=5).status_code == 200: