Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion jenkins/L0_Test.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -6512,7 +6512,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16",
2,
3,
40,
10
)
Expand Down
19 changes: 11 additions & 8 deletions jenkins/scripts/perf/local/submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,16 @@ def parse_test_string(test_case_name: str):

Test name formats:
- Disagg e2e: disagg_upload-e2e-{config_base}
- Disagg e2e + lifecycle breakdown: disagg_upload-e2e_time_breakdown-{config_base}
- Disagg gen_only: disagg_upload-gen_only-{config_base}
- ctx_only: aggr_upload-ctx_only-{config_base} (runs aggr mode but reads disagg config)
- Regular aggr: aggr_upload-{config}-{server_name}

Returns:
tuple: (config_base_name, select_pattern, runtime_mode, benchmark_mode)
- runtime_mode: "aggregated" or "disaggregated"
- benchmark_mode: "e2e", "gen_only", "ctx_only", or None (for normal aggr)
- benchmark_mode: "e2e", "e2e_time_breakdown", "gen_only",
"ctx_only", or None (for normal aggr)
"""
labels = test_case_name.split("-")

Expand All @@ -108,10 +110,10 @@ def parse_test_string(test_case_name: str):
is_aggr_prefix = "aggr" in prefix

if is_disagg_prefix:
# Disagg format: disagg_upload-{e2e|gen_only}-{config_base}
# Disagg format: disagg_upload-{e2e|e2e_time_breakdown|gen_only}-{config_base}
assert len(labels) > 2, "Disagg test must have benchmark_mode and config!"
benchmark_mode = labels[1] # e2e or gen_only
assert benchmark_mode in ("e2e", "gen_only"), (
benchmark_mode = labels[1] # e2e, e2e_time_breakdown, or gen_only
assert benchmark_mode in ("e2e", "e2e_time_breakdown", "gen_only"), (
f"Invalid benchmark_mode for disagg: {benchmark_mode}"
)
runtime_mode = "disaggregated"
Expand Down Expand Up @@ -145,12 +147,13 @@ def get_config_yaml_path(llm_src, config_base_name, benchmark_mode):
Args:
llm_src: Path to LLM source code
config_base_name: Base name of config file (without .yaml extension)
benchmark_mode: "e2e", "gen_only", "ctx_only", or None (for normal aggr)
benchmark_mode: "e2e", "e2e_time_breakdown", "gen_only", "ctx_only", or
None (for normal aggr)

Returns:
str: Full path to config yaml file
"""
if benchmark_mode in ("e2e", "gen_only", "ctx_only"):
if benchmark_mode in ("e2e", "e2e_time_breakdown", "gen_only", "ctx_only"):
config_dir = DISAGG_CONFIG_FOLDER
else:
config_dir = AGG_CONFIG_FOLDER
Expand Down Expand Up @@ -545,7 +548,7 @@ def generate_pytest_command(
"""Generate pytest command and test list."""
# Generate test list content based on runtime_mode and benchmark_mode
if runtime_mode == "disaggregated":
# disagg_upload-{e2e|gen_only}-{config_base}
# disagg_upload-{e2e|e2e_time_breakdown|gen_only}-{config_base}
test_list_content = (
f"perf/test_perf_sanity.py::test_e2e[disagg-{benchmark_mode}-{config_file_base_name}]"
)
Expand Down Expand Up @@ -653,7 +656,7 @@ def main():
parser.add_argument(
"--benchmark-mode",
default="",
choices=["", "e2e", "gen_only", "ctx_only"],
choices=["", "e2e", "e2e_time_breakdown", "gen_only", "ctx_only"],
help="Benchmark mode for disagg config (when --config-file is provided)",
)
parser.add_argument(
Expand Down
16 changes: 10 additions & 6 deletions jenkins/scripts/perf/submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@
runtime_mode = "aggregated", benchmark_mode = "ctx_only"
(reads disagg yaml, but launches via the aggregated single-pytest path
using the ctx worker's parallel sizes)
3. Multi-node disagg e2e/gen: disagg[_upload]-{e2e|gen_only}-{config_base}
runtime_mode = "disaggregated", benchmark_mode in {"e2e", "gen_only"}
3. Multi-node disagg e2e/gen: disagg[_upload]-{e2e|e2e_time_breakdown|gen_only}-{config_base}
runtime_mode = "disaggregated", benchmark_mode in
{"e2e", "e2e_time_breakdown", "gen_only"}
(e2e_time_breakdown launches exactly like e2e -- it differs only in what
the harness asks the servers and the client to record)

Test name → yaml folder mapping mirrors test_perf_sanity.py:parse_test_string.
"""
Expand Down Expand Up @@ -336,13 +339,14 @@ def parse_test_case_name(llm_src, selected_line):
if "disagg" in prefix:
if len(parts) < 3:
raise ValueError(
f"Invalid disagg test format. Expected disagg[_upload]-{{e2e|gen_only}}-"
f"{{config_base}}, got: {bracket_content}"
f"Invalid disagg test format. Expected disagg[_upload]-"
f"{{e2e|e2e_time_breakdown|gen_only}}-{{config_base}}, got: {bracket_content}"
)
benchmark_mode = parts[1]
if benchmark_mode not in ("e2e", "gen_only"):
if benchmark_mode not in ("e2e", "e2e_time_breakdown", "gen_only"):
raise ValueError(
f"Invalid disagg benchmark_mode: {benchmark_mode}. Expected 'e2e' or 'gen_only'."
f"Invalid disagg benchmark_mode: {benchmark_mode}. Expected 'e2e', "
f"'e2e_time_breakdown' or 'gen_only'."
)
runtime_mode = "disaggregated"
server_name = None
Expand Down
33 changes: 31 additions & 2 deletions tensorrt_llm/serve/perf_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,18 @@
HTTP/1.1 200 OK
Content-Type: application/json
Server-Timing: server_queue;dur=1.250000, server_ttft;dur=8.500000, server_e2e;dur=24.000000
X-TRTLLM-Start-End-Time: server-start;ts=12345.123456, server-end;ts=12345.147456
X-TRTLLM-Start-End-Time: server-start;ts=12345.123456, server-end;ts=12345.147456,
server-srv-start;ts=12345.122000, server-srv-ttft;ts=12345.132000
X-TRTLLM-Step-Metrics: server-step-0-forward;dur=2.100000, server-step-0-sample;dur=0.400000
X-TRTLLM-Ctx-Chunk-Metrics: server-ctx-chunk-0-forward;dur=4.200000

``X-TRTLLM-Start-End-Time`` carries absolute timestamps. ``start``/``end`` are the
executor's arrival and last-token times; ``srv-start``/``srv-ttft`` are the HTTP
server's arrival and first-token times; ``kv-start``/``kv-end`` bracket the
KV-cache transfer on a disaggregated generation worker. A disaggregated server
needs all of them to reconstruct a request's full lifecycle from a worker
response -- see :func:`build_metrics_record_from_headers`.

Streaming responses carry the same fields in a named SSE event after ``[DONE]``::

data: [DONE]
Expand Down Expand Up @@ -300,9 +308,19 @@ def build_metrics_headers(records: List[Dict[str, Any]]) -> Dict[str, str]:
for record in records:
for phase, phase_record in record.get("phases", {}).items():
timing = phase_record.get("timing_metrics", {})
# Absolute timestamps forwarded verbatim. The four "srv-"/"kv-" names
# are what let a disagg server reconstruct the full request lifecycle
# from a worker response; without them the per-phase breakdown
# silently collapses to zero-width spans. Names must not contain a
# second "server-"/"server_" substring, because the receiving side
# rewrites the phase prefix with an unqualified str.replace().
for name, field in (
("start", "arrival_time"),
("end", "last_token_time"),
("srv-start", "server_arrival_time"),
("srv-ttft", "server_first_token_time"),
("kv-start", "kv_cache_transfer_start"),
("kv-end", "kv_cache_transfer_end"),
):
timestamp = timing.get(field)
if timestamp is not None:
Expand Down Expand Up @@ -378,6 +396,10 @@ def build_metrics_record_from_headers(
fields = {
f"{phase}-start": "arrival_time",
f"{phase}-end": "last_token_time",
f"{phase}-srv-start": "server_arrival_time",
f"{phase}-srv-ttft": "server_first_token_time",
f"{phase}-kv-start": "kv_cache_transfer_start",
f"{phase}-kv-end": "kv_cache_transfer_end",
}
for item in metrics_headers.get(START_END_TIME_HEADER, "").split(","):
name, separator, timestamp = item.strip().partition(";ts=")
Expand Down Expand Up @@ -484,7 +506,14 @@ def _jsonl_perf_metrics(phase_record: Dict[str, Any]) -> PerfMetrics:

timing_metrics = dict(perf_metrics.get("timing_metrics", {}))
if not timing_metrics.get("kv_cache_size"):
for name in ("kv_cache_size", "kv_cache_transfer_start", "kv_cache_transfer_end"):
timing_metrics.pop("kv_cache_size", None)
# Drop the KV-transfer timestamps only when they were never populated. Keying
# this off kv_cache_size instead discards timestamps that the Server-Timing
# header transport carried successfully, because kv_cache_size is worker-local
# and never reaches a header-derived record -- which zeroed the KV-transfer
# span for every disaggregated request.
for name in ("kv_cache_transfer_start", "kv_cache_transfer_end"):
if timing_metrics.get(name) is None:
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved
timing_metrics.pop(name, None)
perf_metrics["timing_metrics"] = timing_metrics

Expand Down
30 changes: 24 additions & 6 deletions tensorrt_llm/serve/scripts/benchmark_serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -1107,12 +1107,30 @@ def create_dataset_and_sample(dataset_name: str):

analyzer = RequestTimeBreakdown()
timing_data = analyzer.parse_json_file(perf_filename)
if timing_data:
diagram_filename = f"{output_stem}-time_diagram.html"
analyzer.create_timing_diagram(timing_data, diagram_filename)
print(f"Time diagram saved to: {diagram_filename}")
else:
print("No time data found; skipping time breakdown diagram.")
if not timing_data:
print("No time data found; skipping time breakdown report.")
return
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved

# These "Time Breakdown ..." lines are the machine-readable output of the
# breakdown -- the perf-sanity harness scrapes them out of this process's
# stdout the same way it scrapes "Mean TTFT (ms)" -- whereas the JSON and
# the HTML diagram are human aids that need a writable path (and, for the
# diagram, plotly). Print first so that neither one failing can cost us
# the measurement: output_stem is relative to the current directory
# unless --result-dir was given.
span_stats = analyzer.compute_statistics(timing_data)
for span in sorted(span_stats):
for stat in ("mean", "median", "p75", "p99"):
print(f"Time Breakdown {span} {stat} (ms): "
f"{span_stats[span][stat]:.4f}")

stats_filename = f"{output_stem}-time_breakdown_stats.json"
analyzer.export_statistics_json(timing_data, stats_filename)
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved
print(f"Span statistics saved to: {stats_filename}")

diagram_filename = f"{output_stem}-time_diagram.html"
analyzer.create_timing_diagram(timing_data, diagram_filename)
print(f"Time diagram saved to: {diagram_filename}")


if __name__ == "__main__":
Expand Down
52 changes: 52 additions & 0 deletions tensorrt_llm/serve/scripts/time_breakdown/time_breakdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -2163,6 +2163,46 @@ def show_statistics(self, timing_data: List[Dict]):
)
print(f" Median: {np.median(valid_times):.3f}")

def compute_statistics(
self, timing_data: List[Dict]) -> Dict[str, Dict[str, float]]:
"""Aggregate every span across all requests.

Returns ``{span_name: {mean, median, p75, p99, count}}`` with durations in
**milliseconds** (the unit every other serving benchmark metric uses).

A span that is zero for every request is omitted rather than reported as
``0.0``: :meth:`TimingMetric.calculate_duration` returns 0 when an endpoint
timestamp is missing, so 0 means "not measured", not "took no time".
Reporting it as 0.0 would silently fabricate a data point.
"""
stats: Dict[str, Dict[str, float]] = {}
for metric in self.config.metrics:
key = f'{metric.name}_time'
valid = [
data[key] * 1000 for data in timing_data if data.get(key, 0) > 0
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved
]
if not valid:
continue
stats[metric.name] = {
'mean': float(np.mean(valid)),
'median': float(np.median(valid)),
'p75': float(np.percentile(valid, 75)),
'p99': float(np.percentile(valid, 99)),
'count': len(valid),
}
return stats

def export_statistics_json(self, timing_data: List[Dict],
output_path: str) -> Dict[str, Any]:
"""Write :meth:`compute_statistics` output to ``output_path`` as JSON."""
payload = {
'total_requests': len(timing_data),
'spans': self.compute_statistics(timing_data),
}
with open(output_path, 'w', encoding='utf-8') as out_file:
json.dump(payload, out_file, indent=2, sort_keys=True)
return payload


def main():
"""Main CLI entry point."""
Expand All @@ -2176,6 +2216,7 @@ def main():
python time_breakdown.py perf_metrics.jsonl --stats-only
python time_breakdown.py perf_metrics.jsonl --max-requests 50 --sort-by e2e
python time_breakdown.py perf_metrics.jsonl --max-requests 100 --sort-by arrival
python time_breakdown.py perf_metrics.jsonl --stats-only --export-stats-json stats.json
""")

parser.add_argument(
Expand All @@ -2193,6 +2234,13 @@ def main():
parser.add_argument('--show-stats',
action='store_true',
help='Show statistics with diagram')
parser.add_argument(
'--export-stats-json',
type=str,
default=None,
metavar='PATH',
help='Write per-span mean/median/P75/P99 (in milliseconds) to PATH as '
'JSON. Combine with --stats-only to skip rendering the HTML diagram')
parser.add_argument(
'--max-requests',
type=int,
Expand Down Expand Up @@ -2227,6 +2275,10 @@ def main():
if args.stats_only or args.show_stats:
analyzer.show_statistics(timing_data)

if args.export_stats_json:
analyzer.export_statistics_json(timing_data, args.export_stats_json)
print(f"Span statistics saved to: {args.export_stats_json}")

if not args.stats_only:
analyzer.create_timing_diagram(timing_data,
args.output,
Expand Down
Loading
Loading