Skip to content

[None][feat] Add production-level Prometheus metrics (iteration stats, config info, token counters, phase histograms) #12545

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pcastonguay merged 25 commits into from
Apr 13, 2026
Merged

[None][feat] Add production-level Prometheus metrics (iteration stats, config info, token counters, phase histograms) #12545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
8ef3957
[None][feat] Expose iteration stats as Prometheus metrics
nvyutwu Mar 13, 2026
448d6e2
[None][feat] Add config info gauges to Prometheus metrics
nvyutwu Mar 14, 2026
d45cc8c
[None][fix] change spec decode draft/accepted token metrics from Gaug…
nvyutwu Mar 14, 2026
a771c57
[None][feat] add token counters, phase histograms, and label fix to P…
nvyutwu Mar 20, 2026
bd6506d
[None][fix] address code review findings in Step 1 metrics
nvyutwu Mar 20, 2026
7c5122e
[None][fix] guard TPOT computation against missing timestamps
nvyutwu Mar 30, 2026
398f9cf
[None][fix] narrow exception and add debug logging for GPU type detec…
nvyutwu Mar 31, 2026
934d288
[None][feat] Record per-candidate GENERATION_TOKENS and TPOT for n>1
nvyutwu Apr 1, 2026
c352674
[None][fix] Use Optional type hints for log_config_info params
nvyutwu Apr 1, 2026
3924fba
[None][fix] fix config metric labels: add cp_size, fix gpu_count, ren…
nvyutwu Apr 1, 2026
aa4d85e
[None][fix] use getattr for quant_algo access for consistency
nvyutwu Apr 2, 2026
5c85832
[None][fix] rename kv_cache_cfg to kv_cache_config_obj for naming con…
nvyutwu Apr 2, 2026
92e99ef
[None][test] expand metrics unit tests: success counter, base latency…
nvyutwu Apr 2, 2026
b4f3b89
[None][test] add n>1 test for prompt/generation token counting
nvyutwu Apr 3, 2026
3d32492
style: apply pre-commit formatting fixes (isort, yapf, ruff)
nvyutwu Apr 6, 2026
0ad7554
[None][fix] update api_stability snapshot for record_stats sequence_i…
nvyutwu Apr 6, 2026
7aedfaf
style: fix D212 docstring violations in collector.py
nvyutwu Apr 6, 2026
034edc7
[None][fix] add Defaults to 0 in record_stats docstring for api_stabi…
nvyutwu Apr 7, 2026
f0379e1
[None][feat] make Prometheus histogram buckets configurable via LLM args
nvyutwu Apr 7, 2026
64e994b
[None][fix] validate bucket params and add unit tests for configurabl…
nvyutwu Apr 7, 2026
57872ba
[None][refactor] group Prometheus metrics params into PrometheusMetri…
nvyutwu Apr 8, 2026
22c0b95
[None][fix] keep enable_energy_metrics on BaseLlmArgs and fix strictl…
nvyutwu Apr 8, 2026
0a17139
Merge remote-tracking branch 'origin/main' into yutwu/add-metrics-rebase
nvyutwu Apr 11, 2026
d620491
[None][chore] update ruff-legacy baseline after rebase on main
nvyutwu Apr 11, 2026
8083d5c
[None][fix] unify test helper convention to match main's string-based…
nvyutwu Apr 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 40 additions & 92 deletions ruff-legacy-baseline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{
"_meta": {
"generated_by": "scripts/legacy_utils.py lint-update-violations",
"total_violations": 5588,

@venkywonka venkywonka Apr 22, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like these changes slipped in by mistake

"total_files": 500
"total_violations": 5341,
"total_files": 493
},
".github/scripts/label_community_user.py": {
"D212": 1
Expand Down Expand Up @@ -485,7 +485,7 @@
"tensorrt_llm/__init__.py": {
"D202": 1,
"D212": 1,
"E402": 28
"E402": 27
},
"tensorrt_llm/_torch/attention_backend/interface.py": {
"F821": 2
Expand Down Expand Up @@ -933,7 +933,7 @@
"D205": 11,
"D208": 2,
"D210": 7,
"D212": 36,
"D212": 37,
"D300": 8,
"D415": 2
},
Expand Down Expand Up @@ -992,12 +992,6 @@
"D300": 1,
"D415": 4
},
"tensorrt_llm/metrics/__init__.py": {
"F405": 3
},
"tensorrt_llm/metrics/collector.py": {
"D212": 3
},
"tensorrt_llm/models/baichuan/config.py": {
"F821": 2
},
Expand Down Expand Up @@ -1581,7 +1575,8 @@
},
"tensorrt_llm/serve/openai_server.py": {
"D205": 1,
"D212": 2
"D212": 2,
"F821": 2
},
"tensorrt_llm/serve/responses_utils.py": {
"D200": 2,
Expand Down Expand Up @@ -1686,10 +1681,6 @@
"D205": 1,
"D212": 1
},
"tests/integration/defs/accuracy/test_cli_flow.py": {
"D300": 1,
"D415": 1
},
"tests/integration/defs/accuracy/test_disaggregated_serving.py": {
"D212": 1,
"F601": 1
Expand All @@ -1705,10 +1696,9 @@
"F811": 2
},
"tests/integration/defs/accuracy/test_llm_api_pytorch.py": {
"D205": 3,
"D212": 4,
"D212": 1,
"D300": 3,
"D415": 5,
"D415": 3,
"E402": 7
},
"tests/integration/defs/common.py": {
Expand Down Expand Up @@ -1770,34 +1760,26 @@
"D415": 1
},
"tests/integration/defs/examples/test_bindings.py": {
"D300": 2,
"D415": 2
"D300": 1,
"D415": 1
},
"tests/integration/defs/examples/test_chatglm.py": {
"D300": 1
},
"tests/integration/defs/examples/test_commandr.py": {
"D300": 2
},
"tests/integration/defs/examples/test_draft_target_model.py": {
"D300": 2,
"D415": 2
"D300": 1
},
"tests/integration/defs/examples/test_gemma.py": {
"D202": 1,
"D300": 7,
"D403": 6,
"D415": 6
"D300": 6,
"D403": 5,
"D415": 5
},
"tests/integration/defs/examples/test_gpt.py": {
"D202": 5,
"D205": 2,
"D209": 1,
"D210": 1,
"D212": 1,
"D300": 22,
"D202": 3,
"D300": 5,
"D403": 1,
"D415": 23
"D415": 4
},
"tests/integration/defs/examples/test_granite.py": {
"D202": 1,
Expand All @@ -1808,64 +1790,50 @@
"D415": 1
},
"tests/integration/defs/examples/test_llama.py": {
"D200": 1,
"D202": 6,
"D205": 6,
"D208": 10,
"D209": 1,
"D210": 1,
"D212": 6,
"D300": 30,
"D403": 12,
"D415": 26,
"D202": 2,
"D300": 6,
"D403": 4,
"D415": 4,
"E712": 1
},
"tests/integration/defs/examples/test_mamba.py": {
"D300": 2,
"D415": 2
},
"tests/integration/defs/examples/test_mistral.py": {
"D202": 2,
"D300": 2,
"D403": 1,
"D415": 1
"D202": 1
},
"tests/integration/defs/examples/test_mixtral.py": {
"D300": 3,
"D403": 2,
"D415": 1
"D300": 1,
"D403": 1
},
"tests/integration/defs/examples/test_multimodal.py": {
"D202": 1,
"D300": 2,
"D415": 2
},
"tests/integration/defs/examples/test_nemotron_nas.py": {
"D202": 3
},
"tests/integration/defs/examples/test_openai.py": {
"D300": 3,
"D403": 1,
"D415": 3
},
"tests/integration/defs/examples/test_phi.py": {
"D202": 3,
"D300": 6,
"D202": 1,
"D300": 3,
"D403": 1,
"D415": 3
},
"tests/integration/defs/examples/test_qwen.py": {
"D202": 1,
"D300": 14,
"D403": 3,
"D415": 7
"D300": 2,
"D403": 1
},
"tests/integration/defs/examples/test_qwen2audio.py": {
"D300": 2,
"D415": 1
},
"tests/integration/defs/examples/test_qwenvl.py": {
"D300": 2,
"D300": 1,
"D415": 1
},
"tests/integration/defs/examples/test_recurrentgemma.py": {
Expand Down Expand Up @@ -1909,12 +1877,9 @@
"D212": 3
},
"tests/integration/defs/perf/open_search_db_utils.py": {
"D200": 7,
"D205": 3,
"D212": 10,
"D415": 5,
"E402": 1,
"E712": 1
"D200": 2,
"D212": 4,
"E402": 1
},
"tests/integration/defs/perf/pytorch_model_config.py": {
"D200": 1,
Expand Down Expand Up @@ -1978,14 +1943,14 @@
"F821": 2
},
"tests/integration/defs/test_e2e.py": {
"D200": 6,
"D202": 4,
"D200": 5,
"D202": 3,
"D205": 3,
"D210": 1,
"D212": 9,
"D300": 10,
"D403": 5,
"D415": 13,
"D212": 8,
"D300": 9,
"D403": 4,
"D415": 10,
"F811": 2
},
"tests/integration/defs/test_list_parser.py": {
Expand All @@ -2003,11 +1968,9 @@
},
"tests/integration/defs/test_mlpf_results.py": {
"D200": 1,
"D202": 1,
"D205": 1,
"D208": 4,
"D212": 2,
"D300": 1,
"D415": 2
},
"tests/integration/defs/triton_server/common.py": {
Expand Down Expand Up @@ -2105,19 +2068,6 @@
"tests/scripts/allreduce_perf/allreduce_perf_viz.py": {
"D214": 3
},
"tests/scripts/perf-sanity/parse_benchmark_results.py": {
"D200": 2,
"D205": 1,
"D212": 3,
"D415": 5
},
"tests/scripts/perf-sanity/run_benchmark_serve.py": {
"D200": 2,
"D202": 1,
"D212": 2,
"D415": 8,
"F601": 1
},
"tests/unittest/_torch/misc/test_autotuner.py": {
"E731": 1
},
Expand Down Expand Up @@ -2199,7 +2149,7 @@
"F811": 1
},
"tests/unittest/llmapi/test_executor.py": {
"D205": 3,
"D205": 5,
"D209": 3
},
"tests/unittest/llmapi/test_llm.py": {
Expand All @@ -2215,9 +2165,7 @@
"tests/unittest/llmapi/test_llm_args.py": {
"D200": 1,
"D202": 2,
"D205": 3,
"D209": 1,
"D212": 3,
"D212": 2,
"E712": 20,
"F811": 1
},
Expand Down
57 changes: 26 additions & 31 deletions tensorrt_llm/executor/result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
from ..llmapi.tracer import global_tracer
from ..llmapi.utils import AsyncQueue, print_traceback_on_error
from ..metrics import MetricNames, MetricsCollector, RequestEventTiming
from ..metrics.perf_utils import \
process_req_perf_metrics as _process_req_perf_metrics
from ..sampling_params import LogprobParams, SamplingParams
from .utils import ErrorResponse, has_event_loop, is_llm_response

Expand Down Expand Up @@ -179,6 +181,7 @@ def __init__(self,
self._done = False
self._aborted = False
self.metrics_dict = {}
self.candidate_metrics: list[dict] = []
self.trace_headers: Optional[dict[str, str]] = None
# torch backend will use trtllm sampler in beam search mode, but it does not support return logprobs incrementally
self.use_trtllm_sampler = sampling_params.use_beam_search and sampling_params.best_of > 1
Expand Down Expand Up @@ -393,9 +396,14 @@ def _handle_sequence(self,
raise ValueError(
f"Unknown finish reason: {finish_reasons[src_idx]}")

# Only record stats and do tracing when the entire request is done
# Record per-candidate metrics as each sequence finishes so that
# GENERATION_TOKENS and TPOT are captured for every candidate when
# sampling_params.n > 1.
if sequence_is_finished:
self.record_stats(output, req_perf_metrics_dict, seq_idx)

# Tracing is recorded once when the entire request is done.
if self._done:
self.record_stats(output, req_perf_metrics_dict)
self.do_tracing(output, req_perf_metrics_dict)

@print_traceback_on_error
Expand Down Expand Up @@ -547,12 +555,19 @@ def _handle_response(self,

def record_stats(self,
output: CompletionOutput,
stats: Optional[dict[str, float]] = None) -> None:
stats: Optional[dict[str, float]] = None,
sequence_index: int = 0) -> None:
"""Record the stats of the generation result.

Called once per candidate when it finishes. When ``n > 1`` each
candidate has its own timestamps so TPOT and GENERATION_TOKENS are
computed independently per candidate. PROMPT_TOKENS are only recorded
for ``sequence_index == 0`` to avoid double-counting the shared prompt.

Args:
output (CompletionOutput): The output of the generation result.
stats (Optional[dict[str, float]]): The stats of the generation result. Defaults to None.
sequence_index (int): Index of this candidate (0 for the first / only sequence). Defaults to 0.
"""
if not stats:
return
Expand All @@ -563,9 +578,16 @@ def record_stats(self,
output.finish_reason
})
processed_metrics_stat = _process_req_perf_metrics(
stats, len(output.token_ids), self.sampling_params.n > 1)
stats, len(output.token_ids))
if processed_metrics_stat:
metrics_stats.update(processed_metrics_stat)
# Record prompt tokens only for the first candidate to avoid
# double-counting the shared prompt across n candidates.
if output.finish_reason and sequence_index == 0:
prompt_token_ids = getattr(self, "prompt_token_ids", None)
if prompt_token_ids is not None and len(prompt_token_ids) > 0:
metrics_stats[MetricNames.PROMPT_TOKENS] = len(prompt_token_ids)
self.candidate_metrics.append(metrics_stats)
self.metrics_dict.update(metrics_stats)

def do_tracing(
Expand Down Expand Up @@ -1042,30 +1064,3 @@ def _topk_logprobs(logits: torch.Tensor, top_k: int,

return LogProbsResult(prompt=prompt_logprobs,
generation=generation_logprobs)


def _process_req_perf_metrics(
req_perf_metrics_dict: Optional[dict[str, float]],
output_length: int,
is_multiple_response: bool = False) -> dict[MetricNames, float]:
stat = {}
if not req_perf_metrics_dict:
return stat
ttft = req_perf_metrics_dict.get(RequestEventTiming.FIRST_TOKEN_TIME, 0) - \
req_perf_metrics_dict.get(RequestEventTiming.ARRIVAL_TIME, 0)
e2e = req_perf_metrics_dict.get(RequestEventTiming.LAST_TOKEN_TIME, 0) - \
req_perf_metrics_dict.get(RequestEventTiming.ARRIVAL_TIME, 0)
request_queue_time = req_perf_metrics_dict.get(RequestEventTiming.FIRST_SCHEDULED_TIME, 0) - \
req_perf_metrics_dict.get(RequestEventTiming.ARRIVAL_TIME, 0)
stat = {
MetricNames.TTFT: ttft,
MetricNames.E2E: e2e,
MetricNames.REQUEST_QUEUE_TIME: request_queue_time
}
if output_length > 1 and not is_multiple_response:
tpot = (req_perf_metrics_dict.get(
RequestEventTiming.LAST_TOKEN_TIME, 0) - req_perf_metrics_dict.get(
RequestEventTiming.FIRST_TOKEN_TIME, 0)) / (output_length - 1)
stat.update({MetricNames.TPOT: tpot})
stat = dict(filter(lambda item: item[1] > 0, stat.items()))
return stat
Loading
Loading