Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions tensorrt_llm/_torch/pyexecutor/adp_iter_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ class ADPIterStatsRecord:
req_stats: Optional[List[RequestStats]]
kv_iter_stats: Optional[Dict[int, object]]
attention_dp_rank: int
# Per-loop CPU wall and GPU forward time captured on rank 0 alongside
# the IterationStats at queue time. Currently broadcast unchanged to all
# rank rows during fanout (true per-rank timing would require widening
# the rank-state allgather payload). Under steady-state ADP all ranks
# step in lockstep, so this is a reasonable approximation.
host_step_time_ms: Optional[float] = None
prev_device_step_time_ms: Optional[float] = None


class ADPIterStatsBuffer:
Expand All @@ -66,6 +73,11 @@ def __init__(self) -> None:
self._rank0_req_stats: Dict[int, Optional[List[RequestStats]]] = {}
# Rank 0 only: KV iteration stats captured with pending IterationStats.
self._rank0_kv_iter_stats: Dict[int, Optional[Dict[int, object]]] = {}
# Rank 0 only: per-loop CPU and GPU timings captured with the
# IterationStats. Broadcast to all rank rows at fanout (see
# _make_rank_iter_stats / finalize).
self._rank0_host_step_time_ms: Dict[int, Optional[float]] = {}
self._rank0_prev_device_step_time_ms: Dict[int, Optional[float]] = {}
self._oldest_iter: Optional[int] = None

@staticmethod
Expand All @@ -91,6 +103,8 @@ def queue(
*,
kv_iter_stats: Optional[Dict[int, object]] = None,
is_rank0: bool,
host_step_time_ms: Optional[float] = None,
prev_device_step_time_ms: Optional[float] = None,
) -> None:
"""Queue local stats; rank 0 also keeps objects needed for fanout."""
payload = self.make_payload(stats)
Expand All @@ -109,6 +123,8 @@ def queue(
self._rank0_iter_stats[iter_id] = stats
self._rank0_req_stats[iter_id] = req_stats
self._rank0_kv_iter_stats[iter_id] = kv_iter_stats
self._rank0_host_step_time_ms[iter_id] = host_step_time_ms
self._rank0_prev_device_step_time_ms[iter_id] = prev_device_step_time_ms

def next_payload(self) -> Optional[RankIterStatsPayload]:
"""Return the oldest pending stats payload to piggyback."""
Expand Down Expand Up @@ -140,6 +156,8 @@ def _discard(self, iter_id: int, *, recompute_oldest: bool = True) -> None:
self._rank0_iter_stats.pop(iter_id, None)
self._rank0_req_stats.pop(iter_id, None)
self._rank0_kv_iter_stats.pop(iter_id, None)
self._rank0_host_step_time_ms.pop(iter_id, None)
self._rank0_prev_device_step_time_ms.pop(iter_id, None)
if recompute_oldest and iter_id == self._oldest_iter:
self._recompute_oldest_iter()

Expand Down Expand Up @@ -268,6 +286,8 @@ def finalize(

req_stats = self._rank0_req_stats.get(iter_stats_iter)
kv_iter_stats = self._rank0_kv_iter_stats.get(iter_stats_iter)
host_step_time_ms = self._rank0_host_step_time_ms.get(iter_stats_iter)
prev_device_step_time_ms = self._rank0_prev_device_step_time_ms.get(iter_stats_iter)

for rank_state in sorted(matching_states, key=lambda s: s.rank):
rank = rank_state.rank
Expand All @@ -277,6 +297,8 @@ def finalize(
req_stats=req_stats if rank == 0 else None,
kv_iter_stats=kv_iter_stats if rank == 0 else None,
attention_dp_rank=rank,
host_step_time_ms=host_step_time_ms,
prev_device_step_time_ms=prev_device_step_time_ms,
)
)

Expand Down
165 changes: 137 additions & 28 deletions tensorrt_llm/_torch/pyexecutor/py_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,15 @@ def on_detected():
getattr(self.llm_args, 'kv_cache_config', None),
'iteration_stats_interval', 1)
self._adp_iter_stats = ADPIterStatsBuffer()
# Per-loop CPU wall and GPU forward time captured by the profile_step
# closure (see _profiler). Populated whenever enable_iter_perf_stats or
# print_iter_log is on so the /metrics serializer can read them
# without depending on the log line. host_step_time describes the loop
# body that just finished; prev_device_step_time is the GPU forward
# time read via the ping-pong CUDA event pair (lags host by one loop
# under steady state — see ping-pong comment in _profiler).
self._latest_host_step_time_ms: Optional[float] = None
self._latest_prev_device_step_time_ms: Optional[float] = None
self.gather_all_responses = False

self.kv_cache_transceiver = kv_cache_transceiver
Expand Down Expand Up @@ -1036,8 +1045,15 @@ def profile_step():
calibrator.stop()
enabled = False

if start_time is not None and self.print_log and (
log_all_ranks or self.dist.rank in log_ranks):
# Capture per-loop timing whenever stats or the iter log are
# enabled. The reading of the OTHER parity's event pair (the
# ping-pong) is what keeps synchronize() from blocking the GPU
# — the events being read have already passed by the time we
# read them. Stashing on self lets the /metrics serializer pick
# up the values without going through the log line.
should_capture_timing = start_time is not None and (
self.print_log or self.enable_iter_perf_stats)
if should_capture_timing:
end_time = time.time()
if it % 2 == 0:
end_event_1.record()
Expand All @@ -1052,24 +1068,29 @@ def profile_step():
prev_device_step_time = start_event_1.elapsed_time(
end_event_1)

if prev_device_step_time is None:
prev_device_step_time = "N/A" # Handle first iteration
else:
prev_device_step_time = f"{prev_device_step_time}ms"
host_step_time = (end_time - start_time) * 1000 # milliseconds
formatted_timestamp = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
logger.info(
f"iter = {self.iter_counter}, "
f"global_rank = {self.global_rank}, "
f"rank = {self.dist.rank}, "
f"currank_total_requests = {self.num_fetch_requests_cur_rank}/"
f"{self.num_fetch_requests}, "
f"host_step_time = {host_step_time}ms, "
f"prev_device_step_time = {prev_device_step_time}, "
f"timestamp = {formatted_timestamp}, "
f"num_scheduled_requests: {self.num_scheduled_requests}, "
f"states = {self.model_engine.iter_states}")
self._latest_host_step_time_ms = host_step_time
self._latest_prev_device_step_time_ms = prev_device_step_time

if self.print_log and (log_all_ranks
or self.dist.rank in log_ranks):
if prev_device_step_time is None:
prev_device_step_time_str = "N/A" # Handle first iteration
else:
prev_device_step_time_str = f"{prev_device_step_time}ms"
formatted_timestamp = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
logger.info(
f"iter = {self.iter_counter}, "
f"global_rank = {self.global_rank}, "
f"rank = {self.dist.rank}, "
f"currank_total_requests = {self.num_fetch_requests_cur_rank}/"
f"{self.num_fetch_requests}, "
f"host_step_time = {host_step_time}ms, "
f"prev_device_step_time = {prev_device_step_time_str}, "
f"timestamp = {formatted_timestamp}, "
f"num_scheduled_requests: {self.num_scheduled_requests}, "
f"states = {self.model_engine.iter_states}")

it += 1

Expand Down Expand Up @@ -1116,6 +1137,12 @@ def profile_step():
def _get_init_iter_stats(self, num_new_active_requests,
new_active_requests_queue_latency_ms):
stats = IterationStats()
# Stamp iter at construction time so the JSON `iter` field identifies
# the loop that CONSTRUCTED this batch. Under the overlap and PP
# schedulers _process_iter_stats consumes this record in a later
# loop, so stamping at consumption time (via self.iter_counter)
# would mislabel the record by 1+ iterations.
stats.iter = self.iter_counter
stats.timestamp = datetime.datetime.now().strftime(
"%m-%d-%Y %H:%M:%S.%f")

Expand Down Expand Up @@ -1212,7 +1239,11 @@ def _update_iter_stats(self, stats, iter_latency_ms, num_completed_requests,
stats.cpu_mem_usage = 0
stats.pinned_mem_usage = 0

stats.iter = self.iter_counter
# NOTE: stats.iter was stamped at construction time in
# _get_init_iter_stats. Do NOT re-stamp here from self.iter_counter
# — under the overlap and PP schedulers this method runs in a later
# loop than the one that built the batch, so the live counter would
# mislabel the record.

kv_cache_manager = self.resource_manager.resource_managers.get(
ResourceManagerType.KV_CACHE_MANAGER)
Expand Down Expand Up @@ -1433,7 +1464,9 @@ def _append_iter_stats(self,
stats: IterationStats,
req_stats: Optional[List[RequestStats]] = None,
kv_iter_stats: Optional[Dict[int, object]] = None,
attention_dp_rank: Optional[int] = None):
attention_dp_rank: Optional[int] = None,
host_step_time_ms: Optional[float] = None,
prev_device_step_time_ms: Optional[float] = None):
"""Append one iteration's finalized stats to the export buffer.

The normal Attention-DP path fans out rank-local rows before calling
Expand All @@ -1445,12 +1478,42 @@ def _append_iter_stats(self,
req_stats: Optional per-request stats.
kv_iter_stats: Optional KV iteration stats captured with ``stats``.
attention_dp_rank: Optional ADP rank for fanned-out rank-local rows.
host_step_time_ms: Per-loop CPU wall captured by profile_step
for the loop that built this batch. Surfaces as
``hostStepTimeMS`` in the /metrics JSON. Always a clean
single-loop measurement (matches the log line's
``host_step_time``).
prev_device_step_time_ms: GPU forward time captured by the
ping-pong CUDA event pair in profile_step. Surfaces as
``prevDeviceStepTimeMS`` in the /metrics JSON. Note the
value lags by one loop relative to ``host_step_time_ms``
(its sibling on the same record describes a slightly
older batch); see _profiler ping-pong comment.
"""
# Non-ADP appends immediately, so the latest KV stats belong to this
# IterationStats. ADP appends later and passes the saved iter-matched
# KV stats explicitly.
if not self.enable_attention_dp:
kv_iter_stats = self._latest_kv_iter_stats
# Same reasoning for the per-loop timings: non-ADP captures them
# at append time. ADP passes the values that were saved with the
# IterationStats at queue time.
if host_step_time_ms is None:
Comment thread
eopXD marked this conversation as resolved.
Resolved
host_step_time_ms = self._latest_host_step_time_ms
if prev_device_step_time_ms is None:
prev_device_step_time_ms = self._latest_prev_device_step_time_ms

# Per-record indicator so consumers can interpret iterLatencyMS
# without inspecting server config. iterLatencyMS spans ~2 loops
# under "overlap" and a clean single loop under "non_overlap"; the
# always-clean alternative is hostStepTimeMS.
# PP (pp_size > 1) is structurally overlap-style regardless of
# disable_overlap_scheduler: _executor_loop_pp queues a BatchStatePP
# in iter N and _process_previous_batch_pp consumes it in a later
# loop, so iterLatencyMS spans ~2 loops there too.
is_overlap_like = (not self.disable_overlap_scheduler
or self.dist.pp_size > 1)
scheduler_mode = "overlap" if is_overlap_like else "non_overlap"

tp_size = getattr(self.dist, "tp_size", 1)
gather_all_ranks = os.environ.get("TLLM_METRICS_ALL_RANKS", "0") == "1"
Expand Down Expand Up @@ -1490,6 +1553,11 @@ def _append_iter_stats(self,
}
for window_size, s in kv_iter_stats.items()
}
if host_step_time_ms is not None:
local_dict["hostStepTimeMS"] = host_step_time_ms
if prev_device_step_time_ms is not None:
local_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms
local_dict["schedulerMode"] = scheduler_mode
local_dict["rank"] = self.dist.tp_rank

gathered = self.dist.tp_allgather(local_dict)
Expand All @@ -1511,11 +1579,21 @@ def _append_iter_stats(self,
return

# Legacy path: rank-0-only (single-rank or iter stats disabled).
# Tuple layout (length is sniffed by _stats_serializer with
# len() > N guards so older 4-tuples remain readable):
# [0] stats: IterationStats
# [1] req_stats: Optional[List[RequestStats]]
# [2] kv_iter_stats: Optional[Dict[int, KvCacheIterationStats]]
# [3] attention_dp_rank: Optional[int]
# [4] host_step_time_ms: Optional[float]
# [5] prev_device_step_time_ms: Optional[float]
# [6] scheduler_mode: "overlap" | "non_overlap"
with self.stats_lock:
if len(self.stats) > self.max_stats_len:
self.stats.pop(0)
self.stats.append(
(stats, req_stats, kv_iter_stats, attention_dp_rank))
(stats, req_stats, kv_iter_stats, attention_dp_rank,
host_step_time_ms, prev_device_step_time_ms, scheduler_mode))

def _process_iter_stats(
self,
Expand All @@ -1526,10 +1604,31 @@ def _process_iter_stats(
):
"""All ranks: build local stats; ADP queues them for later fanout."""
iter_end_time = time.time()
# iterLatencyMS semantics differ by scheduler:
# - Overlap / PP: batch_state.iter_start_time was captured at the top
# of the loop that BUILT this batch (one or more loops ago). The
# end timestamp is taken here, during the consuming loop. So
# iter_latency_ms spans roughly two loops and tracks the batch's
# full lifecycle, not the loop body that produced it. Under steady
# state, sum(iterLatencyMS) ≈ 2 × wall_time.
# - Non-overlap: iter_start_time was captured at the top of THIS
# loop and the end is taken at the bottom of the same loop, so
# iterLatencyMS is the clean per-loop CPU wall.
# The always-clean alternative is host_step_time_ms (set on every
# IterationStats record); the per-record schedulerMode field tells
# consumers which interpretation applies.
iter_latency_ms = (iter_end_time - batch_state.iter_start_time) * 1e3
if batch_state.iter_stats is None:
return

# Snapshot the per-loop CPU and GPU timings captured by the most
# recent profile_step. These belong to the loop that BUILT this
# batch (host wall) and the loop one earlier than that for the GPU
# forward (per the ping-pong design). Save them now so the values
# ride along with the IterationStats through any ADP fanout delay.
host_step_time_ms = self._latest_host_step_time_ms
prev_device_step_time_ms = self._latest_prev_device_step_time_ms
Comment thread
eopXD marked this conversation as resolved.
Resolved

req_stats = self._populate_req_stats(
finished_requests, active_requests,
batch_state.scheduled_requests) if (
Expand All @@ -1541,12 +1640,19 @@ def _process_iter_stats(
batch_state.scheduled_requests,
micro_batch_id)
if self.enable_attention_dp:
self._adp_iter_stats.queue(stats,
req_stats,
kv_iter_stats=self._latest_kv_iter_stats,
is_rank0=self.dist.rank == 0)
self._adp_iter_stats.queue(
stats,
req_stats,
kv_iter_stats=self._latest_kv_iter_stats,
is_rank0=self.dist.rank == 0,
host_step_time_ms=host_step_time_ms,
prev_device_step_time_ms=prev_device_step_time_ms)
else:
self._append_iter_stats(stats, req_stats)
self._append_iter_stats(
stats,
req_stats,
host_step_time_ms=host_step_time_ms,
prev_device_step_time_ms=prev_device_step_time_ms)

def _executor_loop_cleanup(self):

Expand Down Expand Up @@ -3208,7 +3314,10 @@ def _fetch_new_requests(
record.stats,
record.req_stats,
kv_iter_stats=record.kv_iter_stats,
attention_dp_rank=record.attention_dp_rank)
attention_dp_rank=record.attention_dp_rank,
host_step_time_ms=record.host_step_time_ms,
prev_device_step_time_ms=record.prev_device_step_time_ms
)
all_ranks_num_active_requests = [
s.num_active_requests for s in all_rank_states
]
Expand Down
Loading
Loading