Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -927,6 +927,9 @@ def report_decode_stats(
if x := self.scheduler_status_logger:
x.maybe_dump(batch, self.scheduler.waiting_queue)

# Routers build their cache view from KV events, so flush them every step.
self.scheduler.kv_events_publisher.publish_kv_events()

# Periodic work: log + heavy metrics at decode_log_interval
if self.forward_ct_decode % self.decode_log_interval != 0:
return
Expand Down Expand Up @@ -1139,7 +1142,6 @@ def report_decode_stats(
self._log_hicache_stats()
self.metrics_collector.log_stats(self.stats)
self.scheduler.kv_events_publisher.emit_kv_metrics()
self.scheduler.kv_events_publisher.publish_kv_events()

def log_batch_result_stats(
self,
Expand Down
30 changes: 30 additions & 0 deletions test/registered/unit/observability/test_forward_pass_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
from sglang.srt.managers.scheduler_components.batch_result_processor import (
SchedulerBatchResultProcessor,
)
from sglang.srt.managers.scheduler_components.kv_events_publisher import (
SchedulerKvEventsPublisher,
)
from sglang.srt.managers.scheduler_components.metrics_reporter import (
PrefillStats,
SchedulerMetricsReporter,
Expand Down Expand Up @@ -1140,6 +1143,33 @@ def test_idle_resets_forward_timing_when_metrics_are_disabled(self):
self.assertEqual(self.published_occupancies, [])


class TestDecodeKvEvents(CustomTestCase):
def test_decode_step_publishes_queued_kv_events(self):
"""KV events queued during decode must reach the publisher in the same
step, not only on decode_log_interval steps."""
event = object()
kv_events_publisher = SchedulerKvEventsPublisher(
kv_events_config=None,
tree_cache=types.SimpleNamespace(take_events=lambda: [event]),
send_metrics_from_scheduler=None,
max_running_requests=1,
max_total_num_tokens=1,
get_stats=lambda: None,
)
kv_events_publisher.enable_kv_cache_events = True
kv_events_publisher.kv_event_publisher = _CollectingPublisher()
scheduler = types.SimpleNamespace(kv_events_publisher=kv_events_publisher)
reporter = _make_reporter(self, scheduler)
reporter.forward_ct_decode = 1

reporter.report_decode_stats(
can_run_cuda_graph=False, running_batch=types.SimpleNamespace()
)

batches = kv_events_publisher.kv_event_publisher.metrics
self.assertEqual([batch.events for batch in batches], [[event]])


class TestSchedulerTimeAccounting(CustomTestCase):
def setUp(self):
enter_scope(
Expand Down
Loading