Skip to content

hicache: restore per-tier (L1/L2/L3) KV cache hit metrics #31883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TianDi101 wants to merge 2 commits into
base: main
Choose a base branch
from
+308 −0
Open

hicache: restore per-tier (L1/L2/L3) KV cache hit metrics #31883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c767055
hicache: restore per-tier (L1/L2/L3) KV cache hit metrics
TianDi101 Jul 6, 2026
620016c
monitoring: add per-tier Cache Hit Rate + Cached Tokens Rate panels
TianDi101 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
229 changes: 229 additions & 0 deletions examples/monitoring/grafana/dashboards/json/sglang-dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,235 @@
],
"title": "Number Queued Requests",
"type": "timeseries"
},
{
"datasource": {
"default": true,
"type": "prometheus"
},
"description": "Per-tier cache hit rate. Total hit rate = (L1+L2+L3) / (L1+L2+L3+Miss). Each tier's share uses raw token counts without page alignment.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 32
},
"id": 21,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"default": true,
"type": "prometheus"
},
"editorMode": "code",
"expr": "(sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))) / clamp_min((sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:uncached_prompt_tokens_histogram_sum{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))), 1e-9)",
"instant": false,
"legendFormat": "Total Hit Rate (L1+L2+L3)",
"range": true,
"refId": "A"
},
{
"datasource": {
"default": true,
"type": "prometheus"
},
"editorMode": "code",
"expr": "sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) / clamp_min((sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:uncached_prompt_tokens_histogram_sum{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))), 1e-9)",
"instant": false,
"legendFormat": "L1 Hit Rate (GPU)",
"range": true,
"refId": "B"
},
{
"datasource": {
"default": true,
"type": "prometheus"
},
"editorMode": "code",
"expr": "sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) / clamp_min((sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:uncached_prompt_tokens_histogram_sum{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))), 1e-9)",
"instant": false,
"legendFormat": "L2 Hit Rate (Host DRAM)",
"range": true,
"refId": "C"
},
{
"datasource": {
"default": true,
"type": "prometheus"
},
"editorMode": "code",
"expr": "sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) / clamp_min((sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l1_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l2_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:cache_hit_tokens_l3_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) + sum without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (rate(sglang:uncached_prompt_tokens_histogram_sum{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))), 1e-9)",
"instant": false,
"legendFormat": "L3 Hit Rate (Storage)",
"range": true,
"refId": "D"
}
],
"title": "Cache Hit Rate (Per-Tier)",
"type": "timeseries"
},
{
"datasource": {
"default": true,
"type": "prometheus"
},
"description": "Rate of tokens served from cache, tokens/s, including HiCache hits.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 57
},
"id": 22,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"default": true,
"type": "prometheus"
},
"editorMode": "code",
"expr": "rate(sglang:cached_tokens_total{instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])",
"instant": false,
"legendFormat": "\u547d\u4e2d tokens/s ({{cache_source}})",
"range": true,
"refId": "A"
}
],
"title": "Cached Tokens Rate (hit tokens/s)",
"type": "timeseries"
}
],
"preload": false,
Expand Down
32 changes: 32 additions & 0 deletions python/sglang/srt/managers/schedule_policy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,11 @@ def __init__(
# TODO(lsyin): report the real input tokens excluding page alignment
self.log_input_tokens = 0
self.reprocessed_log_input_tokens = 0
# Per-tier cache hit breakdown (raw token counts, no page alignment)
self.log_l1_hit_tokens = 0
self.log_l2_hit_tokens = 0
self.log_l3_hit_tokens = 0
self.log_miss_tokens = 0

if running_batch is not None:
# Estimate the offset in the remaining token space
Expand Down Expand Up @@ -750,6 +755,12 @@ def _update_prefill_budget(
self.reprocessed_log_hit_tokens += prefix_len
self.reprocessed_log_input_tokens += extend_input_len

def _accumulate_per_tier_hits(self, l1: int, l2: int, l3: int, miss: int) -> None:
self.log_l1_hit_tokens += l1
self.log_l2_hit_tokens += l2
self.log_l3_hit_tokens += l3
self.log_miss_tokens += miss

def _get_dllm_remain_tokens(self) -> int:
_rem_tokens = min(
self.rem_dllm_tokens,
Expand Down Expand Up @@ -1042,6 +1053,7 @@ def add_one_req(
real_input_tokens = cand_extend_input_len - req.host_hit_length
real_input_tokens = self.ceil_paged_tokens(real_input_tokens)
prefix_len = len(req.prefix_indices)
l1_hit = prefix_len # L1 GPU device hits before host load-back

if total_tokens >= self.rem_total_tokens:
return AddReqResult.NO_TOKEN
Expand Down Expand Up @@ -1086,6 +1098,7 @@ def add_one_req(
return AddReqResult.NO_TOKEN
chunk_tokens_limit = min(self.rem_chunk_tokens, swa_cap)

loaded_back = 0
if req.needs_host_load_back():
new_indices, req.last_node = self.tree_cache.init_load_back(
InitLoadBackParams(
Expand All @@ -1097,6 +1110,12 @@ def add_one_req(
req.prefix_indices = torch.cat([req.prefix_indices, new_indices])
prefix_len = len(req.prefix_indices)
req.cache_protected_len = prefix_len
loaded_back = len(new_indices)

# L3 storage hits are the promoted portion of the load-back; the
# remainder came from L2 host DRAM.
l3_hit = min(req.storage_hit_length, loaded_back)
l2_hit = loaded_back - l3_hit

input_tokens = self.ceil_paged_tokens(
len(req.full_untruncated_fill_ids) - len(req.prefix_indices)
Expand All @@ -1122,6 +1141,12 @@ def add_one_req(

self._add_dllm_req(req, prefix_len)
self._req_inc_lock_ref(req)
self._accumulate_per_tier_hits(
l1_hit,
l2_hit,
l3_hit,
req.extend_range.end - req.extend_range.start,
)
elif chunk_tokens_limit is None or input_tokens <= chunk_tokens_limit:
# Non-chunked prefill — the whole sequence is committed this iter.
req.set_extend_range(
Expand All @@ -1140,6 +1165,12 @@ def add_one_req(
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
)
self._accumulate_per_tier_hits(
l1_hit,
l2_hit,
l3_hit,
req.extend_range.end - req.extend_range.start,
)
else:
# Make sure at least one page is available
trunc_len = chunk_tokens_limit // self.page_size * self.page_size
Expand Down Expand Up @@ -1181,6 +1212,7 @@ def add_one_req(
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
)
self._accumulate_per_tier_hits(l1_hit, l2_hit, l3_hit, trunc_len)

return self.budget_state()

Expand Down
13 changes: 13 additions & 0 deletions python/sglang/srt/managers/scheduler_components/metrics_reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ class PrefillStats:
reprocessed_log_input_tokens: int = 0
reprocessed_log_hit_tokens: int = 0
num_pending_tokens: int = 0
# Per-tier cache hit breakdown (raw token counts, no page alignment)
log_l1_hit_tokens: int = 0
log_l2_hit_tokens: int = 0
log_l3_hit_tokens: int = 0
log_miss_tokens: int = 0

@classmethod
def from_adder(
Expand All @@ -83,6 +88,10 @@ def from_adder(
),
num_new_seqs=len(adder.can_run_list),
num_pending_tokens=num_pending_tokens,
log_l1_hit_tokens=adder.log_l1_hit_tokens,
log_l2_hit_tokens=adder.log_l2_hit_tokens,
log_l3_hit_tokens=adder.log_l3_hit_tokens,
log_miss_tokens=adder.log_miss_tokens,
)


Expand Down Expand Up @@ -651,6 +660,10 @@ def report_prefill_stats(
)
self.stats.num_grammar_queue_reqs = len(self.scheduler.grammar_manager)
self.stats.cache_hit_rate = cache_hit_rate
self.stats.l1_hit_tokens = prefill_stats.log_l1_hit_tokens
self.stats.l2_hit_tokens = prefill_stats.log_l2_hit_tokens
self.stats.l3_hit_tokens = prefill_stats.log_l3_hit_tokens
self.stats.cache_miss_tokens = prefill_stats.log_miss_tokens

# Memory pool usage ratios / Absolute token counts
pool_stats.update_scheduler_stats(self.stats)
Expand Down
Loading
Loading