diff --git a/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json b/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json index 4463ac503b02..e13b8a5595d4 100644 --- a/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json +++ b/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json @@ -918,6 +918,922 @@ ], "title": "Number Queued Requests", "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "Host-tier (L2) KV cache utilization (0-1). Only populated when HiCache is enabled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sglang:hicache_host_used_tokens{instance=~\"$instance\",model_name=~\"$model_name\"} / sglang:hicache_host_total_tokens{instance=~\"$instance\",model_name=~\"$model_name\"}", + "instant": false, + "legendFormat": "HiCache host \u4f7f\u7528\u7387", + "range": true, + "refId": "A" + } + ], + "title": "HiCache Host Utilization", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "Host-tier used / total capacity, in tokens. Only populated when HiCache is enabled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sglang:hicache_host_used_tokens{instance=~\"$instance\",model_name=~\"$model_name\"}", + "instant": false, + "legendFormat": "Host \u5df2\u7528 (tokens)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sglang:hicache_host_total_tokens{instance=~\"$instance\",model_name=~\"$model_name\"}", + "instant": false, + "legendFormat": "Host \u603b\u5bb9\u91cf (tokens)", + "range": true, + "refId": "B" + } + ], + "title": "HiCache Host Used / Total Capacity (tokens)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "L3 storage prefetch (L3->Host DRAM) and writeback (Host DRAM->L3) rates, tokens/s.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tokens/s", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:prefetched_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "\u9884\u53d6 L3\u2192Host", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:backuped_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "\u56de\u5199 Host\u2192L3", + "range": true, + "refId": "B" + } + ], + "title": "HiCache L3 Prefetch / Writeback Rate (Host<->Storage)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "L1 (GPU) KV cache eviction rate (dropped without HiCache, written to L2 with it) and load-back rate from L2 (Host) to GPU, tokens/s.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tokens/s", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:evicted_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "L1 \u6dd8\u6c70 (total)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:evicted_backuped_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "L1 \u6dd8\u6c70-backuped (\u6570\u636e\u7559L2)", + "range": true, + "refId": "C" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:evicted_regular_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "L1 \u6dd8\u6c70-regular (\u76f4\u63a5\u5220\u9664)", + "range": true, + "refId": "D" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(sglang:load_back_tokens_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "L2\u2192L1 \u52a0\u8f7d\u56de GPU", + "range": true, + "refId": "B" + } + ], + "title": "HiCache L1<->L2 Eviction / Load-back to GPU Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "GPU KV cache utilization (0-1). Higher utilization means HiCache is more likely to kick in.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (sglang:gpu_kv_cache_occupancy{instance=~\"$instance\",model_name=~\"$model_name\",engine_type=~\"decode|unified\"})", + "instant": false, + "legendFormat": "L1 \u5360\u7528\u7387 (locked+cached)", + "range": true, + "refId": "C" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg without (dp_rank, tp_rank, pp_rank, moe_ep_rank, instance, job) (sglang:token_usage{instance=~\"$instance\",model_name=~\"$model_name\",engine_type=~\"decode|unified\"})", + "instant": false, + "legendFormat": "L1 \u9501\u5b9a\u7387 (locked only)", + "range": true, + "refId": "A" + } + ], + "title": "GPU KV Cache Utilization", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "Distribution of L1->L2 PCIe write bandwidth (logical KV bytes / CUDA event duration), GB/s. Reflects actual GPU->Host transfer rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "GBs", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:eviction_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:eviction_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:eviction_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(sglang:eviction_bandwidth_gb_s_sum{instance=~\"$instance\"}[$__rate_interval])) / sum(rate(sglang:eviction_bandwidth_gb_s_count{instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D" + } + ], + "title": "HiCache L1->L2 PCIe Write Bandwidth (GB/s)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "Distribution of L2->L1 PCIe load bandwidth (logical KV bytes / CUDA event duration), GB/s. Reflects actual Host->GPU transfer rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "GBs", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:load_back_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:load_back_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:load_back_bandwidth_gb_s_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(sglang:load_back_bandwidth_gb_s_sum{instance=~\"$instance\"}[$__rate_interval])) / sum(rate(sglang:load_back_bandwidth_gb_s_count{instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D" + } + ], + "title": "HiCache L2->L1 PCIe Load Bandwidth (GB/s)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "description": "L3 storage read/write bandwidth. Write = Host->Storage (backup), Read = Storage->Host (prefetch).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(sglang:backuped_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Write (Host\u2192Storage)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(sglang:prefetched_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Read (Storage\u2192Host)", + "range": true, + "refId": "B" + } + ], + "title": "HiCache L3 Storage Read/Write Bandwidth", + "type": "timeseries" } ], "preload": false, diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 0f3234b3b358..9201b6cd6a8d 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -70,8 +70,14 @@ def make_timing_event_pair(): class LayerLoadingEvent: def __init__(self, num_layers: int): self._num_layers = num_layers - self.load_events = [device_module.Event() for _ in range(num_layers)] - self.start_event = device_module.Event() # start event on controller stream + # enable_timing=True: needed for start_event.elapsed_time(finish_event) + # to derive load-back PCIe bandwidth (sglang:load_back_bandwidth_gb_s). + self.load_events = [ + device_module.Event(enable_timing=True) for _ in range(num_layers) + ] + self.start_event = device_module.Event( + enable_timing=True + ) # start event on controller stream def complete(self, layer_index: int): assert 0 <= layer_index < self._num_layers @@ -161,6 +167,9 @@ class HiCacheAck(NamedTuple): start_event: device_module.Event finish_event: device_module.Event node_ids: List[int] + # Tokens moved by this batch; paired with start_event.elapsed_time(finish_event) + # to derive PCIe bandwidth (sglang:eviction_bandwidth_gb_s / load_back_bandwidth_gb_s) + # when timing_enabled. num_tokens: int = 0 timing_enabled: bool = False @@ -713,8 +722,9 @@ def start_writing(self) -> None: ) self.write_queue.clear() - start_event = device_module.Event() - finish_event = device_module.Event() + # timing_enabled gates elapsed_time() use downstream: derives eviction + # (L1->L2) PCIe bandwidth (sglang:eviction_bandwidth_gb_s) where supported. + start_event, finish_event, timing_enabled = make_timing_event_pair() start_event.record() with device_module.stream(self.write_stream): @@ -738,7 +748,15 @@ def start_writing(self) -> None: if device_indices.is_cuda: device_indices.record_stream(self.write_stream) - self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids)) + self.ack_write_queue.append( + HiCacheAck( + start_event, + finish_event, + op.node_ids, + num_tokens=len(device_indices), + timing_enabled=timing_enabled, + ) + ) def load( self, @@ -828,7 +846,7 @@ def start_loading(self) -> int: start_event=ack_start_event, finish_event=ack_finish_event, node_ids=op.node_ids, - num_tokens=len(op.device_indices), + num_tokens=len(device_indices), timing_enabled=timing_enabled, ) ) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 392802451966..7f3ae7fd8a12 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -664,6 +664,10 @@ def _drain_backup(): self.storage_metrics_collector.log_backuped_tokens( operation.completed_tokens ) + self.storage_metrics_collector.log_backuped_bytes( + operation.completed_tokens + * self.cache_controller.mem_pool_host.get_size_per_token() + ) def _drain_release(): host_indices_list = [] @@ -978,12 +982,30 @@ def _inc_hit_count(self, node: TreeNode, chunked=False): # write to host if the node is not backuped self.write_backup(node) + def _report_transfer_bandwidth(self, ack, is_eviction: bool) -> None: + # PCIe bandwidth per ack'd batch, via CUDA event timing bracketing the + # actual D2H (write-back) / H2D (load-back) copy -- see start_writing()/ + # start_loading() in cache_controller.py. timing_enabled is False on + # backends where the CUDA/HIP event timing API is unavailable. + if self.metrics_collector is None or ack.num_tokens <= 0 or not ack.timing_enabled: + return + elapsed_ms = ack.start_event.elapsed_time(ack.finish_event) + if elapsed_ms <= 0: + return + bytes_per_token = self.cache_controller.mem_pool_host.get_size_per_token() + gb_per_s = (ack.num_tokens * bytes_per_token / 1e9) / (elapsed_ms / 1000.0) + if is_eviction: + self.metrics_collector.observe_eviction_bandwidth_gb_s(gb_per_s) + else: + self.metrics_collector.observe_load_back_bandwidth_gb_s(gb_per_s) + def writing_check(self, write_back=False): if write_back: # blocking till all write back complete while len(self.ongoing_write_through) > 0: for ack in self.cache_controller.ack_write_queue: ack.finish_event.synchronize() + self._report_transfer_bandwidth(ack, is_eviction=True) for ack_id in ack.node_ids: self._finish_write_through_ack(ack_id, release_lock=False) self.cache_controller.ack_write_queue.clear() @@ -1009,6 +1031,7 @@ def writing_check(self, write_back=False): while finish_count > 0: ack = self.cache_controller.ack_write_queue.pop(0) ack.finish_event.synchronize() + self._report_transfer_bandwidth(ack, is_eviction=True) for ack_id in ack.node_ids: self._finish_write_through_ack(ack_id, release_lock=True) finish_count -= 1 @@ -1029,6 +1052,7 @@ def loading_check(self): while finish_count > 0: ack = self.cache_controller.ack_load_queue.pop(0) ack.finish_event.synchronize() + self._report_transfer_bandwidth(ack, is_eviction=False) for ack_id in ack.node_ids: end_node = self.ongoing_load_back.pop(ack_id) self.dec_lock_ref(end_node) @@ -1116,10 +1140,21 @@ def evict(self, params: EvictParams) -> EvictResult: start_time = time.perf_counter() num_tokens = params.num_tokens if self.cache_controller.write_policy == "write_back": - num_evicted = self._evict_write_back(num_tokens) + num_evicted, num_backuped_evicted = self._evict_write_back(num_tokens) else: - num_evicted = self._evict_write_through(num_tokens) + num_evicted, num_backuped_evicted = self._evict_write_through(num_tokens) self.update_eviction_metrics(num_evicted, start_time) + if self.metrics_collector is not None and num_evicted > 0: + # Split of evicted_tokens_total: "backuped" nodes were already on + # host (cheap detach, no data loss); "regular" nodes are dropped + # without a host copy (write-through under pressure, or plain + # write_through mode with no host tier at all). + self.metrics_collector.increment_evicted_backuped_tokens( + num_backuped_evicted + ) + self.metrics_collector.increment_evicted_regular_tokens( + num_evicted - num_backuped_evicted + ) return EvictResult(num_tokens_evicted=num_evicted) def _make_eviction_heap(self): @@ -1136,30 +1171,38 @@ def _promote_parent(self, node: TreeNode, heap) -> None: if p is not self.root_node and all(c.evicted for c in p.children.values()): heapq.heappush(heap, (self.eviction_strategy.get_priority(p), p)) - def _evict_write_through(self, num_tokens: int) -> int: + def _evict_write_through(self, num_tokens: int) -> Tuple[int, int]: """write_through / write_through_selective: drop non-backuped leaves, demote already-backuped ones. Nothing is staged to host during eviction, so this is a plain on-the-fly pass. + + Returns (num_evicted, num_backuped_evicted). """ heap = self._make_eviction_heap() num_evicted = 0 + num_backuped_evicted = 0 while num_evicted < num_tokens and heap: _, x = heapq.heappop(heap) if x.lock_ref > 0: continue if x.backuped: - num_evicted += self._evict_backuped(x) + n = self._evict_backuped(x) + num_evicted += n + num_backuped_evicted += n else: num_evicted += self._evict_regular(x) self._promote_parent(x, heap) - return num_evicted + return num_evicted, num_backuped_evicted - def _evict_write_back(self, num_tokens: int) -> int: + def _evict_write_back(self, num_tokens: int) -> Tuple[int, int]: """eviction for write_back mode: demote already-backuped leaves, stage non-backuped ones to host if possible, otherwise drop them. note this path will be deprecated in the future. + + Returns (num_evicted, num_backuped_evicted). """ heap = self._make_eviction_heap() num_evicted = 0 + num_backuped_evicted = 0 staged: List[Tuple[TreeNode, torch.Tensor]] = [] def flush_staged() -> None: @@ -1176,17 +1219,21 @@ def flush_staged() -> None: if x.lock_ref > 0: continue if x.backuped: - num_evicted += self._evict_backuped(x) + n = self._evict_backuped(x) + num_evicted += n + num_backuped_evicted += n elif self.write_backup(x, write_back=True) > 0: x.protect_host() staged.append((x, x.value)) - num_evicted += self._detach_backuped(x) + n = self._detach_backuped(x) + num_evicted += n + num_backuped_evicted += n else: flush_staged() num_evicted += self._drop_subtree_no_host(x) self._promote_parent(x, heap) flush_staged() - return num_evicted + return num_evicted, num_backuped_evicted def _detach_backuped(self, node: TreeNode) -> int: # detach nodes from tree while keeping device slots, for write-back eviction @@ -1592,6 +1639,10 @@ def check_prefetch_progress(self, req_id: str) -> bool: if self.enable_storage_metrics: self.storage_metrics_collector.log_prefetched_tokens(loaded_from_storage) + self.storage_metrics_collector.log_prefetched_bytes( + loaded_from_storage + * self.cache_controller.mem_pool_host.get_size_per_token() + ) return True diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index d6c20b821c4d..1bdcb1eeb12c 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -411,8 +411,7 @@ def start_writing(self) -> None: self.move_hybrid_indices(op) ) self.write_queue.clear() - start_event = device_module.Event() - finish_event = device_module.Event() + start_event, finish_event, timing_enabled = make_timing_event_pair() start_event.record() with device_module.stream(self.write_stream): start_event.wait(self.write_stream) @@ -437,7 +436,15 @@ def start_writing(self) -> None: device_indices, resolved_pool_transfers, ) - self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids)) + self.ack_write_queue.append( + HiCacheAck( + start_event, + finish_event, + op.node_ids, + num_tokens=len(device_indices), + timing_enabled=timing_enabled, + ) + ) def load( self, @@ -533,7 +540,7 @@ def start_loading(self) -> int: ack_start_event, ack_finish_event, op.node_ids, - num_tokens=len(op.device_indices), + num_tokens=len(device_indices), timing_enabled=timing_enabled, ) ) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 2e6759bb9b17..bb6592ce4ccb 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -312,6 +312,12 @@ def __init__( labelnames=labels.keys(), multiprocess_mode="mostrecent", ) + self.gpu_kv_cache_occupancy = Gauge( + name="sglang:gpu_kv_cache_occupancy", + documentation="GPU KV cache occupancy (0-1). Alias of token_usage kept for HiCache dashboards.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) self.full_token_usage = Gauge( name="sglang:full_token_usage", documentation="The token usage for full attention layers.", @@ -1282,6 +1288,7 @@ def log_stats(self, stats: SchedulerStats) -> None: # Memory pool usage ratios self._log_gauge(self.token_usage, stats.token_usage) + self._log_gauge(self.gpu_kv_cache_occupancy, stats.token_usage) self._log_gauge(self.full_token_usage, stats.full_token_usage) self._log_gauge(self.swa_token_usage, stats.swa_token_usage) self._log_gauge(self.mamba_usage, stats.mamba_usage) @@ -1769,6 +1776,18 @@ def __init__( labelnames=labels.keys(), ) + self.prefetched_bytes_total = Counter( + name="sglang:prefetched_bytes_total", + documentation="Number of bytes prefetched from L3 storage to host.", + labelnames=labels.keys(), + ) + + self.backuped_bytes_total = Counter( + name="sglang:backuped_bytes_total", + documentation="Number of bytes backed up from host to L3 storage.", + labelnames=labels.keys(), + ) + bucket_io = [ 1, 5, @@ -1823,6 +1842,14 @@ def log_backuped_tokens(self, backuped_tokens: int): if backuped_tokens > 0: self.backuped_tokens_total.labels(**self.labels).inc(backuped_tokens) + def log_prefetched_bytes(self, num_bytes: int): + if num_bytes > 0: + self.prefetched_bytes_total.labels(**self.labels).inc(num_bytes) + + def log_backuped_bytes(self, num_bytes: int): + if num_bytes > 0: + self.backuped_bytes_total.labels(**self.labels).inc(num_bytes) + def _log_histogram(self, histogram, data: Union[int, float]): histogram.labels(**self.labels).observe(data) @@ -1932,6 +1959,21 @@ def __init__( labelnames=labels.keys(), ) + # Split of evicted_tokens_total by whether the node had already been + # backed up to host (cheap detach) vs dropped without a host copy + # (write-through under host memory pressure, or write_through mode + # without a host tier at all). + self.evicted_backuped_tokens_total = Counter( + name="sglang:evicted_backuped_tokens_total", + documentation="Number of evicted GPU tokens that had already been backed up to host (cheap detach, no data loss).", + labelnames=labels.keys(), + ) + self.evicted_regular_tokens_total = Counter( + name="sglang:evicted_regular_tokens_total", + documentation="Number of evicted GPU tokens dropped without a host backup (data is gone, must be recomputed).", + labelnames=labels.keys(), + ) + self.load_back_duration_seconds = Histogram( name="sglang:load_back_duration_seconds", documentation="Time taken to load KV cache from CPU back to GPU in seconds.", @@ -1945,9 +1987,36 @@ def __init__( labelnames=labels.keys(), ) + bucket_bandwidth_gb_s = [0.1, 0.5, 1, 5, 10, 20, 40, 80, 160, 320] + + # PCIe bandwidth, measured via CUDA start/finish events bracketing the + # actual D2H (eviction/write-back) and H2D (load-back) copy on the + # dedicated write/load streams -- not wall-clock time around the + # (often async) Python call. + self.eviction_bandwidth_gb_s = Histogram( + name="sglang:eviction_bandwidth_gb_s", + documentation="L1(GPU)->L2(Host) PCIe write bandwidth in GB/s, measured per write-back batch via CUDA events.", + labelnames=labels.keys(), + buckets=bucket_bandwidth_gb_s, + ) + self.load_back_bandwidth_gb_s = Histogram( + name="sglang:load_back_bandwidth_gb_s", + documentation="L2(Host)->L1(GPU) PCIe load bandwidth in GB/s, measured per load-back batch via CUDA events.", + labelnames=labels.keys(), + buckets=bucket_bandwidth_gb_s, + ) + def increment_eviction_num_tokens(self, num_tokens: int) -> None: self.eviction_num_tokens.labels(**self.labels).inc(num_tokens) + def increment_evicted_backuped_tokens(self, num_tokens: int) -> None: + if num_tokens > 0: + self.evicted_backuped_tokens_total.labels(**self.labels).inc(num_tokens) + + def increment_evicted_regular_tokens(self, num_tokens: int) -> None: + if num_tokens > 0: + self.evicted_regular_tokens_total.labels(**self.labels).inc(num_tokens) + def increment_load_back_num_tokens(self, num_tokens: int) -> None: self.load_back_num_tokens.labels(**self.labels).inc(num_tokens) @@ -1957,6 +2026,12 @@ def observe_eviction_duration(self, duration_seconds: float) -> None: def observe_load_back_duration(self, duration_seconds: float) -> None: self.load_back_duration_seconds.labels(**self.labels).observe(duration_seconds) + def observe_eviction_bandwidth_gb_s(self, gb_per_s: float) -> None: + self.eviction_bandwidth_gb_s.labels(**self.labels).observe(gb_per_s) + + def observe_load_back_bandwidth_gb_s(self, gb_per_s: float) -> None: + self.load_back_bandwidth_gb_s.labels(**self.labels).observe(gb_per_s) + class EncoderMetricsCollector(_StatLoggerDIMixin): """Metrics collector for the EPD encoder server (--encoder-only)."""