diff --git a/README.md b/README.md index 8984ea6fbd..f94a8ef18b 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ The default monitor port is `31993`. You can change it with Helm values such as HAMi also provides: - [HAMi-WebUI](https://github.com/Project-HAMi/HAMi-WebUI) for visual cluster and device management. -- Grafana dashboard examples for accelerator monitoring. +- [Grafana dashboards](dashboards/) for accelerator monitoring. - Benchmark material for evaluating workload behavior and scheduling effects. ![HAMi WebUI](imgs/hami-webui-overview.png) diff --git a/dashboards/README.md b/dashboards/README.md new file mode 100644 index 0000000000..ee4c5021d8 --- /dev/null +++ b/dashboards/README.md @@ -0,0 +1,76 @@ +# HAMi Grafana dashboards + +Importable Grafana dashboards for the metrics HAMi exposes through Prometheus. + +| File | Dashboard | Data source | +| --- | --- | --- | +| [`hami-vgpu-dashboard.json`](hami-vgpu-dashboard.json) | HAMi vGPU metrics | Prometheus | + +![HAMi vGPU metrics dashboard](../imgs/hami-vgpu-metrics-dashboard.png) + +## Prerequisites + +- A Prometheus instance scraping HAMi's components. HAMi exposes metrics from the + **scheduler** and the **vGPU monitor**; the Helm chart can create `ServiceMonitor` + objects for them when `prometheus.enabled=true` (see `charts/hami/values.yaml`). +- Grafana 9.x or newer with that Prometheus configured as a data source. + +## Import + +**From the Grafana UI** + +1. Go to **Dashboards → New → Import**. +2. Upload `hami-vgpu-dashboard.json` (or paste its contents). +3. When prompted, select your Prometheus data source and click **Import**. + +**From the API** + +```bash +curl -sS -X POST "$GRAFANA_URL/api/dashboards/db" \ + -H "Authorization: Bearer $GRAFANA_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"dashboard\": $(cat hami-vgpu-dashboard.json), \"overwrite\": true}" +``` + +The dashboard uses a templated data-source variable, so it is not tied to any +specific Prometheus UID — Grafana asks which data source to bind on import. + +## Variables + +| Variable | Meaning | +| --- | --- | +| `datasource` | The Prometheus data source to query. | +| `node` | Filter host/scheduler panels to one or more nodes (defaults to all). | +| `namespace` | Filter vGPU/container panels to one or more namespaces (defaults to all). | + +## Panels + +- **Cluster overview** — physical GPU count, total and allocated GPU memory, + cluster memory-allocated %, and shared-container count. +- **Physical GPUs (host)** — per-device memory used and utilization, as measured by + the vGPU monitor. +- **Scheduler / allocation** — allocated vs limit GPU memory, per-node memory and + core allocation ratios, and per-device shared count. +- **vGPU / container workloads** — per-container vGPU memory used vs limit, + container utilization, memory used as a % of limit, and a top-10 table. + +## Metrics used + +The dashboard is built on these HAMi metrics (the current `hami_*` names, exported +on the metrics port; the scheduler and the vGPU monitor each expose a subset): + +| Metric | Source | Notes | +| --- | --- | --- | +| `hami_gpu_memory_limit_bytes` | scheduler | Schedulable GPU memory per device. | +| `hami_gpu_memory_allocated_bytes` | scheduler | GPU memory allocated to pods. | +| `hami_gpu_core_allocated_ratio` | scheduler | Allocated compute cores (0-100). | +| `hami_gpu_shared_count` | scheduler | Containers sharing a device. | +| `hami_node_gpu_memory_allocated_ratio` | scheduler | Per-node memory allocated (0-100). | +| `hami_host_gpu_memory_used_bytes` | vGPU monitor | Real memory in use per device. | +| `hami_host_gpu_utilization_ratio` | vGPU monitor | Physical GPU utilization (0-100). | +| `hami_vgpu_memory_used_bytes` | vGPU monitor | Per-container vGPU memory used. | +| `hami_vgpu_memory_limit_bytes` | vGPU monitor | Per-container vGPU memory limit. | +| `hami_container_device_utilization_ratio` | vGPU monitor | Per-container utilization (0-100). | + +> Utilization and allocation-ratio metrics are reported on a 0-100 scale, so the +> percentage panels display them directly without rescaling. diff --git a/dashboards/hami-vgpu-dashboard.json b/dashboards/hami-vgpu-dashboard.json new file mode 100644 index 0000000000..f1e8336a40 --- /dev/null +++ b/dashboards/hami-vgpu-dashboard.json @@ -0,0 +1,842 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "GPU virtualization metrics exported by HAMi (scheduler + vGPU monitor). Cluster capacity and allocation, per-host physical GPU usage, and per-container vGPU usage.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "HAMi project", + "tooltip": "", + "type": "link", + "url": "https://github.com/Project-HAMi/HAMi" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "panels": [], + "title": "Cluster overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Number of physical GPUs known to the scheduler for the selected node(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "count(hami_gpu_memory_limit_bytes{node=~\"$node\"})", + "instant": true, + "legendFormat": "GPUs", + "refId": "A" + } + ], + "title": "Physical GPUs", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Total schedulable GPU memory across the selected node(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 4, "y": 1 }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(hami_gpu_memory_limit_bytes{node=~\"$node\"})", + "instant": true, + "legendFormat": "Total", + "refId": "A" + } + ], + "title": "Total GPU memory", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "GPU memory the scheduler has allocated to pods across the selected node(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 9, "y": 1 }, + "id": 3, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"})", + "instant": true, + "legendFormat": "Allocated", + "refId": "A" + } + ], + "title": "Allocated GPU memory", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Allocated GPU memory as a share of total schedulable GPU memory. Green below 75%, amber 75-90%, red above 90%.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 14, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "100 * sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"}) / sum(hami_gpu_memory_limit_bytes{node=~\"$node\"})", + "instant": true, + "legendFormat": "Allocated %", + "refId": "A" + } + ], + "title": "GPU memory allocated %", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Number of containers sharing GPUs across the selected node(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(hami_gpu_shared_count{node=~\"$node\"})", + "instant": true, + "legendFormat": "Shared", + "refId": "A" + } + ], + "title": "Shared containers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "panels": [], + "title": "Physical GPUs (host)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Real GPU memory in use on each physical device, as read by the vGPU monitor.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 10, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_host_gpu_memory_used_bytes and on (device_uuid) hami_gpu_memory_limit_bytes{node=~\"$node\"}", + "legendFormat": "{{device_type}} idx{{device_index}} ({{device_uuid}})", + "refId": "A" + } + ], + "title": "Host GPU memory used", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Physical GPU utilization per device (0-100%).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 11, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_host_gpu_utilization_ratio and on (device_uuid) hami_gpu_memory_limit_bytes{node=~\"$node\"}", + "legendFormat": "{{device_type}} idx{{device_index}} ({{device_uuid}})", + "refId": "A" + } + ], + "title": "Host GPU utilization", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "panels": [], + "title": "Scheduler / allocation", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Scheduler view of GPU memory: allocated versus the configured limit, summed per node.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "/limit/" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, + { "id": "custom.fillOpacity", "value": 0 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "id": 20, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (node) (hami_gpu_memory_allocated_bytes{node=~\"$node\"})", + "legendFormat": "allocated {{node}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (node) (hami_gpu_memory_limit_bytes{node=~\"$node\"})", + "legendFormat": "limit {{node}}", + "refId": "B" + } + ], + "title": "GPU memory: allocated vs limit", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Per-node GPU memory allocated as a share of capacity. Green below 75%, amber 75-90%, red above 90%.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "dashed" } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "id": 21, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_node_gpu_memory_allocated_ratio{node=~\"$node\"}", + "legendFormat": "{{node}} idx{{device_index}}", + "refId": "A" + } + ], + "title": "Node GPU memory allocated ratio", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Percentage of GPU compute cores the scheduler has allocated per device.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "id": 22, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_gpu_core_allocated_ratio{node=~\"$node\"}", + "legendFormat": "{{node}} idx{{device_index}} ({{device_uuid}})", + "refId": "A" + } + ], + "title": "GPU core allocated ratio", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Number of containers sharing each GPU device.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, + "id": 23, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_gpu_shared_count{node=~\"$node\"}", + "legendFormat": "{{node}} idx{{device_index}} ({{device_uuid}})", + "refId": "A" + } + ], + "title": "GPU shared count per device", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, + "id": 103, + "panels": [], + "title": "vGPU / container workloads", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Per-container vGPU memory used versus its limit, for pods in the selected namespace(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "/limit/" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, + { "id": "custom.fillOpacity", "value": 0 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "id": 30, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"}", + "legendFormat": "used {{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_vgpu_memory_limit_bytes{namespace=~\"$namespace\"}", + "legendFormat": "limit {{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", + "refId": "B" + } + ], + "title": "vGPU memory: used vs limit", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Per-container GPU compute utilization inside the vGPU (0-100%).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "mappings": [], + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, + "id": 31, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "hami_container_device_utilization_ratio{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", + "refId": "A" + } + ], + "title": "Container device utilization", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "vGPU memory used as a share of each container's limit. Green below 75%, amber 75-90%, red above 90%.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "dashed" } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "id": 32, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "100 * hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"} / hami_vgpu_memory_limit_bytes{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}/{{pod}}/{{container}} v{{vdevice_index}}", + "refId": "A" + } + ], + "title": "vGPU memory used % of limit", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Containers using the most vGPU memory right now, in the selected namespace(s).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, + { "id": "custom.width", "value": 160 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "displayName": "Value", "desc": true }] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "topk(10, hami_vgpu_memory_used_bytes{namespace=~\"$namespace\"})", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "refId": "A" + } + ], + "title": "Top 10 containers by vGPU memory", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true, "device_uuid": false }, + "indexByName": {}, + "renameByName": { + "container": "Container", + "device_uuid": "GPU UUID", + "namespace": "Namespace", + "pod": "Pod", + "vdevice_index": "vGPU", + "Value": "Used" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["hami", "gpu", "vgpu"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(hami_gpu_memory_limit_bytes, node)", + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(hami_gpu_memory_limit_bytes, node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(hami_vgpu_memory_used_bytes, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(hami_vgpu_memory_used_bytes, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "HAMi vGPU metrics", + "uid": "hami-vgpu-metrics", + "version": 1, + "weekStart": "" +}