From b85f48752b8fc0a1fc3bf2382cd3edb9499f51aa Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 23 Jun 2021 15:19:08 +0900 Subject: [PATCH 01/25] Add minimum required dashboards for RED method --- manifests/pipecd/grafana-dashboards/main.json | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 manifests/pipecd/grafana-dashboards/main.json diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json new file mode 100644 index 0000000000..cb6477f53c --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -0,0 +1,327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"2.*\", job=\"pipecd-gateway\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\"}[5m]))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Success Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:78", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:79", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total[5m]))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:60", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:61", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(instance,job) (rate(envoy_cluster_upstream_rq_time_sum[5m]))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Duration Rate (ms)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:732", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:733", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Main", + "uid": "qubEFcXMk", + "version": 1 +} From 98789bc79d4c8039d0b419e686e06d43bd39b80d Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 23 Jun 2021 22:05:57 +0900 Subject: [PATCH 02/25] Use quantile for duration --- manifests/pipecd/grafana-dashboards/main.json | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json index cb6477f53c..8967a605e2 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -15,6 +15,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, + "id": 2, "links": [], "panels": [ { @@ -91,7 +92,6 @@ }, "yaxes": [ { - "$$hashKey": "object:78", "format": "short", "label": null, "logBase": 1, @@ -100,7 +100,6 @@ "show": true }, { - "$$hashKey": "object:79", "format": "short", "label": null, "logBase": 1, @@ -188,7 +187,6 @@ }, "yaxes": [ { - "$$hashKey": "object:60", "format": "short", "label": null, "logBase": 1, @@ -197,7 +195,6 @@ "show": true }, { - "$$hashKey": "object:61", "format": "short", "label": null, "logBase": 1, @@ -258,7 +255,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(instance,job) (rate(envoy_cluster_upstream_rq_time_sum[5m]))", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket[5m]))\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -269,7 +266,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API Duration Rate (ms)", + "title": "API Duration 99th Quantile", "tooltip": { "shared": true, "sort": 0, @@ -285,7 +282,6 @@ }, "yaxes": [ { - "$$hashKey": "object:732", "format": "short", "label": null, "logBase": 1, @@ -294,7 +290,6 @@ "show": true }, { - "$$hashKey": "object:733", "format": "short", "label": null, "logBase": 1, @@ -322,6 +317,6 @@ "timepicker": {}, "timezone": "", "title": "Main", - "uid": "qubEFcXMk", - "version": 1 + "uid": "3jeba3R7z", + "version": 3 } From 2549cff25b53627a5fc32d8b87701d92b2b446c6 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Mon, 28 Jun 2021 18:30:06 +0900 Subject: [PATCH 03/25] Use variables --- manifests/pipecd/grafana-dashboards/main.json | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json index 8967a605e2..23c02cc6a7 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -12,10 +12,12 @@ } ] }, + "description": "", "editable": true, "gnetId": null, "graphTooltip": 0, "id": 2, + "iteration": 1624872441420, "links": [], "panels": [ { @@ -65,7 +67,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"2.*\", job=\"pipecd-gateway\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\"}[5m]))", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"2.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -160,7 +162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total[5m]))", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=~\"$api\"}[5m]))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -255,7 +257,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket[5m]))\n)", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -308,7 +310,39 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "server-piped-api", + "value": "server-piped-api" + }, + "datasource": "Prometheus", + "definition": "label_values(envoy_cluster_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "api", + "options": [], + "query": { + "query": "label_values(envoy_cluster_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] }, "time": { "from": "now-6h", @@ -318,5 +352,5 @@ "timezone": "", "title": "Main", "uid": "3jeba3R7z", - "version": 3 + "version": 1 } From de3d9d9460c5e6596661e0cda1647a97d640b664 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Mon, 28 Jun 2021 19:35:46 +0900 Subject: [PATCH 04/25] Update y-axis --- manifests/pipecd/grafana-dashboards/main.json | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json index 23c02cc6a7..bc9078b8e2 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1624872441420, + "iteration": 1624876467533, "links": [], "panels": [ { @@ -67,7 +67,9 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"2.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n* 100", + "format": "time_series", + "instant": false, "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -78,7 +80,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API Success Rate", + "title": "API - 5xx Error Percentage", "tooltip": { "shared": true, "sort": 0, @@ -94,8 +96,8 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "format": "percent", + "label": "", "logBase": 1, "max": null, "min": null, @@ -103,7 +105,7 @@ }, { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, @@ -173,7 +175,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API Request Rate", + "title": "API - Request Rate", "tooltip": { "shared": true, "sort": 0, @@ -268,7 +270,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API Duration 99th Quantile", + "title": "API - Duration 99th Quantile", "tooltip": { "shared": true, "sort": 0, @@ -284,7 +286,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -314,7 +316,7 @@ { "allValue": null, "current": { - "selected": true, + "selected": false, "text": "server-piped-api", "value": "server-piped-api" }, From 37a4d561bd47f5f89f7aa7f5f9b18bf8d1f85ea1 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 30 Jun 2021 10:34:35 +0900 Subject: [PATCH 05/25] Add pod resources metrics --- manifests/pipecd/grafana-dashboards/main.json | 220 +++++++++++++++++- 1 file changed, 211 insertions(+), 9 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json index bc9078b8e2..b5903817d9 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1624876467533, + "iteration": 1625016800293, "links": [], "panels": [ { @@ -34,7 +34,7 @@ "fillGradient": 0, "gridPos": { "h": 9, - "w": 12, + "w": 24, "x": 0, "y": 0 }, @@ -117,6 +117,113 @@ "alignLevel": null } }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 10, + "title": "Row title", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API - Duration 99th Quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -133,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 10 }, "hiddenSeries": false, "id": 4, @@ -228,10 +335,10 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 18 }, "hiddenSeries": false, - "id": 6, + "id": 8, "legend": { "avg": false, "current": false, @@ -259,7 +366,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", + "expr": "sum by(instance) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -270,7 +377,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API - Duration 99th Quantile", + "title": "Pod - CPU Usage Rate", "tooltip": { "shared": true, "sort": 0, @@ -286,7 +393,102 @@ }, "yaxes": [ { - "format": "ms", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(instance) (\n container_memory_usage_bytes{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }\n)", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod - Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", "label": null, "logBase": 1, "max": null, @@ -354,5 +556,5 @@ "timezone": "", "title": "Main", "uid": "3jeba3R7z", - "version": 1 + "version": 12 } From 9fcd599ee20a9852c605a7de7cf3112e26947772 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 30 Jun 2021 16:38:02 +0900 Subject: [PATCH 06/25] Grouping --- manifests/pipecd/grafana-dashboards/main.json | 249 ++++++++++++++++-- 1 file changed, 228 insertions(+), 21 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/main.json index b5903817d9..e185981d93 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/main.json @@ -17,9 +17,23 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625016800293, + "iteration": 1625035674641, "links": [], "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Service Metrics", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -36,7 +50,7 @@ "h": 9, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "hiddenSeries": false, "id": 2, @@ -117,18 +131,6 @@ "alignLevel": null } }, - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 10, - "title": "Row title", - "type": "row" - }, { "aliasColors": {}, "bars": false, @@ -319,6 +321,20 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 14, + "panels": [], + "title": "Resource Metrics", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -335,7 +351,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 19 }, "hiddenSeries": false, "id": 8, @@ -366,7 +382,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(instance) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", + "expr": "sum by(pod) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -430,7 +446,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 19 }, "hiddenSeries": false, "id": 12, @@ -461,7 +477,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(instance) (\n container_memory_usage_bytes{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }\n)", + "expr": "sum by(pod) (\n container_memory_usage_bytes{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -508,8 +524,199 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total[5m]))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node - CPU Usage Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(kubernetes_node) (node_memory_MemAvailable_bytes)", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node - Memory Available Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], + "refresh": "10s", "schemaVersion": 27, "style": "dark", "tags": [], @@ -519,8 +726,8 @@ "allValue": null, "current": { "selected": false, - "text": "server-piped-api", - "value": "server-piped-api" + "text": "server-api", + "value": "server-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -556,5 +763,5 @@ "timezone": "", "title": "Main", "uid": "3jeba3R7z", - "version": 12 + "version": 16 } From e3c68ae06fe2db1e03f71c153d76b6c18290309c Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 30 Jun 2021 17:18:30 +0900 Subject: [PATCH 07/25] Devide into two dashboards --- .../{main.json => resource-metrics.json} | 368 +----------------- .../grafana-dashboards/service-metrics.json | 359 +++++++++++++++++ 2 files changed, 369 insertions(+), 358 deletions(-) rename manifests/pipecd/grafana-dashboards/{main.json => resource-metrics.json} (51%) create mode 100644 manifests/pipecd/grafana-dashboards/service-metrics.json diff --git a/manifests/pipecd/grafana-dashboards/main.json b/manifests/pipecd/grafana-dashboards/resource-metrics.json similarity index 51% rename from manifests/pipecd/grafana-dashboards/main.json rename to manifests/pipecd/grafana-dashboards/resource-metrics.json index e185981d93..1f433e9d39 100644 --- a/manifests/pipecd/grafana-dashboards/main.json +++ b/manifests/pipecd/grafana-dashboards/resource-metrics.json @@ -16,121 +16,9 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 2, - "iteration": 1625035674641, + "id": 3, "links": [], "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 16, - "panels": [], - "title": "Service Metrics", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n* 100", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "API - 5xx Error Percentage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "aliasColors": {}, "bars": false, @@ -147,211 +35,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "API - Duration 99th Quantile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=~\"$api\"}[5m]))", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "API - Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 14, - "panels": [], - "title": "Resource Metrics", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 + "y": 0 }, "hiddenSeries": false, "id": 8, @@ -382,7 +66,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(pod) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", + "expr": "sum by(instance) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -446,7 +130,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 0 }, "hiddenSeries": false, "id": 12, @@ -541,7 +225,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 8 }, "hiddenSeries": false, "id": 18, @@ -636,7 +320,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 8 }, "hiddenSeries": false, "id": 20, @@ -721,39 +405,7 @@ "style": "dark", "tags": [], "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "server-api", - "value": "server-api" - }, - "datasource": "Prometheus", - "definition": "label_values(envoy_cluster_name)", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "api", - "options": [], - "query": { - "query": "label_values(envoy_cluster_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] + "list": [] }, "time": { "from": "now-6h", @@ -761,7 +413,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Main", - "uid": "3jeba3R7z", - "version": 16 + "title": "Resource Metrics", + "uid": "4MJR80z7z", + "version": 2 } diff --git a/manifests/pipecd/grafana-dashboards/service-metrics.json b/manifests/pipecd/grafana-dashboards/service-metrics.json new file mode 100644 index 0000000000..af8a3a8bbc --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/service-metrics.json @@ -0,0 +1,359 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "iteration": 1625040982195, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n* 100", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API - 5xx Error Percentage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API - Duration 99th Quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=~\"$api\"}[5m]))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API - Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "server-piped-api", + "value": "server-piped-api" + }, + "datasource": "Prometheus", + "definition": "label_values(envoy_cluster_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "api", + "options": [], + "query": { + "query": "label_values(envoy_cluster_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service Metrics", + "uid": "3jeba3R7z", + "version": 1 +} From 3c8c9789b9c73e3de811443dc84674aa678709cf Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 11:01:32 +0900 Subject: [PATCH 08/25] Update operator --- .../pipecd/grafana-dashboards/service-metrics.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/service-metrics.json b/manifests/pipecd/grafana-dashboards/service-metrics.json index af8a3a8bbc..305a197594 100644 --- a/manifests/pipecd/grafana-dashboards/service-metrics.json +++ b/manifests/pipecd/grafana-dashboards/service-metrics.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625040982195, + "iteration": 1625104834336, "links": [], "panels": [ { @@ -67,7 +67,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=~\"$api\"}[5m]))\n* 100", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100", "format": "time_series", "instant": false, "interval": "", @@ -164,7 +164,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -259,7 +259,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=~\"$api\"}[5m]))", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=\"$api\"}[5m]))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -318,8 +318,8 @@ "allValue": null, "current": { "selected": false, - "text": "server-piped-api", - "value": "server-piped-api" + "text": "server-api", + "value": "server-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -355,5 +355,5 @@ "timezone": "", "title": "Service Metrics", "uid": "3jeba3R7z", - "version": 1 + "version": 20 } From 9dc77a33314ded5e925ff91939538c38a5adbbba Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 11:27:41 +0900 Subject: [PATCH 09/25] Add variable for service --- .../grafana-dashboards/resource-metrics.json | 107 +++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/resource-metrics.json b/manifests/pipecd/grafana-dashboards/resource-metrics.json index 1f433e9d39..8a2b422abf 100644 --- a/manifests/pipecd/grafana-dashboards/resource-metrics.json +++ b/manifests/pipecd/grafana-dashboards/resource-metrics.json @@ -17,6 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 3, + "iteration": 1625106265192, "links": [], "panels": [ { @@ -66,7 +67,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(instance) (\n rate(container_cpu_usage_seconds_total{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }[5m])\n)", + "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m])\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -161,7 +162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(pod) (\n container_memory_usage_bytes{\n pod=~\"pipecd-server-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)\"\n }\n)", + "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n }\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -405,7 +406,105 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "pipecd-server", + "value": "pipecd-server" + }, + "datasource": "Prometheus", + "definition": "label_values(service)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "service", + "options": [ + { + "selected": false, + "text": "pipecd", + "value": "pipecd" + }, + { + "selected": true, + "text": "pipecd-cache", + "value": "pipecd-cache" + }, + { + "selected": false, + "text": "pipecd-gateway", + "value": "pipecd-gateway" + }, + { + "selected": false, + "text": "pipecd-grafana", + "value": "pipecd-grafana" + }, + { + "selected": false, + "text": "pipecd-kube-state-metrics", + "value": "pipecd-kube-state-metrics" + }, + { + "selected": false, + "text": "pipecd-minio", + "value": "pipecd-minio" + }, + { + "selected": false, + "text": "pipecd-mysql", + "value": "pipecd-mysql" + }, + { + "selected": false, + "text": "pipecd-ops", + "value": "pipecd-ops" + }, + { + "selected": false, + "text": "pipecd-prometheus-alertmanager", + "value": "pipecd-prometheus-alertmanager" + }, + { + "selected": false, + "text": "pipecd-prometheus-node", + "value": "pipecd-prometheus-node" + }, + { + "selected": false, + "text": "pipecd-prometheus-node-exporter", + "value": "pipecd-prometheus-node-exporter" + }, + { + "selected": false, + "text": "pipecd-prometheus-server", + "value": "pipecd-prometheus-server" + }, + { + "selected": false, + "text": "pipecd-server", + "value": "pipecd-server" + } + ], + "query": { + "query": "label_values(service)", + "refId": "StandardVariableQuery" + }, + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] }, "time": { "from": "now-6h", @@ -415,5 +514,5 @@ "timezone": "", "title": "Resource Metrics", "uid": "4MJR80z7z", - "version": 2 + "version": 5 } From d83884a247d25ac5892664892697fd9e4038c262 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 12:13:28 +0900 Subject: [PATCH 10/25] Add Go metrics --- .../grafana-dashboards/resource-metrics.json | 204 +++++++++++++++++- 1 file changed, 197 insertions(+), 7 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/resource-metrics.json b/manifests/pipecd/grafana-dashboards/resource-metrics.json index 8a2b422abf..fe5de88e7f 100644 --- a/manifests/pipecd/grafana-dashboards/resource-metrics.json +++ b/manifests/pipecd/grafana-dashboards/resource-metrics.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 3, - "iteration": 1625106265192, + "iteration": 1625109120162, "links": [], "panels": [ { @@ -78,7 +78,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pod - CPU Usage Rate", + "title": "CPU Usage Rate per Pod", "tooltip": { "shared": true, "sort": 0, @@ -173,7 +173,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pod - Memory Usage", + "title": "Memory Usage per Pod", "tooltip": { "shared": true, "sort": 0, @@ -268,7 +268,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Node - CPU Usage Rate", + "title": "CPU Usage Rate per Node", "tooltip": { "shared": true, "sort": 0, @@ -363,7 +363,197 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Node - Memory Available Bytes", + "title": "Memory Available Bytes per Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Allocated Bytes per Service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (go_goroutines{job=\"$service\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Goroutines per Service", "tooltip": { "shared": true, "sort": 0, @@ -512,7 +702,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Resource Metrics", + "title": "PipeCD Resource Metrics", "uid": "4MJR80z7z", - "version": 5 + "version": 7 } From d6c62ee3435611ead4f4b4d52830dbe36707a757 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 12:14:32 +0900 Subject: [PATCH 11/25] Rename dashboard --- .../pipecd/grafana-dashboards/service-metrics.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/service-metrics.json b/manifests/pipecd/grafana-dashboards/service-metrics.json index 305a197594..8e5c788294 100644 --- a/manifests/pipecd/grafana-dashboards/service-metrics.json +++ b/manifests/pipecd/grafana-dashboards/service-metrics.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625104834336, + "iteration": 1625109245456, "links": [], "panels": [ { @@ -318,8 +318,8 @@ "allValue": null, "current": { "selected": false, - "text": "server-api", - "value": "server-api" + "text": "server-piped-api", + "value": "server-piped-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -353,7 +353,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Service Metrics", + "title": "PipeCD Service Metrics", "uid": "3jeba3R7z", - "version": 20 + "version": 22 } From 355553f68d2f759cb2c7c33979854ee4749fefbe Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 14:41:58 +0900 Subject: [PATCH 12/25] Update unit --- .../grafana-dashboards/resource-metrics.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/resource-metrics.json b/manifests/pipecd/grafana-dashboards/resource-metrics.json index fe5de88e7f..4e99111859 100644 --- a/manifests/pipecd/grafana-dashboards/resource-metrics.json +++ b/manifests/pipecd/grafana-dashboards/resource-metrics.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 3, - "iteration": 1625109120162, + "iteration": 1625118048403, "links": [], "panels": [ { @@ -162,7 +162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n }\n)", + "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n } / (1024 * 1024)\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -189,7 +189,7 @@ }, "yaxes": [ { - "format": "short", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, @@ -257,7 +257,7 @@ "targets": [ { "exemplar": true, - "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total[5m]))", + "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m]))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -352,7 +352,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(kubernetes_node) (node_memory_MemAvailable_bytes)", + "expr": "sum by(kubernetes_node) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / (1024 * 1024 * 1024)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -363,7 +363,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Available Bytes per Node", + "title": "Memory Usage per Node", "tooltip": { "shared": true, "sort": 0, @@ -379,7 +379,7 @@ }, "yaxes": [ { - "format": "short", + "format": "decgbytes", "label": null, "logBase": 1, "max": null, @@ -447,7 +447,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"})", + "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"} / (1024 * 1024))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -474,7 +474,7 @@ }, "yaxes": [ { - "format": "short", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, @@ -704,5 +704,5 @@ "timezone": "", "title": "PipeCD Resource Metrics", "uid": "4MJR80z7z", - "version": 7 + "version": 8 } From 9c96de35f556d865c18a331ae0f72ca7f3f2a986 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 17:17:58 +0900 Subject: [PATCH 13/25] Add prometheus dashboard --- .../grafana-dashboards/prometheus-stats.json | 591 ++++++++++++++++++ ...ource-metrics.json => resource-stats.json} | 0 ...ervice-metrics.json => service-stats.json} | 0 3 files changed, 591 insertions(+) create mode 100644 manifests/pipecd/grafana-dashboards/prometheus-stats.json rename manifests/pipecd/grafana-dashboards/{resource-metrics.json => resource-stats.json} (100%) rename manifests/pipecd/grafana-dashboards/{service-metrics.json => service-stats.json} (100%) diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/prometheus-stats.json new file mode 100644 index 0000000000..0e42a257ad --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/prometheus-stats.json @@ -0,0 +1,591 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "prometheus_tsdb_head_series", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Time Series", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 16, + "title": "HTTP Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(prometheus_http_request_duration_seconds_bucket{handler=\"/graph\"}[5m]))\n)", + "interval": "", + "legendFormat": "graph", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "query", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query_range\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "query_range", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "99th Quantile of HTTP Duration [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "format": "time_series", + "interval": "", + "legendFormat": "localhost", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingested samples [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 14, + "title": "Go Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_alloc_bytes{job=\"prometheus\"} / (1024 * 1024)", + "interval": "", + "legendFormat": "prometheus", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocated Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decmbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_goroutines{job=\"prometheus\"}", + "interval": "", + "legendFormat": "prometheus", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Prometheus Stats", + "uid": "7RpHmaz7k", + "version": 10 +} diff --git a/manifests/pipecd/grafana-dashboards/resource-metrics.json b/manifests/pipecd/grafana-dashboards/resource-stats.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/resource-metrics.json rename to manifests/pipecd/grafana-dashboards/resource-stats.json diff --git a/manifests/pipecd/grafana-dashboards/service-metrics.json b/manifests/pipecd/grafana-dashboards/service-stats.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/service-metrics.json rename to manifests/pipecd/grafana-dashboards/service-stats.json From 6e74ad288458f4cdf704a7cb49ed34418372182e Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 18:34:17 +0900 Subject: [PATCH 14/25] Update Node stats --- .../grafana-dashboards/resource-stats.json | 327 +++++++++++++++++- 1 file changed, 311 insertions(+), 16 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/resource-stats.json b/manifests/pipecd/grafana-dashboards/resource-stats.json index 4e99111859..dfd3961046 100644 --- a/manifests/pipecd/grafana-dashboards/resource-stats.json +++ b/manifests/pipecd/grafana-dashboards/resource-stats.json @@ -17,9 +17,21 @@ "gnetId": null, "graphTooltip": 0, "id": 3, - "iteration": 1625118048403, + "iteration": 1625131987042, "links": [], "panels": [ + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "title": "Pod Stats", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -36,7 +48,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "hiddenSeries": false, "id": 8, @@ -78,7 +90,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU Usage Rate per Pod", + "title": "CPU Usage Rate per Pod [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -131,7 +143,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "hiddenSeries": false, "id": 12, @@ -210,6 +222,256 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 30, + "panels": [], + "title": "Node Stats", + "type": "row" + }, + { + "datasource": "Prometheus", + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 34, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "100 - (\n(avg by (instance) (node_memory_MemAvailable_bytes{job=\"kubernetes-service-endpoints\"}) * 100)\n/\n(avg by (instance) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 12, + "y": 10 + }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 60 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by(job) (count(node_cpu_seconds_total{mode=\"system\"}) by (kubernetes_node))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "CPU Cores Total", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 14, + "y": 10 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 16, + "y": 10 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "avg by(job) (node_time_seconds{job=\"kubernetes-service-endpoints\"}) \n-\navg by(job) (node_boot_time_seconds{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, { "aliasColors": {}, "bars": false, @@ -226,7 +488,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 15 }, "hiddenSeries": false, "id": 18, @@ -257,7 +519,7 @@ "targets": [ { "exemplar": true, - "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m]))", + "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m]))", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -268,7 +530,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU Usage Rate per Node", + "title": "CPU Usage Rate per Node [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -321,7 +583,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 15 }, "hiddenSeries": false, "id": 20, @@ -400,12 +662,27 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 28, + "panels": [], + "title": "Go Stats", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "Heap metrics reported by Go", "fieldConfig": { "defaults": {}, "overrides": [] @@ -416,7 +693,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "hiddenSeries": false, "id": 22, @@ -449,16 +726,32 @@ "exemplar": true, "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"} / (1024 * 1024))", "interval": "", - "legendFormat": "", + "legendFormat": "alloc", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_inuse_bytes{job=\"$service\"} / (1024 * 1024))", + "hide": false, + "interval": "", + "legendFormat": "inuse", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_idle_bytes{job=\"$service\"} / (1024 * 1024))", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Allocated Bytes per Service", + "title": "Heap Stats", "tooltip": { "shared": true, "sort": 0, @@ -502,7 +795,9 @@ "dashes": false, "datasource": "Prometheus", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "short" + }, "overrides": [] }, "fill": 1, @@ -511,7 +806,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "hiddenSeries": false, "id": 24, @@ -544,7 +839,7 @@ "exemplar": true, "expr": "sum by (job) (go_goroutines{job=\"$service\"})", "interval": "", - "legendFormat": "", + "legendFormat": "$service", "queryType": "randomWalk", "refId": "A" } @@ -702,7 +997,7 @@ }, "timepicker": {}, "timezone": "", - "title": "PipeCD Resource Metrics", + "title": "PipeCD Resource Stats", "uid": "4MJR80z7z", - "version": 8 + "version": 14 } From 969fed79b07aad939d5fc4cf08c5d6a662e4b71c Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 18:58:16 +0900 Subject: [PATCH 15/25] Update --- .../grafana-dashboards/application-stats.json | 134 +++++++++++ .../grafana-dashboards/prometheus-stats.json | 10 +- .../grafana-dashboards/service-stats.json | 222 +++++++++++++++++- 3 files changed, 352 insertions(+), 14 deletions(-) create mode 100644 manifests/pipecd/grafana-dashboards/application-stats.json diff --git a/manifests/pipecd/grafana-dashboards/application-stats.json b/manifests/pipecd/grafana-dashboards/application-stats.json new file mode 100644 index 0000000000..46fab6be46 --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/application-stats.json @@ -0,0 +1,134 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(source) (rate(pipecd_cache_get_operation_total{status=\"hit\"}[5m]))\n/\nsum by(source) (rate(pipecd_cache_get_operation_total[5m])\n)\n* 100", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache hit ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PipeCD Application Stats", + "uid": "PbavQbknz", + "version": 5 +} diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/prometheus-stats.json index 0e42a257ad..980864df7c 100644 --- a/manifests/pipecd/grafana-dashboards/prometheus-stats.json +++ b/manifests/pipecd/grafana-dashboards/prometheus-stats.json @@ -45,12 +45,8 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(255, 255, 255, 0.79)", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -105,7 +101,7 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "rgba(255, 255, 255, 0.82)", "value": null } ] @@ -587,5 +583,5 @@ "timezone": "", "title": "Prometheus Stats", "uid": "7RpHmaz7k", - "version": 10 + "version": 12 } diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/service-stats.json index 8e5c788294..3b3d627981 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/service-stats.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625109245456, + "iteration": 1625133315933, "links": [], "panels": [ { @@ -80,7 +80,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API - 5xx Error Percentage", + "title": "5xx Error Percentage", "tooltip": { "shared": true, "sort": 0, @@ -124,7 +124,9 @@ "dashes": false, "datasource": "Prometheus", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "ms" + }, "overrides": [] }, "fill": 1, @@ -166,16 +168,32 @@ "exemplar": true, "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", "interval": "", - "legendFormat": "", + "legendFormat": "0.99", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.95", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API - Duration 99th Quantile", + "title": "Request Duration Quantiles", "tooltip": { "shared": true, "sort": 0, @@ -261,7 +279,7 @@ "exemplar": true, "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=\"$api\"}[5m]))", "interval": "", - "legendFormat": "", + "legendFormat": "$api", "queryType": "randomWalk", "refId": "A" } @@ -306,6 +324,196 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "upstream_rq_body_size", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Size Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "upstream_rs_body_size", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": "10s", @@ -355,5 +563,5 @@ "timezone": "", "title": "PipeCD Service Metrics", "uid": "3jeba3R7z", - "version": 22 + "version": 26 } From 70233891b2de468f7dc07fe7913b142a368febc1 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 1 Jul 2021 22:30:16 +0900 Subject: [PATCH 16/25] Add legend --- .../grafana-dashboards/prometheus-stats.json | 26 +++++++++++++++---- .../grafana-dashboards/service-stats.json | 10 +++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/prometheus-stats.json index 980864df7c..ac8053d14f 100644 --- a/manifests/pipecd/grafana-dashboards/prometheus-stats.json +++ b/manifests/pipecd/grafana-dashboards/prometheus-stats.json @@ -232,7 +232,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "99th Quantile of HTTP Duration [rate-5m]", + "title": "99th Quantile of HTTP Duration", "tooltip": { "shared": true, "sort": 0, @@ -248,7 +248,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -426,16 +426,32 @@ "exemplar": true, "expr": "go_memstats_heap_alloc_bytes{job=\"prometheus\"} / (1024 * 1024)", "interval": "", - "legendFormat": "prometheus", + "legendFormat": "alloc", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "go_memstats_heap_inuse_bytes{job=\"prometheus\"} / (1024 * 1024)", + "hide": false, + "interval": "", + "legendFormat": "inuse", + "refId": "B" + }, + { + "exemplar": true, + "expr": "go_memstats_heap_idle_bytes{job=\"prometheus\"} / (1024 * 1024)", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Allocated Memory Usage", + "title": "Heap Stats", "tooltip": { "shared": true, "sort": 0, @@ -583,5 +599,5 @@ "timezone": "", "title": "Prometheus Stats", "uid": "7RpHmaz7k", - "version": 12 + "version": 14 } diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/service-stats.json index 3b3d627981..b1cc246079 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/service-stats.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625133315933, + "iteration": 1625146174294, "links": [], "panels": [ { @@ -277,7 +277,7 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{ envoy_cluster_name=\"$api\"}[5m]))", + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{envoy_cluster_name=\"$api\"}[5m]))", "interval": "", "legendFormat": "$api", "queryType": "randomWalk", @@ -288,7 +288,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "API - Request Rate", + "title": "Requests/sec [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -561,7 +561,7 @@ }, "timepicker": {}, "timezone": "", - "title": "PipeCD Service Metrics", + "title": "PipeCD Service Stats", "uid": "3jeba3R7z", - "version": 26 + "version": 29 } From f985a9a467251b6554300816cfc3ee43b360fae7 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Fri, 2 Jul 2021 09:28:41 +0900 Subject: [PATCH 17/25] Update --- .../grafana-dashboards/prometheus-stats.json | 4 +-- .../grafana-dashboards/resource-stats.json | 25 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/prometheus-stats.json index ac8053d14f..7ef7b1c1ae 100644 --- a/manifests/pipecd/grafana-dashboards/prometheus-stats.json +++ b/manifests/pipecd/grafana-dashboards/prometheus-stats.json @@ -328,7 +328,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Ingested samples [rate-5m]", + "title": "Ingested Samples/sec [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -599,5 +599,5 @@ "timezone": "", "title": "Prometheus Stats", "uid": "7RpHmaz7k", - "version": 14 + "version": 15 } diff --git a/manifests/pipecd/grafana-dashboards/resource-stats.json b/manifests/pipecd/grafana-dashboards/resource-stats.json index dfd3961046..0fffe16e70 100644 --- a/manifests/pipecd/grafana-dashboards/resource-stats.json +++ b/manifests/pipecd/grafana-dashboards/resource-stats.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 3, - "iteration": 1625131987042, + "iteration": 1625185632623, "links": [], "panels": [ { @@ -38,6 +38,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "CPU usage per Pod", "fieldConfig": { "defaults": {}, "overrides": [] @@ -79,7 +80,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m])\n)", + "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m]) * 100\n)", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -90,7 +91,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU Usage Rate per Pod [rate-5m]", + "title": "CPU Usage of $service Pod [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -106,7 +107,7 @@ }, "yaxes": [ { - "format": "short", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -185,7 +186,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Usage per Pod", + "title": "Memory Usage of $service Pod", "tooltip": { "shared": true, "sort": 0, @@ -519,7 +520,7 @@ "targets": [ { "exemplar": true, - "expr": "avg by(kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m]))", + "expr": "sum by (kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m])) * 100", "interval": "", "legendFormat": "", "queryType": "randomWalk", @@ -530,7 +531,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU Usage Rate per Node [rate-5m]", + "title": "Node - CPU Usage Rate [rate-5m]", "tooltip": { "shared": true, "sort": 0, @@ -546,7 +547,7 @@ }, "yaxes": [ { - "format": "short", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -625,7 +626,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Usage per Node", + "title": "Node - Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -751,7 +752,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Heap Stats", + "title": "Heap Stats of $service", "tooltip": { "shared": true, "sort": 0, @@ -848,7 +849,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number of Goroutines per Service", + "title": "Number of Goroutines of $service", "tooltip": { "shared": true, "sort": 0, @@ -999,5 +1000,5 @@ "timezone": "", "title": "PipeCD Resource Stats", "uid": "4MJR80z7z", - "version": 14 + "version": 16 } From 8f47d25b01d18b13a99df13ac404d03b03ed89be Mon Sep 17 00:00:00 2001 From: nakabonne Date: Fri, 2 Jul 2021 16:24:01 +0900 Subject: [PATCH 18/25] Add 4xx error --- .../grafana-dashboards/service-stats.json | 109 ++++++++++++++++-- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/service-stats.json index b1cc246079..5b418c7e93 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/service-stats.json @@ -12,12 +12,12 @@ } ] }, - "description": "", + "description": "For black-box monitoring", "editable": true, "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625146174294, + "iteration": 1625210570785, "links": [], "panels": [ { @@ -34,7 +34,7 @@ "fillGradient": 0, "gridPos": { "h": 9, - "w": 24, + "w": 12, "x": 0, "y": 0 }, @@ -67,11 +67,11 @@ "targets": [ { "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100", + "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", "format": "time_series", "instant": false, "interval": "", - "legendFormat": "", + "legendFormat": "$api", "queryType": "randomWalk", "refId": "A" } @@ -117,6 +117,101 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"4.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", + "interval": "", + "legendFormat": "$api", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "4xx Error Percentage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -525,7 +620,7 @@ { "allValue": null, "current": { - "selected": false, + "selected": true, "text": "server-piped-api", "value": "server-piped-api" }, @@ -563,5 +658,5 @@ "timezone": "", "title": "PipeCD Service Stats", "uid": "3jeba3R7z", - "version": 29 + "version": 33 } From 4b966daf19867d5c64b95eee0aed91583d7a82b4 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Mon, 5 Jul 2021 14:58:45 +0900 Subject: [PATCH 19/25] Add req/res bytes panes --- .../grafana-dashboards/service-stats.json | 178 +++++++++++++++--- 1 file changed, 151 insertions(+), 27 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/service-stats.json index 5b418c7e93..dcd0fea61d 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/service-stats.json @@ -17,9 +17,104 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625210570785, + "iteration": 1625464657260, "links": [], "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{envoy_cluster_name=\"$api\"}[5m]))", + "interval": "", + "legendFormat": "$api", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests/sec [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -36,7 +131,7 @@ "h": 9, "w": 12, "x": 0, - "y": 0 + "y": 8 }, "hiddenSeries": false, "id": 2, @@ -100,7 +195,7 @@ "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -133,7 +228,7 @@ "h": 9, "w": 12, "x": 12, - "y": 0 + "y": 8 }, "hiddenSeries": false, "id": 12, @@ -195,7 +290,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -230,7 +325,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 17 }, "hiddenSeries": false, "id": 6, @@ -330,7 +425,8 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": null, + "description": "Average of request duration by path", "fieldConfig": { "defaults": {}, "overrides": [] @@ -341,10 +437,10 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 17 }, "hiddenSeries": false, - "id": 4, + "id": 14, "legend": { "avg": false, "current": false, @@ -371,10 +467,6 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum without(envoy_response_code,instance,job) (rate(envoy_cluster_upstream_rq_total{envoy_cluster_name=\"$api\"}[5m]))", - "interval": "", - "legendFormat": "$api", "queryType": "randomWalk", "refId": "A" } @@ -383,7 +475,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Requests/sec [rate-5m]", + "title": "Request Duration Average", "tooltip": { "shared": true, "sort": 0, @@ -436,7 +528,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 25 }, "hiddenSeries": false, "id": 8, @@ -467,18 +559,34 @@ "targets": [ { "exemplar": true, - "expr": "upstream_rq_body_size", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", "interval": "", - "legendFormat": "", + "legendFormat": "0.99", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.99", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request Size Average", + "title": "Request Size Quantiles", "tooltip": { "shared": true, "sort": 0, @@ -531,10 +639,10 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 25 }, "hiddenSeries": false, - "id": 10, + "id": 15, "legend": { "avg": false, "current": false, @@ -562,18 +670,34 @@ "targets": [ { "exemplar": true, - "expr": "upstream_rs_body_size", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", "interval": "", - "legendFormat": "", + "legendFormat": "0.99", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.99", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "hide": false, + "interval": "", + "legendFormat": "0.90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Response Size Average", + "title": "Response Size Quantiles", "tooltip": { "shared": true, "sort": 0, @@ -620,9 +744,9 @@ { "allValue": null, "current": { - "selected": true, - "text": "server-piped-api", - "value": "server-piped-api" + "selected": false, + "text": "server-web-api", + "value": "server-web-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -658,5 +782,5 @@ "timezone": "", "title": "PipeCD Service Stats", "uid": "3jeba3R7z", - "version": 33 + "version": 38 } From 5a878650ac30f14607766501576565bbcde998ac Mon Sep 17 00:00:00 2001 From: nakabonne Date: Mon, 5 Jul 2021 18:08:22 +0900 Subject: [PATCH 20/25] Update --- .../grafana-dashboards/prometheus-stats.json | 4 +- .../grafana-dashboards/resource-stats.json | 651 +++++++++++------- .../grafana-dashboards/service-stats.json | 40 +- 3 files changed, 440 insertions(+), 255 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/prometheus-stats.json index 7ef7b1c1ae..72745e8472 100644 --- a/manifests/pipecd/grafana-dashboards/prometheus-stats.json +++ b/manifests/pipecd/grafana-dashboards/prometheus-stats.json @@ -86,7 +86,7 @@ "refId": "A" } ], - "title": "Time Series", + "title": "Number of samples", "type": "stat" }, { @@ -599,5 +599,5 @@ "timezone": "", "title": "Prometheus Stats", "uid": "7RpHmaz7k", - "version": 15 + "version": 16 } diff --git a/manifests/pipecd/grafana-dashboards/resource-stats.json b/manifests/pipecd/grafana-dashboards/resource-stats.json index 0fffe16e70..b1f1704ec8 100644 --- a/manifests/pipecd/grafana-dashboards/resource-stats.json +++ b/manifests/pipecd/grafana-dashboards/resource-stats.json @@ -12,15 +12,16 @@ } ] }, - "description": "", + "description": "For white-box monitoring", "editable": true, "gnetId": null, "graphTooltip": 0, "id": 3, - "iteration": 1625185632623, + "iteration": 1625476002653, "links": [], "panels": [ { + "collapsed": false, "datasource": null, "gridPos": { "h": 1, @@ -28,105 +29,182 @@ "x": 0, "y": 0 }, - "id": 26, - "title": "Pod Stats", + "id": 30, + "panels": [], + "title": "Node Stats", "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "Prometheus", - "description": "CPU usage per Pod", "fieldConfig": { - "defaults": {}, + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + } + }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, + "h": 3, + "w": 4, "x": 0, "y": 1 }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 40, "options": { - "alertThreshold": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 60 + }, + "textMode": "auto" }, - "percentage": false, "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m]) * 100\n)", + "expr": "sum by(job) (count(node_cpu_seconds_total{mode=\"system\"}) by (kubernetes_node))", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage of $service Pod [rate-5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "CPU Cores Total", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 }, - "yaxes": [ + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "exemplar": true, + "expr": "sum by (job) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "exemplar": true, + "expr": "avg by(job) (node_time_seconds{job=\"kubernetes-service-endpoints\"}) \n-\navg by(job) (node_boot_time_seconds{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Uptime", + "type": "stat" }, { "aliasColors": {}, @@ -134,6 +212,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "Normalized by the number of CPUs\n", "fieldConfig": { "defaults": {}, "overrides": [] @@ -147,7 +226,7 @@ "y": 1 }, "hiddenSeries": false, - "id": 12, + "id": 44, "legend": { "avg": false, "current": false, @@ -175,9 +254,9 @@ "targets": [ { "exemplar": true, - "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n } / (1024 * 1024)\n)", + "expr": "sum by (instance) (node_load5{job=\"kubernetes-service-endpoints\"})\n/\ncount(node_cpu_seconds_total{mode=\"system\"}) by (instance) * 100", "interval": "", - "legendFormat": "", + "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -186,7 +265,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Usage of $service Pod", + "title": "Node - CPU Load Average [5m]", "tooltip": { "shared": true, "sort": 0, @@ -202,7 +281,7 @@ }, "yaxes": [ { - "format": "decmbytes", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -223,20 +302,6 @@ "alignLevel": null } }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 30, - "panels": [], - "title": "Node Stats", - "type": "row" - }, { "datasource": "Prometheus", "description": "Non available RAM memory", @@ -272,7 +337,7 @@ "h": 5, "w": 12, "x": 0, - "y": 10 + "y": 4 }, "id": 34, "options": { @@ -293,7 +358,7 @@ "exemplar": true, "expr": "100 - (\n(avg by (instance) (node_memory_MemAvailable_bytes{job=\"kubernetes-service-endpoints\"}) * 100)\n/\n(avg by (instance) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})))", "interval": "", - "legendFormat": "", + "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -302,176 +367,302 @@ "type": "gauge" }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "Prometheus", "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - } - }, + "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 2, - "x": 12, - "y": 10 + "h": 8, + "w": 12, + "x": 0, + "y": 9 }, - "id": 40, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 60 - }, - "textMode": "auto" + "alertThreshold": true }, + "percentage": false, "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum by(job) (count(node_cpu_seconds_total{mode=\"system\"}) by (kubernetes_node))", + "expr": "sum by (instance) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])) * 100", "interval": "", - "legendFormat": "", + "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" } ], - "title": "CPU Cores Total", - "type": "stat" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node - CPU Usage [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "Prometheus", "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "decbytes" - }, + "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 2, - "x": 14, - "y": 10 + "h": 8, + "w": 12, + "x": 12, + "y": 9 }, - "id": 38, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" + "alertThreshold": true }, + "percentage": false, "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum by (job) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})", + "expr": "sum by(instance) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / (1024 * 1024 * 1024)", "interval": "", - "legendFormat": "", + "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" } ], - "title": "RAM Total", - "type": "stat" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node - Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decgbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 26, + "title": "Pod Stats", + "type": "row" }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "Prometheus", + "description": "CPU usage per Pod", "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "s" - }, + "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 2, - "x": 16, - "y": 10 + "h": 8, + "w": 12, + "x": 0, + "y": 18 }, - "id": 32, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" + "alertThreshold": true }, + "percentage": false, "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "avg by(job) (node_time_seconds{job=\"kubernetes-service-endpoints\"}) \n-\navg by(job) (node_boot_time_seconds{job=\"kubernetes-service-endpoints\"})", + "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m]) * 100\n)", "interval": "", - "legendFormat": "", + "legendFormat": "$service", "queryType": "randomWalk", "refId": "A" } ], - "title": "Uptime", - "type": "stat" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage of $service Pod [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -488,11 +679,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 15 + "x": 12, + "y": 18 }, "hiddenSeries": false, - "id": 18, + "id": 12, "legend": { "avg": false, "current": false, @@ -520,9 +711,9 @@ "targets": [ { "exemplar": true, - "expr": "sum by (kubernetes_node) (rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m])) * 100", + "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n } / (1024 * 1024)\n)", "interval": "", - "legendFormat": "", + "legendFormat": "$service", "queryType": "randomWalk", "refId": "A" } @@ -531,7 +722,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Node - CPU Usage Rate [rate-5m]", + "title": "Memory Usage of $service Pod", "tooltip": { "shared": true, "sort": 0, @@ -547,7 +738,7 @@ }, "yaxes": [ { - "format": "percent", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, @@ -583,11 +774,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 15 + "x": 0, + "y": 26 }, "hiddenSeries": false, - "id": 20, + "id": 42, "legend": { "avg": false, "current": false, @@ -615,9 +806,9 @@ "targets": [ { "exemplar": true, - "expr": "sum by(kubernetes_node) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / (1024 * 1024 * 1024)", + "expr": "sum by (service) (rate(kube_pod_container_status_restarts_total{service=\"$service\"}[5m]))", "interval": "", - "legendFormat": "", + "legendFormat": "$service", "queryType": "randomWalk", "refId": "A" } @@ -626,7 +817,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Node - Memory Usage", + "title": "Restart number of $service Pod", "tooltip": { "shared": true, "sort": 0, @@ -642,7 +833,7 @@ }, "yaxes": [ { - "format": "decgbytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -670,7 +861,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 34 }, "id": 28, "panels": [], @@ -694,7 +885,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 35 }, "hiddenSeries": false, "id": 22, @@ -807,7 +998,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 35 }, "hiddenSeries": false, "id": 24, @@ -910,18 +1101,8 @@ "multi": false, "name": "service", "options": [ - { - "selected": false, - "text": "pipecd", - "value": "pipecd" - }, { "selected": true, - "text": "pipecd-cache", - "value": "pipecd-cache" - }, - { - "selected": false, "text": "pipecd-gateway", "value": "pipecd-gateway" }, @@ -1000,5 +1181,5 @@ "timezone": "", "title": "PipeCD Resource Stats", "uid": "4MJR80z7z", - "version": 16 + "version": 26 } diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/service-stats.json index dcd0fea61d..f8478e83cc 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/service-stats.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625464657260, + "iteration": 1625475954182, "links": [], "panels": [ { @@ -162,7 +162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"5.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", + "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"5\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100", "format": "time_series", "instant": false, "interval": "", @@ -259,7 +259,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{envoy_response_code=~\"4.*\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", + "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"4\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", "interval": "", "legendFormat": "$api", "queryType": "randomWalk", @@ -425,7 +425,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": "Prometheus", "description": "Average of request duration by path", "fieldConfig": { "defaults": {}, @@ -467,6 +467,10 @@ "steppedLine": false, "targets": [ { + "exemplar": true, + "expr": "envoy_cluster_upstream_rq_time_sum{envoy_cluster_name=\"$api\"}\n/\nenvoy_cluster_upstream_rq_time_count{envoy_cluster_name=\"$api\"}", + "interval": "", + "legendFormat": "$api", "queryType": "randomWalk", "refId": "A" } @@ -491,7 +495,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -559,7 +563,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "interval": "", "legendFormat": "0.99", "queryType": "randomWalk", @@ -567,7 +571,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "hide": false, "interval": "", "legendFormat": "0.99", @@ -575,7 +579,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "hide": false, "interval": "", "legendFormat": "0.90", @@ -602,11 +606,11 @@ }, "yaxes": [ { - "format": "short", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -670,7 +674,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "interval": "", "legendFormat": "0.99", "queryType": "randomWalk", @@ -678,7 +682,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "hide": false, "interval": "", "legendFormat": "0.99", @@ -686,7 +690,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", + "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "hide": false, "interval": "", "legendFormat": "0.90", @@ -713,11 +717,11 @@ }, "yaxes": [ { - "format": "short", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -745,8 +749,8 @@ "allValue": null, "current": { "selected": false, - "text": "server-web-api", - "value": "server-web-api" + "text": "server-piped-api", + "value": "server-piped-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -782,5 +786,5 @@ "timezone": "", "title": "PipeCD Service Stats", "uid": "3jeba3R7z", - "version": 38 + "version": 42 } From fe5ebe5bca06e77c1186de8402389b717ac0c344 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 7 Jul 2021 11:51:43 +0900 Subject: [PATCH 21/25] Devide into folders --- .../cluster-stats/nodes.json | 589 ++++++++ .../cluster-stats/pods.json | 619 +++++++++ .../prometheus.json} | 4 +- .../cache.json} | 4 +- .../control-plane-stats/go.json | 556 ++++++++ .../incoming-requests.json} | 285 +--- .../grafana-dashboards/resource-stats.json | 1185 ----------------- 7 files changed, 1792 insertions(+), 1450 deletions(-) create mode 100644 manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json create mode 100644 manifests/pipecd/grafana-dashboards/cluster-stats/pods.json rename manifests/pipecd/grafana-dashboards/{prometheus-stats.json => cluster-stats/prometheus.json} (99%) rename manifests/pipecd/grafana-dashboards/{application-stats.json => control-plane-stats/cache.json} (98%) create mode 100644 manifests/pipecd/grafana-dashboards/control-plane-stats/go.json rename manifests/pipecd/grafana-dashboards/{service-stats.json => control-plane-stats/incoming-requests.json} (62%) delete mode 100644 manifests/pipecd/grafana-dashboards/resource-stats.json diff --git a/manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json b/manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json new file mode 100644 index 0000000000..532f522719 --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json @@ -0,0 +1,589 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "For white-box monitoring", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 14, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "avg by(job) (node_time_seconds{job=\"kubernetes-service-endpoints\"}) \n-\navg by(job) (node_boot_time_seconds{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 30, + "panels": [], + "title": "CPU", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 60 + }, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by(job) (count(node_cpu_seconds_total{mode=\"system\"}) by (kubernetes_node))", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "CPU Cores Total", + "type": "stat" + }, + { + "datasource": "Prometheus", + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 20, + "x": 4, + "y": 4 + }, + "id": 34, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "100 - (\n(avg by (instance) (node_memory_MemAvailable_bytes{job=\"kubernetes-service-endpoints\"}) * 100)\n/\n(avg by (instance) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})))", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (instance) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])) * 100", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node - CPU Usage [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Normalized by the number of CPUs\n", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (instance) (node_load5{job=\"kubernetes-service-endpoints\"})\n/\ncount(node_cpu_seconds_total{mode=\"system\"}) by (instance) * 100", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Load Average [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 46, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 18 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})", + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(instance) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / (1024 * 1024 * 1024)", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decgbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Nodes", + "uid": "o1M4vRmnk", + "version": 4 +} diff --git a/manifests/pipecd/grafana-dashboards/cluster-stats/pods.json b/manifests/pipecd/grafana-dashboards/cluster-stats/pods.json new file mode 100644 index 0000000000..0df6d5906f --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/cluster-stats/pods.json @@ -0,0 +1,619 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "For white-box monitoring", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 16, + "iteration": 1625624931831, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "CPU usage per Pod", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m]) * 100\n)", + "interval": "", + "legendFormat": "$service", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage of $service Pod [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n } / (1024 * 1024)\n)", + "interval": "", + "legendFormat": "$service", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage of $service Pod", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decmbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (service) (rate(kube_pod_container_status_restarts_total{service=\"$service\"}[5m]))", + "interval": "", + "legendFormat": "$service", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Restart number of $service Pod [rate-5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "pipecd-server", + "value": "pipecd-server" + }, + "datasource": "Prometheus", + "definition": "label_values(service)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "service", + "options": [ + { + "selected": false, + "text": "artifact-monitor", + "value": "artifact-monitor" + }, + { + "selected": false, + "text": "bluegreen", + "value": "bluegreen" + }, + { + "selected": false, + "text": "bluegreen-canary", + "value": "bluegreen-canary" + }, + { + "selected": false, + "text": "calico", + "value": "calico" + }, + { + "selected": false, + "text": "calico-node-vertical-autoscaler", + "value": "calico-node-vertical-autoscaler" + }, + { + "selected": false, + "text": "calico-typha", + "value": "calico-typha" + }, + { + "selected": false, + "text": "calico-typha-horizontal-autoscaler", + "value": "calico-typha-horizontal-autoscaler" + }, + { + "selected": false, + "text": "calico-typha-vertical-autoscaler", + "value": "calico-typha-vertical-autoscaler" + }, + { + "selected": false, + "text": "canary", + "value": "canary" + }, + { + "selected": false, + "text": "canary-by-config-change", + "value": "canary-by-config-change" + }, + { + "selected": false, + "text": "canary-secondary", + "value": "canary-secondary" + }, + { + "selected": false, + "text": "default-http-backend", + "value": "default-http-backend" + }, + { + "selected": false, + "text": "demo-grafana", + "value": "demo-grafana" + }, + { + "selected": false, + "text": "event-exporter-gke", + "value": "event-exporter-gke" + }, + { + "selected": false, + "text": "fluentbit", + "value": "fluentbit" + }, + { + "selected": false, + "text": "gke-metrics", + "value": "gke-metrics" + }, + { + "selected": false, + "text": "helm-local-chart", + "value": "helm-local-chart" + }, + { + "selected": false, + "text": "helm-remote-chart-helloworld", + "value": "helm-remote-chart-helloworld" + }, + { + "selected": false, + "text": "helm-remote-git-chart-helloworld", + "value": "helm-remote-git-chart-helloworld" + }, + { + "selected": false, + "text": "ip-masq", + "value": "ip-masq" + }, + { + "selected": false, + "text": "kube-dns", + "value": "kube-dns" + }, + { + "selected": false, + "text": "kube-dns-autoscaler", + "value": "kube-dns-autoscaler" + }, + { + "selected": false, + "text": "kube-proxy-gke-pipecd-dev-primary-nodepool", + "value": "kube-proxy-gke-pipecd-dev-primary-nodepool" + }, + { + "selected": false, + "text": "kubernetes", + "value": "kubernetes" + }, + { + "selected": false, + "text": "kustomize-local-base", + "value": "kustomize-local-base" + }, + { + "selected": false, + "text": "kustomize-remote-base-helloworld", + "value": "kustomize-remote-base-helloworld" + }, + { + "selected": false, + "text": "l7-default-backend", + "value": "l7-default-backend" + }, + { + "selected": false, + "text": "master-piped", + "value": "master-piped" + }, + { + "selected": false, + "text": "mesh-istio-bluegreen", + "value": "mesh-istio-bluegreen" + }, + { + "selected": false, + "text": "mesh-istio-bluegreen-canary", + "value": "mesh-istio-bluegreen-canary" + }, + { + "selected": false, + "text": "mesh-istio-canary", + "value": "mesh-istio-canary" + }, + { + "selected": false, + "text": "metrics-server", + "value": "metrics-server" + }, + { + "selected": false, + "text": "metrics-server-v0.3.6", + "value": "metrics-server-v0.3.6" + }, + { + "selected": false, + "text": "pipecd", + "value": "pipecd" + }, + { + "selected": false, + "text": "pipecd-cache", + "value": "pipecd-cache" + }, + { + "selected": false, + "text": "pipecd-gateway", + "value": "pipecd-gateway" + }, + { + "selected": false, + "text": "pipecd-grafana", + "value": "pipecd-grafana" + }, + { + "selected": false, + "text": "pipecd-kube-state-metrics", + "value": "pipecd-kube-state-metrics" + }, + { + "selected": false, + "text": "pipecd-minio", + "value": "pipecd-minio" + }, + { + "selected": false, + "text": "pipecd-mysql", + "value": "pipecd-mysql" + }, + { + "selected": false, + "text": "pipecd-ops", + "value": "pipecd-ops" + }, + { + "selected": false, + "text": "pipecd-prometheus-alertmanager", + "value": "pipecd-prometheus-alertmanager" + }, + { + "selected": false, + "text": "pipecd-prometheus-node", + "value": "pipecd-prometheus-node" + }, + { + "selected": false, + "text": "pipecd-prometheus-node-exporter", + "value": "pipecd-prometheus-node-exporter" + }, + { + "selected": false, + "text": "pipecd-prometheus-server", + "value": "pipecd-prometheus-server" + }, + { + "selected": true, + "text": "pipecd-server", + "value": "pipecd-server" + }, + { + "selected": false, + "text": "piped", + "value": "piped" + }, + { + "selected": false, + "text": "prometheus-to", + "value": "prometheus-to" + }, + { + "selected": false, + "text": "simple", + "value": "simple" + }, + { + "selected": false, + "text": "simple-sealed-secret", + "value": "simple-sealed-secret" + }, + { + "selected": false, + "text": "site", + "value": "site" + }, + { + "selected": false, + "text": "stackdriver-metadata-agent-cluster-level", + "value": "stackdriver-metadata-agent-cluster-level" + } + ], + "query": { + "query": "label_values(service)", + "refId": "StandardVariableQuery" + }, + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Pods", + "uid": "MMo5vRi7k", + "version": 2 +} diff --git a/manifests/pipecd/grafana-dashboards/prometheus-stats.json b/manifests/pipecd/grafana-dashboards/cluster-stats/prometheus.json similarity index 99% rename from manifests/pipecd/grafana-dashboards/prometheus-stats.json rename to manifests/pipecd/grafana-dashboards/cluster-stats/prometheus.json index 72745e8472..504958dc82 100644 --- a/manifests/pipecd/grafana-dashboards/prometheus-stats.json +++ b/manifests/pipecd/grafana-dashboards/cluster-stats/prometheus.json @@ -597,7 +597,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Prometheus Stats", + "title": "Prometheus", "uid": "7RpHmaz7k", - "version": 16 + "version": 18 } diff --git a/manifests/pipecd/grafana-dashboards/application-stats.json b/manifests/pipecd/grafana-dashboards/control-plane-stats/cache.json similarity index 98% rename from manifests/pipecd/grafana-dashboards/application-stats.json rename to manifests/pipecd/grafana-dashboards/control-plane-stats/cache.json index 46fab6be46..78c660994b 100644 --- a/manifests/pipecd/grafana-dashboards/application-stats.json +++ b/manifests/pipecd/grafana-dashboards/control-plane-stats/cache.json @@ -128,7 +128,7 @@ }, "timepicker": {}, "timezone": "", - "title": "PipeCD Application Stats", + "title": "Cache", "uid": "PbavQbknz", - "version": 5 + "version": 7 } diff --git a/manifests/pipecd/grafana-dashboards/control-plane-stats/go.json b/manifests/pipecd/grafana-dashboards/control-plane-stats/go.json new file mode 100644 index 0000000000..cb5d6134eb --- /dev/null +++ b/manifests/pipecd/grafana-dashboards/control-plane-stats/go.json @@ -0,0 +1,556 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "For white-box monitoring", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 17, + "iteration": 1625624990892, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "panels": [], + "title": "Go Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Heap metrics reported by Go", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"} / (1024 * 1024))", + "interval": "", + "legendFormat": "alloc", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_inuse_bytes{job=\"$service\"} / (1024 * 1024))", + "hide": false, + "interval": "", + "legendFormat": "inuse", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by (job) (go_memstats_heap_idle_bytes{job=\"$service\"} / (1024 * 1024))", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Stats of $service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decmbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (job) (go_goroutines{job=\"$service\"})", + "interval": "", + "legendFormat": "$service", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Goroutines of $service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "pipecd-server", + "value": "pipecd-server" + }, + "datasource": "Prometheus", + "definition": "label_values(service)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "service", + "options": [ + { + "selected": false, + "text": "artifact-monitor", + "value": "artifact-monitor" + }, + { + "selected": false, + "text": "bluegreen", + "value": "bluegreen" + }, + { + "selected": false, + "text": "bluegreen-canary", + "value": "bluegreen-canary" + }, + { + "selected": false, + "text": "calico", + "value": "calico" + }, + { + "selected": false, + "text": "calico-node-vertical-autoscaler", + "value": "calico-node-vertical-autoscaler" + }, + { + "selected": false, + "text": "calico-typha", + "value": "calico-typha" + }, + { + "selected": false, + "text": "calico-typha-horizontal-autoscaler", + "value": "calico-typha-horizontal-autoscaler" + }, + { + "selected": false, + "text": "calico-typha-vertical-autoscaler", + "value": "calico-typha-vertical-autoscaler" + }, + { + "selected": false, + "text": "canary", + "value": "canary" + }, + { + "selected": false, + "text": "canary-by-config-change", + "value": "canary-by-config-change" + }, + { + "selected": false, + "text": "canary-secondary", + "value": "canary-secondary" + }, + { + "selected": false, + "text": "default-http-backend", + "value": "default-http-backend" + }, + { + "selected": false, + "text": "demo-grafana", + "value": "demo-grafana" + }, + { + "selected": false, + "text": "event-exporter-gke", + "value": "event-exporter-gke" + }, + { + "selected": false, + "text": "fluentbit", + "value": "fluentbit" + }, + { + "selected": false, + "text": "gke-metrics", + "value": "gke-metrics" + }, + { + "selected": false, + "text": "helm-local-chart", + "value": "helm-local-chart" + }, + { + "selected": false, + "text": "helm-remote-chart-helloworld", + "value": "helm-remote-chart-helloworld" + }, + { + "selected": false, + "text": "helm-remote-git-chart-helloworld", + "value": "helm-remote-git-chart-helloworld" + }, + { + "selected": false, + "text": "ip-masq", + "value": "ip-masq" + }, + { + "selected": false, + "text": "kube-dns", + "value": "kube-dns" + }, + { + "selected": false, + "text": "kube-dns-autoscaler", + "value": "kube-dns-autoscaler" + }, + { + "selected": false, + "text": "kube-proxy-gke-pipecd-dev-primary-nodepool", + "value": "kube-proxy-gke-pipecd-dev-primary-nodepool" + }, + { + "selected": false, + "text": "kubernetes", + "value": "kubernetes" + }, + { + "selected": false, + "text": "kustomize-local-base", + "value": "kustomize-local-base" + }, + { + "selected": false, + "text": "kustomize-remote-base-helloworld", + "value": "kustomize-remote-base-helloworld" + }, + { + "selected": false, + "text": "l7-default-backend", + "value": "l7-default-backend" + }, + { + "selected": false, + "text": "master-piped", + "value": "master-piped" + }, + { + "selected": false, + "text": "mesh-istio-bluegreen", + "value": "mesh-istio-bluegreen" + }, + { + "selected": false, + "text": "mesh-istio-bluegreen-canary", + "value": "mesh-istio-bluegreen-canary" + }, + { + "selected": false, + "text": "mesh-istio-canary", + "value": "mesh-istio-canary" + }, + { + "selected": false, + "text": "metrics-server", + "value": "metrics-server" + }, + { + "selected": false, + "text": "metrics-server-v0.3.6", + "value": "metrics-server-v0.3.6" + }, + { + "selected": false, + "text": "pipecd", + "value": "pipecd" + }, + { + "selected": false, + "text": "pipecd-cache", + "value": "pipecd-cache" + }, + { + "selected": false, + "text": "pipecd-gateway", + "value": "pipecd-gateway" + }, + { + "selected": false, + "text": "pipecd-grafana", + "value": "pipecd-grafana" + }, + { + "selected": false, + "text": "pipecd-kube-state-metrics", + "value": "pipecd-kube-state-metrics" + }, + { + "selected": false, + "text": "pipecd-minio", + "value": "pipecd-minio" + }, + { + "selected": false, + "text": "pipecd-mysql", + "value": "pipecd-mysql" + }, + { + "selected": false, + "text": "pipecd-ops", + "value": "pipecd-ops" + }, + { + "selected": false, + "text": "pipecd-prometheus-alertmanager", + "value": "pipecd-prometheus-alertmanager" + }, + { + "selected": false, + "text": "pipecd-prometheus-node", + "value": "pipecd-prometheus-node" + }, + { + "selected": false, + "text": "pipecd-prometheus-node-exporter", + "value": "pipecd-prometheus-node-exporter" + }, + { + "selected": false, + "text": "pipecd-prometheus-server", + "value": "pipecd-prometheus-server" + }, + { + "selected": true, + "text": "pipecd-server", + "value": "pipecd-server" + }, + { + "selected": false, + "text": "piped", + "value": "piped" + }, + { + "selected": false, + "text": "prometheus-to", + "value": "prometheus-to" + }, + { + "selected": false, + "text": "simple", + "value": "simple" + }, + { + "selected": false, + "text": "simple-sealed-secret", + "value": "simple-sealed-secret" + }, + { + "selected": false, + "text": "site", + "value": "site" + }, + { + "selected": false, + "text": "stackdriver-metadata-agent-cluster-level", + "value": "stackdriver-metadata-agent-cluster-level" + } + ], + "query": { + "query": "label_values(service)", + "refId": "StandardVariableQuery" + }, + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Go", + "uid": "jeq-vRi7z", + "version": 2 +} diff --git a/manifests/pipecd/grafana-dashboards/service-stats.json b/manifests/pipecd/grafana-dashboards/control-plane-stats/incoming-requests.json similarity index 62% rename from manifests/pipecd/grafana-dashboards/service-stats.json rename to manifests/pipecd/grafana-dashboards/control-plane-stats/incoming-requests.json index f8478e83cc..a1f501f9d2 100644 --- a/manifests/pipecd/grafana-dashboards/service-stats.json +++ b/manifests/pipecd/grafana-dashboards/control-plane-stats/incoming-requests.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1625475954182, + "iteration": 1625625005978, "links": [], "panels": [ { @@ -33,7 +33,7 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 10, "w": 24, "x": 0, "y": 0 @@ -121,6 +121,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "Non-4xx Error Percentage for each gRPC method", "fieldConfig": { "defaults": {}, "overrides": [] @@ -128,10 +129,10 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 10 }, "hiddenSeries": false, "id": 2, @@ -162,7 +163,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"5\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100", + "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"5\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", "format": "time_series", "instant": false, "interval": "", @@ -175,7 +176,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "5xx Error Percentage", + "title": "Non-4xx Error Percentage", "tooltip": { "shared": true, "sort": 0, @@ -218,215 +219,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 8 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"4\", job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n/\nsum by(envoy_cluster_name) (rate(envoy_cluster_upstream_rq_xx{job=\"pipecd-gateway\", envoy_cluster_name=\"$api\"}[5m]))\n* 100 or on() vector(0)", - "interval": "", - "legendFormat": "$api", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "4xx Error Percentage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "ms" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", - "interval": "", - "legendFormat": "0.99", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", - "hide": false, - "interval": "", - "legendFormat": "0.95", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", - "hide": false, - "interval": "", - "legendFormat": "0.90", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request Duration Quantiles", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Average of request duration by path", + "description": "99th Quantile of Request Duration for each gRPC method", "fieldConfig": { "defaults": {}, "overrides": [] @@ -437,7 +230,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 10 }, "hiddenSeries": false, "id": 14, @@ -468,9 +261,9 @@ "targets": [ { "exemplar": true, - "expr": "envoy_cluster_upstream_rq_time_sum{envoy_cluster_name=\"$api\"}\n/\nenvoy_cluster_upstream_rq_time_count{envoy_cluster_name=\"$api\"}", + "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(pipecd_gateway_requests_duration_bucket{envoy_cluster_name=\"$api\"}[5m]))\n)", "interval": "", - "legendFormat": "$api", + "legendFormat": "{{method}}", "queryType": "randomWalk", "refId": "A" } @@ -479,7 +272,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request Duration Average", + "title": "99th Quantile of Request Duration", "tooltip": { "shared": true, "sort": 0, @@ -522,6 +315,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "99th quantile of request size per gRPC method", "fieldConfig": { "defaults": {}, "overrides": [] @@ -532,7 +326,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 18 }, "hiddenSeries": false, "id": 8, @@ -565,32 +359,16 @@ "exemplar": true, "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "interval": "", - "legendFormat": "0.99", + "legendFormat": "{{method}}", "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", - "hide": false, - "interval": "", - "legendFormat": "0.99", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rq_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", - "hide": false, - "interval": "", - "legendFormat": "0.90", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request Size Quantiles", + "title": "99th Quantile of Request Size", "tooltip": { "shared": true, "sort": 0, @@ -633,6 +411,7 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "99th quantile of response size per gRPC method", "fieldConfig": { "defaults": {}, "overrides": [] @@ -643,7 +422,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 18 }, "hiddenSeries": false, "id": 15, @@ -676,32 +455,16 @@ "exemplar": true, "expr": "histogram_quantile(\n 0.99,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", "interval": "", - "legendFormat": "0.99", + "legendFormat": "{{method}}", "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.95,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", - "hide": false, - "interval": "", - "legendFormat": "0.99", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(\n 0.90,\n sum without(instance, job)(rate(envoy_cluster_upstream_rs_body_size_bucket{envoy_cluster_name=\"$api\"}[5m]))\n) * (1024 * 1024)", - "hide": false, - "interval": "", - "legendFormat": "0.90", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Response Size Quantiles", + "title": "99th Quantile of Response Size", "tooltip": { "shared": true, "sort": 0, @@ -749,8 +512,8 @@ "allValue": null, "current": { "selected": false, - "text": "server-piped-api", - "value": "server-piped-api" + "text": "server-api", + "value": "server-api" }, "datasource": "Prometheus", "definition": "label_values(envoy_cluster_name)", @@ -784,7 +547,7 @@ }, "timepicker": {}, "timezone": "", - "title": "PipeCD Service Stats", + "title": "Incoming Requests", "uid": "3jeba3R7z", - "version": 42 + "version": 48 } diff --git a/manifests/pipecd/grafana-dashboards/resource-stats.json b/manifests/pipecd/grafana-dashboards/resource-stats.json deleted file mode 100644 index b1f1704ec8..0000000000 --- a/manifests/pipecd/grafana-dashboards/resource-stats.json +++ /dev/null @@ -1,1185 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "For white-box monitoring", - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 3, - "iteration": 1625476002653, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 30, - "panels": [], - "title": "Node Stats", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 40, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 60 - }, - "textMode": "auto" - }, - "pluginVersion": "7.5.3", - "targets": [ - { - "exemplar": true, - "expr": "sum by(job) (count(node_cpu_seconds_total{mode=\"system\"}) by (kubernetes_node))", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "CPU Cores Total", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 38, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.3", - "targets": [ - { - "exemplar": true, - "expr": "sum by (job) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "RAM Total", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.3", - "targets": [ - { - "exemplar": true, - "expr": "avg by(job) (node_time_seconds{job=\"kubernetes-service-endpoints\"}) \n-\navg by(job) (node_boot_time_seconds{job=\"kubernetes-service-endpoints\"})", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Normalized by the number of CPUs\n", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (instance) (node_load5{job=\"kubernetes-service-endpoints\"})\n/\ncount(node_cpu_seconds_total{mode=\"system\"}) by (instance) * 100", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Node - CPU Load Average [5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "Prometheus", - "description": "Non available RAM memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 34, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "7.5.3", - "targets": [ - { - "exemplar": true, - "expr": "100 - (\n(avg by (instance) (node_memory_MemAvailable_bytes{job=\"kubernetes-service-endpoints\"}) * 100)\n/\n(avg by (instance) (node_memory_MemTotal_bytes{job=\"kubernetes-service-endpoints\"})))", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "RAM Used", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (instance) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])) * 100", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Node - CPU Usage [rate-5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by(instance) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / (1024 * 1024 * 1024)", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Node - Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decgbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 26, - "title": "Pod Stats", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "CPU usage per Pod", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by(service) (\n rate(container_cpu_usage_seconds_total{\n service=\"$service\"\n }[5m]) * 100\n)", - "interval": "", - "legendFormat": "$service", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage of $service Pod [rate-5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by(service) (\n container_memory_usage_bytes{\n service=\"$service\"\n } / (1024 * 1024)\n)", - "interval": "", - "legendFormat": "$service", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage of $service Pod", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "hiddenSeries": false, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (service) (rate(kube_pod_container_status_restarts_total{service=\"$service\"}[5m]))", - "interval": "", - "legendFormat": "$service", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Restart number of $service Pod", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 28, - "panels": [], - "title": "Go Stats", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Heap metrics reported by Go", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (job) (go_memstats_heap_alloc_bytes{job=\"$service\"} / (1024 * 1024))", - "interval": "", - "legendFormat": "alloc", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum by (job) (go_memstats_heap_inuse_bytes{job=\"$service\"} / (1024 * 1024))", - "hide": false, - "interval": "", - "legendFormat": "inuse", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum by (job) (go_memstats_heap_idle_bytes{job=\"$service\"} / (1024 * 1024))", - "hide": false, - "interval": "", - "legendFormat": "idle", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Heap Stats of $service", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (job) (go_goroutines{job=\"$service\"})", - "interval": "", - "legendFormat": "$service", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of Goroutines of $service", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 27, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "text": "pipecd-server", - "value": "pipecd-server" - }, - "datasource": "Prometheus", - "definition": "label_values(service)", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "service", - "options": [ - { - "selected": true, - "text": "pipecd-gateway", - "value": "pipecd-gateway" - }, - { - "selected": false, - "text": "pipecd-grafana", - "value": "pipecd-grafana" - }, - { - "selected": false, - "text": "pipecd-kube-state-metrics", - "value": "pipecd-kube-state-metrics" - }, - { - "selected": false, - "text": "pipecd-minio", - "value": "pipecd-minio" - }, - { - "selected": false, - "text": "pipecd-mysql", - "value": "pipecd-mysql" - }, - { - "selected": false, - "text": "pipecd-ops", - "value": "pipecd-ops" - }, - { - "selected": false, - "text": "pipecd-prometheus-alertmanager", - "value": "pipecd-prometheus-alertmanager" - }, - { - "selected": false, - "text": "pipecd-prometheus-node", - "value": "pipecd-prometheus-node" - }, - { - "selected": false, - "text": "pipecd-prometheus-node-exporter", - "value": "pipecd-prometheus-node-exporter" - }, - { - "selected": false, - "text": "pipecd-prometheus-server", - "value": "pipecd-prometheus-server" - }, - { - "selected": false, - "text": "pipecd-server", - "value": "pipecd-server" - } - ], - "query": { - "query": "label_values(service)", - "refId": "StandardVariableQuery" - }, - "refresh": 0, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "PipeCD Resource Stats", - "uid": "4MJR80z7z", - "version": 26 -} From e25935d4e3ede6b136d4c1bc9e4dc6fbb34ec0e2 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 7 Jul 2021 14:01:38 +0900 Subject: [PATCH 22/25] Rename directories --- .../grafana-dashboards/{cluster-stats => cluster}/nodes.json | 0 .../grafana-dashboards/{cluster-stats => cluster}/pods.json | 0 .../grafana-dashboards/{cluster-stats => cluster}/prometheus.json | 0 .../{control-plane-stats => control-plane}/cache.json | 0 .../{control-plane-stats => control-plane}/go.json | 0 .../{control-plane-stats => control-plane}/incoming-requests.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename manifests/pipecd/grafana-dashboards/{cluster-stats => cluster}/nodes.json (100%) rename manifests/pipecd/grafana-dashboards/{cluster-stats => cluster}/pods.json (100%) rename manifests/pipecd/grafana-dashboards/{cluster-stats => cluster}/prometheus.json (100%) rename manifests/pipecd/grafana-dashboards/{control-plane-stats => control-plane}/cache.json (100%) rename manifests/pipecd/grafana-dashboards/{control-plane-stats => control-plane}/go.json (100%) rename manifests/pipecd/grafana-dashboards/{control-plane-stats => control-plane}/incoming-requests.json (100%) diff --git a/manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json b/manifests/pipecd/grafana-dashboards/cluster/nodes.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/cluster-stats/nodes.json rename to manifests/pipecd/grafana-dashboards/cluster/nodes.json diff --git a/manifests/pipecd/grafana-dashboards/cluster-stats/pods.json b/manifests/pipecd/grafana-dashboards/cluster/pods.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/cluster-stats/pods.json rename to manifests/pipecd/grafana-dashboards/cluster/pods.json diff --git a/manifests/pipecd/grafana-dashboards/cluster-stats/prometheus.json b/manifests/pipecd/grafana-dashboards/cluster/prometheus.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/cluster-stats/prometheus.json rename to manifests/pipecd/grafana-dashboards/cluster/prometheus.json diff --git a/manifests/pipecd/grafana-dashboards/control-plane-stats/cache.json b/manifests/pipecd/grafana-dashboards/control-plane/cache.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/control-plane-stats/cache.json rename to manifests/pipecd/grafana-dashboards/control-plane/cache.json diff --git a/manifests/pipecd/grafana-dashboards/control-plane-stats/go.json b/manifests/pipecd/grafana-dashboards/control-plane/go.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/control-plane-stats/go.json rename to manifests/pipecd/grafana-dashboards/control-plane/go.json diff --git a/manifests/pipecd/grafana-dashboards/control-plane-stats/incoming-requests.json b/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/control-plane-stats/incoming-requests.json rename to manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json From ccf062c8eba0c6dd04568db3e2cfe9a3bc6935d0 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 7 Jul 2021 14:07:25 +0900 Subject: [PATCH 23/25] Update --- manifests/pipecd/grafana-dashboards/cluster/nodes.json | 5 ++--- manifests/pipecd/grafana-dashboards/cluster/pods.json | 7 +++---- .../control-plane/incoming-requests.json | 7 +++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifests/pipecd/grafana-dashboards/cluster/nodes.json b/manifests/pipecd/grafana-dashboards/cluster/nodes.json index 532f522719..071be156b2 100644 --- a/manifests/pipecd/grafana-dashboards/cluster/nodes.json +++ b/manifests/pipecd/grafana-dashboards/cluster/nodes.json @@ -16,7 +16,6 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 14, "links": [], "panels": [ { @@ -583,7 +582,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Nodes", + "title": "Node", "uid": "o1M4vRmnk", - "version": 4 + "version": 1 } diff --git a/manifests/pipecd/grafana-dashboards/cluster/pods.json b/manifests/pipecd/grafana-dashboards/cluster/pods.json index 0df6d5906f..8ed52eb337 100644 --- a/manifests/pipecd/grafana-dashboards/cluster/pods.json +++ b/manifests/pipecd/grafana-dashboards/cluster/pods.json @@ -16,8 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 16, - "iteration": 1625624931831, + "iteration": 1625634242562, "links": [], "panels": [ { @@ -613,7 +612,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Pods", + "title": "Pod", "uid": "MMo5vRi7k", - "version": 2 + "version": 1 } diff --git a/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json b/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json index a1f501f9d2..f2f379e4ec 100644 --- a/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json +++ b/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json @@ -16,8 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 2, - "iteration": 1625625005978, + "iteration": 1625634320370, "links": [], "panels": [ { @@ -547,7 +546,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Incoming Requests", + "title": "Incoming Request", "uid": "3jeba3R7z", - "version": 48 + "version": 1 } From 7d542d62f785923854509c9268b72d6a8ced4f92 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 7 Jul 2021 14:08:24 +0900 Subject: [PATCH 24/25] Rename files --- .../pipecd/grafana-dashboards/cluster/{nodes.json => node.json} | 0 .../pipecd/grafana-dashboards/cluster/{pods.json => pod.json} | 0 .../{incoming-requests.json => incoming-request.json} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename manifests/pipecd/grafana-dashboards/cluster/{nodes.json => node.json} (100%) rename manifests/pipecd/grafana-dashboards/cluster/{pods.json => pod.json} (100%) rename manifests/pipecd/grafana-dashboards/control-plane/{incoming-requests.json => incoming-request.json} (100%) diff --git a/manifests/pipecd/grafana-dashboards/cluster/nodes.json b/manifests/pipecd/grafana-dashboards/cluster/node.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/cluster/nodes.json rename to manifests/pipecd/grafana-dashboards/cluster/node.json diff --git a/manifests/pipecd/grafana-dashboards/cluster/pods.json b/manifests/pipecd/grafana-dashboards/cluster/pod.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/cluster/pods.json rename to manifests/pipecd/grafana-dashboards/cluster/pod.json diff --git a/manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json b/manifests/pipecd/grafana-dashboards/control-plane/incoming-request.json similarity index 100% rename from manifests/pipecd/grafana-dashboards/control-plane/incoming-requests.json rename to manifests/pipecd/grafana-dashboards/control-plane/incoming-request.json From cd578a6e3f1500dfce30e9ace818eba4bbd1a9e5 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Wed, 7 Jul 2021 14:20:46 +0900 Subject: [PATCH 25/25] Allow UI Updates --- manifests/pipecd/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/pipecd/values.yaml b/manifests/pipecd/values.yaml index 5aa30c778b..0759fb91f2 100644 --- a/manifests/pipecd/values.yaml +++ b/manifests/pipecd/values.yaml @@ -382,7 +382,9 @@ grafana: folder: 'Cluster' type: file disableDeletion: false - editable: false + # TODO: Disallow to update Grafana UI from the web once got fixed + # The following Control-Plane's one should be reverted as well. + allowUiUpdates: true options: path: /tmp/dashboards/cluster - name: 'Control-Plane' @@ -390,7 +392,7 @@ grafana: folder: 'Control-Plane' type: file disableDeletion: false - editable: false + allowUiUpdates: true options: path: /tmp/dashboards/control-plane sidecar: