diff --git a/Makefile b/Makefile index bcedd573e34..09c4f5b6221 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed for integration tests on kubernetes -CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services +CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) diff --git a/changelog.d/2-features/metrics b/changelog.d/2-features/metrics new file mode 100644 index 00000000000..a4650c4123a --- /dev/null +++ b/changelog.d/2-features/metrics @@ -0,0 +1 @@ +Use kube-prometheus-stack instead of prometheus-operator and update grafana dashboards for compatibility and add federation endpoints to relevant queries. \ No newline at end of file diff --git a/charts/wire-server-metrics/dashboards/message-stats.json b/charts/wire-server-metrics/dashboards/message-stats.json index e84580b8d0f..027a2a55b62 100644 --- a/charts/wire-server-metrics/dashboards/message-stats.json +++ b/charts/wire-server-metrics/dashboards/message-stats.json @@ -8,19 +8,31 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, + "fiscalYearStartMonth": 0, "gnetId": null, "graphTooltip": 0, - "id": 50, - "iteration": 1584951112798, + "iteration": 1637052673609, "links": [], + "liveNow": false, "panels": [ { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, @@ -38,13 +50,16 @@ "dashLength": 10, "dashes": false, "datasource": "default", + "description": "", "fill": 3, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1 }, + "hiddenSeries": false, "id": 17, "legend": { "avg": false, @@ -59,8 +74,12 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -98,7 +117,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Registrations/invitations/code requests (brig)", + "title": "Registrations/invitations/code requests", "tooltip": { "shared": true, "sort": 0, @@ -135,131 +154,256 @@ "alignLevel": null } }, + { + "datasource": null, + "description": "A client can fetch many prekeys with one API call, so the count here just represents how many calls are made to the API", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\", handler=~\"/federation/claim-multi-prekey-bundle|/users/:uid/prekeys/:client|/users/:uid_domain/:uid/prekeys/:client|/users/:uid/prekeys|/users/:uid_domain/:uid/prekeys/users/prekys|/users/list-prekeys\"}[5m]))", + "interval": "", + "legendFormat": "API Calls", + "refId": "A" + } + ], + "title": "Prekey fetch count", + "type": "timeseries" + }, { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, - "id": 23, + "id": 19, "panels": [], - "title": "Cannon Stats", + "title": "Galley Stats", "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#8F3BB8", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "default", - "fill": 2, + "description": "Includes messages that originate from local clients or remote users in local conversations", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 }, - "id": 15, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 6, "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "show": false }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "sum(net_websocket_clients{namespace=\"$namespace\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Connected Clients", - "refId": "A" + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_bucket{handler=~\"(/conversations/:cnv/otr/messages|/conversations/:cnv_domain/:cnv/proteus/messages|/federation/send-message)\", namespace=\"$namespace\"}[5m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{ le }}", + "refId": "B" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Connected Clients (cannon)", + "title": "Message sending latencies for local and remote conversations", "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": false }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null }, { - "collapsed": false, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": null, + "description": "", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 + "h": 7, + "w": 12, + "x": 12, + "y": 9 }, - "id": 30, - "panels": [], - "title": "Cargohold stats", - "type": "row" + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 32, + "legend": { + "show": false + }, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_bucket{handler=\"/federation/on-message-sent\", namespace=\"$namespace\"}[5m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Latencies for messages received from remote conversations", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, + "datasource": "default", + "description": "Messages sent from local clients to remote conversations are duplicated, if there are any other local clients involved", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 17 + "y": 16 }, - "id": 28, + "hiddenSeries": false, + "id": 11, "legend": { "avg": false, "current": false, @@ -273,8 +417,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, + "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -284,25 +432,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(http_request_duration_seconds_count{handler=\"/assets/v3\", status_code=\"201\", namespace=\"$namespace\"}[5m]))", + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_count{handler=~\"(/conversations/:cnv/otr/messages|/conversations/:cnv_domain/:cnv/proteus/messages|/federation/send-message|/federation/on-message-sent)\", status_code=~\"2..\", namespace=\"$namespace\"}[5m]))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "uploaded/sec", + "legendFormat": "Messages sent", "refId": "A" - }, - { - "expr": "sum(rate(http_request_duration_seconds_count{handler=\"/assets/v3/:key\", status_code=\"302\", namespace=\"$namespace\"}[5m]))\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "downloaded/sec", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Assets (cargohold)", + "title": "Messages Sent every 5m", "tooltip": { "shared": true, "sort": 0, @@ -334,90 +477,10 @@ "show": true } ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 21, - "panels": [], - "title": "Gundeck Stats", - "type": "row" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#8F3BB8", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "default", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 7, - "legend": { - "show": false - }, - "links": [], - "options": {}, - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(increase(http_request_duration_seconds_bucket{handler=\"/i/push/v2\", namespace=\"$namespace\"}[5m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 10, - "legendFormat": "{{ le }}", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Message Push Latencies (gundeck)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -425,17 +488,21 @@ "dashLength": 10, "dashes": false, "datasource": "default", - "fill": 1, + "description": "Does not include federation endpoints as they do not express errors in HTTP status", + "fill": 8, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 26 + "y": 16 }, - "id": 9, + "hiddenSeries": false, + "id": 13, "legend": { "avg": false, "current": false, + "hideEmpty": false, "max": false, "min": false, "show": true, @@ -446,55 +513,54 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "paceLength": 10, - "percentage": false, + "percentage": true, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(increase(push_native_success[5m]))", + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=~\"(/conversations/:cnv/otr/messages|/conversations/:cnv_domain/:cnv/proteus/messages)\", status_code=~\"2..\", namespace=\"$namespace\"}[10m]))", "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "Success", + "interval": "", + "intervalFactor": 5, + "legendFormat": "2XX", "refId": "A" }, { - "expr": "-sum(increase(push_native_disabled[5m]))", + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=~\"(/conversations/:cnv/otr/messages|/conversations/:cnv_domain/:cnv/proteus/messages)\", status_code=~\"4..\", namespace=\"$namespace\"}[10m]))", "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "-Disabled", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "4XX", "refId": "B" }, { - "expr": "-sum(increase(push_native_invalid{namespace=\"$namespace\"}[5m]))", + "exemplar": true, + "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=~\"(/conversations/:cnv/otr/messages|/conversations/:cnv_domain/:cnv/proteus/messages)\", status_code=~\"5..\", namespace=\"$namespace\"}[10m]))", "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "-Invalid", + "interval": "", + "intervalFactor": 5, + "legendFormat": "5XX", "refId": "C" - }, - { - "expr": "-sum(increase(push_native_errors{namespace=\"$namespace\"}[5m]))", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "-Errors", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Native Push (Outgoing) (gundeck)", + "title": "Proteus Message Receives as % by Status Code", "tooltip": { "shared": true, "sort": 0, @@ -513,7 +579,7 @@ "format": "short", "label": null, "logBase": 1, - "max": null, + "max": "100", "min": null, "show": true }, @@ -531,20 +597,41 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 23, + "panels": [], + "title": "Cannon Stats", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "default", - "fill": 1, + "description": "", + "fill": 2, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 24 }, - "id": 2, + "hiddenSeries": false, + "id": 15, "legend": { "avg": false, "current": false, @@ -558,9 +645,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -570,10 +660,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(http_request_duration_seconds_count{role=\"gundeck\", handler=\"/i/push/v2\", namespace=\"$namespace\"}[5m])) by (status_code)", + "expr": "sum(net_websocket_clients{namespace=\"$namespace\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{ status_code }}", + "legendFormat": "Connected Clients", "refId": "A" } ], @@ -581,7 +671,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Push Rate (incoming) (gundeck)", + "title": "Connected Clients", "tooltip": { "shared": true, "sort": 0, @@ -618,20 +708,40 @@ "alignLevel": null } }, + { + "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 30, + "panels": [], + "title": "Cargohold stats", + "type": "row" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "default", + "datasource": null, "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 32 }, - "id": 4, + "hiddenSeries": false, + "id": 28, "legend": { "avg": false, "current": false, @@ -645,30 +755,39 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, - "paceLength": 10, - "percentage": true, + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(http_request_duration_seconds_count{handler=\"/i/push/v2\", namespace=\"$namespace\"}) by (status_code)", + "expr": "sum(rate(http_request_duration_seconds_count{handler=\"/assets/v3\", status_code=\"201\", namespace=\"$namespace\"}[5m]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{ status_code }}", + "legendFormat": "uploaded/sec", "refId": "A" + }, + { + "expr": "sum(rate(http_request_duration_seconds_count{handler=\"/assets/v3/:key\", status_code=\"302\", namespace=\"$namespace\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "downloaded/sec", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Push Status Codes as % (gundeck)", + "title": "Assets", "tooltip": { "shared": true, "sort": 0, @@ -685,15 +804,15 @@ "yaxes": [ { "format": "short", - "label": "", + "label": null, "logBase": 1, - "max": "100", - "min": "0", + "max": null, + "min": null, "show": true }, { "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, @@ -707,15 +826,20 @@ }, { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, - "id": 19, + "id": 21, "panels": [], - "title": "Galley Stats", + "title": "Gundeck Stats", "type": "row" }, { @@ -741,16 +865,15 @@ "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 6, + "id": 7, "legend": { "show": false }, "links": [], - "options": {}, "reverseYBuckets": false, "targets": [ { - "expr": "sum(increase(http_request_duration_seconds_bucket{handler=\"/conversations/:cnv/otr/messages\", namespace=\"$namespace\"}[5m])) by (le)", + "expr": "sum(increase(http_request_duration_seconds_bucket{handler=\"/i/push/v2\", namespace=\"$namespace\"}[5m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -761,7 +884,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Message POSTing Latencies (galley)", + "title": "Message Push Latencies", "tooltip": { "show": true, "showHistogram": false @@ -791,18 +914,19 @@ "dashLength": 10, "dashes": false, "datasource": "default", - "fill": 8, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 41 }, - "id": 13, + "hiddenSeries": false, + "id": 9, "legend": { "avg": false, "current": false, - "hideEmpty": false, "max": false, "min": false, "show": true, @@ -813,45 +937,58 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "paceLength": 10, - "percentage": true, + "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=\"/conversations/:cnv/otr/messages\", status_code=~\"2..\", namespace=\"$namespace\"}[10m]))", + "expr": "sum(increase(push_native_success[5m]))", "format": "time_series", - "intervalFactor": 5, - "legendFormat": "2XX", + "instant": false, + "intervalFactor": 1, + "legendFormat": "Success", "refId": "A" }, { - "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=\"/conversations/:cnv/otr/messages\", status_code=~\"4..\", namespace=\"$namespace\"}[10m]))", + "expr": "-sum(increase(push_native_disabled[5m]))", "format": "time_series", - "hide": false, - "intervalFactor": 5, - "legendFormat": "4XX", + "instant": false, + "intervalFactor": 1, + "legendFormat": "-Disabled", "refId": "B" }, { - "expr": "sum(increase(http_request_duration_seconds_count{role=\"galley\", handler=\"/conversations/:cnv/otr/messages\", status_code=~\"5..\", namespace=\"$namespace\"}[10m]))", + "expr": "-sum(increase(push_native_invalid{namespace=\"$namespace\"}[5m]))", "format": "time_series", - "intervalFactor": 5, - "legendFormat": "5XX", + "instant": false, + "intervalFactor": 1, + "legendFormat": "-Invalid", "refId": "C" + }, + { + "expr": "-sum(increase(push_native_errors{namespace=\"$namespace\"}[5m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "-Errors", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "OTR Message Receives as % by Status Code (galley)", + "title": "Native Push (Outgoing)", "tooltip": { "shared": true, "sort": 0, @@ -870,7 +1007,7 @@ "format": "short", "label": null, "logBase": 1, - "max": "100", + "max": null, "min": null, "show": true }, @@ -895,13 +1032,15 @@ "dashes": false, "datasource": "default", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, "y": 48 }, - "id": 11, + "hiddenSeries": false, + "id": 2, "legend": { "avg": false, "current": false, @@ -915,9 +1054,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -927,10 +1069,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(http_request_duration_seconds_count{handler=\"/conversations/:cnv/otr/messages\", status_code=~\"2..\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(increase(http_request_duration_seconds_count{role=\"gundeck\", handler=\"/i/push/v2\", namespace=\"$namespace\"}[5m])) by (status_code)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "{{ status_code }}", "refId": "A" } ], @@ -938,7 +1080,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Messages Sent every 5m (galley)", + "title": "Push Rate (incoming)", "tooltip": { "shared": true, "sort": 0, @@ -974,9 +1116,101 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "default", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": true, + "pluginVersion": "8.2.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(http_request_duration_seconds_count{handler=\"/i/push/v2\", namespace=\"$namespace\"}) by (status_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ status_code }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Push Status Codes as %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "schemaVersion": 18, + "schemaVersion": 31, "style": "dark", "tags": [], "templating": { @@ -984,25 +1218,29 @@ { "allValue": null, "current": { - "tags": [], - "text": "default", - "value": "default" + "selected": true, + "text": "wire-a", + "value": "wire-a" }, "datasource": "Prometheus", "definition": "kube_namespace_labels", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [], - "query": "kube_namespace_labels", - "refresh": 0, + "query": { + "query": "kube_namespace_labels", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 1, "regex": "/namespace=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -1010,7 +1248,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -1041,5 +1279,5 @@ "timezone": "", "title": "Message Stats", "uid": "sqSUztzZz", - "version": 5 + "version": 1 } diff --git a/charts/wire-server-metrics/dashboards/services.json b/charts/wire-server-metrics/dashboards/services.json index 30bc06c349f..23d1111f028 100644 --- a/charts/wire-server-metrics/dashboards/services.json +++ b/charts/wire-server-metrics/dashboards/services.json @@ -8,28 +8,39 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": true, + "fiscalYearStartMonth": 0, "gnetId": null, "graphTooltip": 0, - "id": 11, - "iteration": 1556873852486, + "iteration": 1636449155094, "links": [], + "liveNow": false, "panels": [ { - "content": "# Haskell Service Statistics\n## Change the $(service) and $(namespace) variables to filter stats", + "datasource": null, "gridPos": { - "h": 3, + "h": 4, "w": 24, "x": 0, "y": 0 }, "id": 97, "links": [], - "mode": "markdown", + "options": { + "content": "# Haskell Service Statistics\n## Change the $(service) and $(namespace) variables to filter stats", + "mode": "markdown" + }, + "pluginVersion": "8.2.3", "timeFrom": null, "timeShift": null, "title": "README", @@ -38,11 +49,16 @@ }, { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 4 }, "id": 89, "panels": [], @@ -54,13 +70,16 @@ "bars": true, "dashLength": 10, "dashes": false, + "datasource": null, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 4 + "y": 5 }, + "hiddenSeries": false, "id": 23, "legend": { "avg": false, @@ -75,8 +94,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -86,8 +109,11 @@ "steppedLine": false, "targets": [ { + "exemplar": false, "expr": "sum(increase(http_request_duration_seconds_count{status_code=~\"5..\", namespace=\"$namespace\"}[5m])) by (role)", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 5, "legendFormat": "{{role}}", "refId": "A" @@ -137,12 +163,13 @@ { "cacheTimeout": null, "columns": [], + "datasource": null, "fontSize": "100%", "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 4 + "y": 5 }, "id": 98, "interval": "", @@ -157,12 +184,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -179,6 +208,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -194,9 +224,11 @@ ], "targets": [ { + "exemplar": false, "expr": "sum(increase(http_request_duration_seconds_count{status_code=~\"5..\", namespace=\"$namespace\"}[$__range])) by (role, handler)", "format": "table", "instant": true, + "interval": "", "intervalFactor": 1, "legendFormat": "# 5XXs", "refId": "A" @@ -206,15 +238,20 @@ "timeShift": null, "title": "5XX/s per handler (current time range) - all services", "transform": "table", - "type": "table" + "type": "table-old" }, { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 91, "panels": [], @@ -230,13 +267,16 @@ "bars": false, "dashLength": 10, "dashes": false, + "datasource": null, "fill": 4, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 13 }, + "hiddenSeries": false, "id": 18, "legend": { "avg": false, @@ -251,8 +291,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -333,12 +377,13 @@ { "cacheTimeout": null, "columns": [], + "datasource": null, "fontSize": "100%", "gridPos": { "h": 8, "w": 6, "x": 12, - "y": 12 + "y": 13 }, "id": 20, "interval": "", @@ -353,12 +398,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -375,6 +422,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -402,17 +450,18 @@ "timeShift": null, "title": "4XX/s per handler (current time range) - $service", "transform": "table", - "type": "table" + "type": "table-old" }, { "cacheTimeout": null, "columns": [], + "datasource": null, "fontSize": "100%", "gridPos": { "h": 8, "w": 6, "x": 18, - "y": 12 + "y": 13 }, "id": 21, "interval": "", @@ -427,12 +476,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -449,6 +500,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -476,7 +528,7 @@ "timeShift": null, "title": "5XX/s per handler (current time range) - $service", "transform": "table", - "type": "table" + "type": "table-old" }, { "aliasColors": {}, @@ -487,13 +539,15 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 20 + "y": 21 }, + "hiddenSeries": false, "id": 2, "legend": { "avg": false, @@ -508,8 +562,12 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", @@ -519,9 +577,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_receive_bytes_total{pod_name=~\"$service.*\", namespace=\"$namespace\"}[5m])) by (pod_name)", + "exemplar": true, + "expr": "sum(rate(container_network_receive_bytes_total{pod=~\"$service.*\", namespace=\"$namespace\"}[5m])) by (pod_name)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "refId": "A", @@ -571,11 +631,16 @@ }, { "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 28 }, "id": 27, "panels": [], @@ -592,13 +657,15 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 28 + "y": 29 }, + "hiddenSeries": false, "id": 4, "legend": { "avg": false, @@ -613,8 +680,12 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", @@ -624,10 +695,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_cpu_load_average_10s{pod_name=~\"$service.*\", namespace=\"$namespace\"}) by (pod_name) * 100", + "exemplar": true, + "expr": "sum(container_cpu_load_average_10s{pod=~\"$service.*\", namespace=\"$namespace\"}) by (pod_name) * 100", "format": "time_series", "hide": false, "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{ pod_name }}", "refId": "B", @@ -685,13 +758,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 28 + "y": 29 }, + "hiddenSeries": false, "id": 5, "legend": { "avg": false, @@ -708,8 +783,12 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": true, + "pluginVersion": "8.2.3", "pointradius": 5, "points": false, "renderer": "flot", @@ -728,7 +807,8 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=~\"$service.*\", namespace=\"$namespace\"}) by (pod_name) / (1024 * 1024)", + "exemplar": true, + "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", pod=~\"$service.*\", namespace=\"$namespace\"}) by (pod_name) / (1024 * 1024)", "format": "time_series", "hide": false, "interval": "", @@ -737,18 +817,6 @@ "refId": "A", "step": 2400, "target": "alias(scale(averageSeries(gundeck.*.eu-west-1.compute.internal.memory.memory.used), 0.000001), 'used')" - }, - { - "expr": "sum(container_spec_memory_limit_bytes{namespace=\"staging\", pod_name=~\"brig.*\", namespace=\"$namespace\"}) / (1024 * 1024)", - "format": "time_series", - "hide": true, - "instant": false, - "intervalFactor": 2, - "legendFormat": "Memory Limit (MB)", - "metric": "", - "refId": "C", - "step": 2400, - "target": "alias(scale(averageSeries(gundeck.*.eu-west-1.compute.internal.memory.memory.used), 0.000001), 'used')" } ], "thresholds": [], @@ -799,12 +867,14 @@ "dashes": false, "datasource": "default", "fill": 2, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 35 }, + "hiddenSeries": false, "id": 25, "legend": { "avg": false, @@ -819,8 +889,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "8.2.3", "pointradius": 2, "points": false, "renderer": "flot", @@ -880,7 +954,7 @@ } ], "refresh": false, - "schemaVersion": 18, + "schemaVersion": 31, "style": "dark", "tags": [], "templating": { @@ -888,10 +962,12 @@ { "allValue": null, "current": { - "tags": [], + "selected": true, "text": "brig", "value": "brig" }, + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "service", @@ -899,7 +975,7 @@ "name": "service", "options": [ { - "selected": false, + "selected": true, "text": "brig", "value": "brig" }, @@ -929,36 +1005,47 @@ "value": "spar" }, { - "selected": true, + "selected": false, "text": "proxy", "value": "proxy" + }, + { + "selected": false, + "text": "federator", + "value": "federator" } ], - "query": "brig,galley,cannon,gundeck,cargohold,spar,proxy", + "query": "brig,galley,cannon,gundeck,cargohold,spar,proxy,federator", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": null, "current": { - "text": "staging", - "value": "staging" + "selected": true, + "text": "wire-a", + "value": "wire-a" }, "datasource": "Prometheus", "definition": "kube_namespace_labels", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [], - "query": "kube_namespace_labels", + "query": { + "query": "kube_namespace_labels", + "refId": "Prometheus-namespace-Variable-Query" + }, "refresh": 1, "regex": "/namespace=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -966,7 +1053,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-30m", "to": "now" }, "timepicker": { @@ -1003,5 +1090,5 @@ "timezone": "browser", "title": "Wire Services", "uid": "000000043", - "version": 17 -} \ No newline at end of file + "version": 1 +} diff --git a/charts/wire-server-metrics/requirements.yaml b/charts/wire-server-metrics/requirements.yaml index 90617e0eab2..5b7925e3e14 100644 --- a/charts/wire-server-metrics/requirements.yaml +++ b/charts/wire-server-metrics/requirements.yaml @@ -1,6 +1,6 @@ --- dependencies: - - name: prometheus-operator - version: 6.7.2 - repository: https://charts.helm.sh/stable + - name: kube-prometheus-stack + version: 19.2.3 + repository: https://prometheus-community.github.io/helm-charts diff --git a/charts/wire-server-metrics/templates/wire-server-dashboards.yaml b/charts/wire-server-metrics/templates/wire-server-dashboards.yaml new file mode 100644 index 00000000000..6be08a4def2 --- /dev/null +++ b/charts/wire-server-metrics/templates/wire-server-dashboards.yaml @@ -0,0 +1,23 @@ +{{- $files := .Files.Glob "dashboards/*.json" }} +{{- $relName := .Release.Name }} +{{- $namespace := .Release.Namespace }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" $relName $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $namespace }} + labels: + {{- if index $.Values "kube-prometheus-stack" "grafana" "sidecar" "dashboards" "label" }} + {{ index $.Values "kube-prometheus-stack" "grafana" "sidecar" "dashboards" "label" }}: "1" + {{- end }} + app: {{ $.Release.Namespace }}-grafana + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/charts/wire-server-metrics/values.yaml b/charts/wire-server-metrics/values.yaml index 37cbc3785d1..44bac846a35 100644 --- a/charts/wire-server-metrics/values.yaml +++ b/charts/wire-server-metrics/values.yaml @@ -1,4 +1,4 @@ -prometheus-operator: +kube-prometheus-stack: prometheus: additionalServiceMonitors: - name: wire-services @@ -38,29 +38,9 @@ prometheus-operator: enabled: true accessModes: ["ReadWriteOnce"] size: 10Gi - - dashboardProviders: - dashboardproviders.yaml: - apiVersion: 1 - providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards/default - - dashboards: - default: - # grafana can only access files from within its own chart directory - # which we don't have access to here; we can either dump the json - # directly into the values file, or load from a url - messages: - url: https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/charts/wire-server-metrics/dashboards/message-stats.json - services: - url: https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/charts/wire-server-metrics/dashboards/services.json + sidecar: + dashboards: + label: grafana_dashboard prometheusSpec: storageSpec: