diff --git a/example/docker-compose/grafana/dashboards/agent-operational.json b/example/docker-compose/grafana/dashboards/agent-operational.json index 11ae541ce9da..45b1c24f83ac 100644 --- a/example/docker-compose/grafana/dashboards/agent-operational.json +++ b/example/docker-compose/grafana/dashboards/agent-operational.json @@ -7,7 +7,7 @@ "graphTooltip": 0, "hideControls": false, "links": [ ], - "refresh": "10s", + "refresh": "1m", "rows": [ { "collapse": false, @@ -117,7 +117,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { @@ -673,7 +673,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { @@ -749,7 +749,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { @@ -825,7 +825,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { @@ -901,7 +901,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { @@ -977,7 +977,7 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 2, - "stack": "true", + "stack": false, "steppedLine": false, "targets": [ { diff --git a/example/docker-compose/grafana/dashboards/agent-remote-write.json b/example/docker-compose/grafana/dashboards/agent-remote-write.json index 8c86c1ababd5..feda23fd3bd3 100644 --- a/example/docker-compose/grafana/dashboards/agent-remote-write.json +++ b/example/docker-compose/grafana/dashboards/agent-remote-write.json @@ -10,7 +10,7 @@ "hideControls": false, "id": null, "links": [ ], - "refresh": "", + "refresh": "1m", "rows": [ { "collapse": false, @@ -218,6 +218,85 @@ "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(agent_wal_samples_appended_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n-\n ignoring(remote_name, url) group_right(pod)\n (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))\n-\n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, @@ -274,7 +353,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 5, + "id": 6, "legend": { "alignAsTable": false, "avg": false, @@ -353,7 +432,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 6, + "id": 7, "legend": { "alignAsTable": false, "avg": false, @@ -432,7 +511,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 7, + "id": 8, "legend": { "alignAsTable": false, "avg": false, @@ -524,7 +603,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 8, + "id": 9, "legend": { "alignAsTable": false, "avg": false, @@ -604,7 +683,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 9, + "id": 10, "legend": { "alignAsTable": false, "avg": false, @@ -683,7 +762,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 10, + "id": 11, "legend": { "alignAsTable": false, "avg": false, @@ -762,7 +841,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 11, + "id": 12, "legend": { "alignAsTable": false, "avg": false, @@ -854,7 +933,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 12, + "id": 13, "legend": { "alignAsTable": false, "avg": false, @@ -946,7 +1025,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 13, + "id": 14, "legend": { "alignAsTable": false, "avg": false, @@ -1038,7 +1117,7 @@ "datasource": "$datasource", "fill": 1, "gridPos": { }, - "id": 14, + "id": 15, "legend": { "alignAsTable": false, "avg": false, @@ -1283,7 +1362,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/example/docker-compose/grafana/dashboards/agent.json b/example/docker-compose/grafana/dashboards/agent.json index a7f95ea97543..eff115f10737 100644 --- a/example/docker-compose/grafana/dashboards/agent.json +++ b/example/docker-compose/grafana/dashboards/agent.json @@ -7,7 +7,7 @@ "graphTooltip": 0, "hideControls": false, "links": [ ], - "refresh": "10s", + "refresh": "1m", "rows": [ { "collapse": false, @@ -309,10 +309,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(prometheus_sd_discovered_targets{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"})", + "expr": "sum by (pod) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Targets", + "legendFormat": "{{pod}}", "legendLink": null, "step": 10 } @@ -573,7 +573,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (job, instance_group_name) (rate(agent_wal_storage_created_series_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))", + "expr": "sum by (job, instance_group_name) (rate(agent_wal_samples_appended_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{instance_group_name}}", @@ -584,7 +584,7 @@ "thresholds": [ ], "timeFrom": null, "timeShift": null, - "title": "Created Series", + "title": "Appended Samples", "tooltip": { "shared": true, "sort": 0, diff --git a/production/grafana-agent-mixin/dashboards.libsonnet b/production/grafana-agent-mixin/dashboards.libsonnet index 34470094d2e8..1b3f57693ac2 100644 --- a/production/grafana-agent-mixin/dashboards.libsonnet +++ b/production/grafana-agent-mixin/dashboards.libsonnet @@ -43,7 +43,7 @@ local template = grafana.template; ) .addPanel( g.panel('Targets') + - g.queryPanel('sum(prometheus_sd_discovered_targets{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"})', 'Targets') + + g.queryPanel('sum by (pod) (prometheus_sd_discovered_targets{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"})', '{{pod}}') + g.stack ) ) @@ -75,8 +75,8 @@ local template = grafana.template; g.stack ) .addPanel( - g.panel('Created Series') + - g.queryPanel('sum by (job, instance_group_name) (rate(agent_wal_storage_created_series_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]))', '{{job}} {{instance_group_name}}') + + g.panel('Appended Samples') + + g.queryPanel('sum by (job, instance_group_name) (rate(agent_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]))', '{{job}} {{instance_group_name}}') + g.stack ) ), @@ -119,6 +119,24 @@ local template = grafana.template; legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}', )); + local samplesRate = + graphPanel.new( + 'Rate, in vs. succeeded or dropped [5m]', + datasource='$datasource', + span=12, + ) + .addTarget(prometheus.target( + ||| + rate(agent_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]) + - + ignoring(remote_name, url) group_right(pod) + (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])) + - + (rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])) + |||, + legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}', + )); + local currentShards = graphPanel.new( 'Current Shards', @@ -242,7 +260,7 @@ local template = grafana.template; legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}', )); - dashboard.new('Agent Prometheus Remote Write', tags=['grafana-agent-mixin'], editable=true) + dashboard.new('Agent Prometheus Remote Write', tags=['grafana-agent-mixin'], editable=true, refresh='30s', time_from='now-1h') .addTemplate( { hide: 0, @@ -327,6 +345,7 @@ local template = grafana.template; ) .addRow( row.new('Samples') + .addPanel(samplesRate) .addPanel(pendingSamples) .addPanel(droppedSamples) .addPanel(failedSamples) diff --git a/production/grafana-agent-mixin/mixin.libsonnet b/production/grafana-agent-mixin/mixin.libsonnet index d7ecd8dd360a..82a9256d0e6e 100644 --- a/production/grafana-agent-mixin/mixin.libsonnet +++ b/production/grafana-agent-mixin/mixin.libsonnet @@ -1,7 +1,8 @@ local dashboards = import 'dashboards.libsonnet'; +local debugging = import 'debugging.libsonnet'; { grafanaDashboards+:: std.mapWithKey(function(field, obj) obj { grafanaDashboardFolder: 'Grafana Agent', - }, dashboards.grafanaDashboards), + }, dashboards.grafanaDashboards + debugging.grafanaDashboards), } diff --git a/production/grafana-agent-mixin/utils.libsonnet b/production/grafana-agent-mixin/utils.libsonnet index 111b7f59e07e..5467553a7d52 100644 --- a/production/grafana-agent-mixin/utils.libsonnet +++ b/production/grafana-agent-mixin/utils.libsonnet @@ -1,6 +1,7 @@ { injectUtils(dashboard):: dashboard { tags: ['grafana-agent-mixin'], + refresh: '30s', addMultiTemplateWithAll(name, metric_name, label_name, all='.*', hide=0):: self { templating+: { list+: [{