From 18c2032df8404853b21b20557728e45fe78f0735 Mon Sep 17 00:00:00 2001
From: Kyle Eckhart <kgeckhart@users.noreply.github.com>
Date: Wed, 1 Sep 2021 16:57:29 -0400
Subject: [PATCH 1/4] Add namespace to some queries which were missing it

---
 production/grafana-agent-mixin/debugging.libsonnet | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/production/grafana-agent-mixin/debugging.libsonnet b/production/grafana-agent-mixin/debugging.libsonnet
index 52c12f5a0367..df992f1dacad 100644
--- a/production/grafana-agent-mixin/debugging.libsonnet
+++ b/production/grafana-agent-mixin/debugging.libsonnet
@@ -37,14 +37,14 @@ local utils = import './utils.libsonnet';
         .addPanel(
           g.panel('CPU') +
           g.queryPanel(
-            'rate(container_cpu_usage_seconds_total{cluster=~"$cluster", container=~"$container", pod=~"$pod"}[5m])',
+            'rate(container_cpu_usage_seconds_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"}[5m])',
             '{{pod}}',
           )
         )
         .addPanel(
           g.panel('WSS') +
           g.queryPanel(
-            'container_memory_working_set_bytes{cluster=~"$cluster", container=~"$container", pod=~"$pod"}',
+            'container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container=~"$container", pod=~"$pod"}',
             '{{pod}}',
           )
         )

From 65d84c6e07905c491b57bb6dfb7a16d927a59c44 Mon Sep 17 00:00:00 2001
From: Kyle Eckhart <kgeckhart@users.noreply.github.com>
Date: Wed, 1 Sep 2021 17:00:54 -0400
Subject: [PATCH 2/4] Move samples queries to samples row, update to current
 metric names, and allow some graphs to work with multiple instance groups

---
 .../grafana-agent-mixin/dashboards.libsonnet  | 50 ++++++-------------
 1 file changed, 15 insertions(+), 35 deletions(-)

diff --git a/production/grafana-agent-mixin/dashboards.libsonnet b/production/grafana-agent-mixin/dashboards.libsonnet
index 81b9957ffbc7..b1c1244e891f 100644
--- a/production/grafana-agent-mixin/dashboards.libsonnet
+++ b/production/grafana-agent-mixin/dashboards.libsonnet
@@ -76,7 +76,7 @@ local template = grafana.template;
         )
         .addPanel(
           g.panel('Appended Samples') +
-          g.queryPanel('sum by (job, instance_group_name) (rate(agent_wal_storage_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]))', '{{job}} {{instance_group_name}}') +
+          g.queryPanel('sum by (job, instance_group_name) (rate(agent_wal_storage_created_series_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]))', '{{job}} {{instance_group_name}}') +
           g.stack
         )
       ),
@@ -94,7 +94,7 @@ local template = grafana.template;
             (
               prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}
               -
-              ignoring(url, instance_group_name, remote_name) group_right(pod)
+              ignoring(url, remote_name) group_right(pod)
               prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}
             )
           |||,
@@ -112,32 +112,13 @@ local template = grafana.template;
             (
               rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])
               -
-              ignoring(url, instance_group_name, remote_name) group_right(pod)
+              ignoring(url, remote_name) group_right(pod)
               rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])
             )
           |||,
           legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
         ));
 
-      local samplesRate =
-        graphPanel.new(
-          'Rate, in vs. succeeded or dropped [5m]',
-          datasource='$datasource',
-          span=12,
-        )
-        .addTarget(prometheus.target(
-          |||
-            rate(
-              prometheus_remote_storage_samples_in_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])
-            -
-              ignoring(url, instance_group_name, remote_name) group_right(pod)
-              rate(prometheus_remote_storage_succeeded_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])
-            -
-              rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])
-          |||,
-          legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
-        ));
-
       local currentShards =
         graphPanel.new(
           'Current Shards',
@@ -201,7 +182,7 @@ local template = grafana.template;
           span=6,
         )
         .addTarget(prometheus.target(
-          'prometheus_remote_storage_pending_samples{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}',
+          'prometheus_remote_storage_samples_pending{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}',
           legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
         ));
 
@@ -221,10 +202,10 @@ local template = grafana.template;
         graphPanel.new(
           'Dropped Samples',
           datasource='$datasource',
-          span=3,
+          span=6,
         )
         .addTarget(prometheus.target(
-          'rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
+          'rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
           legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
         ));
 
@@ -232,10 +213,10 @@ local template = grafana.template;
         graphPanel.new(
           'Failed Samples',
           datasource='$datasource',
-          span=3,
+          span=6,
         )
         .addTarget(prometheus.target(
-          'rate(prometheus_remote_storage_failed_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
+          'rate(prometheus_remote_storage_samples_failed_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
           legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
         ));
 
@@ -243,10 +224,10 @@ local template = grafana.template;
         graphPanel.new(
           'Retried Samples',
           datasource='$datasource',
-          span=3,
+          span=6,
         )
         .addTarget(prometheus.target(
-          'rate(prometheus_remote_storage_retried_samples_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
+          'rate(prometheus_remote_storage_samples_retried_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
           legendFormat='{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}',
         ));
 
@@ -254,7 +235,7 @@ local template = grafana.template;
         graphPanel.new(
           'Enqueue Retries',
           datasource='$datasource',
-          span=3,
+          span=6,
         )
         .addTarget(prometheus.target(
           'rate(prometheus_remote_storage_enqueue_retries_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m])',
@@ -346,7 +327,10 @@ local template = grafana.template;
       )
       .addRow(
         row.new('Samples')
-        .addPanel(samplesRate)
+        .addPanel(pendingSamples)
+        .addPanel(droppedSamples)
+        .addPanel(failedSamples)
+        .addPanel(retriedSamples)
       )
       .addRow(
         row.new('Shards')
@@ -358,7 +342,6 @@ local template = grafana.template;
       .addRow(
         row.new('Shard Details')
         .addPanel(shardsCapacity)
-        .addPanel(pendingSamples)
       )
       .addRow(
         row.new('Segments')
@@ -366,9 +349,6 @@ local template = grafana.template;
       )
       .addRow(
         row.new('Misc. Rates')
-        .addPanel(droppedSamples)
-        .addPanel(failedSamples)
-        .addPanel(retriedSamples)
         .addPanel(enqueueRetries)
       ),
   },

From 9000f734234c699f17513f954f6422e7673ab8d7 Mon Sep 17 00:00:00 2001
From: Kyle Eckhart <kgeckhart@users.noreply.github.com>
Date: Wed, 1 Sep 2021 17:01:30 -0400
Subject: [PATCH 3/4] Update example dashboards with changes

---
 .../grafana/dashboards/agent-operational.json |   8 +-
 .../dashboards/agent-remote-write.json        | 247 ++++++------------
 .../grafana/dashboards/agent.json             |  14 +-
 3 files changed, 98 insertions(+), 171 deletions(-)

diff --git a/example/docker-compose/grafana/dashboards/agent-operational.json b/example/docker-compose/grafana/dashboards/agent-operational.json
index d68a7012d103..11ae541ce9da 100644
--- a/example/docker-compose/grafana/dashboards/agent-operational.json
+++ b/example/docker-compose/grafana/dashboards/agent-operational.json
@@ -273,7 +273,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", container=~\"$container\", pod=~\"$pod\"}[5m])",
+                     "expr": "rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[5m])",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{pod}}",
@@ -349,7 +349,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "container_memory_working_set_bytes{cluster=~\"$cluster\", container=~\"$container\", pod=~\"$pod\"}",
+                     "expr": "container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\", pod=~\"$pod\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{pod}}",
@@ -1036,7 +1036,9 @@
    ],
    "schemaVersion": 14,
    "style": "dark",
-   "tags": [ ],
+   "tags": [
+      "grafana-agent-mixin"
+   ],
    "templating": {
       "list": [
          {
diff --git a/example/docker-compose/grafana/dashboards/agent-remote-write.json b/example/docker-compose/grafana/dashboards/agent-remote-write.json
index 69609c8716a9..8c86c1ababd5 100644
--- a/example/docker-compose/grafana/dashboards/agent-remote-write.json
+++ b/example/docker-compose/grafana/dashboards/agent-remote-write.json
@@ -52,7 +52,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "(\n  prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}\n  -\n  ignoring(url, instance_group_name, remote_name) group_right(pod)\n  prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}\n)\n",
+                     "expr": "(\n  prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}\n  -\n  ignoring(url, remote_name) group_right(pod)\n  prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}\n)\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -131,7 +131,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "(\n  rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n  -\n  ignoring(url, instance_group_name, remote_name) group_right(pod)\n  rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n)\n",
+                     "expr": "(\n  rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n  -\n  ignoring(url, remote_name) group_right(pod)\n  rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n)\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -218,12 +218,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 12,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "rate(\n  prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n-\n  ignoring(url, instance_group_name, remote_name) group_right(pod)\n  rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n-\n  rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])\n",
+                     "expr": "prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -233,7 +233,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Rate, in vs. succeeded or dropped [5m]",
+               "title": "Pending Samples",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -265,20 +265,7 @@
                      "show": true
                   }
                ]
-            }
-         ],
-         "repeat": null,
-         "repeatIteration": null,
-         "repeatRowId": null,
-         "showTitle": true,
-         "title": "Samples",
-         "titleSize": "h6",
-         "type": "row"
-      },
-      {
-         "collapse": false,
-         "collapsed": false,
-         "panels": [
+            },
             {
                "aliasColors": { },
                "bars": false,
@@ -302,7 +289,6 @@
                "lines": true,
                "linewidth": 1,
                "links": [ ],
-               "minSpan": 6,
                "nullPointMode": "null",
                "percentage": false,
                "pointradius": 5,
@@ -311,12 +297,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 12,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -326,7 +312,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Current Shards",
+               "title": "Dropped Samples",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -390,12 +376,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 4,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -405,7 +391,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Max Shards",
+               "title": "Failed Samples",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -469,12 +455,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 4,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -484,7 +470,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Min Shards",
+               "title": "Retried Samples",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -516,7 +502,20 @@
                      "show": true
                   }
                ]
-            },
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Samples",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "panels": [
             {
                "aliasColors": { },
                "bars": false,
@@ -540,6 +539,7 @@
                "lines": true,
                "linewidth": 1,
                "links": [ ],
+               "minSpan": 6,
                "nullPointMode": "null",
                "percentage": false,
                "pointradius": 5,
@@ -548,12 +548,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 4,
+               "span": 12,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -563,7 +563,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Desired Shards",
+               "title": "Current Shards",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -595,20 +595,7 @@
                      "show": true
                   }
                ]
-            }
-         ],
-         "repeat": null,
-         "repeatIteration": null,
-         "repeatRowId": null,
-         "showTitle": true,
-         "title": "Shards",
-         "titleSize": "h6",
-         "type": "row"
-      },
-      {
-         "collapse": false,
-         "collapsed": false,
-         "panels": [
+            },
             {
                "aliasColors": { },
                "bars": false,
@@ -640,12 +627,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 6,
+               "span": 4,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -655,7 +642,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Shard Capacity",
+               "title": "Max Shards",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -719,12 +706,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 6,
+               "span": 4,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -734,7 +721,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Pending Samples",
+               "title": "Min Shards",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -766,20 +753,7 @@
                      "show": true
                   }
                ]
-            }
-         ],
-         "repeat": null,
-         "repeatIteration": null,
-         "repeatRowId": null,
-         "showTitle": true,
-         "title": "Shard Details",
-         "titleSize": "h6",
-         "type": "row"
-      },
-      {
-         "collapse": false,
-         "collapsed": false,
-         "panels": [
+            },
             {
                "aliasColors": { },
                "bars": false,
@@ -811,12 +785,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 6,
+               "span": 4,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
+                     "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -826,7 +800,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Remote Write Current Segment",
+               "title": "Desired Shards",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -842,7 +816,7 @@
                },
                "yaxes": [
                   {
-                     "format": "none",
+                     "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
@@ -864,7 +838,7 @@
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
-         "title": "Segments",
+         "title": "Shards",
          "titleSize": "h6",
          "type": "row"
       },
@@ -903,12 +877,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 3,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
+                     "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -918,7 +892,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Dropped Samples",
+               "title": "Shard Capacity",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -950,7 +924,20 @@
                      "show": true
                   }
                ]
-            },
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Shard Details",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "panels": [
             {
                "aliasColors": { },
                "bars": false,
@@ -982,12 +969,12 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 3,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
+                     "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
@@ -997,7 +984,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Failed Samples",
+               "title": "Remote Write Current Segment",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
@@ -1013,7 +1000,7 @@
                },
                "yaxes": [
                   {
-                     "format": "short",
+                     "format": "none",
                      "label": null,
                      "logBase": 1,
                      "max": null,
@@ -1029,7 +1016,20 @@
                      "show": true
                   }
                ]
-            },
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Segments",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "panels": [
             {
                "aliasColors": { },
                "bars": false,
@@ -1061,86 +1061,7 @@
                "repeat": null,
                "seriesOverrides": [ ],
                "spaceLength": 10,
-               "span": 3,
-               "stack": false,
-               "steppedLine": false,
-               "targets": [
-                  {
-                     "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m])",
-                     "format": "time_series",
-                     "intervalFactor": 2,
-                     "legendFormat": "{{cluster}}:{{pod}}-{{instance_group_name}}-{{url}}",
-                     "refId": "A"
-                  }
-               ],
-               "thresholds": [ ],
-               "timeFrom": null,
-               "timeShift": null,
-               "title": "Retried Samples",
-               "tooltip": {
-                  "shared": true,
-                  "sort": 0,
-                  "value_type": "individual"
-               },
-               "type": "graph",
-               "xaxis": {
-                  "buckets": null,
-                  "mode": "time",
-                  "name": null,
-                  "show": true,
-                  "values": [ ]
-               },
-               "yaxes": [
-                  {
-                     "format": "short",
-                     "label": null,
-                     "logBase": 1,
-                     "max": null,
-                     "min": null,
-                     "show": true
-                  },
-                  {
-                     "format": "short",
-                     "label": null,
-                     "logBase": 1,
-                     "max": null,
-                     "min": null,
-                     "show": true
-                  }
-               ]
-            },
-            {
-               "aliasColors": { },
-               "bars": false,
-               "dashLength": 10,
-               "dashes": false,
-               "datasource": "$datasource",
-               "fill": 1,
-               "gridPos": { },
-               "id": 15,
-               "legend": {
-                  "alignAsTable": false,
-                  "avg": false,
-                  "current": false,
-                  "max": false,
-                  "min": false,
-                  "rightSide": false,
-                  "show": true,
-                  "total": false,
-                  "values": false
-               },
-               "lines": true,
-               "linewidth": 1,
-               "links": [ ],
-               "nullPointMode": "null",
-               "percentage": false,
-               "pointradius": 5,
-               "points": false,
-               "renderer": "flot",
-               "repeat": null,
-               "seriesOverrides": [ ],
-               "spaceLength": 10,
-               "span": 3,
+               "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
@@ -1200,7 +1121,9 @@
    ],
    "schemaVersion": 14,
    "style": "dark",
-   "tags": [ ],
+   "tags": [
+      "grafana-agent-mixin"
+   ],
    "templating": {
       "list": [
          {
diff --git a/example/docker-compose/grafana/dashboards/agent.json b/example/docker-compose/grafana/dashboards/agent.json
index 69a6408fc8ae..520f6507544f 100644
--- a/example/docker-compose/grafana/dashboards/agent.json
+++ b/example/docker-compose/grafana/dashboards/agent.json
@@ -473,7 +473,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))",
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "exceeded sample limit: {{job}}",
@@ -481,7 +481,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))",
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "duplicate timestamp: {{job}}",
@@ -489,7 +489,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))",
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "out of bounds: {{job}}",
@@ -497,7 +497,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))",
+                     "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[1m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "out of order: {{job}}",
@@ -573,7 +573,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum by (job, instance_group_name) (rate(agent_wal_storage_samples_appended_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))",
+                     "expr": "sum by (job, instance_group_name) (rate(agent_wal_storage_created_series_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\"}[5m]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{job}} {{instance_group_name}}",
@@ -628,7 +628,9 @@
    ],
    "schemaVersion": 14,
    "style": "dark",
-   "tags": [ ],
+   "tags": [
+      "grafana-agent-mixin"
+   ],
    "templating": {
       "list": [
          {

From cb2876d06085eddb16f2a7d627f930da49f0f660 Mon Sep 17 00:00:00 2001
From: Kyle Eckhart <kgeckhart@users.noreply.github.com>
Date: Wed, 1 Sep 2021 17:04:31 -0400
Subject: [PATCH 4/4] Rename panel to match metric

---
 example/docker-compose/grafana/dashboards/agent.json | 2 +-
 production/grafana-agent-mixin/dashboards.libsonnet  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/example/docker-compose/grafana/dashboards/agent.json b/example/docker-compose/grafana/dashboards/agent.json
index 520f6507544f..a7f95ea97543 100644
--- a/example/docker-compose/grafana/dashboards/agent.json
+++ b/example/docker-compose/grafana/dashboards/agent.json
@@ -584,7 +584,7 @@
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
-               "title": "Appended Samples",
+               "title": "Created Series",
                "tooltip": {
                   "shared": true,
                   "sort": 0,
diff --git a/production/grafana-agent-mixin/dashboards.libsonnet b/production/grafana-agent-mixin/dashboards.libsonnet
index b1c1244e891f..34470094d2e8 100644
--- a/production/grafana-agent-mixin/dashboards.libsonnet
+++ b/production/grafana-agent-mixin/dashboards.libsonnet
@@ -75,7 +75,7 @@ local template = grafana.template;
           g.stack
         )
         .addPanel(
-          g.panel('Appended Samples') +
+          g.panel('Created Series') +
           g.queryPanel('sum by (job, instance_group_name) (rate(agent_wal_storage_created_series_total{cluster=~"$cluster", namespace=~"$namespace", container=~"$container"}[5m]))', '{{job}} {{instance_group_name}}') +
           g.stack
         )