grafana · pracucci · May 14, 2020 · May 8, 2020
diff --git a/cortex-mixin/dashboards.libsonnet b/cortex-mixin/dashboards.libsonnet
@@ -7,7 +7,9 @@
     (import 'dashboards/writes.libsonnet') +
 
     (if std.setMember('tsdb', $._config.storage_engine)
-     then import 'dashboards/compactor.libsonnet'
+     then
+       (import 'dashboards/compactor.libsonnet') +
+       (import 'dashboards/object-store.libsonnet')
      else {}) +
 
     (if std.setMember('chunks', $._config.storage_engine)

diff --git a/cortex-mixin/dashboards/compactor.libsonnet b/cortex-mixin/dashboards/compactor.libsonnet
@@ -84,6 +84,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher('compactor')),
       )
     )
-    .addRow($.objectStorePanels1('Object Store', 'cortex_compactor'))
-    .addRow($.objectStorePanels2('', 'cortex_compactor')),
+    .addRow($.objectStorePanels1('Object Store', 'compactor'))
+    .addRow($.objectStorePanels2('', 'compactor')),
 }
diff --git a/cortex-mixin/dashboards/dashboard-utils.libsonnet b/cortex-mixin/dashboards/dashboard-utils.libsonnet
@@ -123,51 +123,45 @@ local utils = import 'mixin-utils/utils.libsonnet';
     type: 'text',
   } + options,
 
-  objectStorePanels1(title, metricPrefix)::
-    local opsTotal = '%s_thanos_objstore_bucket_operations_total' % [metricPrefix];
-    local opsTotalFailures = '%s_thanos_objstore_bucket_operation_failures_total' % [metricPrefix];
-    local operationDuration = '%s_thanos_objstore_bucket_operation_duration_seconds' % [metricPrefix];
+  objectStorePanels1(title, component)::
     super.row(title)
     .addPanel(
-      // We use 'up' to add 0 if there are no failed operations.
-      self.successFailurePanel(
-        'Operations/sec',
-        'sum(rate(%s{%s}[$__interval])) - sum(rate(%s{%s}[$__interval]) or (up{%s}*0))' % [opsTotal, $.namespaceMatcher(), opsTotalFailures, $.namespaceMatcher(), $.namespaceMatcher()],
-        'sum(rate(%s{%s}[$__interval]) or (up{%s}*0))' % [opsTotalFailures, $.namespaceMatcher(), $.namespaceMatcher()]
-      )
+      $.panel('Operations / sec') +
+      $.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s,component="%s"}[$__interval]))' % [$.namespaceMatcher(), component], '{{operation}}') +
+      $.stack +
+      { yaxes: $.yaxes('rps') },
     )
     .addPanel(
-      $.panel('Op: ObjectSize') +
-      $.latencyPanel(operationDuration, '{%s, operation="objectsize"}' % $.namespaceMatcher()),
+      $.panel('Error rate') +
+      $.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{%s,component="%s"}[$__interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s,component="%s"}[$__interval]))' % [$.namespaceMatcher(), component, $.namespaceMatcher(), component], '{{operation}}') +
+      { yaxes: $.yaxes('percentunit') },
     )
     .addPanel(
-      // Cortex (Thanos) doesn't track timing for 'iter', so we use ops/sec instead.
-      $.panel('Op: Iter') +
-      $.queryPanel('sum(rate(%s{%s, operation="iter"}[$__interval]))' % [opsTotal, $.namespaceMatcher()], 'ops/sec')
+      $.panel('Op: ObjectSize') +
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="objectsize"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
       $.panel('Op: Exists') +
-      $.latencyPanel(operationDuration, '{%s, operation="exists"}' % $.namespaceMatcher()),
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="exists"}' % [$.namespaceMatcher(), component]),
     ),
 
   // Second row of Object Store stats
-  objectStorePanels2(title, metricPrefix)::
-    local operationDuration = '%s_thanos_objstore_bucket_operation_duration_seconds' % [metricPrefix];
+  objectStorePanels2(title, component)::
     super.row(title)
     .addPanel(
       $.panel('Op: Get') +
-      $.latencyPanel(operationDuration, '{%s, operation="get"}' % $.namespaceMatcher()),
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
       $.panel('Op: GetRange') +
-      $.latencyPanel(operationDuration, '{%s, operation="get_range"}' % $.namespaceMatcher()),
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get_range"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
       $.panel('Op: Upload') +
-      $.latencyPanel(operationDuration, '{%s, operation="upload"}' % $.namespaceMatcher()),
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="upload"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
       $.panel('Op: Delete') +
-      $.latencyPanel(operationDuration, '{%s, operation="delete"}' % $.namespaceMatcher()),
+      $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="delete"}' % [$.namespaceMatcher(), component]),
     ),
 }
diff --git a/cortex-mixin/dashboards/object-store.libsonnet b/cortex-mixin/dashboards/object-store.libsonnet
@@ -0,0 +1,65 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+  'cortex-object-store.json':
+    $.dashboard('Cortex / Object Store')
+    .addClusterSelectorTemplates()
+    .addRow(
+      $.row('Components')
+      .addPanel(
+        $.panel('RPS / component') +
+        $.queryPanel('sum by(component) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % $.namespaceMatcher(), '{{component}}') +
+        $.stack +
+        { yaxes: $.yaxes('rps') },
+      )
+      .addPanel(
+        $.panel('Error rate / component') +
+        $.queryPanel('sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{%s}[$__interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()], '{{component}}') +
+        { yaxes: $.yaxes('percentunit') },
+      )
+    )
+    .addRow(
+      $.row('Operations')
+      .addPanel(
+        $.panel('RPS / operation') +
+        $.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % $.namespaceMatcher(), '{{operation}}') +
+        $.stack +
+        { yaxes: $.yaxes('rps') },
+      )
+      .addPanel(
+        $.panel('Error rate / operation') +
+        $.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{%s}[$__interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()], '{{operation}}') +
+        { yaxes: $.yaxes('percentunit') },
+      )
+    )
+    .addRow(
+      $.row('')
+      .addPanel(
+        $.panel('Op: Get') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="get"}' % $.namespaceMatcher()),
+      )
+      .addPanel(
+        $.panel('Op: GetRange') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="get_range"}' % $.namespaceMatcher()),
+      )
+      .addPanel(
+        $.panel('Op: Exists') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="exists"}' % $.namespaceMatcher()),
+      )
+    )
+    .addRow(
+      $.row('')
+      .addPanel(
+        $.panel('Op: ObjectSize') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="objectsize"}' % $.namespaceMatcher()),
+      )
+      .addPanel(
+        $.panel('Op: Upload') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="upload"}' % $.namespaceMatcher()),
+      )
+      .addPanel(
+        $.panel('Op: Delete') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="delete"}' % $.namespaceMatcher()),
+      )
+    ),
+}
diff --git a/cortex-mixin/dashboards/reads.libsonnet b/cortex-mixin/dashboards/reads.libsonnet
@@ -169,19 +169,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
     // Object store metrics for the store-gateway.
     .addRowIf(
       std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels1('Store-gateway - Blocks Object Store', 'cortex_storegateway'),
+      $.objectStorePanels1('Store-gateway - Blocks Object Store', 'store-gateway'),
     )
     .addRowIf(
       std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels2('', 'cortex_storegateway'),
+      $.objectStorePanels2('', 'store-gateway'),
     )
     // Object store metrics for the querier.
     .addRowIf(
       std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels1('Querier - Blocks Object Store', 'cortex_querier'),
+      $.objectStorePanels1('Querier - Blocks Object Store', 'querier'),
     )
     .addRowIf(
       std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels2('', 'cortex_querier'),
+      $.objectStorePanels2('', 'querier'),
     ),
 }
diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
@@ -151,21 +151,17 @@ local utils = import 'mixin-utils/utils.libsonnet';
     )
     .addRowIf(
       std.setMember('tsdb', $._config.storage_engine),
-      $.row('Blocks Shipper')
+      $.row('Ingester - Blocks storage - Shipper')
       .addPanel(
         $.successFailurePanel(
           'Uploaded blocks / sec',
-          'sum(rate(cortex_ingester_shipper_uploads_total{%s}[$__interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()],
-          'sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.namespaceMatcher()],
+          'sum(rate(cortex_ingester_shipper_uploads_total{%s}[$__interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.jobMatcher('ingester'), $.jobMatcher('ingester')],
+          'sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % $.jobMatcher('ingester'),
         ),
       )
-    )
-    .addRowIf(
-      std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels1('Blocks Object Store Stats (Ingester)', 'cortex_ingester'),
-    )
-    .addRowIf(
-      std.setMember('tsdb', $._config.storage_engine),
-      $.objectStorePanels2('', 'cortex_ingester'),
+      .addPanel(
+        $.panel('Upload latency') +
+        $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="ingester",operation="upload"}' % $.jobMatcher('ingester')),
+      )
     ),
 }