Merge pull request #145 from grafana/add-tsdb-metrics-to-write-dashboard

Added TSDB metrics to Cortex / Writes dashboard

Author: pracucci

cortex-mixin/dashboards/writes.libsonnet

@@ -163,5 +163,60 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Upload latency') +
         $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="ingester",operation="upload"}' % $.jobMatcher($._config.job_names.ingester)),
       )
+    )
+    .addRowIf(
+      std.setMember('tsdb', $._config.storage_engine),
+      $.row('Ingester - Blocks storage - TSDB Head')
+      .addPanel(
+        $.successFailurePanel(
+          'Compactions / sec',
+          'sum(rate(cortex_ingester_tsdb_compactions_total{%s}[$__interval]))' % [$.jobMatcher($._config.job_names.ingester)],
+          'sum(rate(cortex_ingester_tsdb_compactions_failed_total{%s}[$__interval]))' % $.jobMatcher($._config.job_names.ingester),
+        ),
+      )
+      .addPanel(
+        $.panel('Compactions latency') +
+        $.latencyPanel('cortex_ingester_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.ingester)),
+      )
+    )
+    .addRowIf(
+      std.setMember('tsdb', $._config.storage_engine),
+      $.row('Ingester - Blocks storage - TSDB WAL')
+      .addPanel(
+        $.successFailurePanel(
+          'WAL truncations / sec',
+          'sum(rate(cortex_ingester_tsdb_wal_truncations_total{%s}[$__interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{%s}[$__interval]))' % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester)],
+          'sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{%s}[$__interval]))' % $.jobMatcher($._config.job_names.ingester),
+        ),
+      )
+      .addPanel(
+        $.successFailurePanel(
+          'Checkpoints created / sec',
+          'sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{%s}[$__interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{%s}[$__interval]))' % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester)],
+          'sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{%s}[$__interval]))' % $.jobMatcher($._config.job_names.ingester),
+        ),
+      )
+      .addPanel(
+        $.panel('WAL truncations latency (includes checkpointing)') +
+        $.queryPanel('sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{%s}[$__interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{%s}[$__interval]))' % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester)], 'avg') +
+        { yaxes: $.yaxes('s') },
+      )
+      .addPanel(
+        $.panel('Corruptions / sec') +
+        $.queryPanel([
+          'sum(rate(cortex_ingester_wal_corruptions_total{%s}[$__interval]))' % $.jobMatcher($._config.job_names.ingester),
+          'sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{%s}[$__interval]))' % $.jobMatcher($._config.job_names.ingester),
+        ], [
+          'WAL',
+          'mmap-ed chunks',
+        ]) +
+        $.stack + {
+          yaxes: $.yaxes('ops'),
+          aliasColors: {
+            WAL: '#E24D42',
+            'mmap-ed chunks': '#E28A42',
+          },
+        },
+      )
     ),
 }