Add query-scheduler to dashboards and alerts.

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [CHANGE] Enabled sharding for the blocks storage compactor. #218
 * [ENHANCEMENT] Introduce a resources dashboard for the Alertmanager. #219
 * [ENHANCEMENT] Improves query visibility in the Ruler Dashboard for both chunks and blocks storage. #226
+* [ENHANCEMENT] Add query-scheduler to dashboards. Add alert for queries stuck in scheduler. #228
 
 ## 1.5.0 / 2020-11-12
 

cortex-mixin/alerts/alerts.libsonnet

@@ -157,7 +157,22 @@
           },
           annotations: {
             message: |||
-              There are {{ $value }} queued up queries.
+              There are {{ $value }} queued up queries in query-frontend.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexSchedulerQueriesStuck',
+          expr: |||
+            sum by (%s) (cortex_query_scheduler_queue_length) > 1
+          ||| % $._config.alert_aggregation_labels,
+          'for': '5m',  // We don't want to block for longer.
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              There are {{ $value }} queued up queries in query-scheduler.
             |||,
           },
         },

cortex-mixin/config.libsonnet

@@ -31,6 +31,7 @@
       querier: '(querier|cortex$)',
       ruler: '(ruler|cortex$)',
       query_frontend: '(query-frontend|cortex$)',
+      query_scheduler: 'query-scheduler',  // Not part of single-binary.
       table_manager: '(table-manager|cortex$)',
       store_gateway: '(store-gateway|cortex$)',
       gateway: 'cortex-gw',

cortex-mixin/dashboards/queries.libsonnet

@@ -21,6 +21,17 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.queryPanel('cortex_query_frontend_queue_length{%s}' % $.jobMatcher($._config.job_names.query_frontend), '{{cluster}} / {{namespace}} / {{instance}}'),
       )
     )
+    .addRow(
+      $.row('Query Scheduler')
+      .addPanel(
+        $.panel('Queue Duration') +
+        $.latencyPanel('cortex_query_scheduler_queue_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.query_scheduler)),
+      )
+      .addPanel(
+        $.panel('Queue Length') +
+        $.queryPanel('cortex_query_scheduler_queue_length{%s}' % $.jobMatcher($._config.job_names.query_scheduler), '{{cluster}} / {{namespace}} / {{instance}}'),
+      )
+    )
     .addRow(
       $.row('Query Frontend - Results Cache')
       .addPanel(

cortex-mixin/dashboards/reads-resources.libsonnet

@@ -28,6 +28,18 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.goHeapInUsePanel('Memory (go heap inuse)', 'query-frontend'),
       )
     )
+    .addRow(
+      $.row('Query Scheduler')
+      .addPanel(
+        $.containerCPUUsagePanel('CPU', 'query-scheduler'),
+      )
+      .addPanel(
+        $.containerMemoryWorkingSetPanel('Memory (workingset)', 'query-scheduler'),
+      )
+      .addPanel(
+        $.goHeapInUsePanel('Memory (go heap inuse)', 'query-scheduler'),
+      )
+    )
     .addRow(
       $.row('Querier')
       .addPanel(

cortex-mixin/dashboards/reads.libsonnet

@@ -40,6 +40,17 @@ local utils = import 'mixin-utils/utils.libsonnet';
         { yaxes: $.yaxes('s') }
       )
     )
+    .addRow(
+      $.row('Query Scheduler')
+      .addPanel(
+        $.panel('QPS') +
+        $.qpsPanel('cortex_query_scheduler_queue_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.query_scheduler))
+      )
+      .addPanel(
+        $.panel('Latency (Time in Queue)') +
+        $.latencyPanel('cortex_query_scheduler_queue_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.query_scheduler))
+      )
+    )
     .addRow(
       $.row('Cache - Query Results')
       .addPanel(