Merge pull request #376 from grafana/match-custom-deployment-read-path

Match query-frontend/query-scheduler/querier custom deployments by default

Author: pracucci

CHANGELOG.md

@@ -23,6 +23,7 @@
 * [CHANGE] Increased `CortexIngesterReachingSeriesLimit` critical alert threshold from 80% to 85%. #363
 * [CHANGE] Decreased `-server.grpc-max-concurrent-streams` from 100k to 10k. #369
 * [CHANGE] Decreased blocks storage ingesters graceful termination period from 80m to 20m. #369
+* [CHANGE] Changed default `job_names` for query-frontend, query-scheduler and querier to match custom deployments too. #376
 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317
 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. #328
 * [ENHANCEMENT] Ruler dashboard: added object storage metrics. #354
@@ -48,6 +49,7 @@
 * [BUGFIX] Fixed `CortexInconsistentRuntimeConfig` metric. #335
 * [BUGFIX] Fixed scaling dashboard to correctly work when a Cortex service deployment spans across multiple zones (a zone is expected to have the `zone-[a-z]` suffix). #365
 * [BUGFIX] Fixed rollout progress dashboard to correctly work when a Cortex service deployment spans across multiple zones (a zone is expected to have the `zone-[a-z]` suffix). #366
+* [BUGFIX] Fixed rollout progress dashboard to include query-scheduler too. #376
 * [BUGFIX] Fixed `-distributor.extend-writes` setting on ruler when `unregister_ingesters_on_shutdown` is disabled. #369
 
 ## 1.9.0 / 2021-05-18

cortex-mixin/alerts/alerts.libsonnet

@@ -593,7 +593,7 @@
           expr: |||
             memberlist_client_cluster_members_count
               != on (%s) group_left
-            sum by (%s) (up{job=~".+/(admin-api|compactor|store-gateway|distributor|ingester.*|querier|cortex|ruler)"})
+            sum by (%s) (up{job=~".+/(admin-api|compactor|store-gateway|distributor|ingester.*|querier.*|cortex|ruler)"})
           ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels],
           'for': '5m',
           labels: {

cortex-mixin/config.libsonnet

@@ -26,12 +26,12 @@
     // These are used by the dashboards and allow for the simultaneous display of
     // microservice and single binary cortex clusters.
     job_names: {
-      ingester: '(ingester.*|cortex$)',  // Match also ingester-blocks, which is used during the migration from chunks to blocks.
+      ingester: '(ingester.*|cortex$)',  // Match also custom and per-zone ingester deployments.
       distributor: '(distributor|cortex$)',
-      querier: '(querier|cortex$)',
+      querier: '(querier.*|cortex$)',  // Match also custom querier deployments.
       ruler: '(ruler|cortex$)',
-      query_frontend: '(query-frontend|cortex$)',
-      query_scheduler: 'query-scheduler',  // Not part of single-binary.
+      query_frontend: '(query-frontend.*|cortex$)',  // Match also custom query-frontend deployments.
+      query_scheduler: 'query-scheduler.*',  // Not part of single-binary. Match also custom query-scheduler deployments.
       table_manager: '(table-manager|cortex$)',
       store_gateway: '(store-gateway|cortex$)',
       gateway: '(gateway|cortex-gw|cortex-gw-internal)',

cortex-mixin/dashboards/rollout-progress.libsonnet

@@ -6,7 +6,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     gateway_job_matcher: $.jobMatcher($._config.job_names.gateway),
     gateway_write_routes_regex: 'api_(v1|prom)_push',
     gateway_read_routes_regex: '(prometheus|api_prom)_api_v1_.+',
-    all_services_regex: std.join('|', ['cortex-gw', 'distributor', 'ingester.*', 'query-frontend', 'querier', 'compactor', 'store-gateway', 'ruler', 'alertmanager']),
+    all_services_regex: std.join('|', ['cortex-gw', 'distributor', 'ingester.*', 'query-frontend.*', 'query-scheduler.*', 'querier.*', 'compactor', 'store-gateway', 'ruler', 'alertmanager']),
   },
 
   'cortex-rollout-progress.json':

cortex-mixin/dashboards/slow-queries.libsonnet

@@ -16,7 +16,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           targets: [
             {
               // Filter out the remote read endpoint.
-              expr: '{cluster=~"$cluster",namespace=~"$namespace",name="query-frontend"} |= "query stats" != "/api/v1/read" | logfmt | org_id=~"${tenant_id}" | response_time > ${min_duration}',
+              expr: '{cluster=~"$cluster",namespace=~"$namespace",name=~"query-frontend.*"} |= "query stats" != "/api/v1/read" | logfmt | org_id=~"${tenant_id}" | response_time > ${min_duration}',
               instant: false,
               legendFormat: '',
               range: true,