Merge pull request #236 from grafana/add-option-to-not-unregister-ingesters-on-shutdown

Add option to configure unregister ingesters on shutdown

Author: pracucci

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## master / unreleased
 
+* [ENHANCEMENT] Added `unregister_ingesters_on_shutdown` config option to disable unregistering ingesters on shutdown (default is enabled). #213
+
 ## 1.6.0 / 2021-01-05
 
 * [CHANGE] Add default present for ruler limits on all 'user' types. #221, #222

cortex/config.libsonnet

@@ -13,6 +13,13 @@
     aws_region: error 'must specify AWS region',
     s3_bucket_name: error 'must specify S3 bucket name',
 
+    // If false, ingesters are not unregistered on shutdown and left in the ring with
+    // the LEAVING state. Setting to false prevents series resharding during ingesters rollouts,
+    // but requires to:
+    // 1. Either manually forget ingesters on scale down or invoke the /shutdown endpoint
+    // 2. Ensure ingester ID is preserved during rollouts
+    unregister_ingesters_on_shutdown: true,
+
     // schema is used to generate the storage schema yaml file used by
     // the Cortex chunks storage:
     // - More information: https://github.com/cortexproject/cortex/pull/1072

cortex/distributor.libsonnet

@@ -36,7 +36,11 @@
       // The ingestion rate global limit requires the distributors to form a ring.
       'distributor.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace,
       'distributor.ring.prefix': '',
-    },
+
+      // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown"
+      // is set to false.
+      'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown,
+    ),
 
   distributor_ports:: $.util.defaultPorts,
 

cortex/ingester.libsonnet

@@ -15,6 +15,7 @@
       'ingester.max-transfer-retries': 60,  // Each retry is backed off by 5s, so 5mins for new ingester to come up.
       'ingester.heartbeat-period': '15s',
       'ingester.max-stale-chunk-idle': '5m',
+      'ingester.unregister-on-shutdown': $._config.unregister_ingesters_on_shutdown,
 
       // Chunk building/flushing config.
       'ingester.chunk-encoding': 3,  // Bigchunk encoding