Publish ruler's ring metrics

CHANGELOG.md

@@ -18,6 +18,7 @@
 * [ENHANCEMENT] Query-tee: add support for doing a passthrough of requests to preferred backend for unregistered routes #3018
 * [ENHANCEMENT] Expose `storage.aws.dynamodb.backoff_config` configuration file field. #3026
 * [ENHANCEMENT] Added `cortex_request_message_bytes` and `cortex_response_message_bytes` histograms to track received and sent gRPC message and HTTP request/response sizes. Added `cortex_inflight_requests` gauge to track number of inflight gRPC and HTTP requests. #3064
+* [ENHANCEMENT] Publish ruler's ring metrics. #3074
 * [ENHANCEMENT] Add config validation to the experimental Alertmanager API. Invalid configs are no longer accepted. #3053
 * [ENHANCEMENT] Add "integration" as a label for `cortex_alertmanager_notifications_total` and `cortex_alertmanager_notifications_failed_total` metrics. #3056
 * [ENHANCEMENT] Add `cortex_ruler_config_last_reload_successful` and `cortex_ruler_config_last_reload_successful_seconds` to check status of users rule manager. #3056

docs/configuration/single-process-config-blocks-gossip-1.yaml

@@ -95,5 +95,6 @@ ruler:
     local:
       directory: /tmp/cortex/rules
   ring:
+    num_tokens: 512
     kvstore:
-      store: memberlist
+      store: memberlist

docs/configuration/single-process-config-blocks-gossip-2.yaml

@@ -94,5 +94,6 @@ ruler:
     local:
       directory: /tmp/cortex/rules
   ring:
+    num_tokens: 512
     kvstore:
-      store: memberlist
+      store: memberlist

integration/getting_started_with_gossiped_ring_test.go

@@ -62,14 +62,14 @@ func TestGettingStartedWithGossipedRing(t *testing.T) {
 	require.NoError(t, cortex2.WaitSumMetrics(e2e.Equals(2), "memberlist_client_cluster_members_count"))
 
 	// Both Cortex servers should have 512 tokens for ingesters ring and 512 tokens for store-gateways ring.
-	for _, ringName := range []string{"ingester", "store-gateway"} {
+	for _, ringName := range []string{"ingester", "store-gateway", "ruler"} {
 		ringMatcher := labels.MustNewMatcher(labels.MatchEqual, "name", ringName)
 
 		require.NoError(t, cortex1.WaitSumMetricsWithOptions(e2e.Equals(2*512), []string{"cortex_ring_tokens_total"}, e2e.WithLabelMatchers(ringMatcher)))
 		require.NoError(t, cortex2.WaitSumMetricsWithOptions(e2e.Equals(2*512), []string{"cortex_ring_tokens_total"}, e2e.WithLabelMatchers(ringMatcher)))
 	}
 
-	// We need two "ring members" visible from both Cortex instances
+	// We need two "ring members" visible from both Cortex instances for ingesters
 	require.NoError(t, cortex1.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_ring_members"}, e2e.WithLabelMatchers(
 		labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"),
 		labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"))))
@@ -78,6 +78,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) {
 		labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"),
 		labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"))))
 
+	// We need two "ring members" visible from both Cortex instances for rulers
+	require.NoError(t, cortex1.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_ring_members"}, e2e.WithLabelMatchers(
+		labels.MustNewMatcher(labels.MatchEqual, "name", "ruler"),
+		labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"))))
+
+	require.NoError(t, cortex2.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_ring_members"}, e2e.WithLabelMatchers(
+		labels.MustNewMatcher(labels.MatchEqual, "name", "ruler"),
+		labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"))))
+
 	c1, err := e2ecortex.NewClient(cortex1.HTTPEndpoint(), cortex1.HTTPEndpoint(), "", "", "user-1")
 	require.NoError(t, err)
 

pkg/ruler/ruler.go

@@ -202,6 +202,10 @@ func NewRuler(cfg Config, manager MultiTenantManager, reg prometheus.Registerer,
 		if err = enableSharding(ruler, ringStore); err != nil {
 			return nil, errors.Wrap(err, "setup ruler sharding ring")
 		}
+
+		if reg != nil {
+			reg.MustRegister(ruler.ring)
+		}
 	}
 
 	ruler.Service = services.NewBasicService(ruler.starting, ruler.run, ruler.stopping)
@@ -220,12 +224,13 @@ func enableSharding(r *Ruler, ringStore kv.Client) error {
 	delegate = ring.NewLeaveOnStoppingDelegate(delegate, r.logger)
 	delegate = ring.NewAutoForgetDelegate(r.cfg.Ring.HeartbeatTimeout*ringAutoForgetUnhealthyPeriods, delegate, r.logger)
 
-	r.lifecycler, err = ring.NewBasicLifecycler(lifecyclerCfg, ring.RulerRingKey, ring.RulerRingKey, ringStore, delegate, r.logger, r.registry)
+	rulerRingName := "ruler"
+	r.lifecycler, err = ring.NewBasicLifecycler(lifecyclerCfg, rulerRingName, ring.RulerRingKey, ringStore, delegate, r.logger, r.registry)
 	if err != nil {
 		return errors.Wrap(err, "failed to initialize ruler's lifecycler")
 	}
 
-	r.ring, err = ring.NewWithStoreClientAndStrategy(r.cfg.Ring.ToRingConfig(), ring.RulerRingKey, ring.RulerRingKey, ringStore, &ring.DefaultReplicationStrategy{})
+	r.ring, err = ring.NewWithStoreClientAndStrategy(r.cfg.Ring.ToRingConfig(), rulerRingName, ring.RulerRingKey, ringStore, &ring.DefaultReplicationStrategy{})
 	if err != nil {
 		return errors.Wrap(err, "failed to initialize ruler's ring")
 	}