From 75ecd8dd7654c68ae8a9596b59f5628d4a742309 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 20 Oct 2020 16:53:32 +0100 Subject: [PATCH 01/12] Ruler Limits Adds two limits to the rules creation. The number of rules per rule group and the number of rule groups in total, inherently this creates a limit of maximum number of rules of `max per rule group` x `max rule groups`. Signed-off-by: gotjosh --- pkg/ruler/api.go | 19 ++++++++++ pkg/ruler/api_test.go | 67 +++++++++++++++++++++++++++++++++++ pkg/ruler/compat.go | 2 ++ pkg/ruler/ruler.go | 23 ++++++++++++ pkg/ruler/ruler_test.go | 14 ++++++-- pkg/util/validation/limits.go | 16 +++++++-- 6 files changed, 137 insertions(+), 4 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 53a717d06d9..586b55e92d9 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -472,6 +472,25 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { return } + if err := a.ruler.AssertMaxRulesPerRuleGroupPerUser(userID, len(rg.Rules)); err != nil { + level.Error(logger).Log("msg", "limit validation failure", "err", err.Error(), "user", userID) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rgl, err := a.store.ListRuleGroupsForUserAndNamespace(req.Context(), userID, "") + if err != nil { + level.Error(logger).Log("msg", "unable to fetch current rule groups for validation", "err", err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := a.ruler.AssertMaxRuleGroupsPerUser(userID, len(rgl)); err != nil { + level.Error(logger).Log("msg", "limit validation failure", "err", err.Error(), "user", userID) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + rgProto := store.ToProto(userID, namespace, rg) level.Debug(logger).Log("msg", "attempting to store rulegroup", "userID", userID, "group", rgProto.String()) diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go index 9f03087e8be..ddd8968c135 100644 --- a/pkg/ruler/api_test.go +++ b/pkg/ruler/api_test.go @@ -299,6 +299,73 @@ func TestRuler_DeleteNamespace(t *testing.T) { require.Equal(t, "{\"status\":\"error\",\"data\":null,\"errorType\":\"server_error\",\"error\":\"unable to delete rg\"}", w.Body.String()) } +func TestRuler_Limits(t *testing.T) { + cfg, cleanup := defaultRulerConfig(newMockRuleStore(make(map[string]rules.RuleGroupList))) + defer cleanup() + + r, rcleanup := newTestRuler(t, cfg) + defer rcleanup() + defer services.StopAndAwaitTerminated(context.Background(), r) //nolint:errcheck + + r.limits = &ruleLimits{maxRuleGroups: 1, maxRulesPerRuleGroup: 1} + + a := NewAPI(r, r.store) + + tc := []struct { + name string + input string + output string + err error + status int + }{ + { + name: "when exceeding the rules per rule group limit", + status: 400, + input: ` +name: test +interval: 15s +rules: +- record: up_rule + expr: up{} +- alert: up_alert + expr: sum(up{}) > 1 + for: 30s + annotations: + test: test + labels: + test: test +`, + output: "per-user rules per rule group limit (limit: 1 actual: 2) exceeded\n", + }, + { + name: "when exceeding the rule group limit", + status: 400, + input: ` +name: test +interval: 15s +rules: +- record: up_rule + expr: up{} +`, + output: "per-user rules per rule group limit (limit: 1 actual: 1) exceeded\n", + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + router := mux.NewRouter() + router.Path("/api/v1/rules/{namespace}").Methods("POST").HandlerFunc(a.CreateRuleGroup) + // POST + req := requestFor(t, http.MethodPost, "https://localhost:8080/api/v1/rules/namespace", strings.NewReader(tt.input), "user1") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + require.Equal(t, tt.status, w.Code) + require.Equal(t, tt.output, w.Body.String()) + }) + } +} + func requestFor(t *testing.T, method string, url string, body io.Reader, userID string) *http.Request { t.Helper() diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 0d60f499b6d..5ecabbf91ae 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -76,6 +76,8 @@ func (t *PusherAppendable) Appender(ctx context.Context) storage.Appender { type RulesLimits interface { EvaluationDelay(userID string) time.Duration RulerTenantShardSize(userID string) int + RulerMaxRuleGroupsPerUser(userID string) int + RulerMaxRulesPerRuleGroupPerUser(userID string) int } // engineQueryFunc returns a new query function using the rules.EngineQueryFunc function diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 9b3e7c5c545..da5b35b2110 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -56,6 +56,10 @@ const ( rulerSyncReasonInitial = "initial" rulerSyncReasonPeriodic = "periodic" rulerSyncReasonRingChange = "ring-change" + + // Limit errors + errMaxRuleGroupsPerUserLimitExceeded = "per-user rule groups limit (limit %d actual: %d) exceeded" + errMaxRulesPerRuleGroupPerUserLimitExceeded = "per-user rules per rule group limit (limit: %d actual: %d) exceeded" ) // Config is the configuration for the recording rules server. @@ -736,3 +740,22 @@ func (r *Ruler) Rules(ctx context.Context, in *RulesRequest) (*RulesResponse, er return &RulesResponse{Groups: groupDescs}, nil } + +func (r *Ruler) AssertMaxRuleGroupsPerUser(userID string, rg int) error { + limit := r.limits.RulerMaxRuleGroupsPerUser(userID) + + if rg < limit { + return nil + } + + return fmt.Errorf(errMaxRuleGroupsPerUserLimitExceeded, limit, rg) +} + +func (r *Ruler) AssertMaxRulesPerRuleGroupPerUser(userID string, rules int) error { + limit := r.limits.RulerMaxRulesPerRuleGroupPerUser(userID) + + if rules < limit { + return nil + } + return fmt.Errorf(errMaxRulesPerRuleGroupPerUserLimitExceeded, limit, rules) +} diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 360f8eeb3c5..96dbce2aba7 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -63,8 +63,10 @@ func defaultRulerConfig(store rules.RuleStore) (Config, func()) { } type ruleLimits struct { - evalDelay time.Duration - tenantShard int + evalDelay time.Duration + tenantShard int + maxRulesPerRuleGroup int + maxRuleGroups int } func (r ruleLimits) EvaluationDelay(_ string) time.Duration { @@ -75,6 +77,14 @@ func (r ruleLimits) RulerTenantShardSize(_ string) int { return r.tenantShard } +func (r ruleLimits) RulerMaxRuleGroupsPerUser(_ string) int { + return r.maxRuleGroups +} + +func (r ruleLimits) RulerMaxRulesPerRuleGroupPerUser(_ string) int { + return r.maxRulesPerRuleGroup +} + func testSetup(t *testing.T, cfg Config) (*promql.Engine, storage.QueryableFunc, Pusher, log.Logger, RulesLimits, func()) { dir, err := ioutil.TempDir("", filepath.Base(t.Name())) testutil.Ok(t, err) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index c546d1d1148..8a3c421d27f 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -71,8 +71,10 @@ type Limits struct { MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant"` // Ruler defaults and limits. - RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` - RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` + RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` + RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` + RulerMaxRulesPerRuleGroupPerUser int `yaml:"ruler_max_rules_per_rule_group_per_user"` + RulerMaxRuleGroupsPerUser int `yaml:"ruler_max_rule_groups"` // Store-gateway. StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size"` @@ -124,6 +126,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") + f.IntVar(&l.RulerMaxRulesPerRuleGroupPerUser, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant.") + f.IntVar(&l.RulerMaxRuleGroupsPerUser, "ruler.max-rule-groups", 20, "Maximum number of rule groups per-tenant.") f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]") @@ -377,6 +381,14 @@ func (o *Overrides) RulerTenantShardSize(userID string) int { return o.getOverridesForUser(userID).RulerTenantShardSize } +func (o *Overrides) RulerMaxRulesPerRuleGroupPerUser(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroupPerUser +} + +func (o *Overrides) RulerMaxRuleGroupsPerUser(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerUser +} + // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. func (o *Overrides) StoreGatewayTenantShardSize(userID string) int { return o.getOverridesForUser(userID).StoreGatewayTenantShardSize From cbb9057d204c53a048eaee4278a5f2a2cd1d66a3 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 20 Oct 2020 17:36:12 +0100 Subject: [PATCH 02/12] Appease the linter Signed-off-by: gotjosh --- docs/configuration/config-file-reference.md | 8 ++++++++ pkg/ruler/api.go | 4 ++-- pkg/ruler/ruler_test.go | 2 +- pkg/ruler/store_mock_test.go | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index f9f419a74e4..a5dc0b4c976 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2903,6 +2903,14 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ruler.tenant-shard-size [ruler_tenant_shard_size: | default = 0] +# Maximum number of rules per rule group per-tenant. +# CLI flag: -ruler.max-rules-per-rule-group +[ruler_max_rules_per_rule_group_per_user: | default = 15] + +# Maximum number of rule groups per-tenant. +# CLI flag: -ruler.max-rule-groups +[ruler_max_rule_groups: | default = 20] + # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the # shuffle-sharding strategy. When this setting is specified in the per-tenant diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 586b55e92d9..23977a354b5 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -478,14 +478,14 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { return } - rgl, err := a.store.ListRuleGroupsForUserAndNamespace(req.Context(), userID, "") + rgs, err := a.store.ListRuleGroupsForUserAndNamespace(req.Context(), userID, "") if err != nil { level.Error(logger).Log("msg", "unable to fetch current rule groups for validation", "err", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) return } - if err := a.ruler.AssertMaxRuleGroupsPerUser(userID, len(rgl)); err != nil { + if err := a.ruler.AssertMaxRuleGroupsPerUser(userID, len(rgs)); err != nil { level.Error(logger).Log("msg", "limit validation failure", "err", err.Error(), "user", userID) http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 96dbce2aba7..527b7e793c2 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -111,7 +111,7 @@ func testSetup(t *testing.T, cfg Config) (*promql.Engine, storage.QueryableFunc, l := log.NewLogfmtLogger(os.Stdout) l = level.NewFilter(l, level.AllowInfo()) - return engine, noopQueryable, pusher, l, ruleLimits{evalDelay: 0}, cleanup + return engine, noopQueryable, pusher, l, ruleLimits{evalDelay: 0, maxRuleGroups: 20, maxRulesPerRuleGroup: 15}, cleanup } func newManager(t *testing.T, cfg Config) (*DefaultMultiTenantManager, func()) { diff --git a/pkg/ruler/store_mock_test.go b/pkg/ruler/store_mock_test.go index 88a0ab8de4e..a5ddd89d399 100644 --- a/pkg/ruler/store_mock_test.go +++ b/pkg/ruler/store_mock_test.go @@ -144,7 +144,7 @@ func (m *mockRuleStore) ListRuleGroupsForUserAndNamespace(_ context.Context, use userRules, exists := m.rules[userID] if !exists { - return nil, rules.ErrUserNotFound + return rules.RuleGroupList{}, nil } if namespace == "" { @@ -160,7 +160,7 @@ func (m *mockRuleStore) ListRuleGroupsForUserAndNamespace(_ context.Context, use } if len(namespaceRules) == 0 { - return nil, rules.ErrGroupNamespaceNotFound + return rules.RuleGroupList{}, nil } return namespaceRules, nil From 5a3ffe5c9f6a52490f3c3d60c3713cc5e51b6299 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 20 Oct 2020 18:30:42 +0100 Subject: [PATCH 03/12] Add more comments Signed-off-by: gotjosh --- CHANGELOG.md | 1 + pkg/ruler/ruler.go | 4 ++++ pkg/util/validation/limits.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d0f96f51e..54aae7b672e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217 +* [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups` to control the number of rules per rule group and the total number of rule groups for a given user. #3366 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275 * [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244 - `-blocks-storage.s3.http.idle-conn-timeout` diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index da5b35b2110..c0427643158 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -741,6 +741,8 @@ func (r *Ruler) Rules(ctx context.Context, in *RulesRequest) (*RulesResponse, er return &RulesResponse{Groups: groupDescs}, nil } +// AssertMaxRuleGroupsPerUser limit has not been reached compared to the current +// number of total rule groups in input and returns an error if so. func (r *Ruler) AssertMaxRuleGroupsPerUser(userID string, rg int) error { limit := r.limits.RulerMaxRuleGroupsPerUser(userID) @@ -751,6 +753,8 @@ func (r *Ruler) AssertMaxRuleGroupsPerUser(userID string, rg int) error { return fmt.Errorf(errMaxRuleGroupsPerUserLimitExceeded, limit, rg) } +// AssertMaxRulesPerRuleGroupPerUser limit has not been reached compared to the current +// number of rules in a rule group in input and returns an error if so. func (r *Ruler) AssertMaxRulesPerRuleGroupPerUser(userID string, rules int) error { limit := r.limits.RulerMaxRulesPerRuleGroupPerUser(userID) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 8a3c421d27f..8e8b8116696 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -381,10 +381,12 @@ func (o *Overrides) RulerTenantShardSize(userID string) int { return o.getOverridesForUser(userID).RulerTenantShardSize } +// RulerMaxRulesPerRuleGroupPerUser returns the maximum number of rules per rule group for a given user. func (o *Overrides) RulerMaxRulesPerRuleGroupPerUser(userID string) int { return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroupPerUser } +// RulerMaxRuleGroupsPerUser returns the maximum number of rule groups for a given user. func (o *Overrides) RulerMaxRuleGroupsPerUser(userID string) int { return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerUser } From b471b55ec7d1b6055ca0e03b888ac14ae2dcec74 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 12:23:07 +0100 Subject: [PATCH 04/12] Update pkg/util/validation/limits.go Co-authored-by: Marco Pracucci Signed-off-by: gotjosh --- pkg/util/validation/limits.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 8e8b8116696..1b3d368ce0e 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -74,7 +74,7 @@ type Limits struct { RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` RulerMaxRulesPerRuleGroupPerUser int `yaml:"ruler_max_rules_per_rule_group_per_user"` - RulerMaxRuleGroupsPerUser int `yaml:"ruler_max_rule_groups"` + RulerMaxRuleGroupsPerUser int `yaml:"ruler_max_rule_groups_per_user"` // Store-gateway. StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size"` From e6609374f2c9c396452195889cfc640e15b0da63 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 12:23:19 +0100 Subject: [PATCH 05/12] Update pkg/util/validation/limits.go Co-authored-by: Marco Pracucci Signed-off-by: gotjosh --- pkg/util/validation/limits.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 1b3d368ce0e..6e05ff2dfef 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -127,7 +127,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.IntVar(&l.RulerMaxRulesPerRuleGroupPerUser, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant.") - f.IntVar(&l.RulerMaxRuleGroupsPerUser, "ruler.max-rule-groups", 20, "Maximum number of rule groups per-tenant.") + f.IntVar(&l.RulerMaxRuleGroupsPerUser, "ruler.max-rule-groups-per-user", 20, "Maximum number of rule groups per-tenant.") f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]") From 23953d0a33277d599c4da969f240d16fe48d942b Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 12:23:26 +0100 Subject: [PATCH 06/12] Update pkg/ruler/api.go Co-authored-by: Marco Pracucci Signed-off-by: gotjosh --- pkg/ruler/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 23977a354b5..e2320a69dbf 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -480,7 +480,7 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { rgs, err := a.store.ListRuleGroupsForUserAndNamespace(req.Context(), userID, "") if err != nil { - level.Error(logger).Log("msg", "unable to fetch current rule groups for validation", "err", err.Error()) + level.Error(logger).Log("msg", "unable to fetch current rule groups for validation", "err", err.Error(), "user", userID) http.Error(w, err.Error(), http.StatusInternalServerError) return } From 7bb89069ac8200a0a0fd32c55f72a781901a0041 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 12:23:38 +0100 Subject: [PATCH 07/12] Update pkg/ruler/ruler.go Co-authored-by: Marco Pracucci Signed-off-by: gotjosh --- pkg/ruler/ruler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index c0427643158..51c315932ec 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -58,7 +58,7 @@ const ( rulerSyncReasonRingChange = "ring-change" // Limit errors - errMaxRuleGroupsPerUserLimitExceeded = "per-user rule groups limit (limit %d actual: %d) exceeded" + errMaxRuleGroupsPerUserLimitExceeded = "per-user rule groups limit (limit: %d actual: %d) exceeded" errMaxRulesPerRuleGroupPerUserLimitExceeded = "per-user rules per rule group limit (limit: %d actual: %d) exceeded" ) From ef89c42c18c3ec4c092a07bd35dac66dc39888aa Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 13:07:36 +0100 Subject: [PATCH 08/12] Address review comments Signed-off-by: gotjosh --- docs/configuration/config-file-reference.md | 6 +++--- pkg/ruler/api.go | 4 ++-- pkg/ruler/compat.go | 4 ++-- pkg/ruler/ruler.go | 20 +++++++++++------ pkg/ruler/ruler_test.go | 4 ++-- pkg/util/validation/limits.go | 24 ++++++++++----------- 6 files changed, 35 insertions(+), 27 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index a5dc0b4c976..d48b71337e4 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2903,11 +2903,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ruler.tenant-shard-size [ruler_tenant_shard_size: | default = 0] -# Maximum number of rules per rule group per-tenant. +# Maximum number of rules per rule group per-tenant. 0 to disable. # CLI flag: -ruler.max-rules-per-rule-group -[ruler_max_rules_per_rule_group_per_user: | default = 15] +[ruler_max_rules_per_rule_group: | default = 15] -# Maximum number of rule groups per-tenant. +# Maximum number of rule groups per-tenant. 0 to disable. # CLI flag: -ruler.max-rule-groups [ruler_max_rule_groups: | default = 20] diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index e2320a69dbf..ffa6e928365 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -472,7 +472,7 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { return } - if err := a.ruler.AssertMaxRulesPerRuleGroupPerUser(userID, len(rg.Rules)); err != nil { + if err := a.ruler.AssertMaxRulesPerRuleGroup(userID, len(rg.Rules)); err != nil { level.Error(logger).Log("msg", "limit validation failure", "err", err.Error(), "user", userID) http.Error(w, err.Error(), http.StatusBadRequest) return @@ -485,7 +485,7 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { return } - if err := a.ruler.AssertMaxRuleGroupsPerUser(userID, len(rgs)); err != nil { + if err := a.ruler.AssertMaxRuleGroups(userID, len(rgs)); err != nil { level.Error(logger).Log("msg", "limit validation failure", "err", err.Error(), "user", userID) http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 5ecabbf91ae..e899b0a5863 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -76,8 +76,8 @@ func (t *PusherAppendable) Appender(ctx context.Context) storage.Appender { type RulesLimits interface { EvaluationDelay(userID string) time.Duration RulerTenantShardSize(userID string) int - RulerMaxRuleGroupsPerUser(userID string) int - RulerMaxRulesPerRuleGroupPerUser(userID string) int + RulerMaxRuleGroups(userID string) int + RulerMaxRulesPerRuleGroup(userID string) int } // engineQueryFunc returns a new query function using the rules.EngineQueryFunc function diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 51c315932ec..8594cbfb041 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -741,10 +741,14 @@ func (r *Ruler) Rules(ctx context.Context, in *RulesRequest) (*RulesResponse, er return &RulesResponse{Groups: groupDescs}, nil } -// AssertMaxRuleGroupsPerUser limit has not been reached compared to the current +// AssertMaxRuleGroups limit has not been reached compared to the current // number of total rule groups in input and returns an error if so. -func (r *Ruler) AssertMaxRuleGroupsPerUser(userID string, rg int) error { - limit := r.limits.RulerMaxRuleGroupsPerUser(userID) +func (r *Ruler) AssertMaxRuleGroups(userID string, rg int) error { + limit := r.limits.RulerMaxRuleGroups(userID) + + if limit <= 0 { + return nil + } if rg < limit { return nil @@ -753,10 +757,14 @@ func (r *Ruler) AssertMaxRuleGroupsPerUser(userID string, rg int) error { return fmt.Errorf(errMaxRuleGroupsPerUserLimitExceeded, limit, rg) } -// AssertMaxRulesPerRuleGroupPerUser limit has not been reached compared to the current +// AssertMaxRulesPerRuleGroup limit has not been reached compared to the current // number of rules in a rule group in input and returns an error if so. -func (r *Ruler) AssertMaxRulesPerRuleGroupPerUser(userID string, rules int) error { - limit := r.limits.RulerMaxRulesPerRuleGroupPerUser(userID) +func (r *Ruler) AssertMaxRulesPerRuleGroup(userID string, rules int) error { + limit := r.limits.RulerMaxRulesPerRuleGroup(userID) + + if limit <= 0 { + return nil + } if rules < limit { return nil diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 527b7e793c2..a808f66bc41 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -77,11 +77,11 @@ func (r ruleLimits) RulerTenantShardSize(_ string) int { return r.tenantShard } -func (r ruleLimits) RulerMaxRuleGroupsPerUser(_ string) int { +func (r ruleLimits) RulerMaxRuleGroups(_ string) int { return r.maxRuleGroups } -func (r ruleLimits) RulerMaxRulesPerRuleGroupPerUser(_ string) int { +func (r ruleLimits) RulerMaxRulesPerRuleGroup(_ string) int { return r.maxRulesPerRuleGroup } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 6e05ff2dfef..93788515de3 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -71,10 +71,10 @@ type Limits struct { MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant"` // Ruler defaults and limits. - RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` - RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` - RulerMaxRulesPerRuleGroupPerUser int `yaml:"ruler_max_rules_per_rule_group_per_user"` - RulerMaxRuleGroupsPerUser int `yaml:"ruler_max_rule_groups_per_user"` + RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` + RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` + RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group"` + RulerMaxRuleGroups int `yaml:"ruler_max_rule_groups"` // Store-gateway. StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size"` @@ -126,8 +126,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") - f.IntVar(&l.RulerMaxRulesPerRuleGroupPerUser, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant.") - f.IntVar(&l.RulerMaxRuleGroupsPerUser, "ruler.max-rule-groups-per-user", 20, "Maximum number of rule groups per-tenant.") + f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant. 0 to disable.") + f.IntVar(&l.RulerMaxRuleGroups, "ruler.max-rule-groups", 20, "Maximum number of rule groups per-tenant. 0 to disable.") f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]") @@ -381,14 +381,14 @@ func (o *Overrides) RulerTenantShardSize(userID string) int { return o.getOverridesForUser(userID).RulerTenantShardSize } -// RulerMaxRulesPerRuleGroupPerUser returns the maximum number of rules per rule group for a given user. -func (o *Overrides) RulerMaxRulesPerRuleGroupPerUser(userID string) int { - return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroupPerUser +// RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user. +func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroup } -// RulerMaxRuleGroupsPerUser returns the maximum number of rule groups for a given user. -func (o *Overrides) RulerMaxRuleGroupsPerUser(userID string) int { - return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerUser +// RulerMaxRuleGroups returns the maximum number of rule groups for a given user. +func (o *Overrides) RulerMaxRuleGroups(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRuleGroups } // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. From 8304a583643f7ed3235d99e3fe00663c60995129 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 15:45:12 +0100 Subject: [PATCH 09/12] More review feedback Signed-off-by: gotjosh --- docs/configuration/config-file-reference.md | 2 +- pkg/ruler/compat.go | 2 +- pkg/ruler/ruler.go | 2 +- pkg/ruler/ruler_test.go | 2 +- pkg/util/validation/limits.go | 16 ++++++++-------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index d48b71337e4..95316b14afe 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2908,7 +2908,7 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s [ruler_max_rules_per_rule_group: | default = 15] # Maximum number of rule groups per-tenant. 0 to disable. -# CLI flag: -ruler.max-rule-groups +# CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups: | default = 20] # The default tenant's shard size when the shuffle-sharding strategy is used. diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index e899b0a5863..a3f0dd976d0 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -76,7 +76,7 @@ func (t *PusherAppendable) Appender(ctx context.Context) storage.Appender { type RulesLimits interface { EvaluationDelay(userID string) time.Duration RulerTenantShardSize(userID string) int - RulerMaxRuleGroups(userID string) int + RulerMaxRuleGroupsPerTenant(userID string) int RulerMaxRulesPerRuleGroup(userID string) int } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 8594cbfb041..28ccca83ce3 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -744,7 +744,7 @@ func (r *Ruler) Rules(ctx context.Context, in *RulesRequest) (*RulesResponse, er // AssertMaxRuleGroups limit has not been reached compared to the current // number of total rule groups in input and returns an error if so. func (r *Ruler) AssertMaxRuleGroups(userID string, rg int) error { - limit := r.limits.RulerMaxRuleGroups(userID) + limit := r.limits.RulerMaxRuleGroupsPerTenant(userID) if limit <= 0 { return nil diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index a808f66bc41..778c87d2cee 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -77,7 +77,7 @@ func (r ruleLimits) RulerTenantShardSize(_ string) int { return r.tenantShard } -func (r ruleLimits) RulerMaxRuleGroups(_ string) int { +func (r ruleLimits) RulerMaxRuleGroupsPerTenant(_ string) int { return r.maxRuleGroups } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 93788515de3..fca03ad40e7 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -71,10 +71,10 @@ type Limits struct { MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant"` // Ruler defaults and limits. - RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` - RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` - RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group"` - RulerMaxRuleGroups int `yaml:"ruler_max_rule_groups"` + RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` + RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` + RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group"` + RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups"` // Store-gateway. StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size"` @@ -127,7 +127,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant. 0 to disable.") - f.IntVar(&l.RulerMaxRuleGroups, "ruler.max-rule-groups", 20, "Maximum number of rule groups per-tenant. 0 to disable.") + f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 20, "Maximum number of rule groups per-tenant. 0 to disable.") f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]") @@ -386,9 +386,9 @@ func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int { return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroup } -// RulerMaxRuleGroups returns the maximum number of rule groups for a given user. -func (o *Overrides) RulerMaxRuleGroups(userID string) int { - return o.getOverridesForUser(userID).RulerMaxRuleGroups +// RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user. +func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { + return o.getOverridesForUser(userID).RulerMaxRuleGroupsPerTenant } // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. From d69047fe687a8a52e6061bdd78f206238e40dc94 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 15:48:14 +0100 Subject: [PATCH 10/12] Update changelog Signed-off-by: gotjosh --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54aae7b672e..78b84079d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217 -* [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups` to control the number of rules per rule group and the total number of rule groups for a given user. #3366 +* [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups-per-tenant` to control the number of rules per rule group and the total number of rule groups for a given user. #3366 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275 * [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244 - `-blocks-storage.s3.http.idle-conn-timeout` From 7cfb37bb1d0a40f97e3a548470f111855fb98968 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 17:24:12 +0100 Subject: [PATCH 11/12] Last comment Signed-off-by: gotjosh --- docs/configuration/config-file-reference.md | 2 +- pkg/util/validation/limits.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 95316b14afe..1064ae86e16 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2909,7 +2909,7 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # Maximum number of rule groups per-tenant. 0 to disable. # CLI flag: -ruler.max-rule-groups-per-tenant -[ruler_max_rule_groups: | default = 20] +[ruler_max_rule_groups_per_tenant: | default = 20] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index fca03ad40e7..7fdac31a2a8 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -74,7 +74,7 @@ type Limits struct { RulerEvaluationDelay time.Duration `yaml:"ruler_evaluation_delay_duration"` RulerTenantShardSize int `yaml:"ruler_tenant_shard_size"` RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group"` - RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups"` + RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant"` // Store-gateway. StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size"` From c68cf1079d5f4c77a4833b00581a21668f78cc27 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Wed, 21 Oct 2020 18:18:37 +0100 Subject: [PATCH 12/12] Disable limits by default Signed-off-by: gotjosh --- CHANGELOG.md | 2 +- docs/configuration/config-file-reference.md | 4 ++-- pkg/util/validation/limits.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78b84079d38..cb4581bcb66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217 -* [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups-per-tenant` to control the number of rules per rule group and the total number of rule groups for a given user. #3366 +* [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups-per-tenant` to control the number of rules per rule group and the total number of rule groups for a given user. They are disabled by default. #3366 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275 * [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244 - `-blocks-storage.s3.http.idle-conn-timeout` diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 1064ae86e16..4dcb6d5f093 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2905,11 +2905,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # Maximum number of rules per rule group per-tenant. 0 to disable. # CLI flag: -ruler.max-rules-per-rule-group -[ruler_max_rules_per_rule_group: | default = 15] +[ruler_max_rules_per_rule_group: | default = 0] # Maximum number of rule groups per-tenant. 0 to disable. # CLI flag: -ruler.max-rule-groups-per-tenant -[ruler_max_rule_groups_per_tenant: | default = 20] +[ruler_max_rule_groups_per_tenant: | default = 0] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 7fdac31a2a8..eb4e0ddafdb 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -126,8 +126,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") - f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 15, "Maximum number of rules per rule group per-tenant. 0 to disable.") - f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 20, "Maximum number of rule groups per-tenant. 0 to disable.") + f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") + f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]")