From 5bf2b8cf53a0bded746368e3200aa4c3f9475160 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 29 May 2025 14:42:02 -0700 Subject: [PATCH 1/6] Change status code from 429 to 503 Signed-off-by: Justin Jung --- pkg/ingester/ingester.go | 2 +- pkg/storegateway/gateway.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index f1be43e8009..9d126228172 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2231,7 +2231,7 @@ func (i *Ingester) trackInflightQueryRequest() (func(), error) { if i.resourceBasedLimiter != nil { if err := i.resourceBasedLimiter.AcceptNewRequest(); err != nil { level.Warn(i.logger).Log("msg", "failed to accept request", "err", err) - return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "failed to query: %s", limiter.ErrResourceLimitReachedStr) + return nil, httpgrpc.Errorf(http.StatusServiceUnavailable, "failed to query: %s", limiter.ErrResourceLimitReachedStr) } } diff --git a/pkg/storegateway/gateway.go b/pkg/storegateway/gateway.go index 835e95e8912..157c185a68a 100644 --- a/pkg/storegateway/gateway.go +++ b/pkg/storegateway/gateway.go @@ -437,7 +437,7 @@ func (g *StoreGateway) checkResourceUtilization() error { if err := g.resourceBasedLimiter.AcceptNewRequest(); err != nil { level.Warn(g.logger).Log("msg", "failed to accept request", "err", err) - return httpgrpc.Errorf(http.StatusTooManyRequests, "failed to query: %s", util_limiter.ErrResourceLimitReachedStr) + return httpgrpc.Errorf(http.StatusServiceUnavailable, "failed to query: %s", util_limiter.ErrResourceLimitReachedStr) } return nil From 62bc7c3aa7321fbca72b208bebec71e460082f3c Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 29 May 2025 15:15:20 -0700 Subject: [PATCH 2/6] Change configurations Signed-off-by: Justin Jung --- docs/blocks-storage/store-gateway.md | 36 +++++---- docs/configuration/config-file-reference.md | 73 ++++++++++++------- docs/configuration/v1-guarantees.md | 10 +-- pkg/configs/instance_limits.go | 40 ---------- pkg/configs/query_protection.go | 51 +++++++++++++ ...imits_test.go => query_protection_test.go} | 50 +++++++++---- pkg/ingester/ingester.go | 14 ++-- pkg/ingester/instance_limits.go | 10 --- pkg/storegateway/gateway.go | 14 ++-- 9 files changed, 173 insertions(+), 125 deletions(-) delete mode 100644 pkg/configs/instance_limits.go create mode 100644 pkg/configs/query_protection.go rename pkg/configs/{instance_limits_test.go => query_protection_test.go} (61%) diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 5053ea86ee7..b093822dad8 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -349,20 +349,28 @@ store_gateway: # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: | default = ""] - instance_limits: - # EXPERIMENTAL: Max CPU utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -store-gateway.instance-limits.cpu-utilization - [cpu_utilization: | default = 0] - - # EXPERIMENTAL: Max heap utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -store-gateway.instance-limits.heap-utilization - [heap_utilization: | default = 0] + query_protection: + rejection: + # EXPERIMENTAL: Enable query rejection feature, where the component return + # 503 to all incoming query requests when the configured thresholds are + # breached. + # CLI flag: -store-gateway.query-protection.rejection.enabled + [enabled: | default = false] + + threshold: + # EXPERIMENTAL: Max CPU utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, + # between 0 and 1. monitored_resources config must include the resource + # type. 0 to disable. + # CLI flag: -store-gateway.query-protection.rejection.threshold.cpu-utilization + [cpu_utilization: | default = 0] + + # EXPERIMENTAL: Max heap utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, + # between 0 and 1. monitored_resources config must include the resource + # type. 0 to disable. + # CLI flag: -store-gateway.query-protection.rejection.threshold.heap-utilization + [heap_utilization: | default = 0] hedged_request: # If true, hedged requests are applied to object store calls. It can help diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 29b5b979f86..9d3d2570351 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3204,20 +3204,6 @@ lifecycler: [upload_compacted_blocks_enabled: | default = true] instance_limits: - # EXPERIMENTAL: Max CPU utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -ingester.instance-limits.cpu-utilization - [cpu_utilization: | default = 0] - - # EXPERIMENTAL: Max heap utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -ingester.instance-limits.heap-utilization - [heap_utilization: | default = 0] - # Max ingestion rate (samples/sec) that ingester will accept. This limit is # per-ingester, not per-tenant. Additional push requests will be rejected. # Current ingestion rate is computed as exponentially weighted moving average, @@ -3276,6 +3262,29 @@ instance_limits: # If enabled, the metadata API returns all metadata regardless of the limits. # CLI flag: -ingester.skip-metadata-limits [skip_metadata_limits: | default = true] + +query_protection: + rejection: + # EXPERIMENTAL: Enable query rejection feature, where the component return + # 503 to all incoming query requests when the configured thresholds are + # breached. + # CLI flag: -ingester.query-protection.rejection.enabled + [enabled: | default = false] + + threshold: + # EXPERIMENTAL: Max CPU utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, between + # 0 and 1. monitored_resources config must include the resource type. 0 to + # disable. + # CLI flag: -ingester.query-protection.rejection.threshold.cpu-utilization + [cpu_utilization: | default = 0] + + # EXPERIMENTAL: Max heap utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, between + # 0 and 1. monitored_resources config must include the resource type. 0 to + # disable. + # CLI flag: -ingester.query-protection.rejection.threshold.heap-utilization + [heap_utilization: | default = 0] ``` ### `ingester_client_config` @@ -5897,20 +5906,28 @@ sharding_ring: # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: | default = ""] -instance_limits: - # EXPERIMENTAL: Max CPU utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -store-gateway.instance-limits.cpu-utilization - [cpu_utilization: | default = 0] - - # EXPERIMENTAL: Max heap utilization that this ingester can reach before - # rejecting new query request (across all tenants) in percentage, between 0 - # and 1. monitored_resources config must include the resource type. 0 to - # disable. - # CLI flag: -store-gateway.instance-limits.heap-utilization - [heap_utilization: | default = 0] +query_protection: + rejection: + # EXPERIMENTAL: Enable query rejection feature, where the component return + # 503 to all incoming query requests when the configured thresholds are + # breached. + # CLI flag: -store-gateway.query-protection.rejection.enabled + [enabled: | default = false] + + threshold: + # EXPERIMENTAL: Max CPU utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, between + # 0 and 1. monitored_resources config must include the resource type. 0 to + # disable. + # CLI flag: -store-gateway.query-protection.rejection.threshold.cpu-utilization + [cpu_utilization: | default = 0] + + # EXPERIMENTAL: Max heap utilization that this ingester can reach before + # rejecting new query request (across all tenants) in percentage, between + # 0 and 1. monitored_resources config must include the resource type. 0 to + # disable. + # CLI flag: -store-gateway.query-protection.rejection.threshold.heap-utilization + [heap_utilization: | default = 0] hedged_request: # If true, hedged requests are applied to object store calls. It can help with diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index 77a66e4d293..05a2bfeadb8 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -123,10 +123,8 @@ Currently experimental features are: - Query-frontend: dynamic query splits - `querier.max-shards-per-query` (int) CLI flag - `querier.max-fetched-data-duration-per-query` (duration) CLI flag -- Ingester/Store-Gateway: Resource-based throttling - - `-ingester.instance-limits.cpu-utilization` - - `-ingester.instance-limits.heap-utilization` - - `-store-gateway.instance-limits.cpu-utilization` - - `-store-gateway.instance-limits.heap-utilization` +- Ingester/Store-Gateway: Query rejection + - `-ingester.query-protection.rejection` + - `-store-gateway.query-protection.rejection` - Distributor/Ingester: Stream push connection - - Enable stream push connection between distributor and ingester by setting `-distributor.use-stream-push=true` on Distributor. \ No newline at end of file + - Enable stream push connection between distributor and ingester by setting `-distributor.use-stream-push=true` on Distributor. diff --git a/pkg/configs/instance_limits.go b/pkg/configs/instance_limits.go deleted file mode 100644 index 8273400847c..00000000000 --- a/pkg/configs/instance_limits.go +++ /dev/null @@ -1,40 +0,0 @@ -package configs - -import ( - "errors" - "flag" - "strings" - - "github.com/cortexproject/cortex/pkg/util/flagext" - "github.com/cortexproject/cortex/pkg/util/resource" -) - -type InstanceLimits struct { - CPUUtilization float64 `yaml:"cpu_utilization"` - HeapUtilization float64 `yaml:"heap_utilization"` -} - -func (cfg *InstanceLimits) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { - f.Float64Var(&cfg.CPUUtilization, prefix+"instance-limits.cpu-utilization", 0, "EXPERIMENTAL: Max CPU utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.") - f.Float64Var(&cfg.HeapUtilization, prefix+"instance-limits.heap-utilization", 0, "EXPERIMENTAL: Max heap utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.") -} - -func (cfg *InstanceLimits) Validate(monitoredResources flagext.StringSliceCSV) error { - if cfg.CPUUtilization > 1 || cfg.CPUUtilization < 0 { - return errors.New("cpu_utilization must be between 0 and 1") - } - - if cfg.CPUUtilization > 0 && !strings.Contains(monitoredResources.String(), string(resource.CPU)) { - return errors.New("monitored_resources config must include \"cpu\" as well") - } - - if cfg.HeapUtilization > 1 || cfg.HeapUtilization < 0 { - return errors.New("heap_utilization must be between 0 and 1") - } - - if cfg.HeapUtilization > 0 && !strings.Contains(monitoredResources.String(), string(resource.Heap)) { - return errors.New("monitored_resources config must include \"heap\" as well") - } - - return nil -} diff --git a/pkg/configs/query_protection.go b/pkg/configs/query_protection.go new file mode 100644 index 00000000000..2dd353580db --- /dev/null +++ b/pkg/configs/query_protection.go @@ -0,0 +1,51 @@ +package configs + +import ( + "errors" + "flag" + "strings" + + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/resource" +) + +type QueryProtection struct { + Rejection rejection `json:"rejection"` +} + +type rejection struct { + Enabled bool `yaml:"enabled"` + Threshold threshold `yaml:"threshold"` +} + +type threshold struct { + CPUUtilization float64 `yaml:"cpu_utilization"` + HeapUtilization float64 `yaml:"heap_utilization"` +} + +func (cfg *QueryProtection) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { + f.BoolVar(&cfg.Rejection.Enabled, prefix+"query-protection.rejection.enabled", false, "EXPERIMENTAL: Enable query rejection feature, where the component return 503 to all incoming query requests when the configured thresholds are breached.") + f.Float64Var(&cfg.Rejection.Threshold.CPUUtilization, prefix+"query-protection.rejection.threshold.cpu-utilization", 0, "EXPERIMENTAL: Max CPU utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.") + f.Float64Var(&cfg.Rejection.Threshold.HeapUtilization, prefix+"query-protection.rejection.threshold.heap-utilization", 0, "EXPERIMENTAL: Max heap utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.") +} + +func (cfg *QueryProtection) Validate(monitoredResources flagext.StringSliceCSV) error { + thresholdCfg := cfg.Rejection.Threshold + if thresholdCfg.CPUUtilization > 1 || thresholdCfg.CPUUtilization < 0 { + return errors.New("cpu_utilization must be between 0 and 1") + } + + if thresholdCfg.CPUUtilization > 0 && !strings.Contains(monitoredResources.String(), string(resource.CPU)) { + return errors.New("monitored_resources config must include \"cpu\" as well") + } + + if thresholdCfg.HeapUtilization > 1 || thresholdCfg.HeapUtilization < 0 { + return errors.New("heap_utilization must be between 0 and 1") + } + + if thresholdCfg.HeapUtilization > 0 && !strings.Contains(monitoredResources.String(), string(resource.Heap)) { + return errors.New("monitored_resources config must include \"heap\" as well") + } + + return nil +} diff --git a/pkg/configs/instance_limits_test.go b/pkg/configs/query_protection_test.go similarity index 61% rename from pkg/configs/instance_limits_test.go rename to pkg/configs/query_protection_test.go index a1d3686dba6..f06b4be7f38 100644 --- a/pkg/configs/instance_limits_test.go +++ b/pkg/configs/query_protection_test.go @@ -9,51 +9,71 @@ import ( func Test_Validate(t *testing.T) { for name, tc := range map[string]struct { - instanceLimits InstanceLimits + queryProtection QueryProtection monitoredResources []string err error }{ "correct config should pass validation": { - instanceLimits: InstanceLimits{ - CPUUtilization: 0.5, - HeapUtilization: 0.5, + queryProtection: QueryProtection{ + Rejection: rejection{ + Threshold: threshold{ + CPUUtilization: 0.5, + HeapUtilization: 0.5, + }, + }, }, monitoredResources: []string{"cpu", "heap"}, err: nil, }, "utilization config less than 0 should fail validation": { - instanceLimits: InstanceLimits{ - CPUUtilization: -0.5, - HeapUtilization: 0.5, + queryProtection: QueryProtection{ + Rejection: rejection{ + Threshold: threshold{ + CPUUtilization: -0.5, + HeapUtilization: 0.5, + }, + }, }, monitoredResources: []string{"cpu", "heap"}, err: errors.New("cpu_utilization must be between 0 and 1"), }, "utilization config greater than 1 should fail validation": { - instanceLimits: InstanceLimits{ - CPUUtilization: 0.5, - HeapUtilization: 1.5, + queryProtection: QueryProtection{ + Rejection: rejection{ + Threshold: threshold{ + CPUUtilization: 0.5, + HeapUtilization: 1.5, + }, + }, }, monitoredResources: []string{"cpu", "heap"}, err: errors.New("heap_utilization must be between 0 and 1"), }, "missing cpu in monitored_resources config should fail validation": { - instanceLimits: InstanceLimits{ - CPUUtilization: 0.5, + queryProtection: QueryProtection{ + Rejection: rejection{ + Threshold: threshold{ + CPUUtilization: 0.5, + }, + }, }, monitoredResources: []string{"heap"}, err: errors.New("monitored_resources config must include \"cpu\" as well"), }, "missing heap in monitored_resources config should fail validation": { - instanceLimits: InstanceLimits{ - HeapUtilization: 0.5, + queryProtection: QueryProtection{ + Rejection: rejection{ + Threshold: threshold{ + HeapUtilization: 0.5, + }, + }, }, monitoredResources: []string{"cpu"}, err: errors.New("monitored_resources config must include \"heap\" as well"), }, } { t.Run(name, func(t *testing.T) { - err := tc.instanceLimits.Validate(tc.monitoredResources) + err := tc.queryProtection.Validate(tc.monitoredResources) if tc.err != nil { require.Errorf(t, err, tc.err.Error()) } else { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 9d126228172..45852c3fd68 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -46,6 +46,7 @@ import ( "google.golang.org/grpc/codes" "github.com/cortexproject/cortex/pkg/chunk/encoding" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/querysharding" @@ -159,6 +160,8 @@ type Config struct { // If enabled, the metadata API returns all metadata regardless of the limits. SkipMetadataLimits bool `yaml:"skip_metadata_limits"` + + QueryProtection configs.QueryProtection `yaml:"query_protection"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -183,6 +186,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.SkipMetadataLimits, "ingester.skip-metadata-limits", true, "If enabled, the metadata API returns all metadata regardless of the limits.") cfg.DefaultLimits.RegisterFlagsWithPrefix(f, "ingester.") + cfg.QueryProtection.RegisterFlagsWithPrefix(f, "ingester.") } func (cfg *Config) Validate(monitoredResources flagext.StringSliceCSV) error { @@ -194,7 +198,7 @@ func (cfg *Config) Validate(monitoredResources flagext.StringSliceCSV) error { logutil.WarnExperimentalUse("String interning for metrics labels Enabled") } - if err := cfg.DefaultLimits.Validate(monitoredResources); err != nil { + if err := cfg.QueryProtection.Validate(monitoredResources); err != nil { return err } @@ -789,11 +793,11 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe if resourceMonitor != nil { resourceLimits := make(map[resource.Type]float64) - if cfg.DefaultLimits.CPUUtilization > 0 { - resourceLimits[resource.CPU] = cfg.DefaultLimits.CPUUtilization + if cfg.QueryProtection.Rejection.Threshold.CPUUtilization > 0 { + resourceLimits[resource.CPU] = cfg.QueryProtection.Rejection.Threshold.CPUUtilization } - if cfg.DefaultLimits.HeapUtilization > 0 { - resourceLimits[resource.Heap] = cfg.DefaultLimits.HeapUtilization + if cfg.QueryProtection.Rejection.Threshold.HeapUtilization > 0 { + resourceLimits[resource.Heap] = cfg.QueryProtection.Rejection.Threshold.HeapUtilization } i.resourceBasedLimiter, err = limiter.NewResourceBasedLimiter(resourceMonitor, resourceLimits, registerer, "ingester") if err != nil { diff --git a/pkg/ingester/instance_limits.go b/pkg/ingester/instance_limits.go index 334d1250e60..cc6a8d52b32 100644 --- a/pkg/ingester/instance_limits.go +++ b/pkg/ingester/instance_limits.go @@ -4,9 +4,6 @@ import ( "flag" "github.com/pkg/errors" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/util/flagext" ) var ( @@ -20,8 +17,6 @@ var ( // InstanceLimits describes limits used by ingester. Reaching any of these will result in error response to the call. type InstanceLimits struct { - configs.InstanceLimits `yaml:",inline"` - MaxIngestionRate float64 `yaml:"max_ingestion_rate"` MaxInMemoryTenants int64 `yaml:"max_tenants"` MaxInMemorySeries int64 `yaml:"max_series"` @@ -38,11 +33,6 @@ func (cfg *InstanceLimits) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix strin f.Int64Var(&cfg.MaxInMemorySeries, prefix+"instance-limits.max-series", 0, "Max series that this ingester can hold (across all tenants). Requests to create additional series will be rejected. This limit only works when using blocks engine. 0 = unlimited.") f.Int64Var(&cfg.MaxInflightPushRequests, prefix+"instance-limits.max-inflight-push-requests", 0, "Max inflight push requests that this ingester can handle (across all tenants). Additional requests will be rejected. 0 = unlimited.") f.Int64Var(&cfg.MaxInflightQueryRequests, prefix+"instance-limits.max-inflight-query-requests", 0, "Max inflight query requests that this ingester can handle (across all tenants). Additional requests will be rejected. 0 = unlimited.") - cfg.InstanceLimits.RegisterFlagsWithPrefix(f, prefix) -} - -func (cfg *InstanceLimits) Validate(monitoredResources flagext.StringSliceCSV) error { - return cfg.InstanceLimits.Validate(monitoredResources) } // UnmarshalYAML implements the yaml.Unmarshaler interface. If give diff --git a/pkg/storegateway/gateway.go b/pkg/storegateway/gateway.go index 157c185a68a..9e61d63abf2 100644 --- a/pkg/storegateway/gateway.go +++ b/pkg/storegateway/gateway.go @@ -68,7 +68,7 @@ type Config struct { EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"` DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"` - InstanceLimits configs.InstanceLimits `yaml:"instance_limits"` + QueryProtection configs.QueryProtection `yaml:"query_protection"` // Hedged Request HedgedRequest bucket.HedgedRequestConfig `yaml:"hedged_request"` @@ -83,7 +83,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.EnabledTenants, "store-gateway.enabled-tenants", "Comma separated list of tenants whose store metrics this storegateway can process. If specified, only these tenants will be handled by storegateway, otherwise this storegateway will be enabled for all the tenants in the store-gateway cluster.") f.Var(&cfg.DisabledTenants, "store-gateway.disabled-tenants", "Comma separated list of tenants whose store metrics this storegateway cannot process. If specified, a storegateway that would normally pick the specified tenant(s) for processing will ignore them instead.") cfg.HedgedRequest.RegisterFlagsWithPrefix(f, "store-gateway.") - cfg.InstanceLimits.RegisterFlagsWithPrefix(f, "store-gateway.") + cfg.QueryProtection.RegisterFlagsWithPrefix(f, "store-gateway.") } // Validate the Config. @@ -102,7 +102,7 @@ func (cfg *Config) Validate(limits validation.Limits, monitoredResources flagext return err } - if err := cfg.InstanceLimits.Validate(monitoredResources); err != nil { + if err := cfg.QueryProtection.Validate(monitoredResources); err != nil { return err } @@ -244,11 +244,11 @@ func newStoreGateway(gatewayCfg Config, storageCfg cortex_tsdb.BlocksStorageConf if resourceMonitor != nil { resourceLimits := make(map[resource.Type]float64) - if gatewayCfg.InstanceLimits.CPUUtilization > 0 { - resourceLimits[resource.CPU] = gatewayCfg.InstanceLimits.CPUUtilization + if gatewayCfg.QueryProtection.Rejection.Threshold.CPUUtilization > 0 { + resourceLimits[resource.CPU] = gatewayCfg.QueryProtection.Rejection.Threshold.CPUUtilization } - if gatewayCfg.InstanceLimits.HeapUtilization > 0 { - resourceLimits[resource.Heap] = gatewayCfg.InstanceLimits.HeapUtilization + if gatewayCfg.QueryProtection.Rejection.Threshold.HeapUtilization > 0 { + resourceLimits[resource.Heap] = gatewayCfg.QueryProtection.Rejection.Threshold.HeapUtilization } g.resourceBasedLimiter, err = util_limiter.NewResourceBasedLimiter(resourceMonitor, resourceLimits, reg, "store-gateway") if err != nil { From dcefc87e32fa89b9252238c15a2c86bcdc95d67f Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 29 May 2025 15:43:49 -0700 Subject: [PATCH 3/6] Nit Signed-off-by: Justin Jung --- docs/guides/protecting-cortex-from-heavy-queries.md | 12 ++++++++---- integration/resource_based_limiter_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/guides/protecting-cortex-from-heavy-queries.md b/docs/guides/protecting-cortex-from-heavy-queries.md index 4f76ef18824..d051158699d 100644 --- a/docs/guides/protecting-cortex-from-heavy-queries.md +++ b/docs/guides/protecting-cortex-from-heavy-queries.md @@ -48,9 +48,13 @@ For example, the following configuration will start throttling query requests if ``` target: ingester monitored_resources: cpu,heap -instance_limits: - cpu_utilization: 0.8 - heap_utilization: 0.8 +ingester: + query_protection: + rejection: + enabled: true + threshold: + cpu_utilization: 0.8 + heap_utilization: 0.8 ``` -See https://cortexmetrics.io/docs/configuration/configuration-file/:~:text=instance_limits for details. \ No newline at end of file +See https://cortexmetrics.io/docs/configuration/configuration-file/:~:text=instance_limits for details. diff --git a/integration/resource_based_limiter_test.go b/integration/resource_based_limiter_test.go index fc7e5b80916..4e8336198a3 100644 --- a/integration/resource_based_limiter_test.go +++ b/integration/resource_based_limiter_test.go @@ -29,12 +29,12 @@ func Test_ResourceBasedLimiter_shouldStartWithoutError(t *testing.T) { // Start Cortex components. ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{ - "-ingester.instance-limits.cpu-utilization": "0.8", - "-ingester.instance-limits.heap-utilization": "0.8", + "-ingester.query-protection.rejection.threshold.cpu-utilization": "0.8", + "-ingester.query-protection.rejection.threshold.heap-utilization": "0.8", }), "") storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{ - "-store-gateway.instance-limits.cpu-utilization": "0.8", - "-store-gateway.instance-limits.heap-utilization": "0.8", + "-store-gateway.query-protection.rejection.threshold.cpu-utilization": "0.8", + "-store-gateway.query-protection.rejection.threshold.heap-utilization": "0.8", }), "") require.NoError(t, s.StartAndWaitReady(ingester, storeGateway)) } From f79628ff2ed6ddc187685947dd74344722875d42 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 30 May 2025 12:41:26 -0700 Subject: [PATCH 4/6] Nit Signed-off-by: Justin Jung --- docs/guides/protecting-cortex-from-heavy-queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/protecting-cortex-from-heavy-queries.md b/docs/guides/protecting-cortex-from-heavy-queries.md index d051158699d..dd17b4cfa9f 100644 --- a/docs/guides/protecting-cortex-from-heavy-queries.md +++ b/docs/guides/protecting-cortex-from-heavy-queries.md @@ -57,4 +57,4 @@ ingester: heap_utilization: 0.8 ``` -See https://cortexmetrics.io/docs/configuration/configuration-file/:~:text=instance_limits for details. +See https://cortexmetrics.io/docs/configuration/configuration-file/:~:text=query_protection for details. From bdaa4b131d5b1c6e066681d370f8623f285d3b73 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 30 May 2025 12:49:16 -0700 Subject: [PATCH 5/6] Changelog Signed-off-by: Justin Jung --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34934c971a5..0a7db5acdf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727 +* [CHANGE] Ingester/StoreGateway: Change error code and configuration of resource-based limiter. #6771 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590 From f122b75f3e09af392cbd932b8191907684462efa Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 4 Jun 2025 10:03:07 -0700 Subject: [PATCH 6/6] Revert changelog Signed-off-by: Justin Jung --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a7db5acdf9..34934c971a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,6 @@ * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727 -* [CHANGE] Ingester/StoreGateway: Change error code and configuration of resource-based limiter. #6771 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590