diff --git a/CHANGELOG.md b/CHANGELOG.md index e9380c72c92..98bf78baa04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * [FEATURE] Query Frontend: Add `cortex_rejected_queries_total` metric for throttled queries. #5356 * [FEATURE] Querier: Log query stats when querying store gateway. #5376 * [FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. #5393 +* [FEATURE] Added the flag `-alertmanager.api-concurrency` to configure alert manager api concurrency limit. #5412 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 5c729aa3605..371298d5912 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -416,6 +416,10 @@ cluster: # CLI flag: -experimental.alertmanager.enable-api [enable_api: | default = false] +# Maximum number of concurrent GET API requests before returning an error. +# CLI flag: -alertmanager.api-concurrency +[api_concurrency: | default = 0] + alertmanager_client: # Timeout for downstream alertmanagers. # CLI flag: -alertmanager.alertmanager-client.remote-timeout diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index f900076d38d..6ac39ddfbb6 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -80,6 +80,7 @@ type Config struct { Replicator Replicator Store alertstore.AlertStore PersisterConfig PersisterConfig + APIConcurrency int } // An Alertmanager manages the alerts for one user. @@ -266,6 +267,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) { GroupFunc: func(f1 func(*dispatch.Route) bool, f2 func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[model.Fingerprint][]string) { return am.dispatcher.Groups(f1, f2) }, + Concurrency: am.cfg.APIConcurrency, }) if err != nil { return nil, fmt.Errorf("failed to create api: %v", err) diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 3ed0c99b903..4710f85b48d 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -78,7 +78,8 @@ type MultitenantAlertmanagerConfig struct { Cluster ClusterConfig `yaml:"cluster"` - EnableAPI bool `yaml:"enable_api"` + EnableAPI bool `yaml:"enable_api"` + APIConcurrency int `yaml:"api_concurrency"` // For distributor. AlertmanagerClient ClientConfig `yaml:"alertmanager_client"` @@ -117,6 +118,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.PollInterval, "alertmanager.configs.poll-interval", 15*time.Second, "How frequently to poll Cortex configs") f.BoolVar(&cfg.EnableAPI, "experimental.alertmanager.enable-api", false, "Enable the experimental alertmanager config api.") + f.IntVar(&cfg.APIConcurrency, "alertmanager.api-concurrency", 0, "Maximum number of concurrent GET API requests before returning an error.") f.BoolVar(&cfg.ShardingEnabled, "alertmanager.sharding-enabled", false, "Shard tenants across multiple alertmanager instances.") f.Var(&cfg.EnabledTenants, "alertmanager.enabled-tenants", "Comma separated list of tenants whose alerts this alertmanager can process. If specified, only these tenants will be handled by alertmanager, otherwise this alertmanager can process alerts from all tenants.") @@ -965,6 +967,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco Store: am.store, PersisterConfig: am.cfg.Persister, Limits: am.limits, + APIConcurrency: am.cfg.APIConcurrency, }, reg) if err != nil { return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err)