From 57be974996c91dd2f8d685ee0c91917d8ff38975 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 12 Mar 2021 12:31:42 +0100 Subject: [PATCH 1/2] Deprecate old ruler storage config Signed-off-by: Marco Pracucci --- CHANGELOG.md | 5 +++-- docs/configuration/config-file-reference.md | 2 ++ docs/configuration/v1-guarantees.md | 2 -- pkg/alertmanager/multitenant.go | 2 +- pkg/ruler/ruler.go | 2 +- pkg/ruler/storage.go | 1 + 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59d89a2bcc9..df593b0fecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,9 @@ ## master / unreleased * [CHANGE] Alertmanager now removes local files after Alertmanager is no longer running for removed or resharded user. #3910 -* [CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.10. #3910 -* [FEATURE] Ruler Storage: Added `local` backend support to the ruler storage configuration under the `-ruler-storage.` flag prefix. #3932 +* [CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.11. #3910 +* [CHANGE] Ruler: deprecated `-ruler.storage.*` CLI flags (and their respective YAML config options) in favour of `-ruler-storage.*`. The deprecated config will be removed in Cortex 1.11. #3944 +* [FEATURE] Ruler: added `local` backend support to the ruler storage configuration under the `-ruler-storage.` flag prefix. #3932 * [ENHANCEMENT] Ruler: optimized `/api/v1/rules` and `/api/v1/alerts` when ruler sharding is enabled. #3916 * [ENHANCEMENT] Ruler: added the following metrics when ruler sharding is enabled: #3916 * `cortex_ruler_clients` diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index e4b3cad46f2..395b90d2d05 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -1596,6 +1596,8 @@ ruler_client: # CLI flag: -ruler.poll-interval [poll_interval: | default = 1m] +# Deprecated. Use -ruler-storage.* CLI flags and their respective YAML config +# options instead. storage: # Method to use for backend rule storage (configdb, azure, gcs, s3, swift, # local) diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index ca73229d108..71f44b44ef1 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -70,9 +70,7 @@ Currently experimental features are: - Alertmanager: Sharding of tenants across multiple instances - The thanosconvert tool for converting Thanos block metadata to Cortex - HA Tracker: cleanup of old replicas from KV Store. -- Ruler storage: backend client configuration options using a config fields similar to the blocks storage backend clients. - Alertmanager storage: backend client configuration options using a config fields similar to the blocks storage backend clients. -- Ruler storage: backend client configuration options using a config fields similar to the TSDB object storage clients. - Flags for configuring whether blocks-ingester streams samples or chunks are temporary, and will be removed when feature is tested: - `-ingester.stream-chunks-when-using-blocks` CLI flag - `ingester_stream_chunks_when_using_blocks` (boolean) field in runtime config file diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 8a247168114..66756bef10e 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -508,7 +508,7 @@ func (am *MultitenantAlertmanager) starting(ctx context.Context) (err error) { } // migrateStateFilesToPerTenantDirectories migrates any existing configuration from old place to new hierarchy. -// TODO: Remove in Cortex 1.10. +// TODO: Remove in Cortex 1.11. func (am *MultitenantAlertmanager) migrateStateFilesToPerTenantDirectories() error { migrate := func(from, to string) error { level.Info(am.logger).Log("msg", "migrating alertmanager state", "from", from, "to", to) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index a4fcd20c50a..914da55833d 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -72,7 +72,7 @@ type Config struct { // How frequently to poll for updated rules. PollInterval time.Duration `yaml:"poll_interval"` // Rule Storage and Polling configuration. - StoreConfig RuleStoreConfig `yaml:"storage"` + StoreConfig RuleStoreConfig `yaml:"storage" doc:"description=Deprecated. Use -ruler-storage.* CLI flags and their respective YAML config options instead."` // Path to store rule files for prom manager. RulePath string `yaml:"rule_path"` diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 6193fbdc369..ef008f53255 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -26,6 +26,7 @@ import ( ) // RuleStoreConfig configures a rule store. +// TODO remove this legacy config in Cortex 1.11. type RuleStoreConfig struct { Type string `yaml:"type"` ConfigDB client.Config `yaml:"configdb"` From dd1e093da36430255777aee4ac2e174388a17912 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 12 Mar 2021 12:36:45 +0100 Subject: [PATCH 2/2] Updated doc Signed-off-by: Marco Pracucci --- docs/configuration/config-file-reference.md | 422 ++++++++++---------- docs/guides/sharded_ruler.md | 4 +- tools/doc-generator/main.go | 6 + 3 files changed, 223 insertions(+), 209 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 395b90d2d05..999b34ea3a5 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -153,213 +153,8 @@ tenant_federation: # The ruler_config configures the Cortex ruler. [ruler: ] -ruler_storage: - # Backend storage to use. Supported backends are: s3, gcs, azure, swift, - # filesystem, configdb, local. - # CLI flag: -ruler-storage.backend - [backend: | default = "s3"] - - s3: - # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at - # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an - # S3-compatible service in hostname:port format. - # CLI flag: -ruler-storage.s3.endpoint - [endpoint: | default = ""] - - # S3 region. If unset, the client will issue a S3 GetBucketLocation API call - # to autodetect it. - # CLI flag: -ruler-storage.s3.region - [region: | default = ""] - - # S3 bucket name - # CLI flag: -ruler-storage.s3.bucket-name - [bucket_name: | default = ""] - - # S3 secret access key - # CLI flag: -ruler-storage.s3.secret-access-key - [secret_access_key: | default = ""] - - # S3 access key ID - # CLI flag: -ruler-storage.s3.access-key-id - [access_key_id: | default = ""] - - # If enabled, use http:// for the S3 endpoint instead of https://. This - # could be useful in local dev/test environments while using an - # S3-compatible backend storage, like Minio. - # CLI flag: -ruler-storage.s3.insecure - [insecure: | default = false] - - # The signature version to use for authenticating against S3. Supported - # values are: v4, v2. - # CLI flag: -ruler-storage.s3.signature-version - [signature_version: | default = "v4"] - - # The s3_sse_config configures the S3 server-side encryption. - # The CLI flags prefix for this block config is: ruler-storage - [sse: ] - - http: - # The time an idle connection will remain idle before closing. - # CLI flag: -ruler-storage.s3.http.idle-conn-timeout - [idle_conn_timeout: | default = 1m30s] - - # The amount of time the client will wait for a servers response headers. - # CLI flag: -ruler-storage.s3.http.response-header-timeout - [response_header_timeout: | default = 2m] - - # If the client connects to S3 via HTTPS and this option is enabled, the - # client will accept any certificate and hostname. - # CLI flag: -ruler-storage.s3.http.insecure-skip-verify - [insecure_skip_verify: | default = false] - - # Maximum time to wait for a TLS handshake. 0 means no limit. - # CLI flag: -ruler-storage.s3.tls-handshake-timeout - [tls_handshake_timeout: | default = 10s] - - # The time to wait for a server's first response headers after fully - # writing the request headers if the request has an Expect header. 0 to - # send the request body immediately. - # CLI flag: -ruler-storage.s3.expect-continue-timeout - [expect_continue_timeout: | default = 1s] - - # Maximum number of idle (keep-alive) connections across all hosts. 0 - # means no limit. - # CLI flag: -ruler-storage.s3.max-idle-connections - [max_idle_connections: | default = 100] - - # Maximum number of idle (keep-alive) connections to keep per-host. If 0, - # a built-in default value is used. - # CLI flag: -ruler-storage.s3.max-idle-connections-per-host - [max_idle_connections_per_host: | default = 100] - - # Maximum number of connections per host. 0 means no limit. - # CLI flag: -ruler-storage.s3.max-connections-per-host - [max_connections_per_host: | default = 0] - - gcs: - # GCS bucket name - # CLI flag: -ruler-storage.gcs.bucket-name - [bucket_name: | default = ""] - - # JSON representing either a Google Developers Console - # client_credentials.json file or a Google Developers service account key - # file. If empty, fallback to Google default logic. - # CLI flag: -ruler-storage.gcs.service-account - [service_account: | default = ""] - - azure: - # Azure storage account name - # CLI flag: -ruler-storage.azure.account-name - [account_name: | default = ""] - - # Azure storage account key - # CLI flag: -ruler-storage.azure.account-key - [account_key: | default = ""] - - # Azure storage container name - # CLI flag: -ruler-storage.azure.container-name - [container_name: | default = ""] - - # Azure storage endpoint suffix without schema. The account name will be - # prefixed to this value to create the FQDN - # CLI flag: -ruler-storage.azure.endpoint-suffix - [endpoint_suffix: | default = ""] - - # Number of retries for recoverable errors - # CLI flag: -ruler-storage.azure.max-retries - [max_retries: | default = 20] - - swift: - # OpenStack Swift authentication API version. 0 to autodetect. - # CLI flag: -ruler-storage.swift.auth-version - [auth_version: | default = 0] - - # OpenStack Swift authentication URL - # CLI flag: -ruler-storage.swift.auth-url - [auth_url: | default = ""] - - # OpenStack Swift username. - # CLI flag: -ruler-storage.swift.username - [username: | default = ""] - - # OpenStack Swift user's domain name. - # CLI flag: -ruler-storage.swift.user-domain-name - [user_domain_name: | default = ""] - - # OpenStack Swift user's domain ID. - # CLI flag: -ruler-storage.swift.user-domain-id - [user_domain_id: | default = ""] - - # OpenStack Swift user ID. - # CLI flag: -ruler-storage.swift.user-id - [user_id: | default = ""] - - # OpenStack Swift API key. - # CLI flag: -ruler-storage.swift.password - [password: | default = ""] - - # OpenStack Swift user's domain ID. - # CLI flag: -ruler-storage.swift.domain-id - [domain_id: | default = ""] - - # OpenStack Swift user's domain name. - # CLI flag: -ruler-storage.swift.domain-name - [domain_name: | default = ""] - - # OpenStack Swift project ID (v2,v3 auth only). - # CLI flag: -ruler-storage.swift.project-id - [project_id: | default = ""] - - # OpenStack Swift project name (v2,v3 auth only). - # CLI flag: -ruler-storage.swift.project-name - [project_name: | default = ""] - - # ID of the OpenStack Swift project's domain (v3 auth only), only needed if - # it differs the from user domain. - # CLI flag: -ruler-storage.swift.project-domain-id - [project_domain_id: | default = ""] - - # Name of the OpenStack Swift project's domain (v3 auth only), only needed - # if it differs from the user domain. - # CLI flag: -ruler-storage.swift.project-domain-name - [project_domain_name: | default = ""] - - # OpenStack Swift Region to use (v2,v3 auth only). - # CLI flag: -ruler-storage.swift.region-name - [region_name: | default = ""] - - # Name of the OpenStack Swift container to put chunks in. - # CLI flag: -ruler-storage.swift.container-name - [container_name: | default = ""] - - # Max retries on requests error. - # CLI flag: -ruler-storage.swift.max-retries - [max_retries: | default = 3] - - # Time after which a connection attempt is aborted. - # CLI flag: -ruler-storage.swift.connect-timeout - [connect_timeout: | default = 10s] - - # Time after which an idle request is aborted. The timeout watchdog is reset - # each time some data is received, so the timeout triggers after X time no - # data is received on a request. - # CLI flag: -ruler-storage.swift.request-timeout - [request_timeout: | default = 5s] - - filesystem: - # Local filesystem storage directory. - # CLI flag: -ruler-storage.filesystem.dir - [dir: | default = ""] - - # The configstore_config configures the config database storing rules and - # alerts, and is used by the Cortex alertmanager. - # The CLI flags prefix for this block config is: ruler-storage - [configdb: ] - - local: - # Directory to scan for rules - # CLI flag: -ruler-storage.local.directory - [directory: | default = ""] +# The ruler_storage_config configures the Cortex ruler storage backend. +[ruler_storage: ] # The configs_config configures the Cortex Configs DB and API. [configs: ] @@ -1970,6 +1765,219 @@ ring: [enable_api: | default = false] ``` +### `ruler_storage_config` + +The `ruler_storage_config` configures the Cortex ruler storage backend. + +```yaml +# Backend storage to use. Supported backends are: s3, gcs, azure, swift, +# filesystem, configdb, local. +# CLI flag: -ruler-storage.backend +[backend: | default = "s3"] + +s3: + # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at + # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an + # S3-compatible service in hostname:port format. + # CLI flag: -ruler-storage.s3.endpoint + [endpoint: | default = ""] + + # S3 region. If unset, the client will issue a S3 GetBucketLocation API call + # to autodetect it. + # CLI flag: -ruler-storage.s3.region + [region: | default = ""] + + # S3 bucket name + # CLI flag: -ruler-storage.s3.bucket-name + [bucket_name: | default = ""] + + # S3 secret access key + # CLI flag: -ruler-storage.s3.secret-access-key + [secret_access_key: | default = ""] + + # S3 access key ID + # CLI flag: -ruler-storage.s3.access-key-id + [access_key_id: | default = ""] + + # If enabled, use http:// for the S3 endpoint instead of https://. This could + # be useful in local dev/test environments while using an S3-compatible + # backend storage, like Minio. + # CLI flag: -ruler-storage.s3.insecure + [insecure: | default = false] + + # The signature version to use for authenticating against S3. Supported values + # are: v4, v2. + # CLI flag: -ruler-storage.s3.signature-version + [signature_version: | default = "v4"] + + # The s3_sse_config configures the S3 server-side encryption. + # The CLI flags prefix for this block config is: ruler-storage + [sse: ] + + http: + # The time an idle connection will remain idle before closing. + # CLI flag: -ruler-storage.s3.http.idle-conn-timeout + [idle_conn_timeout: | default = 1m30s] + + # The amount of time the client will wait for a servers response headers. + # CLI flag: -ruler-storage.s3.http.response-header-timeout + [response_header_timeout: | default = 2m] + + # If the client connects to S3 via HTTPS and this option is enabled, the + # client will accept any certificate and hostname. + # CLI flag: -ruler-storage.s3.http.insecure-skip-verify + [insecure_skip_verify: | default = false] + + # Maximum time to wait for a TLS handshake. 0 means no limit. + # CLI flag: -ruler-storage.s3.tls-handshake-timeout + [tls_handshake_timeout: | default = 10s] + + # The time to wait for a server's first response headers after fully writing + # the request headers if the request has an Expect header. 0 to send the + # request body immediately. + # CLI flag: -ruler-storage.s3.expect-continue-timeout + [expect_continue_timeout: | default = 1s] + + # Maximum number of idle (keep-alive) connections across all hosts. 0 means + # no limit. + # CLI flag: -ruler-storage.s3.max-idle-connections + [max_idle_connections: | default = 100] + + # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a + # built-in default value is used. + # CLI flag: -ruler-storage.s3.max-idle-connections-per-host + [max_idle_connections_per_host: | default = 100] + + # Maximum number of connections per host. 0 means no limit. + # CLI flag: -ruler-storage.s3.max-connections-per-host + [max_connections_per_host: | default = 0] + +gcs: + # GCS bucket name + # CLI flag: -ruler-storage.gcs.bucket-name + [bucket_name: | default = ""] + + # JSON representing either a Google Developers Console client_credentials.json + # file or a Google Developers service account key file. If empty, fallback to + # Google default logic. + # CLI flag: -ruler-storage.gcs.service-account + [service_account: | default = ""] + +azure: + # Azure storage account name + # CLI flag: -ruler-storage.azure.account-name + [account_name: | default = ""] + + # Azure storage account key + # CLI flag: -ruler-storage.azure.account-key + [account_key: | default = ""] + + # Azure storage container name + # CLI flag: -ruler-storage.azure.container-name + [container_name: | default = ""] + + # Azure storage endpoint suffix without schema. The account name will be + # prefixed to this value to create the FQDN + # CLI flag: -ruler-storage.azure.endpoint-suffix + [endpoint_suffix: | default = ""] + + # Number of retries for recoverable errors + # CLI flag: -ruler-storage.azure.max-retries + [max_retries: | default = 20] + +swift: + # OpenStack Swift authentication API version. 0 to autodetect. + # CLI flag: -ruler-storage.swift.auth-version + [auth_version: | default = 0] + + # OpenStack Swift authentication URL + # CLI flag: -ruler-storage.swift.auth-url + [auth_url: | default = ""] + + # OpenStack Swift username. + # CLI flag: -ruler-storage.swift.username + [username: | default = ""] + + # OpenStack Swift user's domain name. + # CLI flag: -ruler-storage.swift.user-domain-name + [user_domain_name: | default = ""] + + # OpenStack Swift user's domain ID. + # CLI flag: -ruler-storage.swift.user-domain-id + [user_domain_id: | default = ""] + + # OpenStack Swift user ID. + # CLI flag: -ruler-storage.swift.user-id + [user_id: | default = ""] + + # OpenStack Swift API key. + # CLI flag: -ruler-storage.swift.password + [password: | default = ""] + + # OpenStack Swift user's domain ID. + # CLI flag: -ruler-storage.swift.domain-id + [domain_id: | default = ""] + + # OpenStack Swift user's domain name. + # CLI flag: -ruler-storage.swift.domain-name + [domain_name: | default = ""] + + # OpenStack Swift project ID (v2,v3 auth only). + # CLI flag: -ruler-storage.swift.project-id + [project_id: | default = ""] + + # OpenStack Swift project name (v2,v3 auth only). + # CLI flag: -ruler-storage.swift.project-name + [project_name: | default = ""] + + # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it + # differs the from user domain. + # CLI flag: -ruler-storage.swift.project-domain-id + [project_domain_id: | default = ""] + + # Name of the OpenStack Swift project's domain (v3 auth only), only needed if + # it differs from the user domain. + # CLI flag: -ruler-storage.swift.project-domain-name + [project_domain_name: | default = ""] + + # OpenStack Swift Region to use (v2,v3 auth only). + # CLI flag: -ruler-storage.swift.region-name + [region_name: | default = ""] + + # Name of the OpenStack Swift container to put chunks in. + # CLI flag: -ruler-storage.swift.container-name + [container_name: | default = ""] + + # Max retries on requests error. + # CLI flag: -ruler-storage.swift.max-retries + [max_retries: | default = 3] + + # Time after which a connection attempt is aborted. + # CLI flag: -ruler-storage.swift.connect-timeout + [connect_timeout: | default = 10s] + + # Time after which an idle request is aborted. The timeout watchdog is reset + # each time some data is received, so the timeout triggers after X time no + # data is received on a request. + # CLI flag: -ruler-storage.swift.request-timeout + [request_timeout: | default = 5s] + +filesystem: + # Local filesystem storage directory. + # CLI flag: -ruler-storage.filesystem.dir + [dir: | default = ""] + +# The configstore_config configures the config database storing rules and +# alerts, and is used by the Cortex alertmanager. +# The CLI flags prefix for this block config is: ruler-storage +[configdb: ] + +local: + # Directory to scan for rules + # CLI flag: -ruler-storage.local.directory + [directory: | default = ""] +``` + ### `alertmanager_config` The `alertmanager_config` configures the Cortex alertmanager. diff --git a/docs/guides/sharded_ruler.md b/docs/guides/sharded_ruler.md index d5dd3bd2a2a..f84a85a476c 100644 --- a/docs/guides/sharded_ruler.md +++ b/docs/guides/sharded_ruler.md @@ -35,8 +35,8 @@ The local implementation reads [Prometheus recording rules](https://prometheus.i A typical local config may look something like: ``` - -ruler.storage.type=local - -ruler.storage.local.directory=/tmp/cortex/rules + -ruler-storage.backend=local + -ruler-storage.local.directory=/tmp/cortex/rules ``` With the above configuration the ruler would expect the following layout: diff --git a/tools/doc-generator/main.go b/tools/doc-generator/main.go index fc91e16c703..4987cb88977 100644 --- a/tools/doc-generator/main.go +++ b/tools/doc-generator/main.go @@ -32,6 +32,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/kv/etcd" "github.com/cortexproject/cortex/pkg/ring/kv/memberlist" "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/ruler/rulestore" "github.com/cortexproject/cortex/pkg/storage/bucket/s3" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway" @@ -82,6 +83,11 @@ var ( structType: reflect.TypeOf(ruler.Config{}), desc: "The ruler_config configures the Cortex ruler.", }, + { + name: "ruler_storage_config", + structType: reflect.TypeOf(rulestore.Config{}), + desc: "The ruler_storage_config configures the Cortex ruler storage backend.", + }, { name: "alertmanager_config", structType: reflect.TypeOf(alertmanager.MultitenantAlertmanagerConfig{}),