From 58bd57367a1da2065353ccfe3f8dd4f0a24bfa6d Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 20 Apr 2020 13:10:49 -0400 Subject: [PATCH 01/16] migrate cortex ruler to use Thanos objstore bucket client Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 2 +- pkg/ruler/rules/objectclient/rule_store.go | 64 ++++++++++++---------- pkg/ruler/storage.go | 32 ++++++++--- pkg/storage/backend/azure/config.go | 2 +- pkg/storage/backend/gcs/config.go | 2 +- 5 files changed, 61 insertions(+), 41 deletions(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 54a15d3fe96..3a17834eaee 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -167,7 +167,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable promStorage.Queryable return nil, err } - ruleStore, err := NewRuleStorage(cfg.StoreConfig) + ruleStore, err := NewRuleStorage(cfg.StoreConfig, logger) if err != nil { return nil, err } diff --git a/pkg/ruler/rules/objectclient/rule_store.go b/pkg/ruler/rules/objectclient/rule_store.go index b76074efdef..a312c4d0a08 100644 --- a/pkg/ruler/rules/objectclient/rule_store.go +++ b/pkg/ruler/rules/objectclient/rule_store.go @@ -8,8 +8,8 @@ import ( "github.com/go-kit/kit/log/level" proto "github.com/gogo/protobuf/proto" + "github.com/thanos-io/thanos/pkg/objstore" - "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util" ) @@ -25,19 +25,20 @@ const ( // RuleStore allows cortex rules to be stored using an object store backend. type RuleStore struct { - client chunk.ObjectClient + bucket objstore.Bucket } // NewRuleStore returns a new RuleStore -func NewRuleStore(client chunk.ObjectClient) *RuleStore { +func NewRuleStore(bucket objstore.Bucket) *RuleStore { return &RuleStore{ - client: client, + bucket: bucket, } } func (o *RuleStore) getRuleGroup(ctx context.Context, objectKey string) (*rules.RuleGroupDesc, error) { - reader, err := o.client.GetObject(ctx, objectKey) - if err == chunk.ErrStorageObjectNotFound { + + reader, err := o.bucket.Get(ctx, objectKey) + if o.bucket.IsObjNotFoundErr(err) { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", objectKey) return nil, rules.ErrGroupNotFound } @@ -64,28 +65,28 @@ func (o *RuleStore) getRuleGroup(ctx context.Context, objectKey string) (*rules. // ListAllRuleGroups returns all the active rule groups func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) { - ruleGroupObjects, _, err := o.client.List(ctx, generateRuleObjectKey("", "", "")) - if err != nil { - return nil, err - } - userGroupMap := map[string]rules.RuleGroupList{} - for _, obj := range ruleGroupObjects { - - user := decomposeRuleObjectKey(obj.Key) + err := o.bucket.Iter(ctx, generateRuleObjectKey("", "", ""), func(s string) error { + user := decomposeRuleObjectKey(s) if user == "" { - continue + return nil } - rg, err := o.getRuleGroup(ctx, obj.Key) + rg, err := o.getRuleGroup(ctx, s) if err != nil { - return nil, err + return err } if _, exists := userGroupMap[user]; !exists { userGroupMap[user] = rules.RuleGroupList{} } userGroupMap[user] = append(userGroupMap[user], rg) + + return nil + }) + + if err != nil { + return nil, err } return userGroupMap, nil @@ -93,22 +94,25 @@ func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.Rul // ListRuleGroups returns all the active rule groups for a user func (o *RuleStore) ListRuleGroups(ctx context.Context, userID, namespace string) (rules.RuleGroupList, error) { - ruleGroupObjects, _, err := o.client.List(ctx, generateRuleObjectKey(userID, namespace, "")) - if err != nil { - return nil, err - } - groups := []*rules.RuleGroupDesc{} - for _, obj := range ruleGroupObjects { - level.Debug(util.Logger).Log("msg", "listing rule group", "key", obj.Key) - rg, err := o.getRuleGroup(ctx, obj.Key) + err := o.bucket.Iter(ctx, generateRuleObjectKey(userID, namespace, ""), func(s string) error { + level.Debug(util.Logger).Log("msg", "listing rule group", "key", s) + + rg, err := o.getRuleGroup(ctx, s) if err != nil { - level.Error(util.Logger).Log("msg", "unable to retrieve rule group", "err", err, "key", obj.Key) - return nil, err + level.Error(util.Logger).Log("msg", "unable to retrieve rule group", "err", err, "key", s) + return err } groups = append(groups, rg) + + return nil + }) + + if err != nil { + return nil, err } + return groups, nil } @@ -131,14 +135,14 @@ func (o *RuleStore) SetRuleGroup(ctx context.Context, userID string, namespace s } objectKey := generateRuleObjectKey(userID, namespace, group.Name) - return o.client.PutObject(ctx, objectKey, bytes.NewReader(data)) + return o.bucket.Upload(ctx, objectKey, bytes.NewReader(data)) } // DeleteRuleGroup deletes the specified rule group func (o *RuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, groupName string) error { objectKey := generateRuleObjectKey(userID, namespace, groupName) - err := o.client.DeleteObject(ctx, objectKey) - if err == chunk.ErrStorageObjectNotFound { + err := o.bucket.Delete(ctx, objectKey) + if o.bucket.IsObjNotFoundErr(err) { return rules.ErrGroupNotFound } return err diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index e9da9665918..b2716da4a2b 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" +<<<<<<< HEAD "github.com/pkg/errors" "github.com/cortexproject/cortex/pkg/chunk" @@ -12,9 +13,18 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/azure" "github.com/cortexproject/cortex/pkg/chunk/gcp" "github.com/cortexproject/cortex/pkg/chunk/openstack" +======= + "github.com/go-kit/kit/log" + "github.com/thanos-io/thanos/pkg/objstore" + +>>>>>>> af18111b3... migrate cortex ruler to use Thanos objstore bucket client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/ruler/rules/objectclient" + "github.com/cortexproject/cortex/pkg/storage/backend/azure" + "github.com/cortexproject/cortex/pkg/storage/backend/filesystem" + "github.com/cortexproject/cortex/pkg/storage/backend/gcs" + "github.com/cortexproject/cortex/pkg/storage/backend/s3" ) // RuleStoreConfig conigures a rule store @@ -23,10 +33,10 @@ type RuleStoreConfig struct { ConfigDB client.Config `yaml:"configdb"` // Object Storage Configs - Azure azure.BlobStorageConfig `yaml:"azure"` - GCS gcp.GCSConfig `yaml:"gcs"` - S3 aws.S3Config `yaml:"s3"` - Swift openstack.SwiftConfig `yaml:"swift"` + S3 s3.Config `yaml:"s3"` + GCS gcs.Config `yaml:"gcs"` + Azure azure.Config `yaml:"azure"` + Filesystem filesystem.Config `yaml:"filesystem"` mock rules.RuleStore `yaml:"-"` } @@ -50,7 +60,7 @@ func (cfg *RuleStoreConfig) Validate() error { } // NewRuleStorage returns a new rule storage backend poller and store -func NewRuleStorage(cfg RuleStoreConfig) (rules.RuleStore, error) { +func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, error) { if cfg.mock != nil { return cfg.mock, nil } @@ -65,19 +75,25 @@ func NewRuleStorage(cfg RuleStoreConfig) (rules.RuleStore, error) { return rules.NewConfigRuleStore(c), nil case "azure": - return newObjRuleStore(azure.NewBlobStorage(&cfg.Azure, "")) + return newObjRuleStore(azure.NewBucketClient(cfg.Azure, "cortex-ruler", logger)) case "gcs": - return newObjRuleStore(gcp.NewGCSObjectClient(context.Background(), cfg.GCS, "")) + return newObjRuleStore(gcs.NewBucketClient(context.Background(), cfg.GCS, "cortex-ruler", logger)) case "s3": +<<<<<<< HEAD return newObjRuleStore(aws.NewS3ObjectClient(cfg.S3, "")) case "swift": return newObjRuleStore(openstack.NewSwiftObjectClient(cfg.Swift, "")) +======= + return newObjRuleStore(s3.NewBucketClient(cfg.S3, "cortex-ruler", logger)) + case "filesystem": + return newObjRuleStore(filesystem.NewBucketClient(cfg.Filesystem)) +>>>>>>> af18111b3... migrate cortex ruler to use Thanos objstore bucket client default: return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) } } -func newObjRuleStore(client chunk.ObjectClient, err error) (rules.RuleStore, error) { +func newObjRuleStore(client objstore.Bucket, err error) (rules.RuleStore, error) { if err != nil { return nil, err } diff --git a/pkg/storage/backend/azure/config.go b/pkg/storage/backend/azure/config.go index 98e721160ac..8c35303251e 100644 --- a/pkg/storage/backend/azure/config.go +++ b/pkg/storage/backend/azure/config.go @@ -15,7 +15,7 @@ type Config struct { MaxRetries int `yaml:"max_retries"` } -// RegisterFlags registers the flags for TSDB Azure storage +// RegisterFlags registers the flags for TSDB Azure storage with the TSDB prefix func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.RegisterFlagsWithPrefix("experimental.tsdb.", f) } diff --git a/pkg/storage/backend/gcs/config.go b/pkg/storage/backend/gcs/config.go index d46131ffadc..3025ea8a6d0 100644 --- a/pkg/storage/backend/gcs/config.go +++ b/pkg/storage/backend/gcs/config.go @@ -12,7 +12,7 @@ type Config struct { ServiceAccount flagext.Secret `yaml:"service_account"` } -// RegisterFlags registers the flags for TSDB GCS storage +// RegisterFlags registers the flags for TSDB GCS storage with the TSDB prefix func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.RegisterFlagsWithPrefix("experimental.tsdb.", f) } From d06a9f8e48c931c63d9eda841c3d7413359f0f91 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 20 Apr 2020 13:25:41 -0400 Subject: [PATCH 02/16] remove filesystem backend from the Ruler and update docs Signed-off-by: Jacob Lisi --- docs/configuration/config-file-reference.md | 105 ++++++++------------ pkg/ruler/storage.go | 24 +---- 2 files changed, 48 insertions(+), 81 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index c2c49bff79d..27d8d53b2b0 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -780,78 +780,61 @@ storage: # The CLI flags prefix for this block config is: ruler [configdb: ] - azure: - # Name of the blob container used to store chunks. Defaults to `cortex`. - # This container must be created before running cortex. - # CLI flag: -ruler.storage.azure.container-name - [container_name: | default = "cortex"] - - # The Microsoft Azure account name to be used - # CLI flag: -ruler.storage.azure.account-name - [account_name: | default = ""] - - # The Microsoft Azure account key to use. - # CLI flag: -ruler.storage.azure.account-key - [account_key: | default = ""] + s3: + # S3 endpoint without schema + # CLI flag: -ruler.storage.s3.endpoint + [endpoint: | default = ""] - # Preallocated buffer size for downloads (default is 512KB) - # CLI flag: -ruler.storage.azure.download-buffer-size - [download_buffer_size: | default = 512000] + # S3 bucket name + # CLI flag: -ruler.storage.s3.bucket-name + [bucket_name: | default = ""] - # Preallocated buffer size for up;oads (default is 256KB) - # CLI flag: -ruler.storage.azure.upload-buffer-size - [upload_buffer_size: | default = 256000] + # S3 secret access key + # CLI flag: -ruler.storage.s3.secret-access-key + [secret_access_key: | default = ""] - # Number of buffers used to used to upload a chunk. (defaults to 1) - # CLI flag: -ruler.storage.azure.download-buffer-count - [upload_buffer_count: | default = 1] + # S3 access key ID + # CLI flag: -ruler.storage.s3.access-key-id + [access_key_id: | default = ""] - # Timeout for requests made against azure blob storage. Defaults to 30 - # seconds. - # CLI flag: -ruler.storage.azure.request-timeout - [request_timeout: | default = 30s] + # If enabled, use http:// for the S3 endpoint instead of https://. This + # could be useful in local dev/test environments while using an + # S3-compatible backend storage, like Minio. + # CLI flag: -ruler.storage.s3.insecure + [insecure: | default = false] - # Number of retries for a request which times out. - # CLI flag: -ruler.storage.azure.max-retries - [max_retries: | default = 5] + gcs: + # GCS bucket name + # CLI flag: -ruler.storage.gcs.bucket-name + [bucket_name: | default = ""] - # Minimum time to wait before retrying a request. - # CLI flag: -ruler.storage.azure.min-retry-delay - [min_retry_delay: | default = 10ms] + # JSON representing either a Google Developers Console + # client_credentials.json file or a Google Developers service account key + # file. If empty, fallback to Google default logic. + # CLI flag: -ruler.storage.gcs.service-account + [service_account: | default = ""] - # Maximum time to wait before retrying a request. - # CLI flag: -ruler.storage.azure.max-retry-delay - [max_retry_delay: | default = 500ms] + azure: + # Azure storage account name + # CLI flag: -ruler.storage.azure.account-name + [account_name: | default = ""] - gcs: - # Name of GCS bucket to put chunks in. - # CLI flag: -ruler.storage.gcs.bucketname - [bucket_name: | default = ""] + # Azure storage account key + # CLI flag: -ruler.storage.azure.account-key + [account_key: | default = ""] - # The size of the buffer that GCS client for each PUT request. 0 to disable - # buffering. - # CLI flag: -ruler.storage.gcs.chunk-buffer-size - [chunk_buffer_size: | default = 0] + # Azure storage container name + # CLI flag: -ruler.storage.azure.container-name + [container_name: | default = ""] - # The duration after which the requests to GCS should be timed out. - # CLI flag: -ruler.storage.gcs.request-timeout - [request_timeout: | default = 0s] + # Azure storage endpoint suffix without schema. The account name will be + # prefixed to this value to create the FQDN + # CLI flag: -ruler.storage.azure.endpoint-suffix + [endpoint_suffix: | default = ""] - s3: - # S3 endpoint URL with escaped Key and Secret encoded. If only region is - # specified as a host, proper endpoint will be deduced. Use - # inmemory:/// to use a mock in-memory implementation. - # CLI flag: -ruler.storage.s3.url - [s3: | default = ] - - # Comma separated list of bucket names to evenly distribute chunks over. - # Overrides any buckets specified in s3.url flag - # CLI flag: -ruler.storage.s3.buckets - [bucketnames: | default = ""] - - # Set this to `true` to force the request to use path-style addressing. - # CLI flag: -ruler.storage.s3.force-path-style - [s3forcepathstyle: | default = false] + # Number of retries for recoverable errors + # CLI flag: -ruler.storage.azure.max-retries + [max_retries: | default = 20] swift: # Openstack authentication URL. diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index b2716da4a2b..ee2f6dac875 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -5,19 +5,10 @@ import ( "flag" "fmt" -<<<<<<< HEAD - "github.com/pkg/errors" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/aws" - "github.com/cortexproject/cortex/pkg/chunk/azure" - "github.com/cortexproject/cortex/pkg/chunk/gcp" - "github.com/cortexproject/cortex/pkg/chunk/openstack" -======= "github.com/go-kit/kit/log" + "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" ->>>>>>> af18111b3... migrate cortex ruler to use Thanos objstore bucket client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/ruler/rules/objectclient" @@ -33,10 +24,9 @@ type RuleStoreConfig struct { ConfigDB client.Config `yaml:"configdb"` // Object Storage Configs - S3 s3.Config `yaml:"s3"` - GCS gcs.Config `yaml:"gcs"` - Azure azure.Config `yaml:"azure"` - Filesystem filesystem.Config `yaml:"filesystem"` + S3 s3.Config `yaml:"s3"` + GCS gcs.Config `yaml:"gcs"` + Azure azure.Config `yaml:"azure"` mock rules.RuleStore `yaml:"-"` } @@ -79,15 +69,9 @@ func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, er case "gcs": return newObjRuleStore(gcs.NewBucketClient(context.Background(), cfg.GCS, "cortex-ruler", logger)) case "s3": -<<<<<<< HEAD - return newObjRuleStore(aws.NewS3ObjectClient(cfg.S3, "")) - case "swift": - return newObjRuleStore(openstack.NewSwiftObjectClient(cfg.Swift, "")) -======= return newObjRuleStore(s3.NewBucketClient(cfg.S3, "cortex-ruler", logger)) case "filesystem": return newObjRuleStore(filesystem.NewBucketClient(cfg.Filesystem)) ->>>>>>> af18111b3... migrate cortex ruler to use Thanos objstore bucket client default: return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) } From d41c3b79e72dce2df155c7c9e6a8cab60d567dce Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 20 Apr 2020 13:43:08 -0400 Subject: [PATCH 03/16] update changelog Signed-off-by: Jacob Lisi --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e8c72d306..507a71c3192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## master / unreleased +* [CHANGE] Updated the Ruler storage config to utilize Thanos `objstore.Bucket` instead of `chunk.ObjectClient` based clients. #2489 + * Removed `-ruler.storage.azure.download-buffer-size` + * Removed `-ruler.storage.azure.upload-buffer-size` + * Added `-ruler.storage.azure.endpoint-suffix` + * Added `-ruler.storage.azure.max-retries` + * Changed `-ruler.storage.gcs.bucketname` > `-ruler.storage.gcs.bucket-name` + * Removed `-ruler.storage.gcs.chunk-buffer-size` + * Removed `-ruler.storage.gcs.request-timeout` + * Added `-ruler.storage.gcs.service-account` + * Changed `-ruler.storage.s3.url` > `-ruler.storage.s3.endpoint`, credentials are no longer url encoded and in-memory is no longer an option + * Changed `-ruler.storage.s3.buckets` > `-ruler.storage.s3.bucket-name`, multiple buckets can no longer be specified + * Added `-ruler.storage.s3.secret-access-key` + * Added `-ruler.storage.s3.access-key-id` + * Added `-ruler.storage.s3.insecure` * [CHANGE] Added v1 API routes documented in #2327. #2372 * Added `-http.alertmanager-http-prefix` flag which allows the configuration of the path where the Alertmanager API and UI can be reached. The default is set to `/alertmanager`. * Added `-http.prometheus-http-prefix` flag which allows the configuration of the path where the Prometheus API and UI can be reached. The default is set to `/prometheus`. From 4494538f1d61c9f62bd9492f0cc58705499aa908 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 20 Apr 2020 13:55:34 -0400 Subject: [PATCH 04/16] add some debug logs for integration tests Signed-off-by: Jacob Lisi --- integration/api_ruler_test.go | 2 +- integration/configs.go | 16 +++++++++------- integration/e2ecortex/services.go | 2 +- pkg/ruler/ruler.go | 4 ++++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go index 262fdd0e1c2..fc672a85844 100644 --- a/integration/api_ruler_test.go +++ b/integration/api_ruler_test.go @@ -20,7 +20,7 @@ func TestRulerAPI(t *testing.T) { // Start dependencies. dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, RulerConfigs["-ruler.storage.s3.buckets"]) + minio := e2edb.NewMinio(9000, bucketName) require.NoError(t, s.StartAndWaitReady(minio, dynamo)) // Start Cortex components. diff --git a/integration/configs.go b/integration/configs.go index aab608fb7ec..3e69b6e9e41 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -52,13 +52,15 @@ var ( } RulerConfigs = map[string]string{ - "-ruler.enable-sharding": "false", - "-ruler.poll-interval": "2s", - "-experimental.ruler.enable-api": "true", - "-ruler.storage.type": "s3", - "-ruler.storage.s3.buckets": "cortex-rules", - "-ruler.storage.s3.force-path-style": "true", - "-ruler.storage.s3.url": fmt.Sprintf("s3://%s:%s@%s-minio-9000.:9000", e2edb.MinioAccessKey, e2edb.MinioSecretKey, networkName), + "-ruler.enable-sharding": "false", + "-ruler.poll-interval": "2s", + "-experimental.ruler.enable-api": "true", + "-ruler.storage.type": "s3", + "-ruler.storage.s3.access-key-id": e2edb.MinioAccessKey, + "-ruler.storage.s3.secret-access-key": e2edb.MinioSecretKey, + "-ruler.storage.s3.bucket-name": bucketName, + "-ruler.storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName), + "-ruler.storage.s3.insecure": "true", } BlocksStorageFlags = map[string]string{ diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index b0eed836bb1..bf7e41d287e 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -294,7 +294,7 @@ func NewRuler(name string, flags map[string]string, image string) *CortexService image, e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-target": "ruler", - "-log.level": "warn", + "-log.level": "debug", }, flags))...), e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 3a17834eaee..1cf8908f7c7 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -382,12 +382,16 @@ func (r *Ruler) run(ctx context.Context) error { func (r *Ruler) loadRules(ctx context.Context) { ringHasher := fnv.New32a() + level.Debug(r.logger).Log("msg", "polling for rule groups") + configs, err := r.store.ListAllRuleGroups(ctx) if err != nil { level.Error(r.logger).Log("msg", "unable to poll for rules", "err", err) return } + level.Debug(r.logger).Log("msg", "poll for rule groups success", "num_rules", len(configs)) + // Iterate through each users configuration and determine if the on-disk // configurations need to be updated for user, cfg := range configs { From 9d0905f9fb9376d716ecad349a9a20975263a5b7 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 21 Apr 2020 16:33:05 -0400 Subject: [PATCH 05/16] update to use bas64 encoded paths Signed-off-by: Jacob Lisi --- pkg/ruler/rules/objectclient/rule_store.go | 50 +++++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/pkg/ruler/rules/objectclient/rule_store.go b/pkg/ruler/rules/objectclient/rule_store.go index a312c4d0a08..1487eb17f3f 100644 --- a/pkg/ruler/rules/objectclient/rule_store.go +++ b/pkg/ruler/rules/objectclient/rule_store.go @@ -3,6 +3,7 @@ package objectclient import ( "bytes" "context" + "encoding/base64" "io/ioutil" strings "strings" @@ -36,7 +37,6 @@ func NewRuleStore(bucket objstore.Bucket) *RuleStore { } func (o *RuleStore) getRuleGroup(ctx context.Context, objectKey string) (*rules.RuleGroupDesc, error) { - reader, err := o.bucket.Get(ctx, objectKey) if o.bucket.IsObjNotFoundErr(err) { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", objectKey) @@ -63,10 +63,10 @@ func (o *RuleStore) getRuleGroup(ctx context.Context, objectKey string) (*rules. return rg, nil } -// ListAllRuleGroups returns all the active rule groups -func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) { - userGroupMap := map[string]rules.RuleGroupList{} - err := o.bucket.Iter(ctx, generateRuleObjectKey("", "", ""), func(s string) error { +// getNamespace returns all the groups under the specified namespace path +func (o *RuleStore) getNamespace(ctx context.Context, namespacePath string) (rules.RuleGroupList, error) { + groups := rules.RuleGroupList{} + err := o.bucket.Iter(ctx, namespacePath, func(s string) error { user := decomposeRuleObjectKey(s) if user == "" { return nil @@ -77,10 +77,33 @@ func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.Rul return err } + groups = append(groups, rg) + + return nil + }) + + if err != nil { + return nil, err + } + + return groups, nil +} + +// ListAllRuleGroups returns all the active rule groups +func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) { + userGroupMap := map[string]rules.RuleGroupList{} + err := o.bucket.Iter(ctx, rulePrefix, func(s string) error { + user := strings.TrimPrefix(s, rulePrefix) + + groups, err := o.ListRuleGroups(ctx, user, "") + if err != nil { + return err + } + if _, exists := userGroupMap[user]; !exists { userGroupMap[user] = rules.RuleGroupList{} } - userGroupMap[user] = append(userGroupMap[user], rg) + userGroupMap[user] = append(userGroupMap[user], groups...) return nil }) @@ -94,6 +117,10 @@ func (o *RuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.Rul // ListRuleGroups returns all the active rule groups for a user func (o *RuleStore) ListRuleGroups(ctx context.Context, userID, namespace string) (rules.RuleGroupList, error) { + if namespace != "" { + return o.getNamespace(ctx, generateRuleObjectKey(userID, namespace, "")) + } + groups := []*rules.RuleGroupDesc{} err := o.bucket.Iter(ctx, generateRuleObjectKey(userID, namespace, ""), func(s string) error { @@ -148,15 +175,24 @@ func (o *RuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespac return err } +// generateRuleObjectKey encodes the group name and namespace using +// a base64 URL encoding to ensure the character set does not contain +// forward slashes since that is used as a delimiter. func generateRuleObjectKey(id, namespace, name string) string { if id == "" { return rulePrefix } prefix := rulePrefix + id + "/" + if namespace == "" { return prefix } - return prefix + namespace + "/" + name + prefix = prefix + base64.URLEncoding.EncodeToString([]byte(namespace)) + "/" + if name == "" { + return prefix + } + + return prefix + base64.URLEncoding.EncodeToString([]byte(name)) } func decomposeRuleObjectKey(handle string) string { From 442bd464469f96ec37fcdea847cb971973c1e770 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 21 Apr 2020 16:40:51 -0400 Subject: [PATCH 06/16] correct list rules function Signed-off-by: Jacob Lisi --- pkg/ruler/rules/objectclient/rule_store.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/ruler/rules/objectclient/rule_store.go b/pkg/ruler/rules/objectclient/rule_store.go index 1487eb17f3f..475b20bc22a 100644 --- a/pkg/ruler/rules/objectclient/rule_store.go +++ b/pkg/ruler/rules/objectclient/rule_store.go @@ -126,12 +126,12 @@ func (o *RuleStore) ListRuleGroups(ctx context.Context, userID, namespace string err := o.bucket.Iter(ctx, generateRuleObjectKey(userID, namespace, ""), func(s string) error { level.Debug(util.Logger).Log("msg", "listing rule group", "key", s) - rg, err := o.getRuleGroup(ctx, s) + rgs, err := o.getNamespace(ctx, s) if err != nil { level.Error(util.Logger).Log("msg", "unable to retrieve rule group", "err", err, "key", s) return err } - groups = append(groups, rg) + groups = append(groups, rgs...) return nil }) @@ -187,6 +187,7 @@ func generateRuleObjectKey(id, namespace, name string) string { if namespace == "" { return prefix } + prefix = prefix + base64.URLEncoding.EncodeToString([]byte(namespace)) + "/" if name == "" { return prefix From 2bb3e0a558045ab250a5483855e57b5c81aab9c0 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 22 Apr 2020 13:31:23 -0400 Subject: [PATCH 07/16] update mapper to handle forward slash character Signed-off-by: Jacob Lisi --- integration/api_ruler_test.go | 4 ++-- pkg/ruler/mapper.go | 3 ++- pkg/ruler/mapper_test.go | 1 + pkg/ruler/ruler.go | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go index fc672a85844..8678b2dc0f4 100644 --- a/integration/api_ruler_test.go +++ b/integration/api_ruler_test.go @@ -34,9 +34,9 @@ func TestRulerAPI(t *testing.T) { // Create example namespace and rule group to use for tests, using strings that // require url escaping. - namespace := "test_encoded_+namespace?" + namespace := "test_encoded_+namespace/?" rg := rulefmt.RuleGroup{ - Name: "test_encoded_+\"+group_name?", + Name: "test_encoded_+\"+group_name/?", Interval: 100, Rules: []rulefmt.Rule{ { diff --git a/pkg/ruler/mapper.go b/pkg/ruler/mapper.go index 43a8cdf008f..e628dc4c39a 100644 --- a/pkg/ruler/mapper.go +++ b/pkg/ruler/mapper.go @@ -35,6 +35,7 @@ func (m *mapper) MapRules(user string, ruleConfigs map[string][]legacy_rulefmt.R anyUpdated := false filenames := []string{} + // user rule files will be stored as `///` path := filepath.Join(m.Path, user) err := m.FS.MkdirAll(path, 0777) @@ -46,7 +47,7 @@ func (m *mapper) MapRules(user string, ruleConfigs map[string][]legacy_rulefmt.R for filename, groups := range ruleConfigs { // Store the encoded file name to better handle `/` characters encodedFileName := url.PathEscape(filename) - fullFileName := filepath.Join(path, encodedFileName) + fullFileName := filepath.Join(path, encodedFileName fileUpdated, err := m.writeRuleGroupsIfNewer(groups, fullFileName) if err != nil { diff --git a/pkg/ruler/mapper_test.go b/pkg/ruler/mapper_test.go index e09d5e30575..e2fcf1c0dc5 100644 --- a/pkg/ruler/mapper_test.go +++ b/pkg/ruler/mapper_test.go @@ -22,6 +22,7 @@ var ( fileOnePath = "/rules/user1/" + fileOneEncoded fileTwoPath = "/rules/user1/" + fileTwoEncoded + initialRuleSet = map[string][]legacy_rulefmt.RuleGroup{ "file /one": { { diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 1cf8908f7c7..cfba2057252 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -2,6 +2,7 @@ package ruler import ( "context" + "encoding/base64" "flag" "fmt" "hash/fnv" From ac6467eb754a4b58be5d448f55ca47312e870e71 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 12:55:13 -0400 Subject: [PATCH 08/16] rebase and add swift thanos implementation Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 3 -- pkg/ruler/storage.go | 21 +++++-------- pkg/storage/backend/swift/bucket_client.go | 18 +++++++++++ pkg/storage/backend/swift/config.go | 35 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 pkg/storage/backend/swift/bucket_client.go create mode 100644 pkg/storage/backend/swift/config.go diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index cfba2057252..b56d4c62f20 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -96,9 +96,6 @@ type Config struct { // Validate config and returns error on failure func (cfg *Config) Validate() error { - if err := cfg.StoreConfig.Validate(); err != nil { - return errors.Wrap(err, "invalid storage config") - } return nil } diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index ee2f6dac875..ec995d48200 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/go-kit/kit/log" - "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" "github.com/cortexproject/cortex/pkg/configs/client" @@ -16,6 +15,7 @@ import ( "github.com/cortexproject/cortex/pkg/storage/backend/filesystem" "github.com/cortexproject/cortex/pkg/storage/backend/gcs" "github.com/cortexproject/cortex/pkg/storage/backend/s3" + "github.com/cortexproject/cortex/pkg/storage/backend/swift" ) // RuleStoreConfig conigures a rule store @@ -24,9 +24,11 @@ type RuleStoreConfig struct { ConfigDB client.Config `yaml:"configdb"` // Object Storage Configs - S3 s3.Config `yaml:"s3"` - GCS gcs.Config `yaml:"gcs"` - Azure azure.Config `yaml:"azure"` + S3 s3.Config `yaml:"s3"` + GCS gcs.Config `yaml:"gcs"` + Azure azure.Config `yaml:"azure"` + Swift swift.Config `yaml:"swift"` + FileSystem filesystem.Config `yaml:"filesystem"` mock rules.RuleStore `yaml:"-"` } @@ -38,17 +40,10 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.GCS.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f) + cfg.FileSystem.RegisterFlagsWithPrefix("ruler.storage.", f) f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3)") } -// Validate config and returns error on failure -func (cfg *RuleStoreConfig) Validate() error { - if err := cfg.Swift.Validate(); err != nil { - return errors.Wrap(err, "invalid Swift Storage config") - } - return nil -} - // NewRuleStorage returns a new rule storage backend poller and store func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, error) { if cfg.mock != nil { @@ -70,8 +65,6 @@ func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, er return newObjRuleStore(gcs.NewBucketClient(context.Background(), cfg.GCS, "cortex-ruler", logger)) case "s3": return newObjRuleStore(s3.NewBucketClient(cfg.S3, "cortex-ruler", logger)) - case "filesystem": - return newObjRuleStore(filesystem.NewBucketClient(cfg.Filesystem)) default: return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) } diff --git a/pkg/storage/backend/swift/bucket_client.go b/pkg/storage/backend/swift/bucket_client.go new file mode 100644 index 00000000000..d9b808f0189 --- /dev/null +++ b/pkg/storage/backend/swift/bucket_client.go @@ -0,0 +1,18 @@ +package swift + +import ( + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/swift" + "gopkg.in/yaml.v2" +) + +// NewBucketClient creates a new S3 bucket client +func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { + conf, err := yaml.Marshal(cfg) + if err != nil { + return nil, errors.Wrap(err, "unable to marshal swift config into yaml") + } + return swift.NewContainer(logger, conf) +} diff --git a/pkg/storage/backend/swift/config.go b/pkg/storage/backend/swift/config.go new file mode 100644 index 00000000000..e22e6ce18ae --- /dev/null +++ b/pkg/storage/backend/swift/config.go @@ -0,0 +1,35 @@ +package swift + +import ( + "flag" + + "github.com/thanos-io/thanos/pkg/objstore/swift" +) + +// Config is config for the Swift Chunk Client. +type Config struct { + swift.SwiftConfig `yaml:",inline"` +} + +// RegisterFlags registers flags. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.RegisterFlagsWithPrefix("", f) +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.ContainerName, prefix+"swift.container-name", "cortex", "Name of the Swift container to put chunks in.") + f.StringVar(&cfg.DomainName, prefix+"swift.domain-name", "", "Openstack user's domain name.") + f.StringVar(&cfg.DomainId, prefix+"swift.domain-id", "", "Openstack user's domain id.") + f.StringVar(&cfg.UserDomainName, prefix+"swift.user-domain-name", "", "Openstack user's domain name.") + f.StringVar(&cfg.UserDomainID, prefix+"swift.user-domain-id", "", "Openstack user's domain id.") + f.StringVar(&cfg.Username, prefix+"swift.username", "", "Openstack username for the api.") + f.StringVar(&cfg.UserId, prefix+"swift.user-id", "", "Openstack userid for the api.") + f.StringVar(&cfg.Password, prefix+"swift.password", "", "Openstack api key.") + f.StringVar(&cfg.AuthUrl, prefix+"swift.auth-url", "", "Openstack authentication URL.") + f.StringVar(&cfg.RegionName, prefix+"swift.region-name", "", "Openstack Region to use eg LON, ORD - default is use first region (v2,v3 auth only)") + f.StringVar(&cfg.ProjectName, prefix+"swift.project-name", "", "Openstack project name (v2,v3 auth only).") + f.StringVar(&cfg.ProjectID, prefix+"swift.project-id", "", "Openstack project id (v2,v3 auth only).") + f.StringVar(&cfg.ProjectDomainName, prefix+"swift.project-domain-name", "", "Name of the project's domain (v3 auth only), only needed if it differs from the user domain.") + f.StringVar(&cfg.ProjectDomainID, prefix+"swift.project-domain-id", "", "Id of the project's domain (v3 auth only), only needed if it differs the from user domain.") +} From d2b593be1a9c47b1e8bc6ffe69f30c78cab845ae Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 17:27:11 -0400 Subject: [PATCH 09/16] update docs Signed-off-by: Jacob Lisi --- docs/configuration/config-file-reference.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 27d8d53b2b0..6f7ab2fa898 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -896,6 +896,11 @@ storage: # CLI flag: -ruler.storage.swift.container-name [container_name: | default = "cortex"] + filesystem: + # Local filesystem storage directory. + # CLI flag: -ruler.storage.filesystem.dir + [dir: | default = ""] + # file path to store temporary rule files for the prometheus rule managers # CLI flag: -ruler.rule-path [rule_path: | default = "/rules"] From 6c2bce9bc4581df6849129eee1ccc286093d802d Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 18:14:20 -0400 Subject: [PATCH 10/16] cleanup white noise Signed-off-by: Jacob Lisi --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 507a71c3192..cd2fe3835e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ * Changed `-ruler.storage.s3.buckets` > `-ruler.storage.s3.bucket-name`, multiple buckets can no longer be specified * Added `-ruler.storage.s3.secret-access-key` * Added `-ruler.storage.s3.access-key-id` - * Added `-ruler.storage.s3.insecure` + * Added `-ruler.storage.s3.insecure` * [CHANGE] Added v1 API routes documented in #2327. #2372 * Added `-http.alertmanager-http-prefix` flag which allows the configuration of the path where the Alertmanager API and UI can be reached. The default is set to `/alertmanager`. * Added `-http.prometheus-http-prefix` flag which allows the configuration of the path where the Prometheus API and UI can be reached. The default is set to `/prometheus`. From fbbfc06fa94db7bcf848ffd4b9cdb263c9c5aa01 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 19:44:44 -0400 Subject: [PATCH 11/16] remove swift integration test from the ruler Signed-off-by: Jacob Lisi --- integration/chunks_storage_backends_test.go | 63 --------------------- pkg/ruler/storage.go | 5 +- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/integration/chunks_storage_backends_test.go b/integration/chunks_storage_backends_test.go index 4bf255124be..59f50e4184f 100644 --- a/integration/chunks_storage_backends_test.go +++ b/integration/chunks_storage_backends_test.go @@ -4,7 +4,6 @@ package main import ( "context" - "fmt" "sort" "testing" "time" @@ -24,8 +23,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/openstack" "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ruler" - "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -232,66 +229,6 @@ func TestSwiftChunkStorage(t *testing.T) { } -func TestSwiftRuleStorage(t *testing.T) { - s, err := e2e.NewScenario(networkName) - require.NoError(t, err) - defer s.Close() - swift := e2edb.NewSwiftStorage() - - require.NoError(t, s.StartAndWaitReady(swift)) - - store, err := ruler.NewRuleStorage(ruler.RuleStoreConfig{ - Type: "swift", - Swift: swiftConfig(swift), - }) - require.NoError(t, err) - ctx := context.Background() - - // Add 2 rule group. - r1 := newRule(userID, "1") - err = store.SetRuleGroup(ctx, userID, "foo", r1) - require.NoError(t, err) - - r2 := newRule(userID, "2") - err = store.SetRuleGroup(ctx, userID, "bar", r2) - require.NoError(t, err) - - // Get rules back. - rls, err := store.ListAllRuleGroups(ctx) - require.NoError(t, err) - require.Equal(t, 2, len(rls[userID])) - - userRules := rls[userID] - sort.Slice(userRules, func(i, j int) bool { return userRules[i].Name < userRules[j].Name }) - require.Equal(t, r1, userRules[0]) - require.Equal(t, r2, userRules[1]) - - // Delete the first rule group - err = store.DeleteRuleGroup(ctx, userID, "foo", r1.Name) - require.NoError(t, err) - - //Verify we only have the second rule group - rls, err = store.ListAllRuleGroups(ctx) - require.NoError(t, err) - require.Equal(t, 1, len(rls[userID])) - require.Equal(t, r2, rls[userID][0]) -} - -func newRule(userID, name string) *rules.RuleGroupDesc { - return &rules.RuleGroupDesc{ - Name: name + "rule", - Interval: time.Minute, - Namespace: name + "namespace", - Rules: []*rules.RuleDesc{ - { - Expr: fmt.Sprintf(`{%s="bar"}`, name), - Record: name + ":bar", - }, - }, - User: userID, - } -} - func swiftConfig(s *e2e.HTTPService) openstack.SwiftConfig { return openstack.SwiftConfig{ SwiftConfig: thanos.SwiftConfig{ diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index ec995d48200..8746ca1bafe 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -41,7 +41,7 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.FileSystem.RegisterFlagsWithPrefix("ruler.storage.", f) - f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3)") + f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3, swift)") } // NewRuleStorage returns a new rule storage backend poller and store @@ -57,7 +57,6 @@ func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, er if err != nil { return nil, err } - return rules.NewConfigRuleStore(c), nil case "azure": return newObjRuleStore(azure.NewBucketClient(cfg.Azure, "cortex-ruler", logger)) @@ -65,6 +64,8 @@ func NewRuleStorage(cfg RuleStoreConfig, logger log.Logger) (rules.RuleStore, er return newObjRuleStore(gcs.NewBucketClient(context.Background(), cfg.GCS, "cortex-ruler", logger)) case "s3": return newObjRuleStore(s3.NewBucketClient(cfg.S3, "cortex-ruler", logger)) + case "swift": + return newObjRuleStore(swift.NewBucketClient(cfg.Swift, "cortex-ruler", logger)) default: return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) } From 96b4dcfbde24048d71e02ba8dcbe5177c7b1b758 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 20:39:12 -0400 Subject: [PATCH 12/16] update docs Signed-off-by: Jacob Lisi --- docs/configuration/config-file-reference.md | 2 +- .../rules/objectclient/rule_store_test.go | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 pkg/ruler/rules/objectclient/rule_store_test.go diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 6f7ab2fa898..7a001570020 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -771,7 +771,7 @@ The `ruler_config` configures the Cortex ruler. [poll_interval: | default = 1m] storage: - # Method to use for backend rule storage (configdb, azure, gcs, s3) + # Method to use for backend rule storage (configdb, azure, gcs, s3, swift) # CLI flag: -ruler.storage.type [type: | default = "configdb"] diff --git a/pkg/ruler/rules/objectclient/rule_store_test.go b/pkg/ruler/rules/objectclient/rule_store_test.go new file mode 100644 index 00000000000..425e5d237c8 --- /dev/null +++ b/pkg/ruler/rules/objectclient/rule_store_test.go @@ -0,0 +1,64 @@ +package objectclient + +import ( + "context" + "testing" + "time" + + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/objstore/objtesting" + + "github.com/cortexproject/cortex/pkg/ruler/rules" +) + +func TestBucketsImplementations(t *testing.T) { + objtesting.ForeachStore(t, RuleStoreTest) +} + +func RuleStoreTest(t *testing.T, bkt objstore.Bucket) { + var ( + ctx = context.Background() + userOne = "user1" + userTwo = "user2" + namespace = "namespace/one" + interval, _ = time.ParseDuration("1m") + mockRules = map[string]rules.RuleGroupList{ + userOne: { + &rules.RuleGroupDesc{ + Name: "group/+1", + Namespace: namespace, + User: userOne, + Rules: []*rules.RuleDesc{ + { + Record: "UP_RULE", + Expr: "up", + }, + { + Alert: "UP_ALERT", + Expr: "up < 1", + }, + }, + Interval: interval, + }, + }, + userTwo: { + &rules.RuleGroupDesc{ + Name: "group/+1", + Namespace: namespace, + User: userTwo, + Rules: []*rules.RuleDesc{ + { + Record: "UP_RULE", + Expr: "up", + }, + }, + Interval: interval, + }, + }, + } + ) + + store := NewRuleStore(bkt) + + store.SetRuleGroup(ctx, userOne, namespace, mockRules[userOne][0]) +} From 1f93d583dc4049b8373cb9b7c7eba37866f4fbda Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 20:42:01 -0400 Subject: [PATCH 13/16] remove temporary test file Signed-off-by: Jacob Lisi --- .../rules/objectclient/rule_store_test.go | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 pkg/ruler/rules/objectclient/rule_store_test.go diff --git a/pkg/ruler/rules/objectclient/rule_store_test.go b/pkg/ruler/rules/objectclient/rule_store_test.go deleted file mode 100644 index 425e5d237c8..00000000000 --- a/pkg/ruler/rules/objectclient/rule_store_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package objectclient - -import ( - "context" - "testing" - "time" - - "github.com/thanos-io/thanos/pkg/objstore" - "github.com/thanos-io/thanos/pkg/objstore/objtesting" - - "github.com/cortexproject/cortex/pkg/ruler/rules" -) - -func TestBucketsImplementations(t *testing.T) { - objtesting.ForeachStore(t, RuleStoreTest) -} - -func RuleStoreTest(t *testing.T, bkt objstore.Bucket) { - var ( - ctx = context.Background() - userOne = "user1" - userTwo = "user2" - namespace = "namespace/one" - interval, _ = time.ParseDuration("1m") - mockRules = map[string]rules.RuleGroupList{ - userOne: { - &rules.RuleGroupDesc{ - Name: "group/+1", - Namespace: namespace, - User: userOne, - Rules: []*rules.RuleDesc{ - { - Record: "UP_RULE", - Expr: "up", - }, - { - Alert: "UP_ALERT", - Expr: "up < 1", - }, - }, - Interval: interval, - }, - }, - userTwo: { - &rules.RuleGroupDesc{ - Name: "group/+1", - Namespace: namespace, - User: userTwo, - Rules: []*rules.RuleDesc{ - { - Record: "UP_RULE", - Expr: "up", - }, - }, - Interval: interval, - }, - }, - } - ) - - store := NewRuleStore(bkt) - - store.SetRuleGroup(ctx, userOne, namespace, mockRules[userOne][0]) -} From 09c5061e7f32b8bb0c7a82af4d3b219439ecec7f Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 21:21:45 -0400 Subject: [PATCH 14/16] fix rebase errors Signed-off-by: Jacob Lisi --- pkg/ruler/mapper.go | 3 +-- pkg/ruler/mapper_test.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/ruler/mapper.go b/pkg/ruler/mapper.go index e628dc4c39a..43a8cdf008f 100644 --- a/pkg/ruler/mapper.go +++ b/pkg/ruler/mapper.go @@ -35,7 +35,6 @@ func (m *mapper) MapRules(user string, ruleConfigs map[string][]legacy_rulefmt.R anyUpdated := false filenames := []string{} - // user rule files will be stored as `///` path := filepath.Join(m.Path, user) err := m.FS.MkdirAll(path, 0777) @@ -47,7 +46,7 @@ func (m *mapper) MapRules(user string, ruleConfigs map[string][]legacy_rulefmt.R for filename, groups := range ruleConfigs { // Store the encoded file name to better handle `/` characters encodedFileName := url.PathEscape(filename) - fullFileName := filepath.Join(path, encodedFileName + fullFileName := filepath.Join(path, encodedFileName) fileUpdated, err := m.writeRuleGroupsIfNewer(groups, fullFileName) if err != nil { diff --git a/pkg/ruler/mapper_test.go b/pkg/ruler/mapper_test.go index e2fcf1c0dc5..e09d5e30575 100644 --- a/pkg/ruler/mapper_test.go +++ b/pkg/ruler/mapper_test.go @@ -22,7 +22,6 @@ var ( fileOnePath = "/rules/user1/" + fileOneEncoded fileTwoPath = "/rules/user1/" + fileTwoEncoded - initialRuleSet = map[string][]legacy_rulefmt.RuleGroup{ "file /one": { { From 73d3f2a027376feaa0ebcd72c386b1c88894f3d3 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 29 Apr 2020 21:23:34 -0400 Subject: [PATCH 15/16] fix rebase issues Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b56d4c62f20..29f4736d783 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -2,7 +2,6 @@ package ruler import ( "context" - "encoding/base64" "flag" "fmt" "hash/fnv" @@ -380,16 +379,12 @@ func (r *Ruler) run(ctx context.Context) error { func (r *Ruler) loadRules(ctx context.Context) { ringHasher := fnv.New32a() - level.Debug(r.logger).Log("msg", "polling for rule groups") - configs, err := r.store.ListAllRuleGroups(ctx) if err != nil { level.Error(r.logger).Log("msg", "unable to poll for rules", "err", err) return } - level.Debug(r.logger).Log("msg", "poll for rule groups success", "num_rules", len(configs)) - // Iterate through each users configuration and determine if the on-disk // configurations need to be updated for user, cfg := range configs { From ae750426f15d402c2360df77bb289de1a2c70f4d Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 7 May 2020 14:59:53 -0400 Subject: [PATCH 16/16] reorder object storage config Signed-off-by: Jacob Lisi --- docs/configuration/config-file-reference.md | 66 ++++++++++----------- pkg/ruler/storage.go | 4 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 7a001570020..b6181630737 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -780,6 +780,39 @@ storage: # The CLI flags prefix for this block config is: ruler [configdb: ] + azure: + # Azure storage account name + # CLI flag: -ruler.storage.azure.account-name + [account_name: | default = ""] + + # Azure storage account key + # CLI flag: -ruler.storage.azure.account-key + [account_key: | default = ""] + + # Azure storage container name + # CLI flag: -ruler.storage.azure.container-name + [container_name: | default = ""] + + # Azure storage endpoint suffix without schema. The account name will be + # prefixed to this value to create the FQDN + # CLI flag: -ruler.storage.azure.endpoint-suffix + [endpoint_suffix: | default = ""] + + # Number of retries for recoverable errors + # CLI flag: -ruler.storage.azure.max-retries + [max_retries: | default = 20] + + gcs: + # GCS bucket name + # CLI flag: -ruler.storage.gcs.bucket-name + [bucket_name: | default = ""] + + # JSON representing either a Google Developers Console + # client_credentials.json file or a Google Developers service account key + # file. If empty, fallback to Google default logic. + # CLI flag: -ruler.storage.gcs.service-account + [service_account: | default = ""] + s3: # S3 endpoint without schema # CLI flag: -ruler.storage.s3.endpoint @@ -803,39 +836,6 @@ storage: # CLI flag: -ruler.storage.s3.insecure [insecure: | default = false] - gcs: - # GCS bucket name - # CLI flag: -ruler.storage.gcs.bucket-name - [bucket_name: | default = ""] - - # JSON representing either a Google Developers Console - # client_credentials.json file or a Google Developers service account key - # file. If empty, fallback to Google default logic. - # CLI flag: -ruler.storage.gcs.service-account - [service_account: | default = ""] - - azure: - # Azure storage account name - # CLI flag: -ruler.storage.azure.account-name - [account_name: | default = ""] - - # Azure storage account key - # CLI flag: -ruler.storage.azure.account-key - [account_key: | default = ""] - - # Azure storage container name - # CLI flag: -ruler.storage.azure.container-name - [container_name: | default = ""] - - # Azure storage endpoint suffix without schema. The account name will be - # prefixed to this value to create the FQDN - # CLI flag: -ruler.storage.azure.endpoint-suffix - [endpoint_suffix: | default = ""] - - # Number of retries for recoverable errors - # CLI flag: -ruler.storage.azure.max-retries - [max_retries: | default = 20] - swift: # Openstack authentication URL. # CLI flag: -ruler.storage.swift.auth-url diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 8746ca1bafe..df17dac68cc 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -24,9 +24,9 @@ type RuleStoreConfig struct { ConfigDB client.Config `yaml:"configdb"` // Object Storage Configs - S3 s3.Config `yaml:"s3"` - GCS gcs.Config `yaml:"gcs"` Azure azure.Config `yaml:"azure"` + GCS gcs.Config `yaml:"gcs"` + S3 s3.Config `yaml:"s3"` Swift swift.Config `yaml:"swift"` FileSystem filesystem.Config `yaml:"filesystem"`