From 918d284b1f49d370ca6c6105020e460bba47deb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Apr 2020 11:00:09 +0200 Subject: [PATCH 1/3] Modify default values for deletion delays. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 1 + pkg/compactor/compactor.go | 2 +- pkg/storage/tsdb/config.go | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c89ac904fd..2264454eace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * `cortex_querier_bucket_store_blocks_meta_sync_failures_total` > `cortex_querier_blocks_meta_sync_failures_total` * `cortex_querier_bucket_store_blocks_meta_sync_duration_seconds` > `cortex_querier_blocks_meta_sync_duration_seconds` * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds` +* [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392 diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go index a2cb7e065ad..5cc4d440102 100644 --- a/pkg/compactor/compactor.go +++ b/pkg/compactor/compactor.go @@ -64,7 +64,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.CompactionInterval, "compactor.compaction-interval", time.Hour, "The frequency at which the compaction runs") f.IntVar(&cfg.CompactionRetries, "compactor.compaction-retries", 3, "How many times to retry a failed compaction during a single compaction interval") f.BoolVar(&cfg.ShardingEnabled, "compactor.sharding-enabled", false, "Shard tenants across multiple compactor instances. Sharding is required if you run multiple compactor instances, in order to coordinate compactions and avoid race conditions leading to the same tenant blocks simultaneously compacted by different instances.") - f.DurationVar(&cfg.DeletionDelay, "compactor.deletion-delay", 48*time.Hour, "Time before a block marked for deletion is deleted from bucket. "+ + f.DurationVar(&cfg.DeletionDelay, "compactor.deletion-delay", 12*time.Hour, "Time before a block marked for deletion is deleted from bucket. "+ "If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. "+ "If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, "+ "if store gateway still has the block loaded, or compactor is ignoring the deletion because it's compacting the block at the same time.") diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 8b90bf393a3..cbcd56f9519 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -184,9 +184,9 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MetaSyncConcurrency, "experimental.tsdb.bucket-store.meta-sync-concurrency", 20, "Number of Go routines to use when syncing block meta files from object storage per tenant.") f.BoolVar(&cfg.BinaryIndexHeader, "experimental.tsdb.bucket-store.binary-index-header-enabled", true, "Whether the bucket store should use the binary index header. If false, it uses the JSON index header.") f.DurationVar(&cfg.ConsistencyDelay, "experimental.tsdb.bucket-store.consistency-delay", 0, "Minimum age of a block before it's being read. Set it to safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent.") - f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "experimental.tsdb.bucket-store.ignore-deletion-marks-delay", time.Hour*24, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ + f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "experimental.tsdb.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet."+ - "Default is 24h, half of the default value for -compactor.deletion-delay.") + "Default is 6h, half of the default value for -compactor.deletion-delay.") } // Validate the config. From a2046b11bb22e832dba4dd81fd0649eb3b23ea7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Apr 2020 11:02:48 +0200 Subject: [PATCH 2/3] Updated documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/configuration/config-file-reference.md | 6 +++--- docs/operations/blocks-storage.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 0c68fb87978..32e889f6b80 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2441,10 +2441,10 @@ bucket_store: # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore # blocks that are marked for deletion with some delay. This ensures store can # still serve blocks that are meant to be deleted but do not have a - # replacement yet.Default is 24h, half of the default value for + # replacement yet.Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay - [ignore_deletion_mark_delay: | default = 24h0m0s] + [ignore_deletion_mark_delay: | default = 6h0m0s] # How frequently does Cortex try to compact TSDB head. Block is only created if # data covers smallest block range. Must be greater than 0 and max 5 minutes. @@ -2571,7 +2571,7 @@ The `compactor_config` configures the compactor for the experimental blocks stor # failures, if store gateway still has the block loaded, or compactor is # ignoring the deletion because it's compacting the block at the same time. # CLI flag: -compactor.deletion-delay -[deletion_delay: | default = 48h0m0s] +[deletion_delay: | default = 12h0m0s] # Shard tenants across multiple compactor instances. Sharding is required if you # run multiple compactor instances, in order to coordinate compactions and avoid diff --git a/docs/operations/blocks-storage.md b/docs/operations/blocks-storage.md index f37b736408e..a6ded027b10 100644 --- a/docs/operations/blocks-storage.md +++ b/docs/operations/blocks-storage.md @@ -265,10 +265,10 @@ tsdb: # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have - # a replacement yet.Default is 24h, half of the default value for + # a replacement yet.Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay - [ignore_deletion_mark_delay: | default = 24h0m0s] + [ignore_deletion_mark_delay: | default = 6h0m0s] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 5 @@ -398,7 +398,7 @@ compactor: # compactor is ignoring the deletion because it's compacting the block at the # same time. # CLI flag: -compactor.deletion-delay - [deletion_delay: | default = 48h0m0s] + [deletion_delay: | default = 12h0m0s] # Shard tenants across multiple compactor instances. Sharding is required if # you run multiple compactor instances, in order to coordinate compactions and From d8c26bc78fab860f3a684cb7809c742422e56883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Apr 2020 11:03:26 +0200 Subject: [PATCH 3/3] Updated CHANGELOG.md with PR number. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2264454eace..4e294a830f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * `cortex_querier_bucket_store_blocks_meta_sync_failures_total` > `cortex_querier_blocks_meta_sync_failures_total` * `cortex_querier_bucket_store_blocks_meta_sync_duration_seconds` > `cortex_querier_blocks_meta_sync_duration_seconds` * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds` -* [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. +* [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392