From 1a365dd7d2b2f7b110db98fdb8a57ced5e0a6b74 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Tue, 8 Jun 2021 09:48:34 +0200 Subject: [PATCH 1/2] Increase default store-gateway memory request and limit Signed-off-by: Marco Pracucci --- CHANGELOG.md | 2 ++ cortex/tsdb.libsonnet | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6e4479..773d4d0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ * [CHANGE] `namespace` template variable in dashboards now only selects namespaces for selected clusters. #311 * [CHANGE] Alertmanager: mounted overrides configmap to alertmanager too. #315 * [CHANGE] Memcached: upgraded memcached from `1.5.17` to `1.6.9`. #316 +* [CHANGE] Store-gateway: increased memory request and limit respectively from 6GB / 6GB to 12GB / 18GB. #322 +* [CHANGE] Store-gateway: increased `-blocks-storage.bucket-store.max-chunk-pool-bytes` from 2GB (default) to 12GB. #322 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317 * [BUGFIX] Fixed `CortexIngesterHasNotShippedBlocks` alert false positive in case an ingester instance had ingested samples in the past, then no traffic was received for a long period and then it started receiving samples again. #308 diff --git a/cortex/tsdb.libsonnet b/cortex/tsdb.libsonnet index dbf005a7..6c394a07 100644 --- a/cortex/tsdb.libsonnet +++ b/cortex/tsdb.libsonnet @@ -216,6 +216,8 @@ // Block index-headers are pre-downloaded but lazy mmaped and loaded at query time. 'blocks-storage.bucket-store.index-header-lazy-loading-enabled': 'true', 'blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout': '60m', + + 'blocks-storage.bucket-store.max-chunk-pool-bytes': 12 * 1024 * 1024 * 1024, } + $.blocks_chunks_caching_config + $.blocks_metadata_caching_config + @@ -228,8 +230,8 @@ container.withPorts($.store_gateway_ports) + container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) + container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) + - $.util.resourcesRequests('1', '6Gi') + - $.util.resourcesLimits(null, '6Gi') + + k.util.resourcesRequests('1', '12Gi') + + k.util.resourcesLimits(null, '18Gi') + $.util.readinessProbe + $.jaeger_mixin, From dedcf7f1218bdbe3e92440decd759ed421432136 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Tue, 8 Jun 2021 09:51:53 +0200 Subject: [PATCH 2/2] Fix Signed-off-by: Marco Pracucci --- cortex/tsdb.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cortex/tsdb.libsonnet b/cortex/tsdb.libsonnet index 6c394a07..06ddc20a 100644 --- a/cortex/tsdb.libsonnet +++ b/cortex/tsdb.libsonnet @@ -230,8 +230,8 @@ container.withPorts($.store_gateway_ports) + container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) + container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) + - k.util.resourcesRequests('1', '12Gi') + - k.util.resourcesLimits(null, '18Gi') + + $.util.resourcesRequests('1', '12Gi') + + $.util.resourcesLimits(null, '18Gi') + $.util.readinessProbe + $.jaeger_mixin,