Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 5980441

Browse files
pracuccipracucci
authored
Merge pull request #287 from grafana/add-function-to-create-compactor-statefulset
Add function to customize compactor statefulset
2 parents bcb61e4 + e475268 commit 5980441

File tree

6 files changed

+33
-26
lines changed

6 files changed

+33
-26
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
## master / unreleased
44

5+
* [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287
6+
* [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287
7+
58
## 1.8.0 / 2021-03-25
69

710
* [CHANGE] Updated the trunk branch from `master` to `main`. You need to run the following in your local fork: #265

‎cortex-mixin/alerts/compactor.libsonnet‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@
3838
alert: 'CortexCompactorHasNotUploadedBlocks',
3939
'for': '15m',
4040
expr: |||
41-
(time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 60 * 60 * 24)
41+
(time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 60 * 60 * 24)
4242
and
43-
(thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 0)
44-
|||,
43+
(thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 0)
44+
||| % $._config.job_names,
4545
labels: {
4646
severity: 'critical',
4747
},
@@ -54,8 +54,8 @@
5454
alert: 'CortexCompactorHasNotUploadedBlocksSinceStart',
5555
'for': '24h',
5656
expr: |||
57-
thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} == 0
58-
|||,
57+
thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} == 0
58+
||| % $._config.job_names,
5959
labels: {
6060
severity: 'critical',
6161
},

‎cortex-mixin/config.libsonnet‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
table_manager: '(table-manager|cortex$)',
3636
store_gateway: '(store-gateway|cortex$)',
3737
gateway: 'cortex-gw',
38+
compactor: 'compactor.*', // Match also custom compactor deployments.
3839
},
3940

4041
// Labels used to in alert aggregations - should uniquely identify

‎cortex-mixin/dashboards/compactor-resources.libsonnet‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
1313
$.containerMemoryWorkingSetPanel('Memory (workingset)', 'compactor'),
1414
)
1515
.addPanel(
16-
$.goHeapInUsePanel('Memory (go heap inuse)', 'compactor'),
16+
$.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.compactor),
1717
)
1818
)
1919
.addRow(

‎cortex-mixin/dashboards/compactor.libsonnet‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ local utils = import 'mixin-utils/utils.libsonnet';
1515
.addPanel(
1616
$.startedCompletedFailedPanel(
1717
'Per-instance runs / sec',
18-
'sum(rate(cortex_compactor_runs_started_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
19-
'sum(rate(cortex_compactor_runs_completed_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
20-
'sum(rate(cortex_compactor_runs_failed_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor')
18+
'sum(rate(cortex_compactor_runs_started_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
19+
'sum(rate(cortex_compactor_runs_completed_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
20+
'sum(rate(cortex_compactor_runs_failed_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor)
2121
) +
2222
$.bars +
2323
{ yaxes: $.yaxes('ops') },
@@ -30,7 +30,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
3030
cortex_compactor_tenants_processing_failed{%s} +
3131
cortex_compactor_tenants_skipped{%s}
3232
) / cortex_compactor_tenants_discovered{%s}
33-
||| % [$.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor')], '{{%s}}' % $._config.per_instance_label) +
33+
||| % [$.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor)], '{{%s}}' % $._config.per_instance_label) +
3434
{ yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
3535
)
3636
)
@@ -44,12 +44,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
4444
)
4545
.addPanel(
4646
$.panel('Compacted blocks / sec') +
47-
$.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'), 'blocks') +
47+
$.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor), 'blocks') +
4848
{ yaxes: $.yaxes('ops') },
4949
)
5050
.addPanel(
5151
$.panel('Per-block compaction duration') +
52-
$.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher('compactor'))
52+
$.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor))
5353
)
5454
)
5555
.addRow(
@@ -62,27 +62,27 @@ local utils = import 'mixin-utils/utils.libsonnet';
6262
)
6363
.addPanel(
6464
$.panel('Average blocks / tenant') +
65-
$.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), 'avg'),
65+
$.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), 'avg'),
6666
)
6767
.addPanel(
6868
$.panel('Tenants with largest number of blocks') +
69-
$.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), '{{user}}'),
69+
$.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}'),
7070
)
7171
)
7272
.addRow(
7373
$.row('Garbage Collector')
7474
.addPanel(
7575
$.panel('Blocks marked for deletion / sec') +
76-
$.queryPanel('sum(rate(cortex_compactor_blocks_marked_for_deletion_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'), 'blocks') +
76+
$.queryPanel('sum(rate(cortex_compactor_blocks_marked_for_deletion_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor), 'blocks') +
7777
{ yaxes: $.yaxes('ops') },
7878
)
7979
.addPanel(
8080
$.successFailurePanel(
8181
'Blocks deletions / sec',
8282
// The cortex_compactor_blocks_cleaned_total tracks the number of successfully
8383
// deleted blocks.
84-
'sum(rate(cortex_compactor_blocks_cleaned_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
85-
'sum(rate(cortex_compactor_block_cleanup_failures_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
84+
'sum(rate(cortex_compactor_blocks_cleaned_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
85+
'sum(rate(cortex_compactor_block_cleanup_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
8686
) + { yaxes: $.yaxes('ops') }
8787
)
8888
)
@@ -93,14 +93,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
9393
'Metadata Syncs / sec',
9494
// The cortex_compactor_meta_syncs_total metric is incremented each time a per-tenant
9595
// metadata sync is triggered.
96-
'sum(rate(cortex_compactor_meta_syncs_total{%s}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % [$.jobMatcher('compactor'), $.jobMatcher('compactor')],
97-
'sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
96+
'sum(rate(cortex_compactor_meta_syncs_total{%s}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % [$.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor)],
97+
'sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
9898
) + { yaxes: $.yaxes('ops') }
9999
)
100100
.addPanel(
101101
$.panel('Metadata Sync Duration') +
102102
// This metric tracks the duration of a per-tenant metadata sync.
103-
$.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher('compactor')),
103+
$.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor)),
104104
)
105105
)
106106
.addRow($.objectStorePanels1('Object Store', 'compactor'))

‎cortex/tsdb.libsonnet‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -173,13 +173,13 @@
173173
$.util.readinessProbe +
174174
$.jaeger_mixin,
175175

176-
compactor_statefulset:
177-
statefulSet.new('compactor', 1, [$.compactor_container], compactor_data_pvc) +
178-
statefulSet.mixin.spec.withServiceName('compactor') +
176+
newCompactorStatefulSet(name, container)::
177+
statefulSet.new(name, 1, [container], compactor_data_pvc) +
178+
statefulSet.mixin.spec.withServiceName(name) +
179179
statefulSet.mixin.metadata.withNamespace($._config.namespace) +
180-
statefulSet.mixin.metadata.withLabels({ name: 'compactor' }) +
181-
statefulSet.mixin.spec.template.metadata.withLabels({ name: 'compactor' }) +
182-
statefulSet.mixin.spec.selector.withMatchLabels({ name: 'compactor' }) +
180+
statefulSet.mixin.metadata.withLabels({ name: name }) +
181+
statefulSet.mixin.spec.template.metadata.withLabels({ name: name }) +
182+
statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) +
183183
statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) +
184184
statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') +
185185
statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(900) +
@@ -189,6 +189,9 @@
189189
// ready).
190190
statefulSet.mixin.spec.withPodManagementPolicy('Parallel'),
191191

192+
compactor_statefulset:
193+
$.newCompactorStatefulSet('compactor', $.compactor_container),
194+
192195
// The store-gateway runs a statefulset.
193196
local store_gateway_data_pvc =
194197
pvc.new() +

0 commit comments

Comments
 (0)