Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 958fb89

Browse files
pracuccipracucci
authored
Merge pull request #60 from grafana/update-objstore-metrics
Updated Object Store metrics in dashboards
2 parents c936b32 + 7a3497c commit 958fb89

File tree

6 files changed

+97
-40
lines changed

6 files changed

+97
-40
lines changed

‎cortex-mixin/dashboards.libsonnet‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
(import 'dashboards/writes.libsonnet') +
88

99
(if std.setMember('tsdb', $._config.storage_engine)
10-
then import 'dashboards/compactor.libsonnet'
10+
then
11+
(import 'dashboards/compactor.libsonnet') +
12+
(import 'dashboards/object-store.libsonnet')
1113
else {}) +
1214

1315
(if std.setMember('chunks', $._config.storage_engine)

‎cortex-mixin/dashboards/compactor.libsonnet‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
8484
$.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher('compactor')),
8585
)
8686
)
87-
.addRow($.objectStorePanels1('Object Store', 'cortex_compactor'))
88-
.addRow($.objectStorePanels2('', 'cortex_compactor')),
87+
.addRow($.objectStorePanels1('Object Store', 'compactor'))
88+
.addRow($.objectStorePanels2('', 'compactor')),
8989
}

‎cortex-mixin/dashboards/dashboard-utils.libsonnet‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -161,51 +161,45 @@ local utils = import 'mixin-utils/utils.libsonnet';
161161
type: 'text',
162162
} + options,
163163

164-
objectStorePanels1(title, metricPrefix)::
165-
local opsTotal = '%s_thanos_objstore_bucket_operations_total' % [metricPrefix];
166-
local opsTotalFailures = '%s_thanos_objstore_bucket_operation_failures_total' % [metricPrefix];
167-
local operationDuration = '%s_thanos_objstore_bucket_operation_duration_seconds' % [metricPrefix];
164+
objectStorePanels1(title, component)::
168165
super.row(title)
169166
.addPanel(
170-
// We use 'up' to add 0 if there are no failed operations.
171-
self.successFailurePanel(
172-
'Operations/sec',
173-
'sum(rate(%s{%s}[$__interval])) - sum(rate(%s{%s}[$__interval]) or (up{%s}*0))' % [opsTotal, $.namespaceMatcher(), opsTotalFailures, $.namespaceMatcher(), $.namespaceMatcher()],
174-
'sum(rate(%s{%s}[$__interval]) or (up{%s}*0))' % [opsTotalFailures, $.namespaceMatcher(), $.namespaceMatcher()]
175-
)
167+
$.panel('Operations / sec') +
168+
$.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s,component="%s"}[$__interval]))' % [$.namespaceMatcher(), component], '{{operation}}') +
169+
$.stack +
170+
{ yaxes: $.yaxes('rps') },
176171
)
177172
.addPanel(
178-
$.panel('Op: ObjectSize') +
179-
$.latencyPanel(operationDuration, '{%s, operation="objectsize"}' % $.namespaceMatcher()),
173+
$.panel('Error rate') +
174+
$.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{%s,component="%s"}[$__interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s,component="%s"}[$__interval]))' % [$.namespaceMatcher(), component, $.namespaceMatcher(), component], '{{operation}}') +
175+
{ yaxes: $.yaxes('percentunit') },
180176
)
181177
.addPanel(
182-
// Cortex (Thanos) doesn't track timing for 'iter', so we use ops/sec instead.
183-
$.panel('Op: Iter') +
184-
$.queryPanel('sum(rate(%s{%s, operation="iter"}[$__interval]))' % [opsTotal, $.namespaceMatcher()], 'ops/sec')
178+
$.panel('Op: ObjectSize') +
179+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="objectsize"}' % [$.namespaceMatcher(), component]),
185180
)
186181
.addPanel(
187182
$.panel('Op: Exists') +
188-
$.latencyPanel(operationDuration, '{%s, operation="exists"}' % $.namespaceMatcher()),
183+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="exists"}' % [$.namespaceMatcher(), component]),
189184
),
190185

191186
// Second row of Object Store stats
192-
objectStorePanels2(title, metricPrefix)::
193-
local operationDuration = '%s_thanos_objstore_bucket_operation_duration_seconds' % [metricPrefix];
187+
objectStorePanels2(title, component)::
194188
super.row(title)
195189
.addPanel(
196190
$.panel('Op: Get') +
197-
$.latencyPanel(operationDuration, '{%s, operation="get"}' % $.namespaceMatcher()),
191+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get"}' % [$.namespaceMatcher(), component]),
198192
)
199193
.addPanel(
200194
$.panel('Op: GetRange') +
201-
$.latencyPanel(operationDuration, '{%s, operation="get_range"}' % $.namespaceMatcher()),
195+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get_range"}' % [$.namespaceMatcher(), component]),
202196
)
203197
.addPanel(
204198
$.panel('Op: Upload') +
205-
$.latencyPanel(operationDuration, '{%s, operation="upload"}' % $.namespaceMatcher()),
199+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="upload"}' % [$.namespaceMatcher(), component]),
206200
)
207201
.addPanel(
208202
$.panel('Op: Delete') +
209-
$.latencyPanel(operationDuration, '{%s, operation="delete"}' % $.namespaceMatcher()),
203+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="delete"}' % [$.namespaceMatcher(), component]),
210204
),
211205
}

‎cortex-mixin/dashboards/object-store.libsonnet‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
local utils = import 'mixin-utils/utils.libsonnet';
2+
3+
(import 'dashboard-utils.libsonnet') {
4+
'cortex-object-store.json':
5+
$.dashboard('Cortex / Object Store')
6+
.addClusterSelectorTemplates()
7+
.addRow(
8+
$.row('Components')
9+
.addPanel(
10+
$.panel('RPS / component') +
11+
$.queryPanel('sum by(component) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % $.namespaceMatcher(), '{{component}}') +
12+
$.stack +
13+
{ yaxes: $.yaxes('rps') },
14+
)
15+
.addPanel(
16+
$.panel('Error rate / component') +
17+
$.queryPanel('sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{%s}[$__interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()], '{{component}}') +
18+
{ yaxes: $.yaxes('percentunit') },
19+
)
20+
)
21+
.addRow(
22+
$.row('Operations')
23+
.addPanel(
24+
$.panel('RPS / operation') +
25+
$.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % $.namespaceMatcher(), '{{operation}}') +
26+
$.stack +
27+
{ yaxes: $.yaxes('rps') },
28+
)
29+
.addPanel(
30+
$.panel('Error rate / operation') +
31+
$.queryPanel('sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{%s}[$__interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()], '{{operation}}') +
32+
{ yaxes: $.yaxes('percentunit') },
33+
)
34+
)
35+
.addRow(
36+
$.row('')
37+
.addPanel(
38+
$.panel('Op: Get') +
39+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="get"}' % $.namespaceMatcher()),
40+
)
41+
.addPanel(
42+
$.panel('Op: GetRange') +
43+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="get_range"}' % $.namespaceMatcher()),
44+
)
45+
.addPanel(
46+
$.panel('Op: Exists') +
47+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="exists"}' % $.namespaceMatcher()),
48+
)
49+
)
50+
.addRow(
51+
$.row('')
52+
.addPanel(
53+
$.panel('Op: ObjectSize') +
54+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="objectsize"}' % $.namespaceMatcher()),
55+
)
56+
.addPanel(
57+
$.panel('Op: Upload') +
58+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="upload"}' % $.namespaceMatcher()),
59+
)
60+
.addPanel(
61+
$.panel('Op: Delete') +
62+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,operation="delete"}' % $.namespaceMatcher()),
63+
)
64+
),
65+
}

‎cortex-mixin/dashboards/reads.libsonnet‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,19 +169,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
169169
// Object store metrics for the store-gateway.
170170
.addRowIf(
171171
std.setMember('tsdb', $._config.storage_engine),
172-
$.objectStorePanels1('Store-gateway - Blocks Object Store', 'cortex_storegateway'),
172+
$.objectStorePanels1('Store-gateway - Blocks Object Store', 'store-gateway'),
173173
)
174174
.addRowIf(
175175
std.setMember('tsdb', $._config.storage_engine),
176-
$.objectStorePanels2('', 'cortex_storegateway'),
176+
$.objectStorePanels2('', 'store-gateway'),
177177
)
178178
// Object store metrics for the querier.
179179
.addRowIf(
180180
std.setMember('tsdb', $._config.storage_engine),
181-
$.objectStorePanels1('Querier - Blocks Object Store', 'cortex_querier'),
181+
$.objectStorePanels1('Querier - Blocks Object Store', 'querier'),
182182
)
183183
.addRowIf(
184184
std.setMember('tsdb', $._config.storage_engine),
185-
$.objectStorePanels2('', 'cortex_querier'),
185+
$.objectStorePanels2('', 'querier'),
186186
),
187187
}

‎cortex-mixin/dashboards/writes.libsonnet‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -151,21 +151,17 @@ local utils = import 'mixin-utils/utils.libsonnet';
151151
)
152152
.addRowIf(
153153
std.setMember('tsdb', $._config.storage_engine),
154-
$.row('Blocks Shipper')
154+
$.row('Ingester - Blocks storage - Shipper')
155155
.addPanel(
156156
$.successFailurePanel(
157157
'Uploaded blocks / sec',
158-
'sum(rate(cortex_ingester_shipper_uploads_total{%s}[$__interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.namespaceMatcher(), $.namespaceMatcher()],
159-
'sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.namespaceMatcher()],
158+
'sum(rate(cortex_ingester_shipper_uploads_total{%s}[$__interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % [$.jobMatcher('ingester'), $.jobMatcher('ingester')],
159+
'sum(rate(cortex_ingester_shipper_upload_failures_total{%s}[$__interval]))' % $.jobMatcher('ingester'),
160160
),
161161
)
162-
)
163-
.addRowIf(
164-
std.setMember('tsdb', $._config.storage_engine),
165-
$.objectStorePanels1('Blocks Object Store Stats (Ingester)', 'cortex_ingester'),
166-
)
167-
.addRowIf(
168-
std.setMember('tsdb', $._config.storage_engine),
169-
$.objectStorePanels2('', 'cortex_ingester'),
162+
.addPanel(
163+
$.panel('Upload latency') +
164+
$.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="ingester",operation="upload"}' % $.jobMatcher('ingester')),
165+
)
170166
),
171167
}

0 commit comments

Comments
 (0)