From d6ef1f27742d97939191c9e34c2914c25c977556 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 7 Sep 2023 15:45:30 -0700 Subject: [PATCH 1/3] Creating new metric around chunks bytespool Signed-off-by: Alan Protasio --- pkg/storegateway/chunk_bytes_pool.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/storegateway/chunk_bytes_pool.go b/pkg/storegateway/chunk_bytes_pool.go index fee8ac40266..11d627fb991 100644 --- a/pkg/storegateway/chunk_bytes_pool.go +++ b/pkg/storegateway/chunk_bytes_pool.go @@ -12,6 +12,8 @@ type chunkBytesPool struct { // Metrics. requestedBytes prometheus.Counter returnedBytes prometheus.Counter + + poolByteStats *prometheus.CounterVec } func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint64, reg prometheus.Registerer) (*chunkBytesPool, error) { @@ -30,6 +32,10 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6 Name: "cortex_bucket_store_chunk_pool_returned_bytes_total", Help: "Total bytes returned by the chunk bytes pool.", }), + poolByteStats: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_bucket_store_chunk_pool_bytes_total", + Help: "Total bytes polled by the chunk bytes pool by operation.", + }, []string{"operation", "stats"}), }, nil } @@ -42,9 +48,15 @@ func (p *chunkBytesPool) Get(sz int) (*[]byte, error) { p.requestedBytes.Add(float64(sz)) p.returnedBytes.Add(float64(cap(*buffer))) + p.poolByteStats.WithLabelValues("Get", "requested").Add(float64(sz)) + p.poolByteStats.WithLabelValues("Get", "len").Add(float64(len(*buffer))) + p.poolByteStats.WithLabelValues("Get", "cap").Add(float64(cap(*buffer))) + return buffer, err } func (p *chunkBytesPool) Put(b *[]byte) { + p.poolByteStats.WithLabelValues("Put", "len").Add(float64(len(*b))) + p.poolByteStats.WithLabelValues("Put", "cap").Add(float64(cap(*b))) p.pool.Put(b) } From 6743799b66cb4564d69be09e6445efb620a35f76 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 7 Sep 2023 16:35:18 -0700 Subject: [PATCH 2/3] removing old metrics Signed-off-by: Alan Protasio --- pkg/storegateway/chunk_bytes_pool.go | 25 +++++------------------ pkg/storegateway/chunk_bytes_pool_test.go | 23 ++++++++++++--------- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/pkg/storegateway/chunk_bytes_pool.go b/pkg/storegateway/chunk_bytes_pool.go index 11d627fb991..e6be1003b9d 100644 --- a/pkg/storegateway/chunk_bytes_pool.go +++ b/pkg/storegateway/chunk_bytes_pool.go @@ -10,9 +10,6 @@ type chunkBytesPool struct { pool *pool.BucketedBytes // Metrics. - requestedBytes prometheus.Counter - returnedBytes prometheus.Counter - poolByteStats *prometheus.CounterVec } @@ -24,17 +21,9 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6 return &chunkBytesPool{ pool: upstream, - requestedBytes: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_bucket_store_chunk_pool_requested_bytes_total", - Help: "Total bytes requested to chunk bytes pool.", - }), - returnedBytes: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_bucket_store_chunk_pool_returned_bytes_total", - Help: "Total bytes returned by the chunk bytes pool.", - }), poolByteStats: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_bucket_store_chunk_pool_bytes_total", - Help: "Total bytes polled by the chunk bytes pool by operation.", + Help: "Total bytes number of bytes pooled by operation.", }, []string{"operation", "stats"}), }, nil } @@ -45,18 +34,14 @@ func (p *chunkBytesPool) Get(sz int) (*[]byte, error) { return buffer, err } - p.requestedBytes.Add(float64(sz)) - p.returnedBytes.Add(float64(cap(*buffer))) - - p.poolByteStats.WithLabelValues("Get", "requested").Add(float64(sz)) - p.poolByteStats.WithLabelValues("Get", "len").Add(float64(len(*buffer))) - p.poolByteStats.WithLabelValues("Get", "cap").Add(float64(cap(*buffer))) + p.poolByteStats.WithLabelValues("Get", "Requested").Add(float64(sz)) + p.poolByteStats.WithLabelValues("Get", "Cap").Add(float64(cap(*buffer))) return buffer, err } func (p *chunkBytesPool) Put(b *[]byte) { - p.poolByteStats.WithLabelValues("Put", "len").Add(float64(len(*b))) - p.poolByteStats.WithLabelValues("Put", "cap").Add(float64(cap(*b))) + p.poolByteStats.WithLabelValues("Put", "Len").Add(float64(len(*b))) + p.poolByteStats.WithLabelValues("Put", "Cap").Add(float64(cap(*b))) p.pool.Put(b) } diff --git a/pkg/storegateway/chunk_bytes_pool_test.go b/pkg/storegateway/chunk_bytes_pool_test.go index 3ddb3380b61..74db90b6469 100644 --- a/pkg/storegateway/chunk_bytes_pool_test.go +++ b/pkg/storegateway/chunk_bytes_pool_test.go @@ -19,20 +19,23 @@ func TestChunkBytesPool_Get(t *testing.T) { reg := prometheus.NewPedanticRegistry() p, err := newChunkBytesPool(cortex_tsdb.ChunkPoolDefaultMinBucketSize, cortex_tsdb.ChunkPoolDefaultMaxBucketSize, 0, reg) require.NoError(t, err) - + testBytes := []byte("test") _, err = p.Get(store.EstimatedMaxChunkSize - 1) require.NoError(t, err) - _, err = p.Get(store.EstimatedMaxChunkSize + 1) + b, err := p.Get(store.EstimatedMaxChunkSize + 1) require.NoError(t, err) + *b = append(*b, testBytes...) + + p.Put(b) + assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_bucket_store_chunk_pool_requested_bytes_total Total bytes requested to chunk bytes pool. - # TYPE cortex_bucket_store_chunk_pool_requested_bytes_total counter - cortex_bucket_store_chunk_pool_requested_bytes_total %d - - # HELP cortex_bucket_store_chunk_pool_returned_bytes_total Total bytes returned by the chunk bytes pool. - # TYPE cortex_bucket_store_chunk_pool_returned_bytes_total counter - cortex_bucket_store_chunk_pool_returned_bytes_total %d - `, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*3)))) + # HELP cortex_bucket_store_chunk_pool_bytes_total Total bytes number of bytes pooled by operation. + # TYPE cortex_bucket_store_chunk_pool_bytes_total counter + cortex_bucket_store_chunk_pool_bytes_total{operation="Get",stats="Cap"} %d + cortex_bucket_store_chunk_pool_bytes_total{operation="Get",stats="Requested"} %d + cortex_bucket_store_chunk_pool_bytes_total{operation="Put",stats="Cap"} %d + cortex_bucket_store_chunk_pool_bytes_total{operation="Put",stats="Len"} %d + `, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes))))) } From 1072a89e12602e72617a766b4e1d1395a2bb7b47 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 7 Sep 2023 16:56:22 -0700 Subject: [PATCH 3/3] changelog Signed-off-by: Alan Protasio --- CHANGELOG.md | 1 + pkg/storegateway/chunk_bytes_pool.go | 10 +++++----- pkg/storegateway/chunk_bytes_pool_test.go | 12 ++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5918e4df529..429fae9183a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * [CHANGE] Store Gateway: Remove `idle_timeout`, `max_conn_age`, `pool_size`, `min_idle_conns` fields for Redis index cache and caching bucket. #5448 * [CHANGE] Store Gateway: Add flag `-store-gateway.sharding-ring.zone-stable-shuffle-sharding` to enable store gateway to use zone stable shuffle sharding. #5489 * [CHANGE] Bucket Index: Add `series_max_size` and `chunk_max_size` to bucket index. #5489 +* [CHANGE] StoreGateway: Rename `cortex_bucket_store_chunk_pool_returned_bytes_total` and `cortex_bucket_store_chunk_pool_requested_bytes_total` to `cortex_bucket_store_chunk_pool_operation_bytes_total`. #5552 * [CHANGE] Query Frontend/Querier: Make build info API disabled by default and add feature flag `api.build-info-enabled` to enable it. #5533 * [FEATURE] Store Gateway: Add `max_downloaded_bytes_per_request` to limit max bytes to download per store gateway request. * [FEATURE] Added 2 flags `-alertmanager.alertmanager-client.grpc-max-send-msg-size` and ` -alertmanager.alertmanager-client.grpc-max-recv-msg-size` to configure alert manager grpc client message size limits. #5338 diff --git a/pkg/storegateway/chunk_bytes_pool.go b/pkg/storegateway/chunk_bytes_pool.go index e6be1003b9d..9972e59c4dc 100644 --- a/pkg/storegateway/chunk_bytes_pool.go +++ b/pkg/storegateway/chunk_bytes_pool.go @@ -22,7 +22,7 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6 return &chunkBytesPool{ pool: upstream, poolByteStats: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_bucket_store_chunk_pool_bytes_total", + Name: "cortex_bucket_store_chunk_pool_operation_bytes_total", Help: "Total bytes number of bytes pooled by operation.", }, []string{"operation", "stats"}), }, nil @@ -34,14 +34,14 @@ func (p *chunkBytesPool) Get(sz int) (*[]byte, error) { return buffer, err } - p.poolByteStats.WithLabelValues("Get", "Requested").Add(float64(sz)) - p.poolByteStats.WithLabelValues("Get", "Cap").Add(float64(cap(*buffer))) + p.poolByteStats.WithLabelValues("get", "requested").Add(float64(sz)) + p.poolByteStats.WithLabelValues("get", "cap").Add(float64(cap(*buffer))) return buffer, err } func (p *chunkBytesPool) Put(b *[]byte) { - p.poolByteStats.WithLabelValues("Put", "Len").Add(float64(len(*b))) - p.poolByteStats.WithLabelValues("Put", "Cap").Add(float64(cap(*b))) + p.poolByteStats.WithLabelValues("put", "len").Add(float64(len(*b))) + p.poolByteStats.WithLabelValues("put", "cap").Add(float64(cap(*b))) p.pool.Put(b) } diff --git a/pkg/storegateway/chunk_bytes_pool_test.go b/pkg/storegateway/chunk_bytes_pool_test.go index 74db90b6469..dc8f1e317c3 100644 --- a/pkg/storegateway/chunk_bytes_pool_test.go +++ b/pkg/storegateway/chunk_bytes_pool_test.go @@ -31,11 +31,11 @@ func TestChunkBytesPool_Get(t *testing.T) { p.Put(b) assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_bucket_store_chunk_pool_bytes_total Total bytes number of bytes pooled by operation. - # TYPE cortex_bucket_store_chunk_pool_bytes_total counter - cortex_bucket_store_chunk_pool_bytes_total{operation="Get",stats="Cap"} %d - cortex_bucket_store_chunk_pool_bytes_total{operation="Get",stats="Requested"} %d - cortex_bucket_store_chunk_pool_bytes_total{operation="Put",stats="Cap"} %d - cortex_bucket_store_chunk_pool_bytes_total{operation="Put",stats="Len"} %d + # HELP cortex_bucket_store_chunk_pool_operation_bytes_total Total bytes number of bytes pooled by operation. + # TYPE cortex_bucket_store_chunk_pool_operation_bytes_total counter + cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="cap"} %d + cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="requested"} %d + cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="cap"} %d + cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="len"} %d `, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes))))) }