From 29e8c5da60d23333fee2a98af4e7e3d51a59e26d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 22 May 2023 19:19:26 -0700 Subject: [PATCH 1/6] metrics for query throttling Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 109 ++++++++++++-- pkg/frontend/transport/handler_test.go | 200 +++++++++++++++++++++++-- pkg/util/validation/validate.go | 10 ++ 3 files changed, 289 insertions(+), 30 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 524dca36b72..eddd1ba1df9 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + "github.com/cortexproject/cortex/pkg/querier/tripperware" "io" "net/http" "net/url" @@ -23,14 +24,13 @@ import ( "google.golang.org/grpc/status" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" - "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" util_log "github.com/cortexproject/cortex/pkg/util/log" ) const ( - // StatusClientClosedRequest is the status code for when a client request cancellation of an http request + // StatusClientClosedRequest is the status code for when a client request cancellation of a http request StatusClientClosedRequest = 499 ServiceTimingHeaderName = "Server-Timing" ) @@ -41,6 +41,35 @@ var ( errRequestEntityTooLarge = httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "http: request body too large") ) +const ( + reasonRequestBodyTooLarge = "request_body_too_large" + reasonResponseTooLarge = "response_too_large" + reasonTooManyRequests = "too_many_requests" + reasonTooLongRange = "too_long_range" + reasonTooManySamples = "too_many_samples" + reasonSeriesFetched = "series_fetched" + reasonChunksFetched = "chunks_fetched" + reasonChunkBytesFetched = "chunk_bytes_fetched" + reasonDataBytesFetched = "data_bytes_fetched" + reasonSeriesLimitStoreGateway = "series_limit_store_gateway" + reasonChunksLimitStoreGateway = "chunks_limit_store_gateway" + reasonBytesLimitStoreGateway = "bytes_limit_store_gateway" +) + +var ( + LimitTooManySamples = `query processing would load too many samples into memory` + LimitTooLongRange = `the query time range exceeds the limit` + LimitSeriesFetched = `the query hit the max number of series limit` + LimitChunksFetched = `the query hit the max number of chunks limit` + LimitChunkBytesFetched = `the query hit the aggregated chunks size limit` + LimitDataBytesFetched = `the query hit the aggregated data size limit` + + // Store gateway limits. + LimitSeriesStoreGateway = `exceeded series limit` + LimitChunksStoreGateway = `exceeded chunks limit` + LimitBytesStoreGateway = `exceeded bytes limit` +) + // Config for a Handler. type HandlerConfig struct { LogQueriesLongerThan time.Duration `yaml:"log_queries_longer_than"` @@ -62,12 +91,13 @@ type Handler struct { roundTripper http.RoundTripper // Metrics. - queriesCount *prometheus.CounterVec - querySeconds *prometheus.CounterVec - querySeries *prometheus.CounterVec - queryChunkBytes *prometheus.CounterVec - queryDataBytes *prometheus.CounterVec - activeUsers *util.ActiveUsersCleanupService + queriesCount *prometheus.CounterVec + querySeconds *prometheus.CounterVec + querySeries *prometheus.CounterVec + queryChunkBytes *prometheus.CounterVec + queryDataBytes *prometheus.CounterVec + discardedQueries *prometheus.CounterVec + activeUsers *util.ActiveUsersCleanupService } // NewHandler creates a new frontend handler. @@ -104,12 +134,23 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge Help: "Size of all data fetched to execute a query in bytes.", }, []string{"user"}) + h.discardedQueries = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cortex_discarded_queries_total", + Help: "The total number of queries that were discarded.", + }, + []string{"reason", "user"}, + ) + h.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(func(user string) { h.queriesCount.DeleteLabelValues(user) h.querySeconds.DeleteLabelValues(user) h.querySeries.DeleteLabelValues(user) h.queryChunkBytes.DeleteLabelValues(user) h.queryDataBytes.DeleteLabelValues(user) + if err := util.DeleteMatchingLabels(h.discardedQueries, map[string]string{"user": user}); err != nil { + level.Warn(log).Log("msg", "failed to remove cortex_discarded_queries_total metric for user", "user", user, "err", err) + } }) // If cleaner stops or fail, we will simply not clean the metrics for inactive users. _ = h.activeUsers.StartAsync(context.Background()) @@ -124,6 +165,12 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { queryString url.Values ) + tenantIDs, err := tenant.TenantIDs(r.Context()) + if err != nil { + return + } + userID := tenant.JoinTenantIDs(tenantIDs) + // Initialise the stats in the context and make sure it's propagated // down the request chain. if f.cfg.QueryStatsEnabled { @@ -150,6 +197,9 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.URL.Path, "api/v1/read") { if err := r.ParseForm(); err != nil { writeError(w, err) + if f.cfg.QueryStatsEnabled && util.IsRequestBodyTooLarge(err) { + f.discardedQueries.WithLabelValues(reasonRequestBodyTooLarge, userID).Inc() + } return } r.Body = io.NopCloser(&buf) @@ -168,7 +218,9 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if shouldReportSlowQuery { f.reportSlowQuery(r, queryString, queryResponseTime) } + if f.cfg.QueryStatsEnabled { + // Try to parse error and get status code. var statusCode int if err != nil { statusCode = getStatusCodeFromError(err) @@ -184,7 +236,7 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - f.reportQueryStats(r, queryString, queryResponseTime, stats, err, statusCode, resp) + f.reportQueryStats(r, userID, queryString, queryResponseTime, stats, err, statusCode, resp) } if err != nil { @@ -239,12 +291,7 @@ func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, query level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } -func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.QueryStats, error error, statusCode int, resp *http.Response) { - tenantIDs, err := tenant.TenantIDs(r.Context()) - if err != nil { - return - } - userID := tenant.JoinTenantIDs(tenantIDs) +func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.QueryStats, error error, statusCode int, resp *http.Response) { wallTime := stats.LoadWallTime() numSeries := stats.LoadFetchedSeries() numChunks := stats.LoadFetchedChunks() @@ -311,6 +358,38 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer } else { level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } + + var reason string + // 413, 422 or 429. + if statusCode == http.StatusTooManyRequests { + reason = reasonTooManyRequests + } else if statusCode == http.StatusRequestEntityTooLarge { + reason = reasonResponseTooLarge + } else if statusCode == http.StatusUnprocessableEntity { + errMsg := error.Error() + if strings.Contains(errMsg, LimitTooManySamples) { + reason = reasonTooManySamples + } else if strings.Contains(errMsg, LimitTooLongRange) { + reason = reasonTooLongRange + } else if strings.Contains(errMsg, LimitSeriesFetched) { + reason = reasonSeriesFetched + } else if strings.Contains(errMsg, LimitChunksFetched) { + reason = reasonChunksFetched + } else if strings.Contains(errMsg, LimitChunkBytesFetched) { + reason = reasonChunkBytesFetched + } else if strings.Contains(errMsg, LimitDataBytesFetched) { + reason = reasonDataBytesFetched + } else if strings.Contains(errMsg, LimitSeriesStoreGateway) { + reason = reasonSeriesLimitStoreGateway + } else if strings.Contains(errMsg, LimitChunksStoreGateway) { + reason = reasonChunksLimitStoreGateway + } else if strings.Contains(errMsg, LimitBytesStoreGateway) { + reason = reasonBytesLimitStoreGateway + } + } + if len(reason) > 0 { + f.discardedQueries.WithLabelValues(reason, userID).Inc() + } } func (f *Handler) parseRequestQueryString(r *http.Request, bodyBuf bytes.Buffer) url.Values { diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index e28069cc427..032283634cf 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -33,6 +33,7 @@ func TestWriteError(t *testing.T) { {http.StatusGatewayTimeout, context.DeadlineExceeded}, {StatusClientClosedRequest, context.Canceled}, {http.StatusBadRequest, httpgrpc.Errorf(http.StatusBadRequest, "")}, + {http.StatusRequestEntityTooLarge, errors.New("http: request body too large")}, } { t.Run(test.err.Error(), func(t *testing.T) { w := httptest.NewRecorder() @@ -43,34 +44,203 @@ func TestWriteError(t *testing.T) { } func TestHandler_ServeHTTP(t *testing.T) { + roundTripper := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("{}")), + }, nil + }) + userID := "12345" for _, tt := range []struct { - name string - cfg HandlerConfig - expectedMetrics int + name string + cfg HandlerConfig + expectedMetrics int + roundTripperFunc roundTripperFunc + additionalMetricsCheckFunc func(h *Handler) }{ { - name: "test handler with stats enabled", + name: "test handler with stats enabled", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripper, + }, + { + name: "test handler with stats disabled", + cfg: HandlerConfig{QueryStatsEnabled: false}, + expectedMetrics: 0, + roundTripperFunc: roundTripper, + }, + { + name: "test handler with reasonResponseTooLarge", cfg: HandlerConfig{QueryStatsEnabled: true}, expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusRequestEntityTooLarge, + Body: io.NopCloser(strings.NewReader("{}")), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonResponseTooLarge, userID)) + assert.Equal(t, float64(1), v) + }, }, { - name: "test handler with stats disabled", - cfg: HandlerConfig{QueryStatsEnabled: false}, - expectedMetrics: 0, - }, - } { - t.Run(tt.name, func(t *testing.T) { - roundTripper := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + name: "test handler with reasonTooManyRequests", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ - StatusCode: http.StatusOK, + StatusCode: http.StatusTooManyRequests, Body: io.NopCloser(strings.NewReader("{}")), }, nil - }) - + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooManyRequests, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonTooManySamples", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitTooManySamples)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooManySamples, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonTooLongRange", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitTooLongRange)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooLongRange, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonSeriesFetched", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitSeriesFetched)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonSeriesFetched, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonChunksFetched", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitChunksFetched)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunksFetched, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonChunkBytesFetched", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitChunkBytesFetched)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunkBytesFetched, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonDataBytesFetched", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitDataBytesFetched)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonDataBytesFetched, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonSeriesLimitStoreGateway", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitSeriesStoreGateway)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonSeriesLimitStoreGateway, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonChunksLimitStoreGateway", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitChunksStoreGateway)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunksLimitStoreGateway, userID)) + assert.Equal(t, float64(1), v) + }, + }, + { + name: "test handler with reasonBytesLimitStoreGateway", + cfg: HandlerConfig{QueryStatsEnabled: true}, + expectedMetrics: 3, + roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusUnprocessableEntity, + Body: io.NopCloser(strings.NewReader(LimitBytesStoreGateway)), + }, nil + }), + additionalMetricsCheckFunc: func(h *Handler) { + v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonBytesLimitStoreGateway, userID)) + assert.Equal(t, float64(1), v) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { reg := prometheus.NewPedanticRegistry() handler := NewHandler(tt.cfg, roundTripper, log.NewNopLogger(), reg) - ctx := user.InjectOrgID(context.Background(), "12345") + ctx := user.InjectOrgID(context.Background(), userID) req := httptest.NewRequest("GET", "/", nil) req = req.WithContext(ctx) resp := httptest.NewRecorder() diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 0985e7db75c..202d06d2759 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -95,10 +95,20 @@ var DiscardedMetadata = prometheus.NewCounterVec( []string{discardReasonLabel, "user"}, ) +// DiscardedQueries is a metric of the number of discarded queries, by reason. +var DiscardedQueries = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cortex_discarded_query_total", + Help: "The total number of queries that were discarded.", + }, + []string{discardReasonLabel, "user"}, +) + func init() { prometheus.MustRegister(DiscardedSamples) prometheus.MustRegister(DiscardedExemplars) prometheus.MustRegister(DiscardedMetadata) + prometheus.MustRegister(DiscardedQueries) } // ValidateSample returns an err if the sample is invalid. From b27d429a8cf03e008c05f2e7bdea759e8293280d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 22 May 2023 19:24:29 -0700 Subject: [PATCH 2/6] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a255de7724d..a0bb73231a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [FEATURE] Store Gateway: Add `max_downloaded_bytes_per_request` to limit max bytes to download per store gateway request. * [FEATURE] Added 2 flags `-alertmanager.alertmanager-client.grpc-max-send-msg-size` and ` -alertmanager.alertmanager-client.grpc-max-recv-msg-size` to configure alert manager grpc client message size limits. #5338 * [FEATURE] Query Frontend: Add `cortex_queries_total` metric for total number of queries executed per user. #5360 +* [FEATURE] Query Frontend: Add `cortex_discarded_queries_total` metric for throttled queries. #5356 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323 From 4436943c0014ec9ab22ef862c279a9769a273b17 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 22 May 2023 20:18:49 -0700 Subject: [PATCH 3/6] lint Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index eddd1ba1df9..ce65fef74c4 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -6,7 +6,6 @@ import ( "errors" "flag" "fmt" - "github.com/cortexproject/cortex/pkg/querier/tripperware" "io" "net/http" "net/url" @@ -24,6 +23,7 @@ import ( "google.golang.org/grpc/status" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" + "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" util_log "github.com/cortexproject/cortex/pkg/util/log" From bac7b1f9d7b6fc1c06b63885c2c9e47072d6da4c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 24 May 2023 12:09:13 -0700 Subject: [PATCH 4/6] address comment Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 97 +++++++++++++------------- pkg/frontend/transport/handler_test.go | 40 +++++------ pkg/querier/stats/stats.proto | 2 + pkg/util/validation/validate.go | 10 --- 4 files changed, 69 insertions(+), 80 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index ce65fef74c4..b61091c6478 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -42,32 +42,30 @@ var ( ) const ( - reasonRequestBodyTooLarge = "request_body_too_large" - reasonResponseTooLarge = "response_too_large" - reasonTooManyRequests = "too_many_requests" - reasonTooLongRange = "too_long_range" - reasonTooManySamples = "too_many_samples" - reasonSeriesFetched = "series_fetched" - reasonChunksFetched = "chunks_fetched" - reasonChunkBytesFetched = "chunk_bytes_fetched" - reasonDataBytesFetched = "data_bytes_fetched" - reasonSeriesLimitStoreGateway = "series_limit_store_gateway" - reasonChunksLimitStoreGateway = "chunks_limit_store_gateway" - reasonBytesLimitStoreGateway = "bytes_limit_store_gateway" -) - -var ( - LimitTooManySamples = `query processing would load too many samples into memory` - LimitTooLongRange = `the query time range exceeds the limit` - LimitSeriesFetched = `the query hit the max number of series limit` - LimitChunksFetched = `the query hit the max number of chunks limit` - LimitChunkBytesFetched = `the query hit the aggregated chunks size limit` - LimitDataBytesFetched = `the query hit the aggregated data size limit` + reasonRequestBodySizeExceeded = "request_body_size_exceeded" + reasonResponseBodySizeExceeded = "response_body_size_exceeded" + reasonTooManyRequests = "too_many_requests" + reasonTimeRangeExceeded = "time_range_exceeded" + reasonTooManySamples = "too_many_samples" + reasonSeriesFetched = "series_fetched" + reasonChunksFetched = "chunks_fetched" + reasonChunkBytesFetched = "chunk_bytes_fetched" + reasonDataBytesFetched = "data_bytes_fetched" + reasonSeriesLimitStoreGateway = "store_gateway_series_limit" + reasonChunksLimitStoreGateway = "store_gateway_chunks_limit" + reasonBytesLimitStoreGateway = "store_gateway_bytes_limit" + + limitTooManySamples = `query processing would load too many samples into memory` + limitTimeRangeExceeded = `the query time range exceeds the limit` + limitSeriesFetched = `the query hit the max number of series limit` + limitChunksFetched = `the query hit the max number of chunks limit` + limitChunkBytesFetched = `the query hit the aggregated chunks size limit` + limitDataBytesFetched = `the query hit the aggregated data size limit` // Store gateway limits. - LimitSeriesStoreGateway = `exceeded series limit` - LimitChunksStoreGateway = `exceeded chunks limit` - LimitBytesStoreGateway = `exceeded bytes limit` + limitSeriesStoreGateway = `exceeded series limit` + limitChunksStoreGateway = `exceeded chunks limit` + limitBytesStoreGateway = `exceeded bytes limit` ) // Config for a Handler. @@ -91,13 +89,13 @@ type Handler struct { roundTripper http.RoundTripper // Metrics. - queriesCount *prometheus.CounterVec - querySeconds *prometheus.CounterVec - querySeries *prometheus.CounterVec - queryChunkBytes *prometheus.CounterVec - queryDataBytes *prometheus.CounterVec - discardedQueries *prometheus.CounterVec - activeUsers *util.ActiveUsersCleanupService + queriesCount *prometheus.CounterVec + querySeconds *prometheus.CounterVec + querySeries *prometheus.CounterVec + queryChunkBytes *prometheus.CounterVec + queryDataBytes *prometheus.CounterVec + rejectedQueries *prometheus.CounterVec + activeUsers *util.ActiveUsersCleanupService } // NewHandler creates a new frontend handler. @@ -134,10 +132,10 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge Help: "Size of all data fetched to execute a query in bytes.", }, []string{"user"}) - h.discardedQueries = prometheus.NewCounterVec( + h.rejectedQueries = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "cortex_discarded_queries_total", - Help: "The total number of queries that were discarded.", + Name: "cortex_rejected_queries_total", + Help: "The total number of queries that were rejected.", }, []string{"reason", "user"}, ) @@ -148,8 +146,8 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge h.querySeries.DeleteLabelValues(user) h.queryChunkBytes.DeleteLabelValues(user) h.queryDataBytes.DeleteLabelValues(user) - if err := util.DeleteMatchingLabels(h.discardedQueries, map[string]string{"user": user}); err != nil { - level.Warn(log).Log("msg", "failed to remove cortex_discarded_queries_total metric for user", "user", user, "err", err) + if err := util.DeleteMatchingLabels(h.rejectedQueries, map[string]string{"user": user}); err != nil { + level.Warn(log).Log("msg", "failed to remove cortex_rejected_queries_total metric for user", "user", user, "err", err) } }) // If cleaner stops or fail, we will simply not clean the metrics for inactive users. @@ -198,7 +196,7 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { writeError(w, err) if f.cfg.QueryStatsEnabled && util.IsRequestBodyTooLarge(err) { - f.discardedQueries.WithLabelValues(reasonRequestBodyTooLarge, userID).Inc() + f.rejectedQueries.WithLabelValues(reasonRequestBodySizeExceeded, userID).Inc() } return } @@ -360,35 +358,34 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u } var reason string - // 413, 422 or 429. if statusCode == http.StatusTooManyRequests { reason = reasonTooManyRequests } else if statusCode == http.StatusRequestEntityTooLarge { - reason = reasonResponseTooLarge + reason = reasonResponseBodySizeExceeded } else if statusCode == http.StatusUnprocessableEntity { errMsg := error.Error() - if strings.Contains(errMsg, LimitTooManySamples) { + if strings.Contains(errMsg, limitTooManySamples) { reason = reasonTooManySamples - } else if strings.Contains(errMsg, LimitTooLongRange) { - reason = reasonTooLongRange - } else if strings.Contains(errMsg, LimitSeriesFetched) { + } else if strings.Contains(errMsg, limitTimeRangeExceeded) { + reason = reasonTimeRangeExceeded + } else if strings.Contains(errMsg, limitSeriesFetched) { reason = reasonSeriesFetched - } else if strings.Contains(errMsg, LimitChunksFetched) { + } else if strings.Contains(errMsg, limitChunksFetched) { reason = reasonChunksFetched - } else if strings.Contains(errMsg, LimitChunkBytesFetched) { + } else if strings.Contains(errMsg, limitChunkBytesFetched) { reason = reasonChunkBytesFetched - } else if strings.Contains(errMsg, LimitDataBytesFetched) { + } else if strings.Contains(errMsg, limitDataBytesFetched) { reason = reasonDataBytesFetched - } else if strings.Contains(errMsg, LimitSeriesStoreGateway) { + } else if strings.Contains(errMsg, limitSeriesStoreGateway) { reason = reasonSeriesLimitStoreGateway - } else if strings.Contains(errMsg, LimitChunksStoreGateway) { + } else if strings.Contains(errMsg, limitChunksStoreGateway) { reason = reasonChunksLimitStoreGateway - } else if strings.Contains(errMsg, LimitBytesStoreGateway) { + } else if strings.Contains(errMsg, limitBytesStoreGateway) { reason = reasonBytesLimitStoreGateway } } if len(reason) > 0 { - f.discardedQueries.WithLabelValues(reason, userID).Inc() + f.rejectedQueries.WithLabelValues(reason, userID).Inc() } } diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index 032283634cf..9d51c9336b1 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -81,7 +81,7 @@ func TestHandler_ServeHTTP(t *testing.T) { }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonResponseTooLarge, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonRequestBodySizeExceeded, userID)) assert.Equal(t, float64(1), v) }, }, @@ -96,7 +96,7 @@ func TestHandler_ServeHTTP(t *testing.T) { }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooManyRequests, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonTooManyRequests, userID)) assert.Equal(t, float64(1), v) }, }, @@ -107,11 +107,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitTooManySamples)), + Body: io.NopCloser(strings.NewReader(limitTooManySamples)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooManySamples, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonTooManySamples, userID)) assert.Equal(t, float64(1), v) }, }, @@ -122,11 +122,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitTooLongRange)), + Body: io.NopCloser(strings.NewReader(limitTimeRangeExceeded)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonTooLongRange, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonTimeRangeExceeded, userID)) assert.Equal(t, float64(1), v) }, }, @@ -137,11 +137,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitSeriesFetched)), + Body: io.NopCloser(strings.NewReader(limitSeriesFetched)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonSeriesFetched, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonSeriesFetched, userID)) assert.Equal(t, float64(1), v) }, }, @@ -152,11 +152,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitChunksFetched)), + Body: io.NopCloser(strings.NewReader(limitChunksFetched)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunksFetched, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonChunksFetched, userID)) assert.Equal(t, float64(1), v) }, }, @@ -167,11 +167,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitChunkBytesFetched)), + Body: io.NopCloser(strings.NewReader(limitChunkBytesFetched)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunkBytesFetched, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonChunkBytesFetched, userID)) assert.Equal(t, float64(1), v) }, }, @@ -182,11 +182,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitDataBytesFetched)), + Body: io.NopCloser(strings.NewReader(limitDataBytesFetched)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonDataBytesFetched, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonDataBytesFetched, userID)) assert.Equal(t, float64(1), v) }, }, @@ -197,11 +197,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitSeriesStoreGateway)), + Body: io.NopCloser(strings.NewReader(limitSeriesStoreGateway)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonSeriesLimitStoreGateway, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonSeriesLimitStoreGateway, userID)) assert.Equal(t, float64(1), v) }, }, @@ -212,11 +212,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitChunksStoreGateway)), + Body: io.NopCloser(strings.NewReader(limitChunksStoreGateway)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonChunksLimitStoreGateway, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonChunksLimitStoreGateway, userID)) assert.Equal(t, float64(1), v) }, }, @@ -227,11 +227,11 @@ func TestHandler_ServeHTTP(t *testing.T) { roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, - Body: io.NopCloser(strings.NewReader(LimitBytesStoreGateway)), + Body: io.NopCloser(strings.NewReader(limitBytesStoreGateway)), }, nil }), additionalMetricsCheckFunc: func(h *Handler) { - v := promtest.ToFloat64(h.discardedQueries.WithLabelValues(reasonBytesLimitStoreGateway, userID)) + v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonBytesLimitStoreGateway, userID)) assert.Equal(t, float64(1), v) }, }, diff --git a/pkg/querier/stats/stats.proto b/pkg/querier/stats/stats.proto index 7e8ba2003ea..ef3de5d1b76 100644 --- a/pkg/querier/stats/stats.proto +++ b/pkg/querier/stats/stats.proto @@ -26,4 +26,6 @@ message Stats { uint64 fetched_chunks_count = 6; // The number of samples fetched for the query uint64 fetched_samples_count = 7; + // The limit hit when executing the query + string limit_hit = 8 [(gogoproto.nullable) = true]; } diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 202d06d2759..0985e7db75c 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -95,20 +95,10 @@ var DiscardedMetadata = prometheus.NewCounterVec( []string{discardReasonLabel, "user"}, ) -// DiscardedQueries is a metric of the number of discarded queries, by reason. -var DiscardedQueries = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cortex_discarded_query_total", - Help: "The total number of queries that were discarded.", - }, - []string{discardReasonLabel, "user"}, -) - func init() { prometheus.MustRegister(DiscardedSamples) prometheus.MustRegister(DiscardedExemplars) prometheus.MustRegister(DiscardedMetadata) - prometheus.MustRegister(DiscardedQueries) } // ValidateSample returns an err if the sample is invalid. From cdb55662219234d825019d5658c19e4cbed31994 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 24 May 2023 19:22:28 +0000 Subject: [PATCH 5/6] update proto Signed-off-by: Ben Ye --- pkg/querier/stats/stats.pb.go | 115 +++++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 28 deletions(-) diff --git a/pkg/querier/stats/stats.pb.go b/pkg/querier/stats/stats.pb.go index 405dc3bbee7..9ce42b53dec 100644 --- a/pkg/querier/stats/stats.pb.go +++ b/pkg/querier/stats/stats.pb.go @@ -46,6 +46,8 @@ type Stats struct { FetchedChunksCount uint64 `protobuf:"varint,6,opt,name=fetched_chunks_count,json=fetchedChunksCount,proto3" json:"fetched_chunks_count,omitempty"` // The number of samples fetched for the query FetchedSamplesCount uint64 `protobuf:"varint,7,opt,name=fetched_samples_count,json=fetchedSamplesCount,proto3" json:"fetched_samples_count,omitempty"` + // The limit hit when executing the query + LimitHit string `protobuf:"bytes,8,opt,name=limit_hit,json=limitHit,proto3" json:"limit_hit,omitempty"` } func (m *Stats) Reset() { *m = Stats{} } @@ -129,6 +131,13 @@ func (m *Stats) GetFetchedSamplesCount() uint64 { return 0 } +func (m *Stats) GetLimitHit() string { + if m != nil { + return m.LimitHit + } + return "" +} + func init() { proto.RegisterType((*Stats)(nil), "stats.Stats") proto.RegisterMapType((map[string]string)(nil), "stats.Stats.ExtraFieldsEntry") @@ -137,33 +146,35 @@ func init() { func init() { proto.RegisterFile("stats.proto", fileDescriptor_b4756a0aec8b9d44) } var fileDescriptor_b4756a0aec8b9d44 = []byte{ - // 411 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xcf, 0xae, 0xd2, 0x40, - 0x14, 0xc6, 0x3b, 0x94, 0x22, 0x4c, 0x5d, 0xe0, 0x88, 0x49, 0x21, 0x71, 0x20, 0xae, 0x58, 0x98, - 0x62, 0x70, 0x63, 0x34, 0x31, 0x84, 0x3f, 0x3e, 0x40, 0x71, 0xe5, 0xa6, 0x19, 0xe8, 0x50, 0x1a, - 0x4a, 0x87, 0xb4, 0x53, 0xb5, 0x3b, 0x1f, 0xc1, 0xa5, 0x8f, 0xe0, 0x6b, 0xb8, 0x63, 0xc9, 0x92, - 0x95, 0x4a, 0xd9, 0xb8, 0xe4, 0x11, 0xcc, 0xcc, 0xb4, 0xdc, 0x7b, 0xd9, 0xcd, 0x99, 0xdf, 0xf9, - 0x4e, 0xce, 0xf7, 0xcd, 0x40, 0x33, 0xe1, 0x84, 0x27, 0xf6, 0x2e, 0x66, 0x9c, 0x21, 0x43, 0x16, - 0x9d, 0x96, 0xcf, 0x7c, 0x26, 0x6f, 0x06, 0xe2, 0xa4, 0x60, 0x07, 0xfb, 0x8c, 0xf9, 0x21, 0x1d, - 0xc8, 0x6a, 0x91, 0xae, 0x06, 0x5e, 0x1a, 0x13, 0x1e, 0xb0, 0xa8, 0xe0, 0xed, 0x5b, 0x4e, 0xa2, - 0x4c, 0xa1, 0x17, 0xbf, 0x74, 0x68, 0xcc, 0xc5, 0x68, 0x34, 0x82, 0x8d, 0x2f, 0x24, 0x0c, 0x5d, - 0x1e, 0x6c, 0xa9, 0x05, 0x7a, 0xa0, 0x6f, 0x0e, 0xdb, 0xb6, 0x12, 0xda, 0xa5, 0xd0, 0x9e, 0x16, - 0x83, 0xc7, 0xf5, 0xfd, 0xef, 0xae, 0xf6, 0xe3, 0x4f, 0x17, 0x38, 0x75, 0xa1, 0xfa, 0x18, 0x6c, - 0x29, 0x7a, 0x05, 0x5b, 0x2b, 0xca, 0x97, 0x6b, 0xea, 0xb9, 0x09, 0x8d, 0x03, 0x9a, 0xb8, 0x4b, - 0x96, 0x46, 0xdc, 0xaa, 0xf4, 0x40, 0xbf, 0xea, 0xa0, 0x82, 0xcd, 0x25, 0x9a, 0x08, 0x82, 0x6c, - 0xf8, 0xb4, 0x54, 0x2c, 0xd7, 0x69, 0xb4, 0x71, 0x17, 0x19, 0xa7, 0x89, 0xa5, 0x4b, 0xc1, 0x93, - 0x02, 0x4d, 0x04, 0x19, 0x0b, 0x80, 0x5e, 0xc2, 0x72, 0x8a, 0xeb, 0x11, 0x4e, 0x8a, 0xf6, 0xaa, - 0x6c, 0x6f, 0x16, 0x64, 0x4a, 0x38, 0x51, 0xdd, 0x23, 0xf8, 0x98, 0x7e, 0xe5, 0x31, 0x71, 0x57, - 0x01, 0x0d, 0xbd, 0xc4, 0x32, 0x7a, 0x7a, 0xdf, 0x1c, 0x3e, 0xb7, 0x55, 0xae, 0xd2, 0xb5, 0x3d, - 0x13, 0x0d, 0x1f, 0x24, 0x9f, 0x45, 0x3c, 0xce, 0x1c, 0x93, 0xde, 0xdd, 0xdc, 0x77, 0x24, 0xf7, - 0x2b, 0x1d, 0xd5, 0x1e, 0x38, 0x92, 0x0b, 0x16, 0x8e, 0x86, 0xf0, 0xd9, 0x35, 0x03, 0xb2, 0xdd, - 0x85, 0xd7, 0x10, 0x1e, 0x49, 0x49, 0x69, 0x77, 0xae, 0x98, 0xd4, 0x74, 0xde, 0xc3, 0xe6, 0xed, - 0x1a, 0xa8, 0x09, 0xf5, 0x0d, 0xcd, 0xe4, 0x3b, 0x34, 0x1c, 0x71, 0x44, 0x2d, 0x68, 0x7c, 0x26, - 0x61, 0x4a, 0x65, 0x9c, 0x0d, 0x47, 0x15, 0x6f, 0x2b, 0x6f, 0xc0, 0xf8, 0xdd, 0xe1, 0x84, 0xb5, - 0xe3, 0x09, 0x6b, 0x97, 0x13, 0x06, 0xdf, 0x72, 0x0c, 0x7e, 0xe6, 0x18, 0xec, 0x73, 0x0c, 0x0e, - 0x39, 0x06, 0x7f, 0x73, 0x0c, 0xfe, 0xe5, 0x58, 0xbb, 0xe4, 0x18, 0x7c, 0x3f, 0x63, 0xed, 0x70, - 0xc6, 0xda, 0xf1, 0x8c, 0xb5, 0x4f, 0xea, 0x47, 0x2d, 0x6a, 0xf2, 0x6d, 0x5f, 0xff, 0x0f, 0x00, - 0x00, 0xff, 0xff, 0xda, 0x89, 0xe2, 0x3c, 0x6e, 0x02, 0x00, 0x00, + // 436 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0x31, 0x6f, 0xd3, 0x40, + 0x14, 0xc7, 0x7d, 0x75, 0x5c, 0xe2, 0x33, 0x43, 0x38, 0x82, 0xe4, 0x46, 0xe2, 0x6a, 0x98, 0x3c, + 0x20, 0x07, 0x85, 0x05, 0x81, 0x84, 0x2a, 0xb7, 0x45, 0xcc, 0x0e, 0x13, 0x8b, 0x75, 0x89, 0x2f, + 0xce, 0xa9, 0xb6, 0xaf, 0xb2, 0xcf, 0x80, 0x37, 0x3e, 0x02, 0x23, 0x1f, 0x81, 0x8f, 0x92, 0x31, + 0x63, 0xa7, 0x42, 0x1c, 0x06, 0xc6, 0x7e, 0x04, 0x74, 0x77, 0x76, 0x81, 0x6e, 0x7e, 0xef, 0xf7, + 0xfe, 0x4f, 0xef, 0xff, 0x3f, 0x43, 0xa7, 0x12, 0x44, 0x54, 0xc1, 0x65, 0xc9, 0x05, 0x47, 0x96, + 0x2a, 0x26, 0xe3, 0x94, 0xa7, 0x5c, 0x75, 0xa6, 0xf2, 0x4b, 0xc3, 0x09, 0x4e, 0x39, 0x4f, 0x33, + 0x3a, 0x55, 0xd5, 0xa2, 0x5e, 0x4d, 0x93, 0xba, 0x24, 0x82, 0xf1, 0xa2, 0xe3, 0x47, 0x77, 0x39, + 0x29, 0x1a, 0x8d, 0x9e, 0xfe, 0x32, 0xa1, 0x35, 0x97, 0xab, 0xd1, 0x09, 0xb4, 0x3f, 0x91, 0x2c, + 0x8b, 0x05, 0xcb, 0xa9, 0x0b, 0x3c, 0xe0, 0x3b, 0xb3, 0xa3, 0x40, 0x0b, 0x83, 0x5e, 0x18, 0x9c, + 0x75, 0x8b, 0xc3, 0xe1, 0xe6, 0xfa, 0xd8, 0xf8, 0xf6, 0xe3, 0x18, 0x44, 0x43, 0xa9, 0x7a, 0xcf, + 0x72, 0x8a, 0x9e, 0xc3, 0xf1, 0x8a, 0x8a, 0xe5, 0x9a, 0x26, 0x71, 0x45, 0x4b, 0x46, 0xab, 0x78, + 0xc9, 0xeb, 0x42, 0xb8, 0x07, 0x1e, 0xf0, 0x07, 0x11, 0xea, 0xd8, 0x5c, 0xa1, 0x53, 0x49, 0x50, + 0x00, 0x1f, 0xf6, 0x8a, 0xe5, 0xba, 0x2e, 0x2e, 0xe2, 0x45, 0x23, 0x68, 0xe5, 0x9a, 0x4a, 0xf0, + 0xa0, 0x43, 0xa7, 0x92, 0x84, 0x12, 0xa0, 0x67, 0xb0, 0xdf, 0x12, 0x27, 0x44, 0x90, 0x6e, 0x7c, + 0xa0, 0xc6, 0x47, 0x1d, 0x39, 0x23, 0x82, 0xe8, 0xe9, 0x13, 0x78, 0x9f, 0x7e, 0x16, 0x25, 0x89, + 0x57, 0x8c, 0x66, 0x49, 0xe5, 0x5a, 0x9e, 0xe9, 0x3b, 0xb3, 0xc7, 0x81, 0xce, 0x55, 0xb9, 0x0e, + 0xce, 0xe5, 0xc0, 0x5b, 0xc5, 0xcf, 0x0b, 0x51, 0x36, 0x91, 0x43, 0xff, 0x76, 0xfe, 0x75, 0xa4, + 0xee, 0xeb, 0x1d, 0x1d, 0xfe, 0xe7, 0x48, 0x1d, 0xd8, 0x39, 0x9a, 0xc1, 0x47, 0xb7, 0x19, 0x90, + 0xfc, 0x32, 0xbb, 0x0d, 0xe1, 0x9e, 0x92, 0xf4, 0x76, 0xe7, 0x9a, 0x69, 0xcd, 0x13, 0x68, 0x67, + 0x2c, 0x67, 0x22, 0x5e, 0x33, 0xe1, 0x0e, 0x3d, 0xe0, 0xdb, 0xe1, 0x60, 0x73, 0x2d, 0xa3, 0x55, + 0xed, 0x77, 0x4c, 0x4c, 0xde, 0xc0, 0xd1, 0xdd, 0x4b, 0xd1, 0x08, 0x9a, 0x17, 0xb4, 0x51, 0x4f, + 0x65, 0x47, 0xf2, 0x13, 0x8d, 0xa1, 0xf5, 0x91, 0x64, 0x35, 0x55, 0x89, 0xdb, 0x91, 0x2e, 0x5e, + 0x1d, 0xbc, 0x04, 0xe1, 0xeb, 0xed, 0x0e, 0x1b, 0x57, 0x3b, 0x6c, 0xdc, 0xec, 0x30, 0xf8, 0xd2, + 0x62, 0xf0, 0xbd, 0xc5, 0x60, 0xd3, 0x62, 0xb0, 0x6d, 0x31, 0xf8, 0xd9, 0x62, 0xf0, 0xbb, 0xc5, + 0xc6, 0x4d, 0x8b, 0xc1, 0xd7, 0x3d, 0x36, 0xb6, 0x7b, 0x6c, 0x5c, 0xed, 0xb1, 0xf1, 0x41, 0xff, + 0x74, 0x8b, 0x43, 0xf5, 0xfc, 0x2f, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x8f, 0x00, 0x37, + 0x91, 0x02, 0x00, 0x00, } func (this *Stats) Equal(that interface{}) bool { @@ -211,13 +222,16 @@ func (this *Stats) Equal(that interface{}) bool { if this.FetchedSamplesCount != that1.FetchedSamplesCount { return false } + if this.LimitHit != that1.LimitHit { + return false + } return true } func (this *Stats) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&stats.Stats{") s = append(s, "WallTime: "+fmt.Sprintf("%#v", this.WallTime)+",\n") s = append(s, "FetchedSeriesCount: "+fmt.Sprintf("%#v", this.FetchedSeriesCount)+",\n") @@ -238,6 +252,7 @@ func (this *Stats) GoString() string { } s = append(s, "FetchedChunksCount: "+fmt.Sprintf("%#v", this.FetchedChunksCount)+",\n") s = append(s, "FetchedSamplesCount: "+fmt.Sprintf("%#v", this.FetchedSamplesCount)+",\n") + s = append(s, "LimitHit: "+fmt.Sprintf("%#v", this.LimitHit)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -269,6 +284,13 @@ func (m *Stats) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.LimitHit) > 0 { + i -= len(m.LimitHit) + copy(dAtA[i:], m.LimitHit) + i = encodeVarintStats(dAtA, i, uint64(len(m.LimitHit))) + i-- + dAtA[i] = 0x42 + } if m.FetchedSamplesCount != 0 { i = encodeVarintStats(dAtA, i, uint64(m.FetchedSamplesCount)) i-- @@ -366,6 +388,10 @@ func (m *Stats) Size() (n int) { if m.FetchedSamplesCount != 0 { n += 1 + sovStats(uint64(m.FetchedSamplesCount)) } + l = len(m.LimitHit) + if l > 0 { + n += 1 + l + sovStats(uint64(l)) + } return n } @@ -397,6 +423,7 @@ func (this *Stats) String() string { `ExtraFields:` + mapStringForExtraFields + `,`, `FetchedChunksCount:` + fmt.Sprintf("%v", this.FetchedChunksCount) + `,`, `FetchedSamplesCount:` + fmt.Sprintf("%v", this.FetchedSamplesCount) + `,`, + `LimitHit:` + fmt.Sprintf("%v", this.LimitHit) + `,`, `}`, }, "") return s @@ -693,6 +720,38 @@ func (m *Stats) Unmarshal(dAtA []byte) error { break } } + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LimitHit", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthStats + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthStats + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LimitHit = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) From 90d828f79dea45be1a404069f8507030403c731f Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 24 May 2023 12:30:00 -0700 Subject: [PATCH 6/6] set limit hit in query frontend Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index b61091c6478..eee4a6e7275 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -386,6 +386,7 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u } if len(reason) > 0 { f.rejectedQueries.WithLabelValues(reason, userID).Inc() + stats.LimitHit = reason } }