From 2b86df0a4add357159eed69012c57834c7cb66fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Ortu=C3=B1o?= Date: Wed, 8 Jun 2022 11:20:27 +0200 Subject: [PATCH 1/2] ruler: report failed eval on any 5xx status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Ángel Ortuño --- CHANGELOG.md | 1 + pkg/ruler/compat.go | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1a2f50e3b..a29c2ead783 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * [CHANGE] Config flag category overrides can be set dynamically at runtime. #1934 * [CHANGE] Ingester: deprecated `-ingester.ring.join-after`. Mimir now behaves as this setting is always set to 0s. This configuration option will be removed in Mimir 2.4.0. #1965 * [CHANGE] Blocks uploaded by ingester no longer contain `__org_id__` label. Compactor now ignores this label and will compact blocks with and without this label together. `mimirconvert` tool will remove the label from blocks as "unknown" label. #1972 +* [CHANGE] Ruler: report failed evaluation metric for any 5xx status code returned by the query-frontend when remote operational mode is enabled. #2053 * [ENHANCEMENT] Distributor: Added limit to prevent tenants from sending excessive number of requests: #1843 * The following CLI flags (and their respective YAML config options) have been added: * `-distributor.request-rate-limit` diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 3d62b9a226a..7776e7db637 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -8,7 +8,6 @@ package ruler import ( "context" "errors" - "net/http" "time" "github.com/go-kit/log" @@ -151,9 +150,9 @@ func MetricsQueryFunc(qf rules.QueryFunc, queries, failedQueries prometheus.Coun return result, origErr } else if err != nil { - // When remote querier enabled, only consider failed queries those returning a 500 status code. + // When remote querier enabled, only consider failed queries those returning a 5xx status code. st, ok := status.FromError(err) - if ok && st.Code() == http.StatusInternalServerError { + if ok && st.Code()/100 == 5 { failedQueries.Inc() } } From ec1ca9fea0295dbc1c4c2bc4efc73d906b9f963e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Ortu=C3=B1o?= Date: Thu, 9 Jun 2022 09:08:57 +0200 Subject: [PATCH 2/2] addressed PR suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Ángel Ortuño --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29c2ead783..48872c48178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ * [CHANGE] Config flag category overrides can be set dynamically at runtime. #1934 * [CHANGE] Ingester: deprecated `-ingester.ring.join-after`. Mimir now behaves as this setting is always set to 0s. This configuration option will be removed in Mimir 2.4.0. #1965 * [CHANGE] Blocks uploaded by ingester no longer contain `__org_id__` label. Compactor now ignores this label and will compact blocks with and without this label together. `mimirconvert` tool will remove the label from blocks as "unknown" label. #1972 -* [CHANGE] Ruler: report failed evaluation metric for any 5xx status code returned by the query-frontend when remote operational mode is enabled. #2053 * [ENHANCEMENT] Distributor: Added limit to prevent tenants from sending excessive number of requests: #1843 * The following CLI flags (and their respective YAML config options) have been added: * `-distributor.request-rate-limit` @@ -34,6 +33,7 @@ * [BUGFIX] Ring: fix bug where instances may appear unhealthy in the hash ring web UI even though they are not. #1933 * [BUGFIX] API: gzip is now enforced when identity encoding is explicitly rejected. #1864 * [BUGFIX] Fix panic at startup when Mimir is running in monolithic mode and query sharding is enabled. #2036 +* [BUGFIX] Ruler: report failed evaluation metric for any 5xx status code returned by the query-frontend when remote operational mode is enabled. #2053 ### Mixin