From 7fc1cbaf44720ba758a243afdc918af945f4bac8 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 7 May 2020 18:10:41 +0530 Subject: [PATCH 1/2] Ignore frontend instance on connect error Signed-off-by: Annanay --- pkg/querier/frontend/worker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index 0b095458293..4de4fc8a233 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -117,6 +117,7 @@ func (w *worker) watchDNSLoop(servCtx context.Context) error { client, err := w.connect(servCtx, update.Addr) if err != nil { level.Error(w.log).Log("msg", "error connecting", "addr", update.Addr, "err", err) + continue } w.managers[update.Addr] = newFrontendManager(servCtx, w.log, w.server, client, w.cfg.GRPCClientConfig) From cd6fb1d9ad7d8b3c6281963ddf849636cf45bd96 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 7 May 2020 18:16:58 +0530 Subject: [PATCH 2/2] Update CHANGELOG Signed-off-by: Annanay --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a88c7a0af..a1e8c72d306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ * [BUGFIX] Fixed `version`, `revision` and `branch` labels exported by the `cortex_build_info` metric. #2468 * [BUGFIX] QueryFrontend: fixed a situation where HTTP error is ignored and an incorrect status code is set. #2483 * [BUGFIX] QueryFrontend: fixed a situation where span context missed when downstream_url is used. #2539 +* [BUGFIX] Querier: Fixed a situation where querier would crash because of an unresponsive frontend instance. #2569 ## 1.0.0 / 2020-04-02