From 4240ff35792569ac079fa1f61dd8f0ec4a67d67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 26 Jun 2025 12:40:15 +0300 Subject: [PATCH 1/3] cmd/query_frontend: use original roundtripper + close immediately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's avoid using all the Cortex roundtripper machinery by using the downstream roundtripper directly and then close the body immediately as to not allocate any memory for the body of the response. Signed-off-by: Giedrius Statkevičius --- cmd/thanos/query_frontend.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 7579c4cb92a..9bf13f9e9b5 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -329,13 +329,13 @@ func runQueryFrontend( return err } - roundTripper, err := cortexfrontend.NewDownstreamRoundTripper(cfg.DownstreamURL, downstreamTripper) + downstreamRT, err := cortexfrontend.NewDownstreamRoundTripper(cfg.DownstreamURL, downstreamTripper) if err != nil { return errors.Wrap(err, "setup downstream roundtripper") } // Wrap the downstream RoundTripper into query frontend Tripperware. - roundTripper = tripperWare(roundTripper) + roundTripper := tripperWare(downstreamRT) // Create the query frontend transport. handler := transport.NewHandler(*cfg.CortexHandlerConfig, roundTripper, logger, nil) @@ -422,14 +422,14 @@ func runQueryFrontend( return errors.Wrap(err, "creating request to downstream URL") } - resp, err := roundTripper.RoundTrip(req) + resp, err := downstreamRT.RoundTrip(req) if err != nil { level.Warn(logger).Log("msg", "failed to reach downstream URL", "err", err, "readiness_url", readinessUrl) statusProber.NotReady(err) firstRun = false continue } - runutil.ExhaustCloseWithLogOnErr(logger, resp.Body, "downstream health check response body") + runutil.CloseWithLogOnErr(logger, resp.Body, "downstream health check response body") if resp.StatusCode/100 == 4 || resp.StatusCode/100 == 5 { level.Warn(logger).Log("msg", "downstream URL returned an error", "status_code", resp.StatusCode, "readiness_url", readinessUrl) From 5583757964bcc82431718434ce0f6695baa33e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 1 Jul 2025 10:34:37 +0300 Subject: [PATCH 2/3] qfe: defer properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor this check into a separate function so that defer would run at the end of it and clean up resources properly. Signed-off-by: Giedrius Statkevičius --- cmd/thanos/query_frontend.go | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 9bf13f9e9b5..96722f710f3 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -402,17 +402,9 @@ func runQueryFrontend( ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { - var firstRun = true - for { - if !firstRun { - select { - case <-ctx.Done(): - return nil - case <-time.After(10 * time.Second): - } - } + doCheckDownstream := func() (rerr error) { timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -424,21 +416,31 @@ func runQueryFrontend( resp, err := downstreamRT.RoundTrip(req) if err != nil { - level.Warn(logger).Log("msg", "failed to reach downstream URL", "err", err, "readiness_url", readinessUrl) - statusProber.NotReady(err) - firstRun = false - continue + return errors.Wrapf(err, "roundtripping to downstream URL %s", readinessUrl) } - runutil.CloseWithLogOnErr(logger, resp.Body, "downstream health check response body") + defer runutil.CloseWithErrCapture(&rerr, resp.Body, "downstream health check response body") if resp.StatusCode/100 == 4 || resp.StatusCode/100 == 5 { - level.Warn(logger).Log("msg", "downstream URL returned an error", "status_code", resp.StatusCode, "readiness_url", readinessUrl) - statusProber.NotReady(errors.Errorf("downstream URL %s returned an error: %d", readinessUrl, resp.StatusCode)) - firstRun = false - continue + return errors.Errorf("downstream URL %s returned an error: %d", readinessUrl, resp.StatusCode) + } + + return nil + } + for { + if !firstRun { + select { + case <-ctx.Done(): + return nil + case <-time.After(10 * time.Second): + } } + firstRun = false - statusProber.Ready() + if err := doCheckDownstream(); err != nil { + statusProber.NotReady(err) + } else { + statusProber.Ready() + } } }, func(err error) { cancel() From e9bdd79df2bc72df0fc20a04be61bd4a03c70e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 1 Jul 2025 12:59:58 +0300 Subject: [PATCH 3/3] CHANGELOG: release 0.39.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 9 +++++++++ VERSION | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5cec838228..0cb7dba1dad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.) +### [v0.39.1](https://github.com/thanos-io/thanos/tree/release-0.39) - 2025 07 01 + +Fixes a memory leak issue on query-frontend. The bug only affects v0.39.0. + +### Fixed + +- [#8349](https://github.com/thanos-io/thanos/pull/8349) Query-Frontend: properly clean up resources +- [#8338](https://github.com/thanos-io/thanos/pull/8338) Query-Frontend: use original roundtripper + close immediately + ### [v0.39.0](https://github.com/thanos-io/thanos/tree/release-0.39) - 2025 06 25 In short: there are a bunch of fixes and small improvements. The shining items in this release are memory usage improvements in Thanos Query and shuffle sharding support in Thanos Receiver. Information about shuffle sharding support is available in the documentation. Thank you to all contributors! diff --git a/VERSION b/VERSION index 4ef2eb086f5..d2e2400ee94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.39.0 +0.39.1