From bfbabbb89a3119e904d48028fb2534446e31a642 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 8 Nov 2024 10:57:04 +0100 Subject: [PATCH 01/11] Fix ExternalLabels() for Prometheus v3.0 (#7893) Prometheus v3.0.0-rc.0 introduces a new scrape protocol (`PrometheusText1.0.0`) which is present by default in the global configuration. It breaks the Thanos sidecar when it wants to retrieve the external labels. This change replaces the use of the Prometheus `GlobalConfig` struct by a minimal struct which unmarshals only the `external_labels` key. See also https://github.com/prometheus-operator/prometheus-operator/issues/7078 Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/promclient/promclient.go | 7 ++-- pkg/promclient/promclient_test.go | 67 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 pkg/promclient/promclient_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 24206ec32bf..36d6e89aa7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7852](https://github.com/thanos-io/thanos/pull/7852) Query Frontend: pass "stats" parameter forward to queriers and fix Prometheus stats merging. - [#7832](https://github.com/thanos-io/thanos/pull/7832) Query Frontend: Fix cache keys for dynamic split intervals. - [#7885](https://github.com/thanos-io/thanos/pull/7885) Store: Return chunks to the pool after completing a Series call. +- [#7893](https://github.com/thanos-io/thanos/pull/7893) Sidecar: Fix retrieval of external labels for Prometheus v3.0.0. ### Added - [#7763](https://github.com/thanos-io/thanos/pull/7763) Ruler: use native histograms for client latency metrics. diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 271f3f0dafe..eeca44db925 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -27,7 +27,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" @@ -201,13 +200,15 @@ func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labe return labels.EmptyLabels(), errors.Wrapf(err, "unmarshal response: %v", string(body)) } var cfg struct { - GlobalConfig config.GlobalConfig `yaml:"global"` + GlobalConfig struct { + ExternalLabels map[string]string `yaml:"external_labels"` + } `yaml:"global"` } if err := yaml.Unmarshal([]byte(d.Data.YAML), &cfg); err != nil { return labels.EmptyLabels(), errors.Wrapf(err, "parse Prometheus config: %v", d.Data.YAML) } - return cfg.GlobalConfig.ExternalLabels, nil + return labels.FromMap(cfg.GlobalConfig.ExternalLabels), nil } type Flags struct { diff --git a/pkg/promclient/promclient_test.go b/pkg/promclient/promclient_test.go new file mode 100644 index 00000000000..00173d3302b --- /dev/null +++ b/pkg/promclient/promclient_test.go @@ -0,0 +1,67 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Package promclient offers helper client function for various API endpoints. + +package promclient + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/efficientgo/core/testutil" +) + +func TestExternalLabels(t *testing.T) { + for _, tc := range []struct { + name string + response string + err bool + labels map[string]string + }{ + { + name: "invalid payload", + response: `{`, + err: true, + }, + { + name: "unknown scrape protocol", + response: `{"status":"success","data":{"yaml":"global:\n scrape_interval: 1m\n scrape_timeout: 10s\n scrape_protocols:\n - OpenMetricsText1.0.0\n - OpenMetricsText0.0.1\n - PrometheusText1.0.0\n - PrometheusText0.0.4\n - UnknownScrapeProto\n evaluation_interval: 1m\n external_labels:\n az: \"1\"\n region: eu-west\nruntime:\n gogc: 75\n"}}`, + labels: map[string]string{ + "region": "eu-west", + "az": "1", + }, + }, + { + name: "no external labels", + response: `{"status":"success","data":{"yaml":"global:\n scrape_interval: 1m\n scrape_timeout: 10s\n scrape_protocols:\n - OpenMetricsText1.0.0\n - OpenMetricsText0.0.1\n - PrometheusText1.0.0\n - PrometheusText0.0.4\n - UnknownScrapeProto\n evaluation_interval: 1m\nruntime:\n gogc: 75\n"}}`, + labels: map[string]string{}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, tc.response) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + testutil.Ok(t, err) + + ext, err := NewDefaultClient().ExternalLabels(context.Background(), u) + if tc.err { + testutil.NotOk(t, err) + return + } + + testutil.Ok(t, err) + testutil.Equals(t, len(tc.labels), ext.Len()) + for k, v := range tc.labels { + testutil.Equals(t, v, ext.Get(k)) + } + }) + } +} From f9da21ec0b28073875520159fe72ab744c255b2e Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 13 Nov 2024 06:49:48 -0700 Subject: [PATCH 02/11] Fix store debug matchers panic on regex matcher (#7903) * fix store debug matchers panic on regex Signed-off-by: Ben Ye * add test Signed-off-by: Ben Ye * changelog Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + pkg/api/query/v1_test.go | 25 ++++++++++++++++++++++++- pkg/extpromql/parser.go | 11 +---------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36d6e89aa7e..c06fd391cef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7832](https://github.com/thanos-io/thanos/pull/7832) Query Frontend: Fix cache keys for dynamic split intervals. - [#7885](https://github.com/thanos-io/thanos/pull/7885) Store: Return chunks to the pool after completing a Series call. - [#7893](https://github.com/thanos-io/thanos/pull/7893) Sidecar: Fix retrieval of external labels for Prometheus v3.0.0. +- [#7903](https://github.com/thanos-io/thanos/pull/7903) Query: Fix panic on regex store matchers. ### Added - [#7763](https://github.com/thanos-io/thanos/pull/7763) Ruler: use native histograms for client latency metrics. diff --git a/pkg/api/query/v1_test.go b/pkg/api/query/v1_test.go index 923a9cf4113..0c08a0ebca9 100644 --- a/pkg/api/query/v1_test.go +++ b/pkg/api/query/v1_test.go @@ -1722,6 +1722,16 @@ func TestParseStoreDebugMatchersParam(t *testing.T) { labels.MustNewMatcher(labels.MatchEqual, "cluster", "test"), }}, }, + { + storeMatchers: `{__address__=~"localhost:.*"}`, + fail: false, + result: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchRegexp, "__address__", "localhost:.*")}}, + }, + { + storeMatchers: `{__address__!~"localhost:.*"}`, + fail: false, + result: [][]*labels.Matcher{{labels.MustNewMatcher(labels.MatchNotRegexp, "__address__", "localhost:.*")}}, + }, } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { api := QueryAPI{ @@ -1736,11 +1746,24 @@ func TestParseStoreDebugMatchersParam(t *testing.T) { storeMatchers, err := api.parseStoreDebugMatchersParam(r) if !tc.fail { - testutil.Equals(t, tc.result, storeMatchers) + testutil.Equals(t, len(tc.result), len(storeMatchers)) + for i, m := range tc.result { + testutil.Equals(t, len(m), len(storeMatchers[i])) + for j, n := range m { + testutil.Equals(t, n.String(), storeMatchers[i][j].String()) + } + } testutil.Equals(t, (*baseAPI.ApiError)(nil), err) } else { testutil.NotOk(t, err) } + + // We don't care about results but checking for panic. + for _, matchers := range storeMatchers { + for _, matcher := range matchers { + _ = matcher.Matches("") + } + } }) } } diff --git a/pkg/extpromql/parser.go b/pkg/extpromql/parser.go index 43d7188fdc5..a4e92e9c557 100644 --- a/pkg/extpromql/parser.go +++ b/pkg/extpromql/parser.go @@ -43,16 +43,7 @@ func ParseMetricSelector(input string) ([]*labels.Matcher, error) { return nil, fmt.Errorf("expected type *parser.VectorSelector, got %T", expr) } - matchers := make([]*labels.Matcher, len(vs.LabelMatchers)) - for i, lm := range vs.LabelMatchers { - matchers[i] = &labels.Matcher{ - Type: lm.Type, - Name: lm.Name, - Value: lm.Value, - } - } - - return matchers, nil + return vs.LabelMatchers, nil } func isEmptyNameMatcherErr(err error) bool { From dc4c49f2492f9fe88f9187cd33535b4f31f56f86 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:36:56 +0530 Subject: [PATCH 03/11] store: support hedged requests (#7860) * support hedged requests in store Signed-off-by: milinddethe15 * hedged roundtripper with tdigest for dynamic delay Signed-off-by: milinddethe15 * refactor struct and fix lint Signed-off-by: milinddethe15 * Improve hedging implementation Signed-off-by: milinddethe15 * Improved hedging implementation Signed-off-by: milinddethe15 * Update store doc Signed-off-by: milinddethe15 * fix white space Signed-off-by: milinddethe15 * add enabled field Signed-off-by: milinddethe15 --------- Signed-off-by: milinddethe15 --- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 9 ++- cmd/thanos/tools_bucket.go | 20 +++---- docs/components/store.md | 27 +++++++++ go.mod | 7 ++- go.sum | 14 +++-- pkg/exthttp/hedging.go | 96 +++++++++++++++++++++++++++++++ pkg/replicate/replicator.go | 4 +- test/e2e/compact_test.go | 6 +- test/e2e/query_test.go | 6 +- test/e2e/store_gateway_test.go | 16 +++--- test/e2e/tools_bucket_web_test.go | 2 +- 16 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 pkg/exthttp/hedging.go diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index fe39803cbda..3d955ee3949 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -206,7 +206,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index a5cd5c38edf..556369b0a1f 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -84,7 +84,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 56e8a35203d..fffcf6bbdeb 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -193,7 +193,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, comp.String()) + bkt, err = client.NewBucket(logger, confContentYaml, comp.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index e0780452fd9..ab36cf0f66c 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -842,7 +842,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index f799b9b98ca..127584ea947 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -380,7 +380,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 92fcbad34b8..795c108cd3f 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus" commonmodel "github.com/prometheus/common/model" "github.com/prometheus/common/route" + "gopkg.in/yaml.v2" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/client" @@ -32,6 +33,7 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" hidden "github.com/thanos-io/thanos/pkg/extflag" + "github.com/thanos-io/thanos/pkg/exthttp" "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" @@ -308,8 +310,11 @@ func runStore( if err != nil { return err } - - bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String()) + customBktConfig := exthttp.DefaultCustomBucketConfig() + if err := yaml.Unmarshal(confContentYaml, &customBktConfig); err != nil { + return errors.Wrap(err, "parsing config YAML file") + } + bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String(), exthttp.CreateHedgedTransportWithConfig(customBktConfig)) if err != nil { return err } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 7a09dd23a2b..e0391af15b5 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -327,7 +327,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -346,7 +346,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } } else { // nil Prometheus registerer: don't create conflicting metrics. - backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String()) + backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -411,7 +411,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -519,7 +519,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -629,7 +629,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return errors.Wrap(err, "bucket client") } @@ -826,7 +826,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String(), nil) if err != nil { return err } @@ -1084,7 +1084,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String(), nil) if err != nil { return err } @@ -1164,7 +1164,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String(), nil) if err != nil { return err } @@ -1372,7 +1372,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String(), nil) if err != nil { return err } @@ -1462,7 +1462,7 @@ func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extfla return errors.Wrap(err, "unable to parse objstore config") } - bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String(), nil) if err != nil { return errors.Wrap(err, "unable to create bucket") } diff --git a/docs/components/store.md b/docs/components/store.md index f9f70b42a17..a6219e5872e 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -576,6 +576,33 @@ Note that there must be no trailing slash in the `peers` configuration i.e. one If timeout is set to zero then there is no timeout for fetching and fetching's lifetime is equal to the lifetime to the original request's lifetime. It is recommended to keep it higher than zero. It is generally preferred to keep this value higher because the fetching operation potentially includes loading of data from remote object storage. +## Hedged Requests + +Thanos Store Gateway supports `hedged requests` to enhance performance and reliability, particularly in high-latency environments. This feature addresses `long-tail latency issues` that can occur between the Thanos Store Gateway and an external cache, reducing the impact of slower response times on overall performance. + +The configuration options for hedged requests allow for tuning based on latency tolerance and cost considerations, as some providers may charge per request. + +In the `bucket.yml` file, you can specify the following fields under `hedging_config`: + +- `enabled`: bool to enable hedged requests. +- `up_to`: maximum number of hedged requests allowed for each initial request. + - **Purpose**: controls the redundancy level of hedged requests to improve response times. + - **Cost vs. Benefit**: increasing up_to can reduce latency but may increase costs, as some providers charge per request. Higher values provide diminishing returns on latency beyond a certain level. +- `quantile`: latency threshold, specified as a quantile (e.g., percentile), which determines when additional hedged requests should be sent. + - **Purpose**: controls when hedged requests are triggered based on response time distribution. + - **Cost vs. Benefit**: lower quantile (e.g., 0.7) initiates hedged requests sooner, potentially raising costs while lowering latency variance. A higher quantile (e.g., 0.95) will initiate hedged requests later, reducing cost by limiting redundancy. + +By default, `hedging_config` is set as follows: + +```yaml +hedging_config: + enabled: false + up_to: 3 + quantile: 0.9 +``` + +This configuration sends up to three additional requests if the initial request response time exceeds the 90th percentile. + ## Index Header In order to query series inside blocks from object storage, Store Gateway has to know certain initial info from each block index. In order to achieve so, on startup the Gateway builds an `index-header` for each block and stores it on local disk; such `index-header` is build by downloading specific pieces of original block's index, stored on local disk and then mmaped and used by Store Gateway. diff --git a/go.mod b/go.mod index e5f3dd0dae7..7b9dbbadd2e 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/alertmanager v0.27.0 - github.com/prometheus/client_golang v1.20.4 + github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.60.0 github.com/prometheus/exporter-toolkit v0.12.0 @@ -61,7 +61,7 @@ require ( github.com/prometheus/prometheus v0.55.1-0.20241102120812-a6fd22b9d2c8 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 + github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect @@ -144,6 +144,7 @@ require ( github.com/google/s2a-go v0.1.8 // indirect github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect github.com/jcchavezs/porto v0.1.0 // indirect + github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/mdlayher/vsock v1.2.1 // indirect github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect @@ -191,10 +192,12 @@ require ( github.com/aws/smithy-go v1.11.1 // indirect github.com/baidubce/bce-sdk-go v0.9.111 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/caio/go-tdigest v3.1.0+incompatible github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cristalhq/hedgedhttp v0.9.1 github.com/dennwc/varint v1.0.0 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elastic/go-sysinfo v1.8.1 // indirect diff --git a/go.sum b/go.sum index 260d225d49b..73a3764f95d 100644 --- a/go.sum +++ b/go.sum @@ -1452,6 +1452,8 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds= +github.com/caio/go-tdigest v3.1.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -1508,6 +1510,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e h1:nOWmgQD3L/Z0bmm29iDxB7nlqjMnh7yD/PNOx9rnZmA= github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e/go.mod h1:+bSqRETXJ1uk2S93m//htzTVqu8DJPvlGEb3bSE9PzI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs= +github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -1970,6 +1974,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= +github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 h1:YjW+hUb8Fh2S58z4av4t/0cBMK/Q0aP48RocCFsC8yI= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7/go.mod h1:Spd59icnvRxSKuyijbbwe5AemzvcyXAUBgApa7VybMw= @@ -2136,8 +2142,8 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= -github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -2249,8 +2255,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= +github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20 h1:NmVMYAsXPnj9zRG5dDj0SGqrHfbs/1parMRZTvwB8YE= +github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/exthttp/hedging.go b/pkg/exthttp/hedging.go new file mode 100644 index 00000000000..09a1b3e8a29 --- /dev/null +++ b/pkg/exthttp/hedging.go @@ -0,0 +1,96 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package exthttp + +import ( + "fmt" + "net/http" + "sync" + "time" + + "github.com/caio/go-tdigest" + "github.com/cristalhq/hedgedhttp" +) + +type CustomBucketConfig struct { + HedgingConfig HedgingConfig `yaml:"hedging_config"` +} + +type HedgingConfig struct { + Enabled bool `yaml:"enabled"` + UpTo uint `yaml:"up_to"` + Quantile float64 `yaml:"quantile"` +} + +func DefaultCustomBucketConfig() CustomBucketConfig { + return CustomBucketConfig{ + HedgingConfig: HedgingConfig{ + Enabled: false, + UpTo: 3, + Quantile: 0.9, + }, + } +} + +type hedgingRoundTripper struct { + Transport http.RoundTripper + TDigest *tdigest.TDigest + mu sync.RWMutex + config HedgingConfig +} + +func (hrt *hedgingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := hrt.Transport.RoundTrip(req) + if err != nil { + return nil, err + } + duration := float64(time.Since(start).Milliseconds()) + hrt.mu.Lock() + err = hrt.TDigest.Add(duration) + if err != nil { + return nil, err + } + hrt.mu.Unlock() + return resp, err +} + +func (hrt *hedgingRoundTripper) nextFn() (int, time.Duration) { + hrt.mu.RLock() + defer hrt.mu.RUnlock() + + delayMs := hrt.TDigest.Quantile(hrt.config.Quantile) + delay := time.Duration(delayMs) * time.Millisecond + upto := int(hrt.config.UpTo) + return upto, delay +} + +func CreateHedgedTransportWithConfig(config CustomBucketConfig) func(rt http.RoundTripper) http.RoundTripper { + if !config.HedgingConfig.Enabled { + return func(rt http.RoundTripper) http.RoundTripper { + return rt + } + } + return func(rt http.RoundTripper) http.RoundTripper { + td, err := tdigest.New() + if err != nil { + panic(fmt.Sprintf("BUG: Failed to initialize T-Digest: %v", err)) + } + hrt := &hedgingRoundTripper{ + Transport: rt, + TDigest: td, + config: config.HedgingConfig, + } + cfg := hedgedhttp.Config{ + Transport: hrt, + Upto: int(config.HedgingConfig.UpTo), + Next: hrt.nextFn, + } + hedgedrt, err := hedgedhttp.New(cfg) + if err != nil { + panic(fmt.Sprintf("BUG: Failed to create hedged transport: %v", err)) + } + return hedgedrt + } +} diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 668d64afce9..dc9e804f563 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -115,7 +115,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate from") } - bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String()) + bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } @@ -136,7 +136,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate to") } - toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String()) + toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 54c04fe91af..975edb51c50 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -351,7 +351,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) @@ -896,7 +896,7 @@ func TestCompactorDownsampleIgnoresMarked(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bktCfg := e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()) - bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test") + bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test", nil) testutil.Ok(t, err) downsampledBase := blockDesc{ @@ -944,7 +944,7 @@ func TestCompactorIssue6775(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) baseBlockDesc := blockDesc{ diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 763e5cd9cbf..0351cc9bc81 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -619,7 +619,7 @@ func TestQueryStoreMetrics(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Preparing 3 different blocks for the tests. @@ -807,7 +807,7 @@ func TestQueryStoreDedup(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) storeGW := e2ethanos.NewStoreGW( @@ -2035,7 +2035,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Add series from different tenants diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 912712f0a8d..8977aa46bb8 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -127,7 +127,7 @@ metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached")) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -453,7 +453,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -672,7 +672,7 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -807,7 +807,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -934,7 +934,7 @@ config: testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -1068,7 +1068,7 @@ config: l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1180,7 +1180,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1354,7 +1354,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index a72a14a3db8..e679d2b924d 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -156,7 +156,7 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools", nil) testutil.Ok(t, err) // Create share dir for upload. From 20af3eb7dfbe078ddb7d736fbe5945a4c4c2f83a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 15 Nov 2024 11:02:29 +0200 Subject: [PATCH 04/11] receive/capnp: remove close (#7909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I always get this in logs: ``` err: receive capnp conn: close tcp ...: use of closed network connection ``` This is also visible in the e2e test. After Done() returns, the connection is closed either way so no need to close it again. Signed-off-by: Giedrius Statkevičius --- pkg/receive/capnp_server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/receive/capnp_server.go b/pkg/receive/capnp_server.go index 34b406bfa93..3ca359744bc 100644 --- a/pkg/receive/capnp_server.go +++ b/pkg/receive/capnp_server.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/receive/writecapnp" - "github.com/thanos-io/thanos/pkg/runutil" ) type CapNProtoServer struct { @@ -39,7 +38,6 @@ func (c *CapNProtoServer) ListenAndServe() error { } go func() { - defer runutil.CloseWithLogOnErr(c.logger, conn, "receive capnp conn") rpcConn := rpc.NewConn(rpc.NewPackedStreamTransport(conn), &rpc.Options{ // The BootstrapClient is the RPC interface that will be made available // to the remote endpoint by default. From caa972ffd197618f19c050801b9636f98ae3ec5f Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Fri, 15 Nov 2024 10:50:10 +0100 Subject: [PATCH 05/11] store, query: remote engine bug (#7904) * Fix a storage GW bug that loses TSDB infos when joining them * E2E test demonstrating a bug in the MinT calculation in distributed Engine Signed-off-by: Michael Hoffmann --- pkg/query/remote_engine.go | 3 -- pkg/store/bucket.go | 6 +-- pkg/store/bucket_test.go | 12 +++++ test/e2e/query_test.go | 89 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/pkg/query/remote_engine.go b/pkg/query/remote_engine.go index c625cab5ba5..77a74c9a6fa 100644 --- a/pkg/query/remote_engine.go +++ b/pkg/query/remote_engine.go @@ -126,12 +126,10 @@ func (r *remoteEngine) MinT() int64 { highestMintByLabelSet[key] = lset.MinTime continue } - if lset.MinTime > lsetMinT { highestMintByLabelSet[key] = lset.MinTime } } - var mint int64 = math.MaxInt64 for _, m := range highestMintByLabelSet { if m < mint { @@ -190,7 +188,6 @@ func (r *remoteEngine) adjustedInfos() infopb.TSDBInfos { labelpb.ZLabelsFromPromLabels(builder.Labels())), ) } - return infos } diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index eec1de10053..963de83fbe5 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -922,17 +922,15 @@ func (s *BucketStore) TSDBInfos() []infopb.TSDBInfo { sort.Slice(infos, func(i, j int) bool { return infos[i].MinTime < infos[j].MinTime }) cur := infos[0] - for i, info := range infos { + for _, info := range infos { if info.MinTime > cur.MaxTime { res = append(res, cur) cur = info continue } cur.MaxTime = info.MaxTime - if i == len(infos)-1 { - res = append(res, cur) - } } + res = append(res, cur) } return res diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index ec3e0bee7e7..e8dffd093b1 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -682,6 +682,8 @@ func TestBucketStore_TSDBInfo(t *testing.T) { {mint: 3500, maxt: 5000, extLabels: labels.FromStrings("a", "b")}, {mint: 0, maxt: 1000, extLabels: labels.FromStrings("a", "c")}, {mint: 500, maxt: 2000, extLabels: labels.FromStrings("a", "c")}, + {mint: 0, maxt: 1000, extLabels: labels.FromStrings("a", "d")}, + {mint: 2000, maxt: 3000, extLabels: labels.FromStrings("a", "d")}, } { id1, err := e2eutil.CreateBlock(ctx, dir, series, 10, tt.mint, tt.maxt, tt.extLabels, 0, metadata.NoneFunc) testutil.Ok(t, err) @@ -738,6 +740,16 @@ func TestBucketStore_TSDBInfo(t *testing.T) { MinTime: 0, MaxTime: 2000, }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{{Name: "a", Value: "d"}}}, + MinTime: 0, + MaxTime: 1000, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{{Name: "a", Value: "d"}}}, + MinTime: 2000, + MaxTime: 3000, + }, }) } diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 0351cc9bc81..cdb98f3782c 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -2376,3 +2376,92 @@ func TestDistributedEngineWithExtendedFunctions(t *testing.T) { }, time.Now, promclient.QueryOptions{}, 1) testutil.Equals(t, model.SampleValue(0), result[0].Value) } + +func TestDistributedEngineWithDisjointTSDBs(t *testing.T) { + e, err := e2e.New(e2e.WithName("dist-disj-tsdbs")) + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + ctx := context.Background() + l := log.NewLogfmtLogger(os.Stdout) + now := time.Now() + + bucket1 := "dist-disj-tsdbs-test1" + minio1 := e2edb.NewMinio(e, "1", bucket1, e2edb.WithMinioTLS()) + testutil.Ok(t, e2e.StartAndWaitReady(minio1)) + + bkt1, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket1, minio1.Endpoint("http"), minio1.Dir()), "test") + testutil.Ok(t, err) + + // Setup a storage GW with 2 blocks that have a gap to trigger distributed query MinT bug + dir1 := filepath.Join(e.SharedDir(), "tmp1") + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), dir1), os.ModePerm)) + blockID1, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir1, + []labels.Labels{labels.FromStrings("__name__", "foo", "instance", "foo_1")}, + 1000, + timestamp.FromTime(now.Add(-10*time.Hour)), + timestamp.FromTime(now.Add(-8*time.Hour)), + 30*time.Minute, + labels.FromStrings("prometheus", "p1", "replica", "0"), + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt1, path.Join(dir1, blockID1.String()), blockID1.String())) + + blockID2, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir1, + []labels.Labels{labels.FromStrings("__name__", "foo", "instance", "foo_1")}, + 1000, + timestamp.FromTime(now.Add(-4*time.Hour)), + timestamp.FromTime(now.Add(-2*time.Hour)), + 30*time.Minute, + labels.FromStrings("prometheus", "p1", "replica", "0"), + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt1, path.Join(dir1, blockID2.String()), blockID2.String())) + store1 := e2ethanos.NewStoreGW( + e, + "s1", + client.BucketConfig{ + Type: client.S3, + Config: e2ethanos.NewS3Config(bucket1, minio1.InternalEndpoint("http"), minio1.InternalDir()), + }, + "", + "", + nil, + ) + testutil.Ok(t, e2e.StartAndWaitReady(store1)) + + querierLeaf1 := e2ethanos.NewQuerierBuilder(e, "1", store1.InternalEndpoint("grpc")).Init() + + // We need another querier to circumvent the passthrough optimizer + promConfig2 := e2ethanos.DefaultPromConfig("p2", 0, "", "", e2ethanos.LocalPrometheusTarget) + prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "p2", promConfig2, "", e2ethanos.DefaultPrometheusImage(), "") + querierLeaf2 := e2ethanos.NewQuerierBuilder(e, "2", sidecar2.InternalEndpoint("grpc")).Init() + + querierDistributed := e2ethanos.NewQuerierBuilder(e, "3", + querierLeaf1.InternalEndpoint("grpc"), + querierLeaf2.InternalEndpoint("grpc"), + ). + WithEngine(v1.PromqlEngineThanos). + WithQueryMode("distributed"). + Init() + + testutil.Ok(t, e2e.StartAndWaitReady(querierLeaf1, prom2, sidecar2, querierLeaf2, querierDistributed)) + + // We would expect 2x2h ranges for the 2 blocks containing foo samples. That would be around 240 expected sample pairs in the result matrix. + // We assert on more then 200 to reduce flakiness + rangeQuery(t, ctx, querierDistributed.Endpoint("http"), func() string { return "foo" }, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now), 60, promclient.QueryOptions{}, func(res model.Matrix) error { + if res.Len() < 1 { + return errors.New("No result series returned") + } + if nvals := len(res[0].Values); nvals < 200 { + return errors.Errorf("Too few values in result matrix, got %d, expected > 200", nvals) + } + return nil + }) +} From 2a975d33662bf16b47e82680c9ac085909214353 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Fri, 15 Nov 2024 10:16:59 +0000 Subject: [PATCH 06/11] Skip TestDistributedEngineWithDisjointTSDBs (#7911) Signed-off-by: Saswata Mukherjee --- test/e2e/query_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index cdb98f3782c..11bb203247b 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -2378,6 +2378,7 @@ func TestDistributedEngineWithExtendedFunctions(t *testing.T) { } func TestDistributedEngineWithDisjointTSDBs(t *testing.T) { + t.Skip("skipping test as this replicates a bug") e, err := e2e.New(e2e.WithName("dist-disj-tsdbs")) testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -2390,7 +2391,7 @@ func TestDistributedEngineWithDisjointTSDBs(t *testing.T) { minio1 := e2edb.NewMinio(e, "1", bucket1, e2edb.WithMinioTLS()) testutil.Ok(t, e2e.StartAndWaitReady(minio1)) - bkt1, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket1, minio1.Endpoint("http"), minio1.Dir()), "test") + bkt1, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket1, minio1.Endpoint("http"), minio1.Dir()), "test", nil) testutil.Ok(t, err) // Setup a storage GW with 2 blocks that have a gap to trigger distributed query MinT bug From 8c49344fea90b7838dde3b99078bc396bfae8901 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 18 Nov 2024 09:42:38 +0000 Subject: [PATCH 07/11] Changelog: Mark v0.37 release in progress (#7920) Signed-off-by: Saswata Mukherjee --- CHANGELOG.md | 10 ++++++++++ VERSION | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c06fd391cef..f89c47c8552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,16 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed +### Added + +### Changed + +### Removed + +## [v0.37.0 - ](https://github.com/thanos-io/thanos/tree/release-0.37) + +### Fixed + - [#7511](https://github.com/thanos-io/thanos/pull/7511) Query Frontend: fix doubled gzip compression for response body. - [#7592](https://github.com/thanos-io/thanos/pull/7592) Ruler: Only increment `thanos_rule_evaluation_with_warnings_total` metric for non PromQL warnings. - [#7614](https://github.com/thanos-io/thanos/pull/7614) *: fix debug log formatting. diff --git a/VERSION b/VERSION index 8aa7336bf08..c05b0f07860 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.37.0-dev +0.38.0-dev From f998fc59c1711a876a7fa2b6860b2a3157adee2c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 18 Nov 2024 03:36:52 -0800 Subject: [PATCH 08/11] Close block series client at the end to not reuse chunk buf (#7915) * always close block series client at the end Signed-off-by: Ben Ye * add back close for loser tree Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- pkg/store/bucket.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 963de83fbe5..d9940221ffd 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1572,6 +1572,8 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, seriesSrv storepb.Store tenant, ) + defer blockClient.Close() + g.Go(func() error { span, _ := tracing.StartSpan(gctx, "bucket_store_block_series", tracing.Tags{ @@ -3379,7 +3381,6 @@ func (r *bucketIndexReader) Close() error { } func (b *blockSeriesClient) CloseSend() error { - b.Close() return nil } From df9cca7d31d5df8dffd39f56dc66b5b5ee43db62 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 18 Nov 2024 14:44:42 +0000 Subject: [PATCH 09/11] Update objstore and promql-engine to latest (#7924) * Update objstore and promql-engine to latest Signed-off-by: Saswata Mukherjee * Fixes after upgrade Signed-off-by: Saswata Mukherjee --------- Signed-off-by: Saswata Mukherjee --- cmd/thanos/main_test.go | 10 ++++++++++ go.mod | 12 ++++++------ go.sum | 22 ++++++++++++---------- pkg/api/query/v1.go | 2 +- pkg/block/fetcher.go | 2 +- pkg/cache/groupcache.go | 2 +- 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index fcc394a6149..6cbd84cb152 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -105,6 +105,16 @@ func (b *erroringBucket) Name() string { return b.bkt.Name() } +// IterWithAttributes allows to iterate over objects in the bucket with their attributes. +func (b *erroringBucket) IterWithAttributes(ctx context.Context, dir string, f func(objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + return b.bkt.IterWithAttributes(ctx, dir, f, options...) +} + +// SupportedIterOptions returns the supported iteration options. +func (b *erroringBucket) SupportedIterOptions() []objstore.IterOptionType { + return b.bkt.SupportedIterOptions() +} + // Ensures that downsampleBucket() stops its work properly // after an error occurs with some blocks in the backlog. // Testing for https://github.com/thanos-io/thanos/issues/4960. diff --git a/go.mod b/go.mod index 7b9dbbadd2e..309e0f6432a 100644 --- a/go.mod +++ b/go.mod @@ -36,12 +36,12 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 github.com/jpillora/backoff v1.0.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.17.9 + github.com/klauspost/compress v1.17.11 github.com/leanovate/gopter v0.2.9 github.com/lightstep/lightstep-tracer-go v0.25.0 github.com/lovoo/gcloud-opentracing v0.3.0 github.com/miekg/dns v1.1.62 - github.com/minio/minio-go/v7 v7.0.72 // indirect + github.com/minio/minio-go/v7 v7.0.80 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.1.0 github.com/oklog/ulid v1.3.1 @@ -61,8 +61,8 @@ require ( github.com/prometheus/prometheus v0.55.1-0.20241102120812-a6fd22b9d2c8 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20 - github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 + github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97 + github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a @@ -137,6 +137,7 @@ require ( github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/go-licenser v0.3.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-openapi/runtime v0.27.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect @@ -250,7 +251,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/redis/rueidis v1.0.45-alpha.1 github.com/rivo/uniseg v0.2.0 // indirect - github.com/rs/xid v1.5.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/shirou/gopsutil/v3 v3.22.9 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -278,7 +279,6 @@ require ( golang.org/x/tools v0.24.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/protobuf v1.35.1 - gopkg.in/ini.v1 v1.67.0 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect ) diff --git a/go.sum b/go.sum index 73a3764f95d..9250fff8b87 100644 --- a/go.sum +++ b/go.sum @@ -1608,6 +1608,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -1945,8 +1947,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -2027,8 +2029,8 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA= -github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= +github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -2193,8 +2195,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= @@ -2255,10 +2257,10 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20 h1:NmVMYAsXPnj9zRG5dDj0SGqrHfbs/1parMRZTvwB8YE= -github.com/thanos-io/objstore v0.0.0-20241024120700-168679cbbf20/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= -github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= -github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= +github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97 h1:VjG0mwhN1DkncwDHFvrpd12/2TLfgYNRmEQA48ikp+0= +github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97/go.mod h1:vyzFrBXgP+fGNG2FopEGWOO/zrIuoy7zt3LpLeezRsw= +github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a h1:BhWU58VHOxkxQEMByih9fM2WwuwCGtk5AulIcSRSr0A= +github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 7d6d85e28d8..49b40b15974 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -494,7 +494,7 @@ func processAnalysis(a *engine.AnalyzeOutputNode) queryTelemetry { analysis.PeakSamples = a.PeakSamples() analysis.TotalSamples = a.TotalSamples() for _, c := range a.Children { - analysis.Children = append(analysis.Children, processAnalysis(&c)) + analysis.Children = append(analysis.Children, processAnalysis(c)) } return analysis } diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 772d37a48bd..02e9dec513e 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -213,7 +213,7 @@ func (f *RecursiveLister) GetActiveAndPartialBlockIDs(ctx context.Context, ch ch case ch <- id: } return nil - }, objstore.WithRecursiveIter) + }, objstore.WithRecursiveIter()) return partialBlocks, err } diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 43bf1aeefe2..c68828863fe 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -208,7 +208,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { list = append(list, s) return nil - }, objstore.WithRecursiveIter); err != nil { + }, objstore.WithRecursiveIter()); err != nil { return err } From 6a2be98876f24c2d91140ca8b6b63b94dd4548dd Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 18 Nov 2024 14:45:18 +0000 Subject: [PATCH 10/11] docs: Add link to ignore (#7926) Signed-off-by: Saswata Mukherjee --- .mdox.validate.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mdox.validate.yaml b/.mdox.validate.yaml index ba983aadc21..4a3edc4dbb1 100644 --- a/.mdox.validate.yaml +++ b/.mdox.validate.yaml @@ -47,3 +47,6 @@ validators: # Expired certificate - regex: 'bestpractices\.coreinfrastructure\.org\/projects\/3048' type: 'ignore' + # Frequent DNS issues. + - regex: 'build\.thebeat\.co' + type: 'ignore' From fd0643206a95832ee9785641fcb94981e41a9e50 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 18 Nov 2024 17:25:35 +0000 Subject: [PATCH 11/11] docs: Fix formatting again (#7928) Signed-off-by: Saswata Mukherjee --- docs/components/receive.md | 1 + docs/components/sidecar.md | 1 + docs/components/store.md | 1 + docs/components/tools.md | 3 +++ docs/storage.md | 3 +++ 5 files changed, 9 insertions(+) diff --git a/docs/components/receive.md b/docs/components/receive.md index 4082bee625b..cd0dc322c73 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -113,6 +113,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index d8e9cb930d4..01295a5d4db 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -75,6 +75,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` diff --git a/docs/components/store.md b/docs/components/store.md index a6219e5872e..34250001469 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -34,6 +34,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` diff --git a/docs/components/tools.md b/docs/components/tools.md index 426861fdff8..7e34ab4c275 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -129,6 +129,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` @@ -700,6 +701,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` @@ -803,6 +805,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` diff --git a/docs/storage.md b/docs/storage.md index 78aacb38e98..48033629a2b 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -103,6 +103,7 @@ config: kms_encryption_context: {} encryption_key: "" sts_endpoint: "" + max_retries: 0 prefix: "" ``` @@ -305,6 +306,7 @@ config: insecure_skip_verify: false disable_compression: false chunk_size_bytes: 0 + max_retries: 0 prefix: "" ``` @@ -495,6 +497,7 @@ config: endpoint: "" secret_key: "" secret_id: "" + max_retries: 0 http_config: idle_conn_timeout: 1m30s response_header_timeout: 2m