From 8461dc18675ae341ac760174313722e6ac70937c Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jun 2020 12:00:58 +0200 Subject: [PATCH 1/3] Higher granularity for the query-tee duration metrics Signed-off-by: Marco Pracucci --- tools/querytee/proxy_metrics.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/querytee/proxy_metrics.go b/tools/querytee/proxy_metrics.go index 5fda10bbfbe..ea126d12a65 100644 --- a/tools/querytee/proxy_metrics.go +++ b/tools/querytee/proxy_metrics.go @@ -3,7 +3,6 @@ package querytee import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/weaveworks/common/instrument" ) const ( @@ -23,7 +22,7 @@ func NewProxyMetrics(registerer prometheus.Registerer) *ProxyMetrics { Namespace: "cortex_querytee", Name: "request_duration_seconds", Help: "Time (in seconds) spent serving HTTP requests.", - Buckets: instrument.DefBuckets, + Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 0.75, 1, 1.5, 2, 3, 4, 5, 10, 25, 50, 100}, }, []string{"backend", "method", "route", "status_code"}), responsesTotal: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: "cortex_querytee", From bcb0afa72eff5545ca3fef42624ae7a48fb6c663 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jun 2020 12:09:28 +0200 Subject: [PATCH 2/3] Fast fail if the preferred backend is misconfigured Signed-off-by: Marco Pracucci --- tools/querytee/proxy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/querytee/proxy.go b/tools/querytee/proxy.go index 7d2802e2cf5..8a5b0a458b2 100644 --- a/tools/querytee/proxy.go +++ b/tools/querytee/proxy.go @@ -107,6 +107,21 @@ func NewProxy(cfg ProxyConfig, logger log.Logger, routes []Route, registerer pro return nil, errMinBackends } + // If the preferred backend is configured, then it must exists among the actual backends. + if cfg.PreferredBackend != "" { + exists := false + for _, b := range p.backends { + if b.preferred { + exists = true + break + } + } + + if !exists { + return nil, fmt.Errorf("the preferred backend (hostname) has not been found among the list of configured backends") + } + } + if cfg.CompareResponses && len(p.backends) != 2 { return nil, fmt.Errorf("when enabling comparison of results number of backends should be 2 exactly") } @@ -159,6 +174,7 @@ func (p *Proxy) Start() error { } }() + level.Info(p.logger).Log("msg", "The proxy is up and running.") return nil } From f2bf134a5ed7b3300efaecee791482cdcc9ae65c Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jun 2020 12:12:55 +0200 Subject: [PATCH 3/3] Added CHANGELOG entry Signed-off-by: Marco Pracucci --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 734a2241218..e0baf97e284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * [FEATURE] Introduced `ruler.for-grace-period`, Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. #2783 * [FEATURE] Introduced `ruler.for-resend-delay`, Minimum amount of time to wait before resending an alert to Alertmanager. #2783 * [ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using `-querier.second-store-engine` option, with values `chunks` or `tsdb`. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than `-querier.use-second-store-before-time`. Default value of zero will always query secondary store. #2747 +* [ENHANCEMENT] Query-tee: increased the `cortex_querytee_request_duration_seconds` metric buckets granularity. #2799 +* [ENHANCEMENT] Query-tee: fail to start if the configured `-backend.preferred` is unknown. #2799 * [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796 * [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764 * [BUGFIX] Fix race in processing of headers in sharded queries. #2762