diff --git a/CHANGELOG.md b/CHANGELOG.md index 984bb80b92d..7e32d241d49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * [CHANGE] Metric `cortex_overrides_last_reload_successful` has been renamed to `cortex_runtime_config_last_reload_successful`. #2874 * [CHANGE] HipChat support has been removed from the alertmanager (because removed from the Prometheus upstream too). #2902 * [CHANGE] Add constant label `name` to metric `cortex_cache_request_duration_seconds`. #2903 +* [CHANGE] Add `user` label to metric `cortex_query_frontend_queue_length`. #2939 * [FEATURE] Introduced `ruler.for-outage-tolerance`, Max time to tolerate outage for restoring "for" state of alert. #2783 * [FEATURE] Introduced `ruler.for-grace-period`, Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. #2783 * [FEATURE] Introduced `ruler.resend-delay`, Minimum amount of time to wait before resending an alert to Alertmanager. #2783 diff --git a/pkg/querier/frontend/frontend.go b/pkg/querier/frontend/frontend.go index f629a0aafe5..d6beed38322 100644 --- a/pkg/querier/frontend/frontend.go +++ b/pkg/querier/frontend/frontend.go @@ -71,7 +71,7 @@ type Frontend struct { // Metrics. queueDuration prometheus.Histogram - queueLength prometheus.Gauge + queueLength *prometheus.GaugeVec } type request struct { @@ -96,11 +96,11 @@ func New(cfg Config, log log.Logger, registerer prometheus.Registerer) (*Fronten Help: "Time spend by requests queued.", Buckets: prometheus.DefBuckets, }), - queueLength: promauto.With(registerer).NewGauge(prometheus.GaugeOpts{ + queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ Namespace: "cortex", Name: "query_frontend_queue_length", Help: "Number of queries in the queue.", - }), + }, []string{"user"}), connectedClients: atomic.NewInt32(0), } f.cond = sync.NewCond(&f.mtx) @@ -363,7 +363,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error { select { case queue <- req: - f.queueLength.Add(1) + f.queueLength.WithLabelValues(userID).Inc() f.cond.Broadcast() return nil default: @@ -416,7 +416,7 @@ FindQueue: f.cond.Broadcast() f.queueDuration.Observe(time.Since(request.enqueueTime).Seconds()) - f.queueLength.Add(-1) + f.queueLength.WithLabelValues(userID).Dec() request.queueSpan.Finish() // Ensure the request has not already expired.