From 5ba3bfdbffb70aa6bd0e852d2b3a3a12cf4ac004 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Tue, 30 Nov 2021 10:56:59 +0100 Subject: [PATCH] promhttp: count hard request failures in roundtripper metrics Right now InstrumentRoundTripper{Counter,Duration} don't increase any metrics in case of hard request failures (e.g., TCP, TLS failures). This makes these functions unattractive to use, as those failures are typically among the most interesting ones to alert on. This change extends these functions to count such requests properly. To distinguish them from requests that did yield an actual response, we leave the "code" label empty. This seems the correct thing to do, because hard request failures don't yield a code from the remote side. --- prometheus/promhttp/instrument_client.go | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go index 83c49b66a..7e115076b 100644 --- a/prometheus/promhttp/instrument_client.go +++ b/prometheus/promhttp/instrument_client.go @@ -32,6 +32,26 @@ func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return rt(r) } +// labelsForError obtains the values of the "code" and "method" labels +// for the case where a HTTP request failed without getting a server +// response. The "code" label is left empty to distinguish from cases +// where a server response was obtained. +func labelsForError(code, method bool, reqMethod string) prometheus.Labels { + if !(code || method) { + return emptyLabels + } + labels := prometheus.Labels{} + + if code { + labels["code"] = "" + } + if method { + labels["method"] = sanitizeMethod(reqMethod) + } + + return labels +} + // InstrumentRoundTripperInFlight is a middleware that wraps the provided // http.RoundTripper. It sets the provided prometheus.Gauge to the number of // requests currently handled by the wrapped http.RoundTripper. @@ -53,8 +73,8 @@ func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripp // and/or HTTP method if the respective instance label names are present in the // CounterVec. For unpartitioned counting, use a CounterVec with zero labels. // -// If the wrapped RoundTripper panics or returns a non-nil error, the Counter -// is not incremented. +// If the wrapped RoundTripper returns a non-nil error, the "code" label is +// empty. If it panics, the Counter is not incremented. // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc { @@ -64,6 +84,8 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou resp, err := next.RoundTrip(r) if err == nil { counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc() + } else { + counter.With(labelsForError(code, method, r.Method)).Inc() } return resp, err }) @@ -80,8 +102,8 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou // unpartitioned observations, use an ObserverVec with zero labels. Note that // partitioning of Histograms is expensive and should be used judiciously. // -// If the wrapped RoundTripper panics or returns a non-nil error, no values are -// reported. +// If the wrapped RoundTripper returns a non-nil error, the "code" label is +// empty. If it panics, no values are reported. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. @@ -93,6 +115,8 @@ func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundT resp, err := next.RoundTrip(r) if err == nil { obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds()) + } else { + obs.With(labelsForError(code, method, r.Method)).Observe(time.Since(start).Seconds()) } return resp, err })