diff --git a/CHANGELOG.md b/CHANGELOG.md index e7c599a7fef..15da95868fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * [ENHANCEMENT] Query-tee: added a small tolerance to floating point sample values comparison. #2994 * [ENHANCEMENT] Query-tee: add support for doing a passthrough of requests to preferred backend for unregistered routes #3018 * [ENHANCEMENT] Expose `storage.aws.dynamodb.backoff_config` configuration file field. #3026 +* [ENHANCEMENT] Added `cortex_request_message_bytes` and `cortex_response_message_bytes` histograms to track received and sent gRPC message and HTTP request/response sizes. Added `cortex_inflight_requests` gauge to track number of inflight gRPC and HTTP requests. #3064 * [BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. #2990 * [BUGFIX] Querier: query /series from ingesters regardless the `-querier.query-ingesters-within` setting. #3035 * [BUGFIX] Experimental blocks storage: Ingester is less likely to hit gRPC message size limit when streaming data to queriers. #3015 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 5438f0f4b04..159d6b3f9f6 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -286,6 +286,22 @@ grpc_tls_config: # CLI flag: -log.level [log_level: | default = "info"] +# Optionally log the source IPs. +# CLI flag: -server.log-source-ips-enabled +[log_source_ips_enabled: | default = false] + +# Header field storing the source IPs. Only used if +# server.log-source-ips-enabled is true. If not set the default Forwarded, +# X-Real-IP and X-Forwarded-For headers are used +# CLI flag: -server.log-source-ips-header +[log_source_ips_header: | default = ""] + +# Regex for matching the source IPs. Only used if server.log-source-ips-enabled +# is true. If not set the default Forwarded, X-Real-IP and X-Forwarded-For +# headers are used +# CLI flag: -server.log-source-ips-regex +[log_source_ips_regex: | default = ""] + # Base path to serve all API routes from (e.g. /v1/) # CLI flag: -server.path-prefix [http_path_prefix: | default = ""] diff --git a/go.mod b/go.mod index 96e67cc0a83..8883596a884 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/stretchr/testify v1.5.1 github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763 github.com/uber/jaeger-client-go v2.25.0+incompatible - github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9 + github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f go.uber.org/atomic v1.6.0 diff --git a/go.sum b/go.sum index 43d323ce9da..633b25af35b 100644 --- a/go.sum +++ b/go.sum @@ -292,6 +292,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -1073,6 +1075,8 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9 h1:dNVIG9aKQHR9T4uYAC4YxmkHHryOsfTwsL54WrS7u28= github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY= +github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e h1:t/as1iFw9iI6s0q9ESR2tTn2qGhI42LjBkPuQLuLzM8= +github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e/go.mod h1:hz10LOsAdzC3K/iXaKoFxOKTDRgxJl+BTGX1GY+TzO4= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= diff --git a/pkg/api/api.go b/pkg/api/api.go index 45db017daf4..85443628c5a 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -266,6 +266,9 @@ func (a *API) RegisterQuerier( registerRoutesExternally bool, tombstonesLoader *purger.TombstonesLoader, querierRequestDuration *prometheus.HistogramVec, + receivedMessageSize *prometheus.HistogramVec, + sentMessageSize *prometheus.HistogramVec, + inflightRequests *prometheus.GaugeVec, ) http.Handler { api := v1.NewAPI( engine, @@ -305,8 +308,11 @@ func (a *API) RegisterQuerier( // Use a separate metric for the querier in order to differentiate requests from the query-frontend when // running Cortex as a single binary. inst := middleware.Instrument{ - Duration: querierRequestDuration, - RouteMatcher: router, + RouteMatcher: router, + Duration: querierRequestDuration, + RequestBodySize: receivedMessageSize, + ResponseBodySize: sentMessageSize, + InflightRequests: inflightRequests, } promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1") diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index ba14888fb25..fb25eb9370e 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -12,6 +12,7 @@ import ( prom_storage "github.com/prometheus/prometheus/storage" httpgrpc_server "github.com/weaveworks/common/httpgrpc/server" "github.com/weaveworks/common/instrument" + "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" "github.com/cortexproject/cortex/pkg/alertmanager" @@ -181,9 +182,29 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) { Buckets: instrument.DefBuckets, }, []string{"method", "route", "status_code", "ws"}) + receivedMessageSize := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_request_message_bytes", + Help: "Size (in bytes) of messages received in the request to the querier.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + + sentMessageSize := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_response_message_bytes", + Help: "Size (in bytes) of messages sent in response by the querier.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + + inflightRequests := promauto.With(prometheus.DefaultRegisterer).NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "cortex", + Name: "querier_inflight_requests", + Help: "Current number of inflight requests to the querier.", + }, []string{"method", "route"}) + // if we are not configured for single binary mode then the querier needs to register its paths externally registerExternally := t.Cfg.Target != All - handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration) + handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration, receivedMessageSize, sentMessageSize, inflightRequests) // single binary mode requires a properly configured worker. if the operator did not attempt to configure the // worker we will attempt an automatic configuration here diff --git a/vendor/github.com/felixge/httpsnoop/.gitignore b/vendor/github.com/felixge/httpsnoop/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vendor/github.com/felixge/httpsnoop/.travis.yml b/vendor/github.com/felixge/httpsnoop/.travis.yml new file mode 100644 index 00000000000..bfc421200d0 --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/.travis.yml @@ -0,0 +1,6 @@ +language: go + +go: + - 1.6 + - 1.7 + - 1.8 diff --git a/vendor/github.com/felixge/httpsnoop/LICENSE.txt b/vendor/github.com/felixge/httpsnoop/LICENSE.txt new file mode 100644 index 00000000000..e028b46a9b0 --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/vendor/github.com/felixge/httpsnoop/Makefile b/vendor/github.com/felixge/httpsnoop/Makefile new file mode 100644 index 00000000000..2d84889aed7 --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/Makefile @@ -0,0 +1,10 @@ +.PHONY: ci generate clean + +ci: clean generate + go test -v ./... + +generate: + go generate . + +clean: + rm -rf *_generated*.go diff --git a/vendor/github.com/felixge/httpsnoop/README.md b/vendor/github.com/felixge/httpsnoop/README.md new file mode 100644 index 00000000000..ae44137e9b0 --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/README.md @@ -0,0 +1,94 @@ +# httpsnoop + +Package httpsnoop provides an easy way to capture http related metrics (i.e. +response time, bytes written, and http status code) from your application's +http.Handlers. + +Doing this requires non-trivial wrapping of the http.ResponseWriter interface, +which is also exposed for users interested in a more low-level API. + +[![GoDoc](https://godoc.org/github.com/felixge/httpsnoop?status.svg)](https://godoc.org/github.com/felixge/httpsnoop) +[![Build Status](https://travis-ci.org/felixge/httpsnoop.svg?branch=master)](https://travis-ci.org/felixge/httpsnoop) + +## Usage Example + +```go +// myH is your app's http handler, perhaps a http.ServeMux or similar. +var myH http.Handler +// wrappedH wraps myH in order to log every request. +wrappedH := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m := httpsnoop.CaptureMetrics(myH, w, r) + log.Printf( + "%s %s (code=%d dt=%s written=%d)", + r.Method, + r.URL, + m.Code, + m.Duration, + m.Written, + ) +}) +http.ListenAndServe(":8080", wrappedH) +``` + +## Why this package exists + +Instrumenting an application's http.Handler is surprisingly difficult. + +However if you google for e.g. "capture ResponseWriter status code" you'll find +lots of advise and code examples that suggest it to be a fairly trivial +undertaking. Unfortunately everything I've seen so far has a high chance of +breaking your application. + +The main problem is that a `http.ResponseWriter` often implements additional +interfaces such as `http.Flusher`, `http.CloseNotifier`, `http.Hijacker`, `http.Pusher`, and +`io.ReaderFrom`. So the naive approach of just wrapping `http.ResponseWriter` +in your own struct that also implements the `http.ResponseWriter` interface +will hide the additional interfaces mentioned above. This has a high change of +introducing subtle bugs into any non-trivial application. + +Another approach I've seen people take is to return a struct that implements +all of the interfaces above. However, that's also problematic, because it's +difficult to fake some of these interfaces behaviors when the underlying +`http.ResponseWriter` doesn't have an implementation. It's also dangerous, +because an application may choose to operate differently, merely because it +detects the presence of these additional interfaces. + +This package solves this problem by checking which additional interfaces a +`http.ResponseWriter` implements, returning a wrapped version implementing the +exact same set of interfaces. + +Additionally this package properly handles edge cases such as `WriteHeader` not +being called, or called more than once, as well as concurrent calls to +`http.ResponseWriter` methods, and even calls happening after the wrapped +`ServeHTTP` has already returned. + +Unfortunately this package is not perfect either. It's possible that it is +still missing some interfaces provided by the go core (let me know if you find +one), and it won't work for applications adding their own interfaces into the +mix. + +However, hopefully the explanation above has sufficiently scared you of rolling +your own solution to this problem. httpsnoop may still break your application, +but at least it tries to avoid it as much as possible. + +Anyway, the real problem here is that smuggling additional interfaces inside +`http.ResponseWriter` is a problematic design choice, but it probably goes as +deep as the Go language specification itself. But that's okay, I still prefer +Go over the alternatives ;). + +## Performance + +``` +BenchmarkBaseline-8 20000 94912 ns/op +BenchmarkCaptureMetrics-8 20000 95461 ns/op +``` + +As you can see, using `CaptureMetrics` on a vanilla http.Handler introduces an +overhead of ~500 ns per http request on my machine. However, the margin of +error appears to be larger than that, therefor it should be reasonable to +assume that the overhead introduced by `CaptureMetrics` is absolutely +negligible. + +## License + +MIT diff --git a/vendor/github.com/felixge/httpsnoop/capture_metrics.go b/vendor/github.com/felixge/httpsnoop/capture_metrics.go new file mode 100644 index 00000000000..4c45b1a8c15 --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/capture_metrics.go @@ -0,0 +1,84 @@ +package httpsnoop + +import ( + "io" + "net/http" + "sync" + "time" +) + +// Metrics holds metrics captured from CaptureMetrics. +type Metrics struct { + // Code is the first http response code passed to the WriteHeader func of + // the ResponseWriter. If no such call is made, a default code of 200 is + // assumed instead. + Code int + // Duration is the time it took to execute the handler. + Duration time.Duration + // Written is the number of bytes successfully written by the Write or + // ReadFrom function of the ResponseWriter. ResponseWriters may also write + // data to their underlaying connection directly (e.g. headers), but those + // are not tracked. Therefor the number of Written bytes will usually match + // the size of the response body. + Written int64 +} + +// CaptureMetrics wraps the given hnd, executes it with the given w and r, and +// returns the metrics it captured from it. +func CaptureMetrics(hnd http.Handler, w http.ResponseWriter, r *http.Request) Metrics { + return CaptureMetricsFn(w, func(ww http.ResponseWriter) { + hnd.ServeHTTP(ww, r) + }) +} + +// CaptureMetricsFn wraps w and calls fn with the wrapped w and returns the +// resulting metrics. This is very similar to CaptureMetrics (which is just +// sugar on top of this func), but is a more usable interface if your +// application doesn't use the Go http.Handler interface. +func CaptureMetricsFn(w http.ResponseWriter, fn func(http.ResponseWriter)) Metrics { + var ( + start = time.Now() + m = Metrics{Code: http.StatusOK} + headerWritten bool + lock sync.Mutex + hooks = Hooks{ + WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc { + return func(code int) { + next(code) + lock.Lock() + defer lock.Unlock() + if !headerWritten { + m.Code = code + headerWritten = true + } + } + }, + + Write: func(next WriteFunc) WriteFunc { + return func(p []byte) (int, error) { + n, err := next(p) + lock.Lock() + defer lock.Unlock() + m.Written += int64(n) + headerWritten = true + return n, err + } + }, + + ReadFrom: func(next ReadFromFunc) ReadFromFunc { + return func(src io.Reader) (int64, error) { + n, err := next(src) + lock.Lock() + defer lock.Unlock() + headerWritten = true + m.Written += n + return n, err + } + }, + } + ) + + fn(Wrap(w, hooks)) + m.Duration = time.Since(start) + return m +} diff --git a/vendor/github.com/felixge/httpsnoop/docs.go b/vendor/github.com/felixge/httpsnoop/docs.go new file mode 100644 index 00000000000..203c35b3c6d --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/docs.go @@ -0,0 +1,10 @@ +// Package httpsnoop provides an easy way to capture http related metrics (i.e. +// response time, bytes written, and http status code) from your application's +// http.Handlers. +// +// Doing this requires non-trivial wrapping of the http.ResponseWriter +// interface, which is also exposed for users interested in a more low-level +// API. +package httpsnoop + +//go:generate go run codegen/main.go diff --git a/vendor/github.com/felixge/httpsnoop/go.mod b/vendor/github.com/felixge/httpsnoop/go.mod new file mode 100644 index 00000000000..73b3946905a --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/go.mod @@ -0,0 +1,3 @@ +module github.com/felixge/httpsnoop + +go 1.13 diff --git a/vendor/github.com/felixge/httpsnoop/wrap_generated_gteq_1.8.go b/vendor/github.com/felixge/httpsnoop/wrap_generated_gteq_1.8.go new file mode 100644 index 00000000000..41a20da9eab --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/wrap_generated_gteq_1.8.go @@ -0,0 +1,385 @@ +// +build go1.8 +// Code generated by "httpsnoop/codegen"; DO NOT EDIT + +package httpsnoop + +import ( + "bufio" + "io" + "net" + "net/http" +) + +// HeaderFunc is part of the http.ResponseWriter interface. +type HeaderFunc func() http.Header + +// WriteHeaderFunc is part of the http.ResponseWriter interface. +type WriteHeaderFunc func(code int) + +// WriteFunc is part of the http.ResponseWriter interface. +type WriteFunc func(b []byte) (int, error) + +// FlushFunc is part of the http.Flusher interface. +type FlushFunc func() + +// CloseNotifyFunc is part of the http.CloseNotifier interface. +type CloseNotifyFunc func() <-chan bool + +// HijackFunc is part of the http.Hijacker interface. +type HijackFunc func() (net.Conn, *bufio.ReadWriter, error) + +// ReadFromFunc is part of the io.ReaderFrom interface. +type ReadFromFunc func(src io.Reader) (int64, error) + +// PushFunc is part of the http.Pusher interface. +type PushFunc func(target string, opts *http.PushOptions) error + +// Hooks defines a set of method interceptors for methods included in +// http.ResponseWriter as well as some others. You can think of them as +// middleware for the function calls they target. See Wrap for more details. +type Hooks struct { + Header func(HeaderFunc) HeaderFunc + WriteHeader func(WriteHeaderFunc) WriteHeaderFunc + Write func(WriteFunc) WriteFunc + Flush func(FlushFunc) FlushFunc + CloseNotify func(CloseNotifyFunc) CloseNotifyFunc + Hijack func(HijackFunc) HijackFunc + ReadFrom func(ReadFromFunc) ReadFromFunc + Push func(PushFunc) PushFunc +} + +// Wrap returns a wrapped version of w that provides the exact same interface +// as w. Specifically if w implements any combination of: +// +// - http.Flusher +// - http.CloseNotifier +// - http.Hijacker +// - io.ReaderFrom +// - http.Pusher +// +// The wrapped version will implement the exact same combination. If no hooks +// are set, the wrapped version also behaves exactly as w. Hooks targeting +// methods not supported by w are ignored. Any other hooks will intercept the +// method they target and may modify the call's arguments and/or return values. +// The CaptureMetrics implementation serves as a working example for how the +// hooks can be used. +func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter { + rw := &rw{w: w, h: hooks} + _, i0 := w.(http.Flusher) + _, i1 := w.(http.CloseNotifier) + _, i2 := w.(http.Hijacker) + _, i3 := w.(io.ReaderFrom) + _, i4 := w.(http.Pusher) + switch { + // combination 1/32 + case !i0 && !i1 && !i2 && !i3 && !i4: + return struct { + http.ResponseWriter + }{rw} + // combination 2/32 + case !i0 && !i1 && !i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Pusher + }{rw, rw} + // combination 3/32 + case !i0 && !i1 && !i2 && i3 && !i4: + return struct { + http.ResponseWriter + io.ReaderFrom + }{rw, rw} + // combination 4/32 + case !i0 && !i1 && !i2 && i3 && i4: + return struct { + http.ResponseWriter + io.ReaderFrom + http.Pusher + }{rw, rw, rw} + // combination 5/32 + case !i0 && !i1 && i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.Hijacker + }{rw, rw} + // combination 6/32 + case !i0 && !i1 && i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Hijacker + http.Pusher + }{rw, rw, rw} + // combination 7/32 + case !i0 && !i1 && i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.Hijacker + io.ReaderFrom + }{rw, rw, rw} + // combination 8/32 + case !i0 && !i1 && i2 && i3 && i4: + return struct { + http.ResponseWriter + http.Hijacker + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw} + // combination 9/32 + case !i0 && i1 && !i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.CloseNotifier + }{rw, rw} + // combination 10/32 + case !i0 && i1 && !i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Pusher + }{rw, rw, rw} + // combination 11/32 + case !i0 && i1 && !i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.CloseNotifier + io.ReaderFrom + }{rw, rw, rw} + // combination 12/32 + case !i0 && i1 && !i2 && i3 && i4: + return struct { + http.ResponseWriter + http.CloseNotifier + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw} + // combination 13/32 + case !i0 && i1 && i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + }{rw, rw, rw} + // combination 14/32 + case !i0 && i1 && i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + http.Pusher + }{rw, rw, rw, rw} + // combination 15/32 + case !i0 && i1 && i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 16/32 + case !i0 && i1 && i2 && i3 && i4: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw, rw} + // combination 17/32 + case i0 && !i1 && !i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + }{rw, rw} + // combination 18/32 + case i0 && !i1 && !i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.Pusher + }{rw, rw, rw} + // combination 19/32 + case i0 && !i1 && !i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + io.ReaderFrom + }{rw, rw, rw} + // combination 20/32 + case i0 && !i1 && !i2 && i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw} + // combination 21/32 + case i0 && !i1 && i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + }{rw, rw, rw} + // combination 22/32 + case i0 && !i1 && i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + http.Pusher + }{rw, rw, rw, rw} + // combination 23/32 + case i0 && !i1 && i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 24/32 + case i0 && !i1 && i2 && i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw, rw} + // combination 25/32 + case i0 && i1 && !i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + }{rw, rw, rw} + // combination 26/32 + case i0 && i1 && !i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Pusher + }{rw, rw, rw, rw} + // combination 27/32 + case i0 && i1 && !i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 28/32 + case i0 && i1 && !i2 && i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw, rw} + // combination 29/32 + case i0 && i1 && i2 && !i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + }{rw, rw, rw, rw} + // combination 30/32 + case i0 && i1 && i2 && !i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + http.Pusher + }{rw, rw, rw, rw, rw} + // combination 31/32 + case i0 && i1 && i2 && i3 && !i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw, rw} + // combination 32/32 + case i0 && i1 && i2 && i3 && i4: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + io.ReaderFrom + http.Pusher + }{rw, rw, rw, rw, rw, rw} + } + panic("unreachable") +} + +type rw struct { + w http.ResponseWriter + h Hooks +} + +func (w *rw) Header() http.Header { + f := w.w.(http.ResponseWriter).Header + if w.h.Header != nil { + f = w.h.Header(f) + } + return f() +} + +func (w *rw) WriteHeader(code int) { + f := w.w.(http.ResponseWriter).WriteHeader + if w.h.WriteHeader != nil { + f = w.h.WriteHeader(f) + } + f(code) +} + +func (w *rw) Write(b []byte) (int, error) { + f := w.w.(http.ResponseWriter).Write + if w.h.Write != nil { + f = w.h.Write(f) + } + return f(b) +} + +func (w *rw) Flush() { + f := w.w.(http.Flusher).Flush + if w.h.Flush != nil { + f = w.h.Flush(f) + } + f() +} + +func (w *rw) CloseNotify() <-chan bool { + f := w.w.(http.CloseNotifier).CloseNotify + if w.h.CloseNotify != nil { + f = w.h.CloseNotify(f) + } + return f() +} + +func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) { + f := w.w.(http.Hijacker).Hijack + if w.h.Hijack != nil { + f = w.h.Hijack(f) + } + return f() +} + +func (w *rw) ReadFrom(src io.Reader) (int64, error) { + f := w.w.(io.ReaderFrom).ReadFrom + if w.h.ReadFrom != nil { + f = w.h.ReadFrom(f) + } + return f(src) +} + +func (w *rw) Push(target string, opts *http.PushOptions) error { + f := w.w.(http.Pusher).Push + if w.h.Push != nil { + f = w.h.Push(f) + } + return f(target, opts) +} diff --git a/vendor/github.com/felixge/httpsnoop/wrap_generated_lt_1.8.go b/vendor/github.com/felixge/httpsnoop/wrap_generated_lt_1.8.go new file mode 100644 index 00000000000..36bb59b837c --- /dev/null +++ b/vendor/github.com/felixge/httpsnoop/wrap_generated_lt_1.8.go @@ -0,0 +1,243 @@ +// +build !go1.8 +// Code generated by "httpsnoop/codegen"; DO NOT EDIT + +package httpsnoop + +import ( + "bufio" + "io" + "net" + "net/http" +) + +// HeaderFunc is part of the http.ResponseWriter interface. +type HeaderFunc func() http.Header + +// WriteHeaderFunc is part of the http.ResponseWriter interface. +type WriteHeaderFunc func(code int) + +// WriteFunc is part of the http.ResponseWriter interface. +type WriteFunc func(b []byte) (int, error) + +// FlushFunc is part of the http.Flusher interface. +type FlushFunc func() + +// CloseNotifyFunc is part of the http.CloseNotifier interface. +type CloseNotifyFunc func() <-chan bool + +// HijackFunc is part of the http.Hijacker interface. +type HijackFunc func() (net.Conn, *bufio.ReadWriter, error) + +// ReadFromFunc is part of the io.ReaderFrom interface. +type ReadFromFunc func(src io.Reader) (int64, error) + +// Hooks defines a set of method interceptors for methods included in +// http.ResponseWriter as well as some others. You can think of them as +// middleware for the function calls they target. See Wrap for more details. +type Hooks struct { + Header func(HeaderFunc) HeaderFunc + WriteHeader func(WriteHeaderFunc) WriteHeaderFunc + Write func(WriteFunc) WriteFunc + Flush func(FlushFunc) FlushFunc + CloseNotify func(CloseNotifyFunc) CloseNotifyFunc + Hijack func(HijackFunc) HijackFunc + ReadFrom func(ReadFromFunc) ReadFromFunc +} + +// Wrap returns a wrapped version of w that provides the exact same interface +// as w. Specifically if w implements any combination of: +// +// - http.Flusher +// - http.CloseNotifier +// - http.Hijacker +// - io.ReaderFrom +// +// The wrapped version will implement the exact same combination. If no hooks +// are set, the wrapped version also behaves exactly as w. Hooks targeting +// methods not supported by w are ignored. Any other hooks will intercept the +// method they target and may modify the call's arguments and/or return values. +// The CaptureMetrics implementation serves as a working example for how the +// hooks can be used. +func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter { + rw := &rw{w: w, h: hooks} + _, i0 := w.(http.Flusher) + _, i1 := w.(http.CloseNotifier) + _, i2 := w.(http.Hijacker) + _, i3 := w.(io.ReaderFrom) + switch { + // combination 1/16 + case !i0 && !i1 && !i2 && !i3: + return struct { + http.ResponseWriter + }{rw} + // combination 2/16 + case !i0 && !i1 && !i2 && i3: + return struct { + http.ResponseWriter + io.ReaderFrom + }{rw, rw} + // combination 3/16 + case !i0 && !i1 && i2 && !i3: + return struct { + http.ResponseWriter + http.Hijacker + }{rw, rw} + // combination 4/16 + case !i0 && !i1 && i2 && i3: + return struct { + http.ResponseWriter + http.Hijacker + io.ReaderFrom + }{rw, rw, rw} + // combination 5/16 + case !i0 && i1 && !i2 && !i3: + return struct { + http.ResponseWriter + http.CloseNotifier + }{rw, rw} + // combination 6/16 + case !i0 && i1 && !i2 && i3: + return struct { + http.ResponseWriter + http.CloseNotifier + io.ReaderFrom + }{rw, rw, rw} + // combination 7/16 + case !i0 && i1 && i2 && !i3: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + }{rw, rw, rw} + // combination 8/16 + case !i0 && i1 && i2 && i3: + return struct { + http.ResponseWriter + http.CloseNotifier + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 9/16 + case i0 && !i1 && !i2 && !i3: + return struct { + http.ResponseWriter + http.Flusher + }{rw, rw} + // combination 10/16 + case i0 && !i1 && !i2 && i3: + return struct { + http.ResponseWriter + http.Flusher + io.ReaderFrom + }{rw, rw, rw} + // combination 11/16 + case i0 && !i1 && i2 && !i3: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + }{rw, rw, rw} + // combination 12/16 + case i0 && !i1 && i2 && i3: + return struct { + http.ResponseWriter + http.Flusher + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 13/16 + case i0 && i1 && !i2 && !i3: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + }{rw, rw, rw} + // combination 14/16 + case i0 && i1 && !i2 && i3: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + io.ReaderFrom + }{rw, rw, rw, rw} + // combination 15/16 + case i0 && i1 && i2 && !i3: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + }{rw, rw, rw, rw} + // combination 16/16 + case i0 && i1 && i2 && i3: + return struct { + http.ResponseWriter + http.Flusher + http.CloseNotifier + http.Hijacker + io.ReaderFrom + }{rw, rw, rw, rw, rw} + } + panic("unreachable") +} + +type rw struct { + w http.ResponseWriter + h Hooks +} + +func (w *rw) Header() http.Header { + f := w.w.(http.ResponseWriter).Header + if w.h.Header != nil { + f = w.h.Header(f) + } + return f() +} + +func (w *rw) WriteHeader(code int) { + f := w.w.(http.ResponseWriter).WriteHeader + if w.h.WriteHeader != nil { + f = w.h.WriteHeader(f) + } + f(code) +} + +func (w *rw) Write(b []byte) (int, error) { + f := w.w.(http.ResponseWriter).Write + if w.h.Write != nil { + f = w.h.Write(f) + } + return f(b) +} + +func (w *rw) Flush() { + f := w.w.(http.Flusher).Flush + if w.h.Flush != nil { + f = w.h.Flush(f) + } + f() +} + +func (w *rw) CloseNotify() <-chan bool { + f := w.w.(http.CloseNotifier).CloseNotify + if w.h.CloseNotify != nil { + f = w.h.CloseNotify(f) + } + return f() +} + +func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) { + f := w.w.(http.Hijacker).Hijack + if w.h.Hijack != nil { + f = w.h.Hijack(f) + } + return f() +} + +func (w *rw) ReadFrom(src io.Reader) (int64, error) { + f := w.w.(io.ReaderFrom).ReadFrom + if w.h.ReadFrom != nil { + f = w.h.ReadFrom(f) + } + return f(src) +} diff --git a/vendor/github.com/weaveworks/common/middleware/grpc_stats.go b/vendor/github.com/weaveworks/common/middleware/grpc_stats.go new file mode 100644 index 00000000000..6c2581c4c2f --- /dev/null +++ b/vendor/github.com/weaveworks/common/middleware/grpc_stats.go @@ -0,0 +1,69 @@ +package middleware + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/stats" +) + +// NewStatsHandler creates handler that can be added to gRPC server options to track received and sent message sizes. +func NewStatsHandler(receivedPayloadSize, sentPayloadSize *prometheus.HistogramVec, inflightRequests *prometheus.GaugeVec) stats.Handler { + return &grpcStatsHandler{ + receivedPayloadSize: receivedPayloadSize, + sentPayloadSize: sentPayloadSize, + inflightRequests: inflightRequests, + } +} + +type grpcStatsHandler struct { + receivedPayloadSize *prometheus.HistogramVec + sentPayloadSize *prometheus.HistogramVec + inflightRequests *prometheus.GaugeVec +} + +// Custom type to hide it from other packages. +type contextKey int + +const ( + contextKeyMethodName contextKey = 1 +) + +func (g *grpcStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + return context.WithValue(ctx, contextKeyMethodName, info.FullMethodName) +} + +func (g *grpcStatsHandler) HandleRPC(ctx context.Context, rpcStats stats.RPCStats) { + // We use full method name from context, because not all RPCStats structs have it. + fullMethodName, ok := ctx.Value(contextKeyMethodName).(string) + if !ok { + return + } + + switch s := rpcStats.(type) { + case *stats.Begin: + g.inflightRequests.WithLabelValues(gRPC, fullMethodName).Inc() + case *stats.End: + g.inflightRequests.WithLabelValues(gRPC, fullMethodName).Dec() + case *stats.InHeader: + // Ignore incoming headers. + case *stats.InPayload: + g.receivedPayloadSize.WithLabelValues(gRPC, fullMethodName).Observe(float64(s.WireLength)) + case *stats.InTrailer: + // Ignore incoming trailers. + case *stats.OutHeader: + // Ignore outgoing headers. + case *stats.OutPayload: + g.sentPayloadSize.WithLabelValues(gRPC, fullMethodName).Observe(float64(s.WireLength)) + case *stats.OutTrailer: + // Ignore outgoing trailers. OutTrailer doesn't have valid WireLength (there is a deprecated field, always set to 0). + } +} + +func (g *grpcStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +func (g *grpcStatsHandler) HandleConn(_ context.Context, _ stats.ConnStats) { + // Not interested. +} diff --git a/vendor/github.com/weaveworks/common/middleware/http_tracing.go b/vendor/github.com/weaveworks/common/middleware/http_tracing.go index 54c99844497..a1ac40f52b0 100644 --- a/vendor/github.com/weaveworks/common/middleware/http_tracing.go +++ b/vendor/github.com/weaveworks/common/middleware/http_tracing.go @@ -17,20 +17,28 @@ var _ = nethttp.MWURLTagFunc // Tracer is a middleware which traces incoming requests. type Tracer struct { RouteMatcher RouteMatcher + SourceIPs *SourceIPExtractor } // Wrap implements Interface func (t Tracer) Wrap(next http.Handler) http.Handler { - opMatcher := nethttp.OperationNameFunc(func(r *http.Request) string { - op := getRouteName(t.RouteMatcher, r) - if op == "" { - return "HTTP " + r.Method - } - - return fmt.Sprintf("HTTP %s - %s", r.Method, op) - }) + options := []nethttp.MWOption{ + nethttp.OperationNameFunc(func(r *http.Request) string { + op := getRouteName(t.RouteMatcher, r) + if op == "" { + return "HTTP " + r.Method + } + + return fmt.Sprintf("HTTP %s - %s", r.Method, op) + }), + } + if t.SourceIPs != nil { + options = append(options, nethttp.MWSpanObserver(func(sp opentracing.Span, r *http.Request) { + sp.SetTag("sourceIPs", t.SourceIPs.Get(r)) + })) + } - return nethttp.Middleware(opentracing.GlobalTracer(), next, opMatcher) + return nethttp.Middleware(opentracing.GlobalTracer(), next, options...) } // ExtractTraceID extracts the trace id, if any from the context. diff --git a/vendor/github.com/weaveworks/common/middleware/instrument.go b/vendor/github.com/weaveworks/common/middleware/instrument.go index ac4edc71b89..06165251b3d 100644 --- a/vendor/github.com/weaveworks/common/middleware/instrument.go +++ b/vendor/github.com/weaveworks/common/middleware/instrument.go @@ -1,19 +1,22 @@ package middleware import ( - "bufio" - "fmt" - "net" + "io" "net/http" "regexp" "strconv" "strings" - "time" + "github.com/felixge/httpsnoop" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" ) +const mb = 1024 * 1024 + +// BodySizeBuckets defines buckets for request/response body sizes. +var BodySizeBuckets = []float64{1 * mb, 2.5 * mb, 5 * mb, 10 * mb, 25 * mb, 50 * mb, 100 * mb, 250 * mb} + // RouteMatcher matches routes type RouteMatcher interface { Match(*http.Request, *mux.RouteMatch) bool @@ -21,8 +24,11 @@ type RouteMatcher interface { // Instrument is a Middleware which records timings for every HTTP request type Instrument struct { - RouteMatcher RouteMatcher - Duration *prometheus.HistogramVec + RouteMatcher RouteMatcher + Duration *prometheus.HistogramVec + RequestBodySize *prometheus.HistogramVec + ResponseBodySize *prometheus.HistogramVec + InflightRequests *prometheus.GaugeVec } // IsWSHandshakeRequest returns true if the given request is a websocket handshake request. @@ -42,16 +48,29 @@ func IsWSHandshakeRequest(req *http.Request) bool { // Wrap implements middleware.Interface func (i Instrument) Wrap(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - begin := time.Now() - isWS := strconv.FormatBool(IsWSHandshakeRequest(r)) - interceptor := &interceptor{ResponseWriter: w, statusCode: http.StatusOK} route := i.getRouteName(r) - next.ServeHTTP(interceptor, r) - var ( - status = strconv.Itoa(interceptor.statusCode) - took = time.Since(begin) - ) - i.Duration.WithLabelValues(r.Method, route, status, isWS).Observe(took.Seconds()) + inflight := i.InflightRequests.WithLabelValues(r.Method, route) + inflight.Inc() + defer inflight.Dec() + + origBody := r.Body + defer func() { + // No need to leak our Body wrapper beyond the scope of this handler. + r.Body = origBody + }() + + rBody := &reqBody{b: origBody} + r.Body = rBody + + isWS := strconv.FormatBool(IsWSHandshakeRequest(r)) + + respMetrics := httpsnoop.CaptureMetricsFn(w, func(ww http.ResponseWriter) { + next.ServeHTTP(ww, r) + }) + + i.Duration.WithLabelValues(r.Method, route, strconv.Itoa(respMetrics.Code), isWS).Observe(respMetrics.Duration.Seconds()) + i.RequestBodySize.WithLabelValues(r.Method, route).Observe(float64(rBody.read)) + i.ResponseBodySize.WithLabelValues(r.Method, route).Observe(float64(respMetrics.Written)) }) } @@ -107,30 +126,19 @@ func MakeLabelValue(path string) string { return result } -// interceptor implements WriteHeader to intercept status codes. WriteHeader -// may not be called on success, so initialize statusCode with the status you -// want to report on success, i.e. http.StatusOK. -// -// interceptor also implements net.Hijacker, to let the downstream Handler -// hijack the connection. This is needed, for example, for working with websockets. -type interceptor struct { - http.ResponseWriter - statusCode int - recorded bool +type reqBody struct { + b io.ReadCloser + read int64 } -func (i *interceptor) WriteHeader(code int) { - if !i.recorded { - i.statusCode = code - i.recorded = true +func (w *reqBody) Read(p []byte) (int, error) { + n, err := w.b.Read(p) + if n > 0 { + w.read += int64(n) } - i.ResponseWriter.WriteHeader(code) + return n, err } -func (i *interceptor) Hijack() (net.Conn, *bufio.ReadWriter, error) { - hj, ok := i.ResponseWriter.(http.Hijacker) - if !ok { - return nil, nil, fmt.Errorf("interceptor: can't cast parent ResponseWriter to Hijacker") - } - return hj.Hijack() +func (w *reqBody) Close() error { + return w.b.Close() } diff --git a/vendor/github.com/weaveworks/common/middleware/logging.go b/vendor/github.com/weaveworks/common/middleware/logging.go index 79368f9daf0..00270df95fd 100644 --- a/vendor/github.com/weaveworks/common/middleware/logging.go +++ b/vendor/github.com/weaveworks/common/middleware/logging.go @@ -15,16 +15,25 @@ import ( type Log struct { Log logging.Interface LogRequestHeaders bool // LogRequestHeaders true -> dump http headers at debug log level + SourceIPs *SourceIPExtractor } // logWithRequest information from the request and context as fields. func (l Log) logWithRequest(r *http.Request) logging.Interface { + localLog := l.Log traceID, ok := ExtractTraceID(r.Context()) if ok { - l.Log = l.Log.WithField("traceID", traceID) + localLog = localLog.WithField("traceID", traceID) } - return user.LogWith(r.Context(), l.Log) + if l.SourceIPs != nil { + ips := l.SourceIPs.Get(r) + if ips != "" { + localLog = localLog.WithField("sourceIPs", ips) + } + } + + return user.LogWith(r.Context(), localLog) } // Wrap implements Middleware diff --git a/vendor/github.com/weaveworks/common/middleware/source_ips.go b/vendor/github.com/weaveworks/common/middleware/source_ips.go new file mode 100644 index 00000000000..17178d427dd --- /dev/null +++ b/vendor/github.com/weaveworks/common/middleware/source_ips.go @@ -0,0 +1,141 @@ +package middleware + +import ( + "fmt" + "net" + "net/http" + "regexp" + "strings" +) + +// Parts copied and changed from gorilla mux proxy_headers.go + +var ( + // De-facto standard header keys. + xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") + xRealIP = http.CanonicalHeaderKey("X-Real-IP") +) + +var ( + // RFC7239 defines a new "Forwarded: " header designed to replace the + // existing use of X-Forwarded-* headers. + // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43 + forwarded = http.CanonicalHeaderKey("Forwarded") + // Allows for a sub-match of the first value after 'for=' to the next + // comma, semi-colon or space. The match is case-insensitive. + forRegex = regexp.MustCompile(`(?i)(?:for=)([^(;|,| )]+)`) +) + +// SourceIPExtractor extracts the source IPs from a HTTP request +type SourceIPExtractor struct { + // The header to search for + header string + // A regex that extracts the IP address from the header. + // It should contain at least one capturing group the first of which will be returned. + regex *regexp.Regexp +} + +// NewSourceIPs creates a new SourceIPs +func NewSourceIPs(header, regex string) (*SourceIPExtractor, error) { + if (header == "" && regex != "") || (header != "" && regex == "") { + return nil, fmt.Errorf("either both a header field and a regex have to be given or neither") + } + re, err := regexp.Compile(regex) + if err != nil { + return nil, fmt.Errorf("invalid regex given") + } + + return &SourceIPExtractor{ + header: header, + regex: re, + }, nil +} + +// extractHost returns the Host IP address without any port information +func extractHost(address string) string { + hostIP := net.ParseIP(address) + if hostIP != nil { + return hostIP.String() + } + var err error + hostStr, _, err := net.SplitHostPort(address) + if err != nil { + // Invalid IP address, just return it so it shows up in the logs + return address + } + return hostStr +} + +// Get returns any source addresses we can find in the request, comma-separated +func (sips SourceIPExtractor) Get(req *http.Request) string { + fwd := extractHost(sips.getIP(req)) + if fwd == "" { + if req.RemoteAddr == "" { + return "" + } + return extractHost(req.RemoteAddr) + } + // If RemoteAddr is empty just return the header + if req.RemoteAddr == "" { + return fwd + } + remoteIP := extractHost(req.RemoteAddr) + if fwd == remoteIP { + return remoteIP + } + // If both a header and RemoteAddr are present return them both, stripping off any port info from the RemoteAddr + return fmt.Sprintf("%v, %v", fwd, remoteIP) +} + +// getIP retrieves the IP from the RFC7239 Forwarded headers, +// X-Real-IP and X-Forwarded-For (in that order) or from the +// custom regex. +func (sips SourceIPExtractor) getIP(r *http.Request) string { + var addr string + + // Use the custom regex only if it was setup + if sips.header != "" { + hdr := r.Header.Get(sips.header) + if hdr == "" { + return "" + } + allMatches := sips.regex.FindAllStringSubmatch(hdr, 1) + if len(allMatches) == 0 { + return "" + } + firstMatch := allMatches[0] + // Check there is at least 1 submatch + if len(firstMatch) < 2 { + return "" + } + return firstMatch[1] + } + + if fwd := r.Header.Get(forwarded); fwd != "" { + // match should contain at least two elements if the protocol was + // specified in the Forwarded header. The first element will always be + // the 'for=' capture, which we ignore. In the case of multiple IP + // addresses (for=8.8.8.8, 8.8.4.4,172.16.1.20 is valid) we only + // extract the first, which should be the client IP. + if match := forRegex.FindStringSubmatch(fwd); len(match) > 1 { + // IPv6 addresses in Forwarded headers are quoted-strings. We strip + // these quotes. + addr = strings.Trim(match[1], `"`) + } + } else if fwd := r.Header.Get(xRealIP); fwd != "" { + // X-Real-IP should only contain one IP address (the client making the + // request). + addr = fwd + } else if fwd := strings.ReplaceAll(r.Header.Get(xForwardedFor), " ", ""); fwd != "" { + // Only grab the first (client) address. Note that '192.168.0.1, + // 10.1.1.1' is a valid key for X-Forwarded-For where addresses after + // the first may represent forwarding proxies earlier in the chain. + s := strings.Index(fwd, ",") + if s == -1 { + s = len(fwd) + } + addr = fwd[:s] + } + + return addr +} diff --git a/vendor/github.com/weaveworks/common/server/server.go b/vendor/github.com/weaveworks/common/server/server.go index 4dc8ae0e9cd..ae04022c691 100644 --- a/vendor/github.com/weaveworks/common/server/server.go +++ b/vendor/github.com/weaveworks/common/server/server.go @@ -76,9 +76,12 @@ type Config struct { GRPCServerTime time.Duration `yaml:"grpc_server_keepalive_time"` GRPCServerTimeout time.Duration `yaml:"grpc_server_keepalive_timeout"` - LogFormat logging.Format `yaml:"log_format"` - LogLevel logging.Level `yaml:"log_level"` - Log logging.Interface `yaml:"-"` + LogFormat logging.Format `yaml:"log_format"` + LogLevel logging.Level `yaml:"log_level"` + Log logging.Interface `yaml:"-"` + LogSourceIPs bool `yaml:"log_source_ips_enabled"` + LogSourceIPsHeader string `yaml:"log_source_ips_header"` + LogSourceIPsRegex string `yaml:"log_source_ips_regex"` // If not set, default signal handler is used. SignalHandler SignalHandler `yaml:"-"` @@ -120,6 +123,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.PathPrefix, "server.path-prefix", "", "Base path to serve all API routes from (e.g. /v1/)") cfg.LogFormat.RegisterFlags(f) cfg.LogLevel.RegisterFlags(f) + f.BoolVar(&cfg.LogSourceIPs, "server.log-source-ips-enabled", false, "Optionally log the source IPs.") + f.StringVar(&cfg.LogSourceIPsHeader, "server.log-source-ips-header", "", "Header field storing the source IPs. Only used if server.log-source-ips-enabled is true. If not set the default Forwarded, X-Real-IP and X-Forwarded-For headers are used") + f.StringVar(&cfg.LogSourceIPsRegex, "server.log-source-ips-regex", "", "Regex for matching the source IPs. Only used if server.log-source-ips-enabled is true. If not set the default Forwarded, X-Real-IP and X-Forwarded-For headers are used") } // Server wraps a HTTP and gRPC server, and some common initialization. @@ -191,6 +197,29 @@ func New(cfg Config) (*Server, error) { }, []string{"method", "route", "status_code", "ws"}) prometheus.MustRegister(requestDuration) + receivedMessageSize := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: cfg.MetricsNamespace, + Name: "request_message_bytes", + Help: "Size (in bytes) of messages received in the request.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + prometheus.MustRegister(receivedMessageSize) + + sentMessageSize := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: cfg.MetricsNamespace, + Name: "response_message_bytes", + Help: "Size (in bytes) of messages sent in response.", + Buckets: middleware.BodySizeBuckets, + }, []string{"method", "route"}) + prometheus.MustRegister(sentMessageSize) + + inflightRequests := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: cfg.MetricsNamespace, + Name: "inflight_requests", + Help: "Current number of inflight requests.", + }, []string{"method", "route"}) + prometheus.MustRegister(inflightRequests) + log.WithField("http", httpListener.Addr()).WithField("grpc", grpcListener.Addr()).Infof("server listening on addresses") // Setup gRPC server @@ -231,6 +260,7 @@ func New(cfg Config) (*Server, error) { grpc.MaxRecvMsgSize(cfg.GPRCServerMaxRecvMsgSize), grpc.MaxSendMsgSize(cfg.GRPCServerMaxSendMsgSize), grpc.MaxConcurrentStreams(uint32(cfg.GPRCServerMaxConcurrentStreams)), + grpc.StatsHandler(middleware.NewStatsHandler(receivedMessageSize, sentMessageSize, inflightRequests)), } grpcOptions = append(grpcOptions, cfg.GRPCOptions...) if grpcTLSConfig != nil { @@ -249,16 +279,28 @@ func New(cfg Config) (*Server, error) { if cfg.RegisterInstrumentation { RegisterInstrumentation(router) } + var sourceIPs *middleware.SourceIPExtractor + if cfg.LogSourceIPs { + sourceIPs, err = middleware.NewSourceIPs(cfg.LogSourceIPsHeader, cfg.LogSourceIPsRegex) + if err != nil { + return nil, fmt.Errorf("error setting up source IP extraction: %v", err) + } + } httpMiddleware := []middleware.Interface{ middleware.Tracer{ RouteMatcher: router, + SourceIPs: sourceIPs, }, middleware.Log{ - Log: log, + Log: log, + SourceIPs: sourceIPs, }, middleware.Instrument{ - Duration: requestDuration, - RouteMatcher: router, + RouteMatcher: router, + Duration: requestDuration, + RequestBodySize: receivedMessageSize, + ResponseBodySize: sentMessageSize, + InflightRequests: inflightRequests, }, } diff --git a/vendor/modules.txt b/vendor/modules.txt index 2c031a4545e..20dc7ffcb80 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -190,6 +190,8 @@ github.com/edsrzf/mmap-go github.com/facette/natsort # github.com/fatih/color v1.9.0 github.com/fatih/color +# github.com/felixge/httpsnoop v1.0.1 +github.com/felixge/httpsnoop # github.com/fsouza/fake-gcs-server v1.7.0 ## explicit github.com/fsouza/fake-gcs-server/fakestorage @@ -695,7 +697,7 @@ github.com/uber/jaeger-client-go/utils # github.com/uber/jaeger-lib v2.2.0+incompatible github.com/uber/jaeger-lib/metrics github.com/uber/jaeger-lib/metrics/prometheus -# github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9 +# github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e ## explicit github.com/weaveworks/common/aws github.com/weaveworks/common/errors