Add support for decompression of HTTP responses

If the module configuration specifies the "compression" option
blackbox_exporter will try to decompress the response using the
specified algorithm. If the response is not compressed using that
algorithm, the probe will fail.

It validates that the "Accept-Encoding" header is either absent, or that
it specifies the same algorithm as the "compression" option. If the
"Accept-Encoding" header is present but it specifies a different
algorithm, the probe will fail.

If the compression option is *not* used, probe_http_content_length and
probe_http_uncompressed_body_length will have the same value
corresponding to the original content length. If the compression option
is used and the content can be decompressed, probe_http_content_length
will report the original content length as it currently does, and
probe_http_uncompressed_body_length will report the length of the body
after decompression as expected.

Fixes #684

Signed-off-by: Marcelo E. Magallon <[email protected]>

CONFIGURATION.md

@@ -49,6 +49,9 @@ The other placeholders are specified separately.
   headers:
     [ <string>: <string> ... ]
 
+  # The compression algorithm to use to decompress the response (gzip, br, deflate, identity).
+  [ compression: <string> | default = "" ]
+
   # Whether or not the probe will follow any redirects.
   [ no_follow_redirects: <boolean> | default = false ]
 

config/config.go

@@ -188,6 +188,7 @@ type HTTPProbe struct {
 	FailIfHeaderNotMatchesRegexp []HeaderMatch           `yaml:"fail_if_header_not_matches,omitempty"`
 	Body                         string                  `yaml:"body,omitempty"`
 	HTTPClientConfig             config.HTTPClientConfig `yaml:"http_client_config,inline"`
+	Compression                  string                  `yaml:"compression,omitempty"`
 }
 
 type HeaderMatch struct {

example.yml

@@ -52,6 +52,18 @@ modules:
       method: GET
       tls_config:
         ca_file: "/certs/my_cert.crt"
+  http_gzip:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+  http_gzip_with_accept_encoding:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+      headers:
+        Accept-Encoding: gzip
   tls_connect:
     prober: tcp
     timeout: 5s

go.mod

@@ -1,6 +1,7 @@
 module github.com/prometheus/blackbox_exporter
 
 require (
+	github.com/andybalholm/brotli v1.0.1
 	github.com/go-kit/kit v0.10.0
 	github.com/miekg/dns v1.1.40
 	github.com/pkg/errors v0.9.1

go.sum

@@ -13,6 +13,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
+github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=

prober/http.go

@@ -14,6 +14,8 @@
 package prober
 
 import (
+	"compress/flate"
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -30,6 +32,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/andybalholm/brotli"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -397,10 +400,23 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 	request = request.WithContext(ctx)
 
 	for key, value := range httpConfig.Headers {
-		if strings.Title(key) == "Host" {
+		normalizedKey := strings.Title(key)
+
+		if normalizedKey == "Host" {
 			request.Host = value
 			continue
 		}
+
+		// If there's a compression setting, and there's also an
+		// accept-encoding header, they MUST match, otherwise we
+		// end up requesting one encoding and trying to process
+		// a different one, which is more likely than not going
+		// to fail.
+		if httpConfig.Compression != "" && normalizedKey == "Accept-Encoding" && !strings.EqualFold(value, httpConfig.Compression) {
+			level.Error(logger).Log("msg", "Invalid configuration", key, value, "compression", httpConfig.Compression)
+			return
+		}
+
 		request.Header.Set(key, value)
 	}
 
@@ -455,6 +471,31 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			}
 		}
 
+		// Since the configuration specifies a compression algorithm, blindly treat the response body as a
+		// compressed payload; if we cannot decompress it it's a failure because the configuration says we
+		// should expect the response to be compressed in that way.
+		if httpConfig.Compression != "" {
+			dec, err := getDecompressionReader(httpConfig.Compression, resp.Body)
+			if err != nil {
+				level.Info(logger).Log("msg", "Failed to get decompressor for HTTP response body", "err", err.Error())
+				success = false
+			} else if dec != nil {
+				// Since we are replacing the original resp.Body with the decoder, we need to make sure
+				// we close the original body. We cannot close it right away because the decompressor
+				// might not have read it yet.
+				defer func(c io.Closer) {
+					err := c.Close()
+					if err != nil {
+						// At this point we cannot really do anything with this error, but log
+						// it in case it contains useful information as to what's the problem.
+						level.Info(logger).Log("msg", "Error while closing response from server", "error", err.Error())
+					}
+				}(resp.Body)
+
+				resp.Body = dec
+			}
+		}
+
 		byteCounter := &byteCounter{ReadCloser: resp.Body}
 
 		if success && (len(httpConfig.FailIfBodyMatchesRegexp) > 0 || len(httpConfig.FailIfBodyNotMatchesRegexp) > 0) {
@@ -476,8 +517,9 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			respBodyBytes = byteCounter.n
 
 			if err := byteCounter.Close(); err != nil {
-				// We have already read everything we could from the server. The error here might be a
-				// TCP error. Log it in case it contains useful information as to what's the problem.
+				// We have already read everything we could from the server, maybe even uncompressed the
+				// body. The error here might be either a decompression error or a TCP error. Log it in
+				// case it contains useful information as to what's the problem.
 				level.Info(logger).Log("msg", "Error while closing response from server", "error", err.Error())
 			}
 		}
@@ -578,3 +620,22 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 	redirectsGauge.Set(float64(redirects))
 	return
 }
+
+func getDecompressionReader(algorithm string, origBody io.ReadCloser) (io.ReadCloser, error) {
+	switch strings.ToLower(algorithm) {
+	case "br":
+		return ioutil.NopCloser(brotli.NewReader(origBody)), nil
+
+	case "deflate":
+		return flate.NewReader(origBody), nil
+
+	case "gzip":
+		return gzip.NewReader(origBody)
+
+	case "identity", "":
+		return origBody, nil
+
+	default:
+		return nil, errors.New("unsupported compression algorithm")
+	}
+}