fixup! Add support for decompression of HTTP responses

Signed-off-by: Marcelo E. Magallon <[email protected]>

config/config.go

@@ -16,9 +16,13 @@ package config
 import (
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"regexp"
 	"runtime"
+	"sort"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -272,6 +276,16 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := s.HTTPClientConfig.Validate(); err != nil {
 		return err
 	}
+
+	for key, value := range s.Headers {
+		switch strings.Title(key) {
+		case "Accept-Encoding":
+			if !isCompressionAcceptEncodingValid(s.Compression, value) {
+				return fmt.Errorf(`invalid configuration "%s: %s", "compression: %s"`, key, value, s.Compression)
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -359,3 +373,70 @@ func (s *HeaderMatch) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return nil
 }
+
+// isCompressionAcceptEncodingValid validates the compression +
+// Accept-Encoding combination.
+//
+// If there's a compression setting, and there's also an accept-encoding
+// header, they MUST match, otherwise we end up requesting something
+// that doesn't include the specified compression, and that's likely to
+// fail, depending on how the server is configured. Testing that the
+// server _ignores_ Accept-Encoding, e.g. by not including a particular
+// compression in the header but expecting it in the response falls out
+// of the scope of the tests we perform.
+//
+// With that logic, this function validates that if a compression
+// algorithm is specified, it's covered by the specified accept encoding
+// header. It doesn't need to be the most prefered encoding, but it MUST
+// be included in the prefered encodings.
+func isCompressionAcceptEncodingValid(encoding, acceptEncoding string) bool {
+	// unspecified compression + any encoding value is valid
+	// any compression + no accept encoding is valid
+	if encoding == "" || acceptEncoding == "" {
+		return true
+	}
+
+	type encodingQuality struct {
+		encoding string
+		quality  float32
+	}
+
+	var encodings []encodingQuality
+
+	for _, parts := range strings.Split(acceptEncoding, ",") {
+		var e encodingQuality
+
+		if idx := strings.LastIndexByte(parts, ';'); idx == -1 {
+			e.encoding = strings.TrimSpace(parts)
+			e.quality = 1.0
+		} else {
+			parseQuality := func(str string) float32 {
+				q, err := strconv.ParseFloat(str, 32)
+				if err != nil {
+					return 0
+				}
+				return float32(math.Round(q*1000) / 1000)
+			}
+
+			e.encoding = strings.TrimSpace(parts[:idx])
+
+			q := strings.TrimSpace(parts[idx+1:])
+			q = strings.TrimPrefix(q, "q=")
+			e.quality = parseQuality(q)
+		}
+
+		encodings = append(encodings, e)
+	}
+
+	sort.SliceStable(encodings, func(i, j int) bool {
+		return encodings[j].quality < encodings[i].quality
+	})
+
+	for _, e := range encodings {
+		if encoding == e.encoding || e.encoding == "*" {
+			return e.quality > 0
+		}
+	}
+
+	return false
+}

config/config_test.go

@@ -75,6 +75,14 @@ func TestLoadBadConfigs(t *testing.T) {
 			input: "testdata/invalid-http-header-match-regexp.yml",
 			want:  `error parsing config file: "Could not compile regular expression" regexp=":["`,
 		},
+		{
+			input: "testdata/invalid-http-compression-mismatch.yml",
+			want:  `error parsing config file: invalid configuration "Accept-Encoding: deflate", "compression: gzip"`,
+		},
+		{
+			input: "testdata/invalid-http-request-compression-reject-all-encodings.yml",
+			want:  `error parsing config file: invalid configuration "Accept-Encoding: *;q=0.0", "compression: gzip"`,
+		},
 		{
 			input: "testdata/invalid-tcp-query-response-regexp.yml",
 			want:  `error parsing config file: "Could not compile regular expression" regexp=":["`,
@@ -111,3 +119,86 @@ func TestHideConfigSecrets(t *testing.T) {
 		t.Fatal("config's String method reveals authentication credentials.")
 	}
 }
+
+func TestIsEncodingAcceptable(t *testing.T) {
+	testcases := map[string]struct {
+		input          string
+		acceptEncoding string
+		expected       bool
+	}{
+		"empty compression": {
+			input:          "",
+			acceptEncoding: "gzip",
+			expected:       true,
+		},
+		"trivial": {
+			input:          "gzip",
+			acceptEncoding: "gzip",
+			expected:       true,
+		},
+		"trivial, quality": {
+			input:          "gzip",
+			acceptEncoding: "gzip;q=1.0",
+			expected:       true,
+		},
+		"first": {
+			input:          "gzip",
+			acceptEncoding: "gzip, compress",
+			expected:       true,
+		},
+		"second": {
+			input:          "gzip",
+			acceptEncoding: "compress, gzip",
+			expected:       true,
+		},
+		"missing": {
+			input:          "br",
+			acceptEncoding: "gzip, compress",
+			expected:       false,
+		},
+		"*": {
+			input:          "br",
+			acceptEncoding: "gzip, compress, *",
+			expected:       true,
+		},
+		"* with quality": {
+			input:          "br",
+			acceptEncoding: "gzip, compress, *;q=0.1",
+			expected:       true,
+		},
+		"rejected": {
+			input:          "br",
+			acceptEncoding: "gzip, compress, br;q=0.0",
+			expected:       false,
+		},
+		"rejected *": {
+			input:          "br",
+			acceptEncoding: "gzip, compress, *;q=0.0",
+			expected:       false,
+		},
+		"complex": {
+			input:          "br",
+			acceptEncoding: "gzip;q=1.0, compress;q=0.5, br;q=0.1, *;q=0.0",
+			expected:       true,
+		},
+		"complex out of order": {
+			input:          "br",
+			acceptEncoding: "*;q=0.0, compress;q=0.5, br;q=0.1, gzip;q=1.0",
+			expected:       true,
+		},
+		"complex with extra blanks": {
+			input:          "br",
+			acceptEncoding: " gzip;q=1.0, compress; q=0.5, br;q=0.1, *; q=0.0 ",
+			expected:       true,
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			actual := isCompressionAcceptEncodingValid(tc.input, tc.acceptEncoding)
+			if actual != tc.expected {
+				t.Errorf("Unexpected result: input=%q acceptEncoding=%q expected=%t actual=%t", tc.input, tc.acceptEncoding, tc.expected, actual)
+			}
+		})
+	}
+}

config/testdata/invalid-http-compression-mismatch.yml

@@ -0,0 +1,8 @@
+modules:
+  http_headers:
+    prober: http
+    timeout: 5s
+    http:
+      compression: gzip
+      headers:
+        "Accept-Encoding": "deflate"

config/testdata/invalid-http-request-compression-reject-all-encodings.yml

@@ -0,0 +1,10 @@
+modules:
+  http_headers:
+    prober: http
+    timeout: 5s
+    http:
+      # this configuration is invalid because it's requesting a
+      # compressed encoding, but it's rejecting every possible encoding
+      compression: gzip
+      headers:
+        "Accept-Encoding": "*;q=0.0"

prober/http.go

@@ -407,20 +407,6 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			continue
 		}
 
-		// If there's a compression setting, and there's also an
-		// accept-encoding header, they MUST match, otherwise we
-		// end up requesting something that doesn't include the
-		// specified compression, and that's likely to fail,
-		// depending on how the server is configured. Testing
-		// that the server _ignores_ Accept-Encoding, e.g. by
-		// not including a particular compression in the header
-		// but expecting it in the response falls out of the
-		// scope of the tests we perform.
-		if httpConfig.Compression != "" && normalizedKey == "Accept-Encoding" && !isEncodingAcceptable(httpConfig.Compression, value) {
-			level.Error(logger).Log("msg", "Invalid configuration", key, value, "compression", httpConfig.Compression)
-			return
-		}
-
 		request.Header.Set(key, value)
 	}
 

prober/http_test.go

@@ -453,56 +453,6 @@ func TestHandlingOfCompressionSetting(t *testing.T) {
 			}
 		}(),
 
-		"header mismatch": func() testdata {
-			msg := testmsg
-			var buf bytes.Buffer
-			enc := gzip.NewWriter(&buf)
-			enc.Write(msg)
-			enc.Close()
-			return testdata{
-				expectFailure:          true,
-				contentLength:          0, // Content won't be fetched because the configuration is invalid.
-				uncompressedBodyLength: 0,
-				handler: func(w http.ResponseWriter, r *http.Request) {
-					w.Header().Add("Content-Encoding", "gzip")
-					w.WriteHeader(http.StatusOK)
-					w.Write(buf.Bytes())
-				},
-				httpConfig: config.HTTPProbe{
-					IPProtocolFallback: true,
-					Compression:        "gzip",
-					Headers: map[string]string{
-						"Accept-Encoding": "deflate",
-					},
-				},
-			}
-		}(),
-
-		"request compression handling but reject everything": func() testdata {
-			msg := testmsg
-			var buf bytes.Buffer
-			enc := gzip.NewWriter(&buf)
-			enc.Write(msg)
-			enc.Close()
-			return testdata{
-				expectFailure:          true,
-				contentLength:          0, // Content won't be fetched because the configuration is invalid.
-				uncompressedBodyLength: 0,
-				handler: func(w http.ResponseWriter, r *http.Request) {
-					w.Header().Add("Content-Encoding", "gzip")
-					w.WriteHeader(http.StatusOK)
-					w.Write(buf.Bytes())
-				},
-				httpConfig: config.HTTPProbe{
-					IPProtocolFallback: true,
-					Compression:        "gzip",
-					Headers: map[string]string{
-						"Accept-Encoding": "*;q=0.0",
-					},
-				},
-			}
-		}(),
-
 		"compressed content without compression setting": func() testdata {
 			msg := testmsg
 			var buf bytes.Buffer

prober/utils.go

@@ -17,11 +17,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
-	"math"
 	"net"
-	"sort"
-	"strconv"
-	"strings"
 	"time"
 
 	"github.com/go-kit/kit/log"
@@ -112,49 +108,3 @@ func ipHash(ip net.IP) float64 {
 	h.Write(ip)
 	return float64(h.Sum32())
 }
-
-func isEncodingAcceptable(encoding, acceptEncoding string) bool {
-	type encodingQuality struct {
-		encoding string
-		quality  float32
-	}
-
-	var encodings []encodingQuality
-
-	for _, parts := range strings.Split(acceptEncoding, ",") {
-		var e encodingQuality
-
-		if idx := strings.LastIndexByte(parts, ';'); idx == -1 {
-			e.encoding = strings.TrimSpace(parts)
-			e.quality = 1.0
-		} else {
-			parseQuality := func(str string) float32 {
-				q, err := strconv.ParseFloat(str, 32)
-				if err != nil {
-					return 0
-				}
-				return float32(math.Round(q*1000) / 1000)
-			}
-
-			e.encoding = strings.TrimSpace(parts[:idx])
-
-			q := strings.TrimSpace(parts[idx+1:])
-			q = strings.TrimPrefix(q, "q=")
-			e.quality = parseQuality(q)
-		}
-
-		encodings = append(encodings, e)
-	}
-
-	sort.SliceStable(encodings, func(i, j int) bool {
-		return encodings[j].quality < encodings[i].quality
-	})
-
-	for _, e := range encodings {
-		if encoding == e.encoding || e.encoding == "*" {
-			return e.quality > 0
-		}
-	}
-
-	return false
-}