Upgrade goproxy to v1.9.0

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.32.36
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.35
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.60.5
-	github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380
+	github.com/elazarl/goproxy v1.9.0
 	github.com/evalphobia/logrus_sentry v0.8.2
 	github.com/getsentry/raven-go v0.2.0
 	github.com/jarcoal/httpmock v1.4.2

go.sum

@@ -32,6 +32,8 @@ github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGL
 github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40 h1:xvUo53O5MRZhVMJAxWCJcS5HHrqAiAG9SJ1LpMu6aAI=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=
+github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -41,10 +43,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0=
-github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/elazarl/goproxy v1.9.0 h1:2j3c13lD5v0QTjxphJSSIHS7w8/m/pzSHtLMPOpznC0=
+github.com/elazarl/goproxy v1.9.0/go.mod h1:THdE5ix2clxX9lZzcICPpZ67d6CdrPZxdOYsNgU5e30=
 github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ=
 github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=

internal/cache/handlers.go

@@ -214,6 +214,18 @@ func (d *DB) OnResponse(resp *http.Response, proxyCtx *goproxy.ProxyCtx) *http.R
 		logrus.Warnln("Received nil response")
 		return resp
 	}
+	if resp.StatusCode == http.StatusSwitchingProtocols {
+		return resp
+	}
+	if responseMustNotHaveBody(resp) {
+		if resp.Body != nil && resp.Body != http.NoBody {
+			_ = resp.Body.Close()
+		}
+		resp.Body = http.NoBody
+		resp.TransferEncoding = nil
+		resp.Header.Del("Transfer-Encoding")
+		return resp
+	}
 	k, ok := proxyctx.GetValue(proxyCtx, keyValue)
 	if !ok {
 		// can't calculate key as response body is empty
@@ -259,6 +271,13 @@ func (d *DB) OnResponse(resp *http.Response, proxyCtx *goproxy.ProxyCtx) *http.R
 	return resp
 }
 
+func responseMustNotHaveBody(resp *http.Response) bool {
+	return resp.StatusCode >= 100 && resp.StatusCode < 200 ||
+		resp.StatusCode == http.StatusNoContent ||
+		resp.StatusCode == http.StatusNotModified ||
+		resp.Request != nil && resp.Request.Method == http.MethodHead
+}
+
 var sanitizeRegex = regexp.MustCompile(`\W`)
 
 func sanitize(host string) string {

internal/cache/handlers_test.go

@@ -18,6 +18,8 @@ import (
 	"github.com/elazarl/goproxy"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/dependabot/proxy/internal/proxyctx"
 )
 
 // None of these tests should make network calls
@@ -111,6 +113,74 @@ func TestCache(t *testing.T) {
 	})
 }
 
+func TestCache_BodyForbiddenResponses(t *testing.T) {
+	tests := []struct {
+		name       string
+		method     string
+		statusCode int
+	}{
+		{name: "informational", method: http.MethodGet, statusCode: http.StatusEarlyHints},
+		{name: "HEAD", method: http.MethodHead, statusCode: http.StatusOK},
+		{name: "no content", method: http.MethodGet, statusCode: http.StatusNoContent},
+		{name: "not modified", method: http.MethodGet, statusCode: http.StatusNotModified},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cacher, err := New(true, t.TempDir())
+			require.NoError(t, err)
+
+			req := httptest.NewRequestWithContext(t.Context(), test.method, URL, nil)
+			proxyCtx := &goproxy.ProxyCtx{Req: req}
+			proxyctx.SetValue(proxyCtx, keyValue, Key{Method: req.Method, URL: req.URL.String()})
+
+			originalBody := &BufferWithClose{}
+			resp := &http.Response{
+				Request:          req,
+				StatusCode:       test.statusCode,
+				Header:           http.Header{"Transfer-Encoding": []string{"chunked"}},
+				Body:             originalBody,
+				TransferEncoding: []string{"chunked"},
+			}
+
+			result := cacher.OnResponse(resp, proxyCtx)
+
+			assert.Same(t, resp, result)
+			assert.Equal(t, http.NoBody, result.Body)
+			assert.True(t, originalBody.WasCloseCalled)
+			assert.Empty(t, result.TransferEncoding)
+			assert.Empty(t, result.Header.Values("Transfer-Encoding"))
+			assert.Empty(t, cacher.cacheDB)
+			assert.Zero(t, cacher.callCursor)
+		})
+	}
+}
+
+func TestCache_SwitchingProtocolsPreservesUpgradedStream(t *testing.T) {
+	cacher, err := New(true, t.TempDir())
+	require.NoError(t, err)
+
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, URL, nil)
+	proxyCtx := &goproxy.ProxyCtx{Req: req}
+	proxyctx.SetValue(proxyCtx, keyValue, Key{Method: req.Method, URL: req.URL.String()})
+
+	upgradedStream := &BufferWithClose{}
+	resp := &http.Response{
+		Request:    req,
+		StatusCode: http.StatusSwitchingProtocols,
+		Header:     http.Header{"Upgrade": []string{"websocket"}},
+		Body:       upgradedStream,
+	}
+
+	result := cacher.OnResponse(resp, proxyCtx)
+
+	assert.Same(t, resp, result)
+	assert.Same(t, upgradedStream, result.Body)
+	assert.False(t, upgradedStream.WasCloseCalled)
+	assert.Empty(t, cacher.cacheDB)
+	assert.Zero(t, cacher.callCursor)
+}
+
 func Test_sanitize(t *testing.T) {
 	var tests = []struct {
 		Input, Expected string

proxy.go

@@ -27,6 +27,10 @@ type Proxy struct {
 }
 
 func newProxy(envSettings config.ProxyEnvSettings, cfg *config.Config, blockedIps []net.IP) *Proxy {
+	return newProxyWithCacheDir(envSettings, cfg, blockedIps, "/cache")
+}
+
+func newProxyWithCacheDir(envSettings config.ProxyEnvSettings, cfg *config.Config, blockedIps []net.IP, cacheDir string) *Proxy {
 	var err error
 
 	if err := setCA([]byte(cfg.CA.Cert), []byte(cfg.CA.Key)); err != nil {
@@ -62,7 +66,7 @@ func newProxy(envSettings config.ProxyEnvSettings, cfg *config.Config, blockedIp
 	proxy.OnResponse().DoFunc(logger.logResponse)
 
 	enableCache := os.Getenv("PROXY_CACHE") == "true"
-	cacher, err := cache.New(enableCache, "/cache")
+	cacher, err := cache.New(enableCache, cacheDir)
 	if err != nil {
 		log.Fatal(err)
 	}

proxy_test.go

@@ -9,10 +9,13 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"io"
 	"math/big"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -46,6 +49,99 @@ func TestProxyHTTPRequest(t *testing.T) {
 	assert.Equal(t, 200, rsp.StatusCode)
 }
 
+func TestProxyHTTPSMITMResponseFraming(t *testing.T) {
+	var fixedGETRequests atomic.Int32
+	upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/fixed":
+			if r.Method == http.MethodGet {
+				fixedGETRequests.Add(1)
+			}
+			_, err := io.WriteString(w, "hello")
+			assert.NoError(t, err)
+		case "/chunked":
+			_, err := io.WriteString(w, "hello ")
+			assert.NoError(t, err)
+			w.(http.Flusher).Flush()
+			_, err = io.WriteString(w, "world")
+			assert.NoError(t, err)
+		case "/trailers":
+			w.Header().Set("Trailer", "X-Checksum")
+			_, err := io.WriteString(w, "trailed")
+			assert.NoError(t, err)
+			w.Header().Set("X-Checksum", "abc123")
+		case "/no-content":
+			w.WriteHeader(http.StatusNoContent)
+		case "/not-modified":
+			w.WriteHeader(http.StatusNotModified)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer upstream.Close()
+
+	t.Setenv("PROXY_CACHE", "true")
+	client, proxy := testProxyServer(t, testProxyConfig, nil, upstream.Certificate())
+	defer proxy.Close()
+	transport := client.Transport.(*http.Transport)
+	var proxyDials atomic.Int32
+	dialer := &net.Dialer{}
+	transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+		proxyDials.Add(1)
+		return dialer.DialContext(ctx, network, address)
+	}
+
+	tests := []struct {
+		name       string
+		method     string
+		path       string
+		statusCode int
+		body       string
+		chunked    bool
+		trailer    string
+	}{
+		{name: "fixed length", method: http.MethodGet, path: "/fixed", statusCode: http.StatusOK, body: "hello"},
+		{name: "unknown length", method: http.MethodGet, path: "/chunked", statusCode: http.StatusOK, body: "hello world", chunked: true},
+		{name: "trailers", method: http.MethodGet, path: "/trailers", statusCode: http.StatusOK, body: "trailed", chunked: true, trailer: "abc123"},
+		{name: "HEAD", method: http.MethodHead, path: "/fixed", statusCode: http.StatusOK},
+		{name: "no content", method: http.MethodGet, path: "/no-content", statusCode: http.StatusNoContent},
+		{name: "not modified", method: http.MethodGet, path: "/not-modified", statusCode: http.StatusNotModified},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req, err := http.NewRequestWithContext(t.Context(), test.method, upstream.URL+test.path, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			body, err := io.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.NoError(t, resp.Body.Close())
+
+			assert.Equal(t, test.statusCode, resp.StatusCode)
+			assert.Equal(t, test.body, string(body))
+			if test.chunked {
+				assert.Equal(t, []string{"chunked"}, resp.TransferEncoding)
+			}
+			if test.trailer != "" {
+				assert.Equal(t, test.trailer, resp.Trailer.Get("X-Checksum"))
+			}
+
+			req, err = http.NewRequestWithContext(t.Context(), http.MethodGet, upstream.URL+"/fixed", nil)
+			require.NoError(t, err)
+			resp, err = client.Do(req)
+			require.NoError(t, err)
+			body, err = io.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.NoError(t, resp.Body.Close())
+			assert.Equal(t, "hello", string(body))
+		})
+	}
+	require.Equal(t, int32(1), fixedGETRequests.Load())
+	require.Equal(t, int32(1), proxyDials.Load())
+}
+
 func TestIPRestrictions(t *testing.T) {
 	blockedIPs = []net.IP{iPV4Localhost, iPV6Localhost}
 	client, proxy := testProxyServer(t, testProxyConfig, blockedIPs)
@@ -149,7 +245,7 @@ func TestMetadataAPIRestriction(t *testing.T) {
 	}
 }
 
-func testProxyServer(t *testing.T, cfg *config.Config, blockedIPs []net.IP) (*http.Client, *http.Server) {
+func testProxyServer(t *testing.T, cfg *config.Config, blockedIPs []net.IP, upstreamRoots ...*x509.Certificate) (*http.Client, *http.Server) {
 	envSettings := config.ProxyEnvSettings{
 		APIEndpoint:    "",
 		PackageManager: "",
@@ -162,7 +258,18 @@ func testProxyServer(t *testing.T, cfg *config.Config, blockedIPs []net.IP) (*ht
 	srv := &http.Server{
 		ReadHeaderTimeout: 10 * time.Second,
 	}
-	srv.Handler = newProxy(envSettings, testProxyConfig, blockedIPs)
+	proxyHandler := newProxyWithCacheDir(envSettings, cfg, blockedIPs, t.TempDir())
+	if len(upstreamRoots) > 0 {
+		rootCAs, err := x509.SystemCertPool()
+		if err != nil {
+			rootCAs = x509.NewCertPool()
+		}
+		for _, certificate := range upstreamRoots {
+			rootCAs.AddCert(certificate)
+		}
+		proxyHandler.Tr.TLSClientConfig.RootCAs = rootCAs
+	}
+	srv.Handler = proxyHandler
 
 	lc := net.ListenConfig{}
 	ln, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0")