diff --git a/lib/kube/proxy/forwarder.go b/lib/kube/proxy/forwarder.go index da94c2184c950..078491a02b597 100644 --- a/lib/kube/proxy/forwarder.go +++ b/lib/kube/proxy/forwarder.go @@ -732,8 +732,21 @@ func (f *Forwarder) formatStatusResponseError(rw http.ResponseWriter, respErr er // should retry the request themselves. if errString := respErr.Error(); strings.HasSuffix(errString, `after Request.Body was written; define Request.GetBody to avoid this error`) && strings.Contains(errString, `http2: Transport: cannot retry err`) { + + data, err := runtime.Encode(globalKubeCodecs.LegacyCodec(), &kubeerrors.NewTooManyRequests("Connection closed by upstream Kubernetes server", 1).ErrStatus) + if err != nil { + f.log.WarnContext(f.ctx, "Failed encoding error into kube Status object", "error", err) + trace.WriteError(rw, respErr) + return + } + rw.Header().Set("Retry-After", "1") + rw.Header().Set(responsewriters.ContentTypeHeader, "application/json") rw.WriteHeader(http.StatusTooManyRequests) + + if _, err := rw.Write(data); err != nil && !utils.IsOKNetworkError(err) { + f.log.WarnContext(f.ctx, "Failed writing kube error response body", "error", err) + } return } @@ -759,7 +772,7 @@ func (f *Forwarder) formatStatusResponseError(rw http.ResponseWriter, respErr er // `Error from server (InternalError): an error on the server ("unknown") // has prevented the request from succeeding`` instead of the correct reason. rw.WriteHeader(trace.ErrorToCode(respErr)) - if _, err := rw.Write(data); err != nil { + if _, err := rw.Write(data); err != nil && !utils.IsOKNetworkError(err) { f.log.WarnContext(f.ctx, "Failed writing kube error response body", "error", err) } } diff --git a/lib/kube/proxy/forwarder_test.go b/lib/kube/proxy/forwarder_test.go index b9e6c2c6b1e62..884aea054a0c6 100644 --- a/lib/kube/proxy/forwarder_test.go +++ b/lib/kube/proxy/forwarder_test.go @@ -24,6 +24,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "errors" "fmt" "io" @@ -44,6 +45,7 @@ import ( "github.com/gravitational/trace" "github.com/jonboulle/clockwork" "github.com/julienschmidt/httprouter" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" "golang.org/x/net/http2" @@ -1771,9 +1773,15 @@ func TestGOAWAYHandling(t *testing.T) { require.NoError(t, err) resp, err := forwarderServer.Client().Do(req) require.NoError(t, err) + + t.Cleanup(func() { assert.NoError(t, resp.Body.Close()) }) require.Equal(t, http.StatusTooManyRequests, resp.StatusCode) require.Equal(t, "1", resp.Header.Get("Retry-After")) - require.NoError(t, resp.Body.Close()) + + var status metav1.Status + err = json.NewDecoder(resp.Body).Decode(&status) + require.NoError(t, err) + require.Equal(t, metav1.StatusReasonTooManyRequests, status.Reason) } // goawayServer is a fake [http2.Server] that terminates all received client