From 3283505f68a8a57d7da3d7027bc0339c810483fb Mon Sep 17 00:00:00 2001 From: user Date: Wed, 6 Jul 2022 19:36:55 +0800 Subject: [PATCH 1/4] impose time limit for rpc shutdown in case it hangs over in edge cases --- node/rpcstack.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/node/rpcstack.go b/node/rpcstack.go index 09692c0a0b19..af5ff2aa8576 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -27,6 +27,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -81,6 +82,10 @@ type httpServer struct { handlerNames map[string]string } +const ( + shutdownTimeout = 5 * time.Second +) + func newHTTPServer(log log.Logger, timeouts rpc.HTTPTimeouts) *httpServer { h := &httpServer{log: log, timeouts: timeouts, handlerNames: make(map[string]string)} @@ -261,7 +266,12 @@ func (h *httpServer) doStop() { h.wsHandler.Store((*rpcHandler)(nil)) wsHandler.server.Stop() } - h.server.Shutdown(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + err := h.server.Shutdown(ctx) + if err != nil { + h.log.Warn("Something wrong with HTTP server graceful shutdown", "error", err) + } h.listener.Close() h.log.Info("HTTP server stopped", "endpoint", h.listener.Addr()) From 5a9615e4645243a694b5d208c424ea54bd676983 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 7 Jul 2022 23:29:58 +0800 Subject: [PATCH 2/4] actively close the connections if they escape the graceful shutdown procedure --- node/rpcstack.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/node/rpcstack.go b/node/rpcstack.go index af5ff2aa8576..10e120028097 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -19,6 +19,7 @@ package node import ( "compress/gzip" "context" + "errors" "fmt" "io" "net" @@ -269,8 +270,9 @@ func (h *httpServer) doStop() { ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() err := h.server.Shutdown(ctx) - if err != nil { - h.log.Warn("Something wrong with HTTP server graceful shutdown", "error", err) + if errors.Is(err, context.DeadlineExceeded) { + h.log.Warn("Some connections are not shutdown gracefully within the time bound, close them actively") + h.server.Close() } h.listener.Close() h.log.Info("HTTP server stopped", "endpoint", h.listener.Addr()) From b0bac8531990aedd2987caf4adc7eebd52485d5a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 7 Jul 2022 21:53:05 +0200 Subject: [PATCH 3/4] Update rpcstack.go --- node/rpcstack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/rpcstack.go b/node/rpcstack.go index 10e120028097..b852820cb882 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -270,8 +270,8 @@ func (h *httpServer) doStop() { ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() err := h.server.Shutdown(ctx) - if errors.Is(err, context.DeadlineExceeded) { - h.log.Warn("Some connections are not shutdown gracefully within the time bound, close them actively") + if err == ctx.Err() { + h.log.Warn("HTTP server graceful shutdown timed out") h.server.Close() } h.listener.Close() From ef6c5f3551acbe94a60398efc41f005217d8626d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 7 Jul 2022 23:21:34 +0200 Subject: [PATCH 4/4] node: gofmt --- node/rpcstack.go | 1 - 1 file changed, 1 deletion(-) diff --git a/node/rpcstack.go b/node/rpcstack.go index b852820cb882..455e29beaf65 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -19,7 +19,6 @@ package node import ( "compress/gzip" "context" - "errors" "fmt" "io" "net"