diff --git a/agreement/gossip/networkFull_test.go b/agreement/gossip/networkFull_test.go
index ee2448a0e0..80b6475b80 100644
--- a/agreement/gossip/networkFull_test.go
+++ b/agreement/gossip/networkFull_test.go
@@ -38,7 +38,7 @@ func TestMain(m *testing.M) {
logging.Base().SetLevel(logging.Debug)
// increase limit on max allowed number of sockets
- err := util.RaiseRlimit(500)
+ err := util.SetFdSoftLimit(500)
if err != nil {
os.Exit(1)
}
@@ -50,7 +50,6 @@ func spinNetwork(t *testing.T, nodesCount int) ([]*networkImpl, []*messageCounte
cfg := config.GetDefaultLocal()
cfg.GossipFanout = nodesCount - 1
cfg.NetAddress = "127.0.0.1:0"
- cfg.IncomingConnectionsLimit = -1
cfg.IncomingMessageFilterBucketCount = 5
cfg.IncomingMessageFilterBucketSize = 32
cfg.OutgoingMessageFilterBucketCount = 3
diff --git a/config/localTemplate.go b/config/localTemplate.go
index efe1cc8e46..53c2c3f778 100644
--- a/config/localTemplate.go
+++ b/config/localTemplate.go
@@ -74,7 +74,7 @@ type Local struct {
CadaverSizeTarget uint64 `version[0]:"1073741824"`
// IncomingConnectionsLimit specifies the max number of long-lived incoming
- // connections. 0 means no connections allowed. -1 is unbounded.
+ // connections. 0 means no connections allowed. Must be non-negative.
// Estimating 5MB per incoming connection, 5MB*800 = 4GB
IncomingConnectionsLimit int `version[0]:"-1" version[1]:"10000" version[17]:"800"`
@@ -99,9 +99,9 @@ type Local struct {
PriorityPeers map[string]bool `version[4]:""`
// To make sure the algod process does not run out of FDs, algod ensures
- // that RLIMIT_NOFILE exceeds the max number of incoming connections (i.e.,
- // IncomingConnectionsLimit) by at least ReservedFDs. ReservedFDs are meant
- // to leave room for short-lived FDs like DNS queries, SQLite files, etc.
+ // that RLIMIT_NOFILE >= IncomingConnectionsLimit + RestConnectionsHardLimit +
+ // ReservedFDs. ReservedFDs are meant to leave room for short-lived FDs like
+ // DNS queries, SQLite files, etc. This parameter shouldn't be changed.
ReservedFDs uint64 `version[2]:"256"`
// local server
@@ -423,6 +423,13 @@ type Local struct {
// ProposalAssemblyTime is the max amount of time to spend on generating a proposal block.
ProposalAssemblyTime time.Duration `version[19]:"250000000"`
+
+ // When the number of http connections to the REST layer exceeds the soft limit,
+ // we start returning http code 429 Too Many Requests.
+ RestConnectionsSoftLimit uint64 `version[20]:"1024"`
+ // The http server does not accept new connections as long we have this many
+ // (hard limit) connections already.
+ RestConnectionsHardLimit uint64 `version[20]:"2048"`
}
// DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers
diff --git a/config/local_defaults.go b/config/local_defaults.go
index 5aa4bdcfae..27bab01584 100644
--- a/config/local_defaults.go
+++ b/config/local_defaults.go
@@ -99,6 +99,8 @@ var defaultLocal = Local{
PublicAddress: "",
ReconnectTime: 60000000000,
ReservedFDs: 256,
+ RestConnectionsHardLimit: 2048,
+ RestConnectionsSoftLimit: 1024,
RestReadTimeoutSeconds: 15,
RestWriteTimeoutSeconds: 120,
RunHosted: false,
diff --git a/network/wsNetwork_windows.go b/daemon/algod/api/server/lib/middlewares/connectionLimiter.go
similarity index 50%
rename from network/wsNetwork_windows.go
rename to daemon/algod/api/server/lib/middlewares/connectionLimiter.go
index 9fe3090d1f..bf27ef2b60 100644
--- a/network/wsNetwork_windows.go
+++ b/daemon/algod/api/server/lib/middlewares/connectionLimiter.go
@@ -14,10 +14,35 @@
// You should have received a copy of the GNU Affero General Public License
// along with go-algorand. If not, see .
-// +build windows
+package middlewares
-package network
+import (
+ "net/http"
-func (wn *WebsocketNetwork) rlimitIncomingConnections() error {
- return nil
+ "github.com/labstack/echo/v4"
+)
+
+// MakeConnectionLimiter makes an echo middleware that limits the number of
+// simultaneous connections. All connections above the limit will be returned
+// the 429 Too Many Requests http error.
+func MakeConnectionLimiter(limit uint64) echo.MiddlewareFunc {
+ sem := make(chan struct{}, limit)
+
+ return func(next echo.HandlerFunc) echo.HandlerFunc {
+ return func(ctx echo.Context) error {
+ select {
+ case sem <- struct{}{}:
+ defer func() {
+ // If we fail to read from `sem`, just continue.
+ select {
+ case <-sem:
+ default:
+ }
+ }()
+ return next(ctx)
+ default:
+ return ctx.NoContent(http.StatusTooManyRequests)
+ }
+ }
+ }
}
diff --git a/daemon/algod/api/server/lib/middlewares/connectionLimiter_test.go b/daemon/algod/api/server/lib/middlewares/connectionLimiter_test.go
new file mode 100644
index 0000000000..6de0a3c207
--- /dev/null
+++ b/daemon/algod/api/server/lib/middlewares/connectionLimiter_test.go
@@ -0,0 +1,100 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package middlewares_test
+
+import (
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/labstack/echo/v4"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/algorand/go-algorand/daemon/algod/api/server/lib/middlewares"
+ "github.com/algorand/go-algorand/test/partitiontest"
+)
+
+func TestConnectionLimiterBasic(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ e := echo.New()
+
+ handlerCh := make(chan struct{})
+ limit := 5
+ handler := func(c echo.Context) error {
+ <-handlerCh
+ return c.String(http.StatusOK, "test")
+ }
+ middleware := middlewares.MakeConnectionLimiter(uint64(limit))
+
+ numConnections := 13
+ for i := 0; i < 3; i++ {
+ var recorders []*httptest.ResponseRecorder
+ doneCh := make(chan int)
+ errCh := make(chan error)
+
+ for index := 0; index < numConnections; index++ {
+ req := httptest.NewRequest(http.MethodGet, "/", nil)
+ rec := httptest.NewRecorder()
+ ctx := e.NewContext(req, rec)
+
+ recorders = append(recorders, rec)
+
+ go func(index int) {
+ err := middleware(handler)(ctx)
+ doneCh <- index
+ errCh <- err
+ }(index)
+ }
+
+ // Check http 429 code.
+ for j := 0; j < numConnections-limit; j++ {
+ index := <-doneCh
+ assert.Equal(t, http.StatusTooManyRequests, recorders[index].Code)
+ }
+
+ // Let handlers finish.
+ for j := 0; j < limit; j++ {
+ handlerCh <- struct{}{}
+ }
+
+ // All other connections must return 200.
+ for j := 0; j < limit; j++ {
+ index := <-doneCh
+ assert.Equal(t, http.StatusOK, recorders[index].Code)
+ }
+
+ // Check that no errors were returned by the middleware.
+ for i := 0; i < numConnections; i++ {
+ assert.NoError(t, <-errCh)
+ }
+ }
+}
+
+func TestConnectionLimiterForwardsError(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ handlerError := errors.New("handler error")
+ handler := func(c echo.Context) error {
+ return handlerError
+ }
+ middleware := middlewares.MakeConnectionLimiter(1)
+
+ err := middleware(handler)(nil)
+ assert.ErrorIs(t, err, handlerError)
+}
diff --git a/daemon/algod/api/server/router.go b/daemon/algod/api/server/router.go
index 13d01f6cf7..57e56f817a 100644
--- a/daemon/algod/api/server/router.go
+++ b/daemon/algod/api/server/router.go
@@ -81,6 +81,8 @@ import (
const (
apiV1Tag = "/v1"
+ // TokenHeader is the header where we put the token.
+ TokenHeader = "X-Algo-API-Token"
)
// wrapCtx passes a common context to each request without a global variable.
@@ -99,11 +101,8 @@ func registerHandlers(router *echo.Echo, prefix string, routes lib.Routes, ctx l
}
}
-// TokenHeader is the header where we put the token.
-const TokenHeader = "X-Algo-API-Token"
-
// NewRouter builds and returns a new router with our REST handlers registered.
-func NewRouter(logger logging.Logger, node *node.AlgorandFullNode, shutdown <-chan struct{}, apiToken string, adminAPIToken string, listener net.Listener) *echo.Echo {
+func NewRouter(logger logging.Logger, node *node.AlgorandFullNode, shutdown <-chan struct{}, apiToken string, adminAPIToken string, listener net.Listener, numConnectionsLimit uint64) *echo.Echo {
if err := tokens.ValidateAPIToken(apiToken); err != nil {
logger.Errorf("Invalid apiToken was passed to NewRouter ('%s'): %v", apiToken, err)
}
@@ -118,9 +117,12 @@ func NewRouter(logger logging.Logger, node *node.AlgorandFullNode, shutdown <-ch
e.Listener = listener
e.HideBanner = true
- e.Pre(middleware.RemoveTrailingSlash())
- e.Use(middlewares.MakeLogger(logger))
- e.Use(middlewares.MakeCORS(TokenHeader))
+ e.Pre(
+ middlewares.MakeConnectionLimiter(numConnectionsLimit),
+ middleware.RemoveTrailingSlash())
+ e.Use(
+ middlewares.MakeLogger(logger),
+ middlewares.MakeCORS(TokenHeader))
// Request Context
ctx := lib.ReqContext{Node: node, Log: logger, Shutdown: shutdown}
diff --git a/daemon/algod/server.go b/daemon/algod/server.go
index 8ced04e816..a3875f05e9 100644
--- a/daemon/algod/server.go
+++ b/daemon/algod/server.go
@@ -18,6 +18,7 @@ package algod
import (
"context"
+ "errors"
"fmt"
"io/ioutil"
"net"
@@ -35,10 +36,13 @@ import (
"github.com/algorand/go-algorand/config"
apiServer "github.com/algorand/go-algorand/daemon/algod/api/server"
"github.com/algorand/go-algorand/daemon/algod/api/server/lib"
+ "github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/logging/telemetryspec"
+ "github.com/algorand/go-algorand/network/limitlistener"
"github.com/algorand/go-algorand/node"
+ "github.com/algorand/go-algorand/util"
"github.com/algorand/go-algorand/util/metrics"
"github.com/algorand/go-algorand/util/tokens"
)
@@ -84,6 +88,34 @@ func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genes
s.log.SetLevel(logging.Level(cfg.BaseLoggerDebugLevel))
setupDeadlockLogger()
+ // Check some config parameters.
+ if cfg.RestConnectionsSoftLimit > cfg.RestConnectionsHardLimit {
+ s.log.Warnf(
+ "RestConnectionsSoftLimit %d exceeds RestConnectionsHardLimit %d",
+ cfg.RestConnectionsSoftLimit, cfg.RestConnectionsHardLimit)
+ cfg.RestConnectionsSoftLimit = cfg.RestConnectionsHardLimit
+ }
+ if cfg.IncomingConnectionsLimit < 0 {
+ return fmt.Errorf(
+ "Initialize() IncomingConnectionsLimit %d must be non-negative",
+ cfg.IncomingConnectionsLimit)
+ }
+
+ // Set large enough soft file descriptors limit.
+ var ot basics.OverflowTracker
+ fdRequired := ot.Add(
+ cfg.ReservedFDs,
+ ot.Add(uint64(cfg.IncomingConnectionsLimit), cfg.RestConnectionsHardLimit))
+ if ot.Overflowed {
+ return errors.New(
+ "Initialize() overflowed when adding up ReservedFDs, IncomingConnectionsLimit " +
+ "RestConnectionsHardLimit; decrease them")
+ }
+ err = util.SetFdSoftLimit(fdRequired)
+ if err != nil {
+ return fmt.Errorf("Initialize() err: %w", err)
+ }
+
// configure the deadlock detector library
switch {
case cfg.DeadlockDetection > 0:
@@ -192,11 +224,12 @@ func (s *Server) Start() {
}
listener, err := makeListener(addr)
-
if err != nil {
fmt.Printf("Could not start node: %v\n", err)
os.Exit(1)
}
+ listener = limitlistener.RejectingLimitListener(
+ listener, cfg.RestConnectionsHardLimit, s.log)
addr = listener.Addr().String()
server = http.Server{
@@ -205,9 +238,9 @@ func (s *Server) Start() {
WriteTimeout: time.Duration(cfg.RestWriteTimeoutSeconds) * time.Second,
}
- tcpListener := listener.(*net.TCPListener)
-
- e := apiServer.NewRouter(s.log, s.node, s.stopping, apiToken, adminAPIToken, tcpListener)
+ e := apiServer.NewRouter(
+ s.log, s.node, s.stopping, apiToken, adminAPIToken, listener,
+ cfg.RestConnectionsSoftLimit)
// Set up files for our PID and our listening address
// before beginning to listen to prevent 'goal node start'
diff --git a/installer/config.json.example b/installer/config.json.example
index 3a77e301bf..17cbfd974f 100644
--- a/installer/config.json.example
+++ b/installer/config.json.example
@@ -78,6 +78,8 @@
"PublicAddress": "",
"ReconnectTime": 60000000000,
"ReservedFDs": 256,
+ "RestConnectionsHardLimit": 2048,
+ "RestConnectionsSoftLimit": 1024,
"RestReadTimeoutSeconds": 15,
"RestWriteTimeoutSeconds": 120,
"RunHosted": false,
diff --git a/network/limitlistener/helper_stub_test.go b/network/limitlistener/helper_stub_test.go
new file mode 100644
index 0000000000..a1430427ca
--- /dev/null
+++ b/network/limitlistener/helper_stub_test.go
@@ -0,0 +1,12 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows
+// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows
+
+package limitlistener_test
+
+func maxOpenFiles() int {
+ return defaultMaxOpenFiles
+}
diff --git a/network/limitlistener/helper_unix_test.go b/network/limitlistener/helper_unix_test.go
new file mode 100644
index 0000000000..9cae789122
--- /dev/null
+++ b/network/limitlistener/helper_unix_test.go
@@ -0,0 +1,18 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package limitlistener_test
+
+import "syscall"
+
+func maxOpenFiles() int {
+ var rlim syscall.Rlimit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
+ return defaultMaxOpenFiles
+ }
+ return int(rlim.Cur)
+}
diff --git a/network/limitlistener/helper_windows_test.go b/network/limitlistener/helper_windows_test.go
new file mode 100644
index 0000000000..c67e20d4ae
--- /dev/null
+++ b/network/limitlistener/helper_windows_test.go
@@ -0,0 +1,9 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package limitlistener_test
+
+func maxOpenFiles() int {
+ return 4 * defaultMaxOpenFiles /* actually it's 16581375 */
+}
diff --git a/network/limitlistener/rejectingLimitListener.go b/network/limitlistener/rejectingLimitListener.go
new file mode 100644
index 0000000000..60d89199c8
--- /dev/null
+++ b/network/limitlistener/rejectingLimitListener.go
@@ -0,0 +1,85 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// RejectingLimitListener is a modification of LimitListener in
+// "golang.org/x/net/netutil". The difference is that when the number of connections
+// exceeds the limit, RejectingLimitListener will accept and immediately close all
+// new connections.
+
+package limitlistener
+
+import (
+ "errors"
+ "fmt"
+ "net"
+ "sync"
+
+ "github.com/algorand/go-algorand/logging"
+)
+
+// RejectingLimitListener returns a Listener that accepts at most n simultaneous
+// connections from the provided Listener. `log` can be nil.
+func RejectingLimitListener(l net.Listener, n uint64, log logging.Logger) net.Listener {
+ return &rejectingLimitListener{
+ Listener: l,
+ log: log,
+ sem: make(chan struct{}, n),
+ done: make(chan struct{}),
+ }
+}
+
+type rejectingLimitListener struct {
+ net.Listener
+ log logging.Logger
+ sem chan struct{}
+ closeOnce sync.Once // ensures the done chan is only closed once
+ done chan struct{} // no values sent; closed when Close is called
+}
+
+func (l *rejectingLimitListener) release() {
+ <-l.sem
+}
+
+func (l *rejectingLimitListener) Accept() (net.Conn, error) {
+ for {
+ select {
+ case <-l.done:
+ return nil, errors.New("Accept() limit listener is closed")
+ default:
+ c, err := l.Listener.Accept()
+ if err != nil {
+ return nil, fmt.Errorf("Accept() accept err: %w", err)
+ }
+ select {
+ case l.sem <- struct{}{}:
+ return &rejectingLimitListenerConn{Conn: c, release: l.release}, nil
+ default:
+ // Close connection immediately.
+ err = c.Close()
+ if (err != nil) && (l.log != nil) {
+ l.log.Debugf(
+ "rejectingLimitListener.Accept() failed to close connection, err %v", err)
+ }
+ }
+ }
+ }
+}
+
+func (l *rejectingLimitListener) Close() error {
+ err := l.Listener.Close()
+ l.closeOnce.Do(func() { close(l.done) })
+ return err
+}
+
+type rejectingLimitListenerConn struct {
+ net.Conn
+ releaseOnce sync.Once
+ release func()
+}
+
+func (l *rejectingLimitListenerConn) Close() error {
+ err := l.Conn.Close()
+ l.releaseOnce.Do(l.release)
+ return err
+}
diff --git a/network/limitlistener/rejectingLimitListener_test.go b/network/limitlistener/rejectingLimitListener_test.go
new file mode 100644
index 0000000000..7f286e13de
--- /dev/null
+++ b/network/limitlistener/rejectingLimitListener_test.go
@@ -0,0 +1,144 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package limitlistener_test
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/algorand/go-algorand/network/limitlistener"
+ "github.com/algorand/go-algorand/test/partitiontest"
+)
+
+const defaultMaxOpenFiles = 256
+const timeout = 5 * time.Second
+
+func TestRejectingLimitListenerBasic(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ const limit = 5
+ attempts := (maxOpenFiles() - limit) / 2
+ if attempts > 256 { // maximum length of accept queue is 128 by default
+ attempts = 256
+ }
+
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ l = limitlistener.RejectingLimitListener(l, limit, nil)
+
+ server := http.Server{}
+ handlerCh := make(chan struct{})
+ server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-handlerCh
+ fmt.Fprint(w, "some body")
+ })
+ go server.Serve(l)
+ defer server.Close()
+
+ for i := 0; i < 3; i++ {
+ queryCh := make(chan error)
+ for j := 0; j < attempts; j++ {
+ go func() {
+ c := http.Client{}
+ r, err := c.Get("http://" + l.Addr().String())
+ if err != nil {
+ queryCh <- err
+ return
+ }
+
+ io.Copy(ioutil.Discard, r.Body)
+ r.Body.Close()
+
+ queryCh <- nil
+ }()
+ }
+
+ for j := 0; j < attempts-limit; j++ {
+ err := <-queryCh
+ if err == nil {
+ t.Errorf("this connection should have failed")
+ }
+ }
+
+ for j := 0; j < limit; j++ {
+ handlerCh <- struct{}{}
+ err := <-queryCh
+ if err != nil {
+ t.Errorf("this connection should have been successful, err: %v", err)
+ }
+ }
+
+ // Give the rejecting limit listener time to update its semaphor.
+ time.Sleep(time.Millisecond)
+ }
+}
+
+type errorListener struct {
+ net.Listener
+}
+
+func (errorListener) Accept() (net.Conn, error) {
+ return nil, errFake
+}
+
+var errFake = errors.New("fake error from errorListener")
+
+func TestRejectingLimitListenerBaseListenerError(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ errCh := make(chan error, 1)
+ go func() {
+ defer close(errCh)
+ const n = 2
+ ll := limitlistener.RejectingLimitListener(errorListener{}, n, nil)
+ for i := 0; i < n+1; i++ {
+ _, err := ll.Accept()
+ if !errors.Is(err, errFake) {
+ errCh <- fmt.Errorf("Accept error %v doesn't contain errFake", err)
+ return
+ }
+ }
+ }()
+
+ select {
+ case err, ok := <-errCh:
+ if ok {
+ t.Fatalf("server: %v", err)
+ }
+ case <-time.After(timeout):
+ t.Fatal("timeout. deadlock?")
+ }
+}
+
+func TestRejectingLimitListenerClose(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer ln.Close()
+ ln = limitlistener.RejectingLimitListener(ln, 1, nil)
+
+ err = ln.Close()
+ if err != nil {
+ t.Errorf("unsuccessful ln.Close()")
+ }
+
+ c, err := ln.Accept()
+ if err == nil {
+ c.Close()
+ t.Errorf("unexpected successful Accept()")
+ }
+}
diff --git a/network/wsNetwork.go b/network/wsNetwork.go
index 68217f727d..92da09f1a6 100644
--- a/network/wsNetwork.go
+++ b/network/wsNetwork.go
@@ -734,11 +734,6 @@ func (wn *WebsocketNetwork) setup() {
// Start makes network connections and threads
func (wn *WebsocketNetwork) Start() {
- var err error
- if wn.config.IncomingConnectionsLimit < 0 {
- wn.config.IncomingConnectionsLimit = MaxInt
- }
-
wn.messagesOfInterestMu.Lock()
defer wn.messagesOfInterestMu.Unlock()
wn.messagesOfInterestEncoded = true
@@ -746,15 +741,6 @@ func (wn *WebsocketNetwork) Start() {
wn.messagesOfInterestEnc = MarshallMessageOfInterestMap(wn.messagesOfInterest)
}
- // Make sure we do not accept more incoming connections than our
- // open file rlimit, with some headroom for other FDs (DNS, log
- // files, SQLite files, telemetry, ...)
- err = wn.rlimitIncomingConnections()
- if err != nil {
- wn.log.Error("ws network start: rlimitIncomingConnections ", err)
- return
- }
-
if wn.config.NetAddress != "" {
listener, err := net.Listen("tcp", wn.config.NetAddress)
if err != nil {
diff --git a/network/wsNetwork_common.go b/network/wsNetwork_common.go
deleted file mode 100644
index 074fd573e7..0000000000
--- a/network/wsNetwork_common.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (C) 2019-2022 Algorand, Inc.
-// This file is part of go-algorand
-//
-// go-algorand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// go-algorand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with go-algorand. If not, see .
-
-// +build !windows
-
-package network
-
-import (
- "runtime"
-
- "golang.org/x/sys/unix"
-)
-
-func (wn *WebsocketNetwork) rlimitIncomingConnections() error {
- var lim unix.Rlimit
- err := unix.Getrlimit(unix.RLIMIT_NOFILE, &lim)
- if err != nil {
- return err
- }
-
- // If rlim_max is not sufficient, reduce IncomingConnectionsLimit
- var rlimitMaxCap uint64
- if lim.Max < wn.config.ReservedFDs {
- rlimitMaxCap = 0
- } else {
- rlimitMaxCap = lim.Max - wn.config.ReservedFDs
- }
- if rlimitMaxCap > uint64(MaxInt) {
- rlimitMaxCap = uint64(MaxInt)
- }
- if wn.config.IncomingConnectionsLimit > int(rlimitMaxCap) {
- wn.log.Warnf("Reducing IncomingConnectionsLimit from %d to %d since RLIMIT_NOFILE is %d",
- wn.config.IncomingConnectionsLimit, rlimitMaxCap, lim.Max)
- wn.config.IncomingConnectionsLimit = int(rlimitMaxCap)
- }
-
- // Set rlim_cur to match IncomingConnectionsLimit
- newLimit := uint64(wn.config.IncomingConnectionsLimit) + wn.config.ReservedFDs
- if newLimit > lim.Cur {
- if runtime.GOOS == "darwin" && newLimit > 10240 && lim.Max == 0x7fffffffffffffff {
- // The max file limit is 10240, even though
- // the max returned by Getrlimit is 1<<63-1.
- // This is OPEN_MAX in sys/syslimits.h.
- // see https://github.com/golang/go/issues/30401
- newLimit = 10240
- }
- lim.Cur = newLimit
- err = unix.Setrlimit(unix.RLIMIT_NOFILE, &lim)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go
index f2f2c6a20d..f7e5db2da3 100644
--- a/network/wsNetwork_test.go
+++ b/network/wsNetwork_test.go
@@ -108,7 +108,6 @@ func init() {
defaultConfig.GossipFanout = 4
defaultConfig.NetAddress = "127.0.0.1:0"
defaultConfig.BaseLoggerDebugLevel = uint32(logging.Debug)
- defaultConfig.IncomingConnectionsLimit = -1
defaultConfig.DNSBootstrapID = ""
defaultConfig.MaxConnectionsPerIP = 30
}
diff --git a/node/node.go b/node/node.go
index 8d497089c0..b7bf95f7a1 100644
--- a/node/node.go
+++ b/node/node.go
@@ -159,7 +159,6 @@ type TxnWithStatus struct {
// MakeFull sets up an Algorand full node
// (i.e., it returns a node that participates in consensus)
func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAddresses []string, genesis bookkeeping.Genesis) (*AlgorandFullNode, error) {
-
node := new(AlgorandFullNode)
node.rootDir = rootDir
node.log = log.With("name", cfg.NetAddress)
diff --git a/node/node_test.go b/node/node_test.go
index b5e7c1e09a..6dfdd1d904 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -57,7 +57,7 @@ var defaultConfig = config.Local{
}
func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationPool execpool.BacklogPool, customConsensus config.ConsensusProtocols) ([]*AlgorandFullNode, []string, []string) {
- util.RaiseRlimit(1000)
+ util.SetFdSoftLimit(1000)
f, _ := os.Create(t.Name() + ".log")
logging.Base().SetJSONFormatter()
logging.Base().SetOutput(f)
diff --git a/test/testdata/configs/config-v20.json b/test/testdata/configs/config-v20.json
index 3a77e301bf..17cbfd974f 100644
--- a/test/testdata/configs/config-v20.json
+++ b/test/testdata/configs/config-v20.json
@@ -78,6 +78,8 @@
"PublicAddress": "",
"ReconnectTime": 60000000000,
"ReservedFDs": 256,
+ "RestConnectionsHardLimit": 2048,
+ "RestConnectionsSoftLimit": 1024,
"RestReadTimeoutSeconds": 15,
"RestWriteTimeoutSeconds": 120,
"RunHosted": false,
diff --git a/util/util.go b/util/util.go
index 90a277c151..c4c7a7385c 100644
--- a/util/util.go
+++ b/util/util.go
@@ -19,23 +19,24 @@
package util
import (
+ "fmt"
"syscall"
)
/* misc */
-// RaiseRlimit increases the number of file descriptors we can have
-func RaiseRlimit(amount uint64) error {
+// SetFdSoftLimit sets a new file descriptors soft limit.
+func SetFdSoftLimit(newLimit uint64) error {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
- return err
+ return fmt.Errorf("SetFdSoftLimit() err: %w", err)
}
- rLimit.Cur = amount
+ rLimit.Cur = newLimit
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
- return err
+ return fmt.Errorf("SetFdSoftLimit() err: %w", err)
}
return nil
}
diff --git a/util/util_windows.go b/util/util_windows.go
index 25d7529b35..5a533b655d 100644
--- a/util/util_windows.go
+++ b/util/util_windows.go
@@ -24,8 +24,8 @@ import (
/* misc */
-// RaiseRlimit increases the number of file descriptors we can have
-func RaiseRlimit(_ uint64) error {
+// SetFdSoftLimit sets a new file descriptors soft limit.
+func SetFdSoftLimit(_ uint64) error {
return nil
}