diff --git a/config/config_test.go b/config/config_test.go index c2ee070fc4..1e24fa4591 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -42,8 +42,9 @@ var defaultConfig = Local{ BaseLoggerDebugLevel: 1, //Info level } -func TestSaveThenLoad(t *testing.T) { +func TestLocal_SaveThenLoad(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() c1, err := loadWithoutDefaults(defaultConfig) require.NoError(t, err) @@ -53,13 +54,10 @@ func TestSaveThenLoad(t *testing.T) { ser1 := json.NewEncoder(&b1) ser1.Encode(c1) - os.RemoveAll("testdir") - err = os.Mkdir("testdir", 0777) - require.NoError(t, err) - - c1.SaveToDisk("testdir") + tempDir := t.TempDir() + c1.SaveToDisk(tempDir) - c2, err := LoadConfigFromDisk("testdir") + c2, err := LoadConfigFromDisk(tempDir) require.NoError(t, err) var b2 bytes.Buffer @@ -67,23 +65,23 @@ func TestSaveThenLoad(t *testing.T) { ser2.Encode(c2) require.True(t, bytes.Equal(b1.Bytes(), b2.Bytes())) - - os.RemoveAll("testdir") } -func TestLoadMissing(t *testing.T) { +func TestConfig_LoadMissing(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() - os.RemoveAll("testdir") - _, err := LoadConfigFromDisk("testdir") + tempDir := t.TempDir() + os.RemoveAll(tempDir) + _, err := LoadConfigFromDisk(tempDir) require.True(t, os.IsNotExist(err)) } -func TestMergeConfig(t *testing.T) { +func TestLocal_MergeConfig(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() - os.RemoveAll("testdir") - err := os.Mkdir("testdir", 0777) + tempDir := t.TempDir() c1 := struct { GossipFanout int @@ -98,7 +96,7 @@ func TestMergeConfig(t *testing.T) { c1.NetAddress = testString // write our reduced version of the Local struct - fileToMerge := filepath.Join("testdir", ConfigFilename) + fileToMerge := filepath.Join(tempDir, ConfigFilename) f, err := os.OpenFile(fileToMerge, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err == nil { enc := json.NewEncoder(f) @@ -110,7 +108,7 @@ func TestMergeConfig(t *testing.T) { // Take defaultConfig and merge with the saved custom settings. // This should result in c2 being the same as defaultConfig except for the value(s) in our custom c1 - c2, err := mergeConfigFromDir("testdir", defaultConfig) + c2, err := mergeConfigFromDir(tempDir, defaultConfig) require.NoError(t, err) require.Equal(t, defaultConfig.Archival || c1.NetAddress != "", c2.Archival) @@ -119,12 +117,10 @@ func TestMergeConfig(t *testing.T) { require.Equal(t, c1.NetAddress, c2.NetAddress) require.Equal(t, c1.GossipFanout, c2.GossipFanout) - - os.RemoveAll("testdir") } -func saveFullPhonebook(phonebook phonebookBlackWhiteList) error { - filename := filepath.Join("testdir", PhonebookFilename) +func saveFullPhonebook(phonebook phonebookBlackWhiteList, saveToDir string) error { + filename := filepath.Join(saveToDir, PhonebookFilename) f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err == nil { defer f.Close() @@ -142,52 +138,48 @@ var phonebookToMerge = phonebookBlackWhiteList{ Include: []string{"test1", "addThisOne"}, } -var expectedMerged = []string{ - "test1", "test2", "addThisOne", -} - func TestLoadPhonebook(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() - os.RemoveAll("testdir") - err := os.Mkdir("testdir", 0777) - require.NoError(t, err) + tempDir := t.TempDir() - err = saveFullPhonebook(phonebook) + err := saveFullPhonebook(phonebook, tempDir) require.NoError(t, err) - phonebookEntries, err := LoadPhonebook("testdir") + phonebookEntries, err := LoadPhonebook(tempDir) require.NoError(t, err) require.Equal(t, 3, len(phonebookEntries)) for index, entry := range phonebookEntries { require.Equal(t, phonebook.Include[index], entry) } - os.RemoveAll("testdir") } func TestLoadPhonebookMissing(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() - os.RemoveAll("testdir") - _, err := LoadPhonebook("testdir") + tempDir := t.TempDir() + _, err := LoadPhonebook(tempDir) require.True(t, os.IsNotExist(err)) } func TestArchivalIfRelay(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() testArchivalIfRelay(t, true) } func TestArchivalIfNotRelay(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() testArchivalIfRelay(t, false) } func testArchivalIfRelay(t *testing.T, relay bool) { - os.RemoveAll("testdir") - err := os.Mkdir("testdir", 0777) + tempDir := t.TempDir() c1 := struct { NetAddress string @@ -197,7 +189,7 @@ func testArchivalIfRelay(t *testing.T, relay bool) { } // write our reduced version of the Local struct - fileToMerge := filepath.Join("testdir", ConfigFilename) + fileToMerge := filepath.Join(tempDir, ConfigFilename) f, err := os.OpenFile(fileToMerge, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err == nil { enc := json.NewEncoder(f) @@ -207,18 +199,16 @@ func testArchivalIfRelay(t *testing.T, relay bool) { require.NoError(t, err) require.False(t, defaultConfig.Archival, "Default should be non-archival") - c2, err := mergeConfigFromDir("testdir", defaultConfig) + c2, err := mergeConfigFromDir(tempDir, defaultConfig) require.NoError(t, err) if relay { require.True(t, c2.Archival, "Relay should be archival") } else { require.False(t, c2.Archival, "Non-relay should still be non-archival") } - - os.RemoveAll("testdir") } -func TestConfigExampleIsCorrect(t *testing.T) { +func TestLocal_ConfigExampleIsCorrect(t *testing.T) { partitiontest.PartitionTest(t) a := require.New(t) @@ -261,7 +251,7 @@ func loadWithoutDefaults(cfg Local) (Local, error) { return cfg, err } -func TestConfigMigrate(t *testing.T) { +func TestLocal_ConfigMigrate(t *testing.T) { partitiontest.PartitionTest(t) a := require.New(t) @@ -288,7 +278,7 @@ func TestConfigMigrate(t *testing.T) { a.NotEqual(defaultLocal, c0Modified) } -func TestConfigMigrateFromDisk(t *testing.T) { +func TestLocal_ConfigMigrateFromDisk(t *testing.T) { partitiontest.PartitionTest(t) a := require.New(t) @@ -311,7 +301,7 @@ func TestConfigMigrateFromDisk(t *testing.T) { } // Verify that nobody is changing the shipping default configurations -func TestConfigInvariant(t *testing.T) { +func TestLocal_ConfigInvariant(t *testing.T) { partitiontest.PartitionTest(t) a := require.New(t) @@ -328,8 +318,9 @@ func TestConfigInvariant(t *testing.T) { } } -func TestConfigLatestVersion(t *testing.T) { +func TestLocal_ConfigLatestVersion(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() a := require.New(t) @@ -391,6 +382,7 @@ func TestConsensusLatestVersion(t *testing.T) { func TestLocal_DNSBootstrapArray(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() type fields struct { DNSBootstrapID string @@ -434,6 +426,7 @@ func TestLocal_DNSBootstrapArray(t *testing.T) { func TestLocal_DNSBootstrap(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() type fields struct { DNSBootstrapID string @@ -480,8 +473,9 @@ func TestLocal_DNSBootstrap(t *testing.T) { } } -func TestLocalStructTags(t *testing.T) { +func TestLocal_StructTags(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() localType := reflect.TypeOf(Local{}) @@ -523,8 +517,9 @@ func TestLocalStructTags(t *testing.T) { } } -func TestGetVersionedDefaultLocalConfig(t *testing.T) { +func TestLocal_GetVersionedDefaultLocalConfig(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() for i := uint32(0); i < getLatestConfigVersion(); i++ { localVersion := getVersionedDefaultLocalConfig(i) @@ -533,8 +528,9 @@ func TestGetVersionedDefaultLocalConfig(t *testing.T) { } // TestLocalVersionField - ensures the Version contains only versions tags, the versions are all contiguous, and that no non-version tags are included there. -func TestLocalVersionField(t *testing.T) { +func TestLocal_VersionField(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() localType := reflect.TypeOf(Local{}) field, ok := localType.FieldByName("Version") @@ -553,8 +549,9 @@ func TestLocalVersionField(t *testing.T) { require.Equal(t, expectedTag, string(field.Tag)) } -func TestGetNonDefaultConfigValues(t *testing.T) { +func TestLocal_GetNonDefaultConfigValues(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() cfg := GetDefaultLocal() @@ -583,6 +580,8 @@ func TestGetNonDefaultConfigValues(t *testing.T) { func TestLocal_TxFiltering(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() + cfg := GetDefaultLocal() // ensure the default @@ -605,3 +604,62 @@ func TestLocal_TxFiltering(t *testing.T) { require.True(t, cfg.TxFilterRawMsgEnabled()) require.True(t, cfg.TxFilterCanonicalEnabled()) } + +func TestLocal_IsGossipServer(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + cfg := GetDefaultLocal() + require.False(t, cfg.IsGossipServer()) + + cfg.NetAddress = ":4160" + require.True(t, cfg.IsGossipServer()) +} + +func TestLocal_RecalculateConnectionLimits(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + var tests = []struct { + maxFDs uint64 + reservedIn uint64 + restSoftIn uint64 + restHardIn uint64 + incomingIn int + + updated bool + restSoftExp uint64 + restHardExp uint64 + incomingExp int + }{ + {100, 10, 20, 40, 50, false, 20, 40, 50}, // no change + {100, 10, 20, 50, 50, true, 20, 40, 50}, // borrow from rest + {100, 10, 25, 50, 50, true, 25, 40, 50}, // borrow from rest + {100, 10, 50, 50, 50, true, 40, 40, 50}, // borrow from rest, update soft + {100, 10, 9, 19, 81, true, 9, 10, 80}, // borrow from both rest and incoming + {100, 10, 10, 20, 80, true, 10, 10, 80}, // borrow from both rest and incoming + {100, 50, 10, 30, 40, true, 10, 10, 40}, // borrow from both rest and incoming + {100, 90, 10, 30, 40, true, 10, 10, 0}, // borrow from both rest and incoming, clear incoming + {4096, 256, 1024, 2048, 2400, true, 1024, 1440, 2400}, // real numbers + {5000, 256, 1024, 2048, 2400, false, 1024, 2048, 2400}, // real numbers + } + + for i, test := range tests { + test := test + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + t.Parallel() + + c := Local{ + RestConnectionsSoftLimit: test.restSoftIn, + RestConnectionsHardLimit: test.restHardIn, + IncomingConnectionsLimit: test.incomingIn, + } + requireFDs := test.reservedIn + test.restHardIn + uint64(test.incomingIn) + res := c.AdjustConnectionLimits(requireFDs, test.maxFDs) + require.Equal(t, test.updated, res) + require.Equal(t, test.restSoftExp, c.RestConnectionsSoftLimit) + require.Equal(t, test.restHardExp, c.RestConnectionsHardLimit) + require.Equal(t, test.incomingExp, c.IncomingConnectionsLimit) + }) + } +} diff --git a/config/localTemplate.go b/config/localTemplate.go index 7caa5e9dee..12df3915b4 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -103,6 +103,8 @@ type Local struct { // that RLIMIT_NOFILE >= IncomingConnectionsLimit + RestConnectionsHardLimit + // ReservedFDs. ReservedFDs are meant to leave room for short-lived FDs like // DNS queries, SQLite files, etc. This parameter shouldn't be changed. + // If RLIMIT_NOFILE < IncomingConnectionsLimit + RestConnectionsHardLimit + ReservedFDs + // then either RestConnectionsHardLimit or IncomingConnectionsLimit decreased. ReservedFDs uint64 `version[2]:"256"` // local server @@ -588,3 +590,37 @@ func (cfg Local) TxFilterRawMsgEnabled() bool { func (cfg Local) TxFilterCanonicalEnabled() bool { return cfg.TxIncomingFilteringFlags&txFilterCanonical != 0 } + +// IsGossipServer returns true if NetAddress is set and this node supposed +// to start websocket server +func (cfg Local) IsGossipServer() bool { + return cfg.NetAddress != "" +} + +// AdjustConnectionLimits updates RestConnectionsSoftLimit, RestConnectionsHardLimit, IncomingConnectionsLimit +// if requiredFDs greater than maxFDs +func (cfg *Local) AdjustConnectionLimits(requiredFDs, maxFDs uint64) bool { + if maxFDs >= requiredFDs { + return false + } + const reservedRESTConns = 10 + diff := requiredFDs - maxFDs + + if cfg.RestConnectionsHardLimit <= diff+reservedRESTConns { + restDelta := diff + reservedRESTConns - cfg.RestConnectionsHardLimit + cfg.RestConnectionsHardLimit = reservedRESTConns + if cfg.IncomingConnectionsLimit > int(restDelta) { + cfg.IncomingConnectionsLimit -= int(restDelta) + } else { + cfg.IncomingConnectionsLimit = 0 + } + } else { + cfg.RestConnectionsHardLimit -= diff + } + + if cfg.RestConnectionsSoftLimit > cfg.RestConnectionsHardLimit { + cfg.RestConnectionsSoftLimit = cfg.RestConnectionsHardLimit + } + + return true +} diff --git a/daemon/algod/server.go b/daemon/algod/server.go index 0c65b2efab..44b8a4dccb 100644 --- a/daemon/algod/server.go +++ b/daemon/algod/server.go @@ -109,18 +109,49 @@ func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genes // Set large enough soft file descriptors limit. var ot basics.OverflowTracker - fdRequired := ot.Add( - cfg.ReservedFDs, - ot.Add(uint64(cfg.IncomingConnectionsLimit), cfg.RestConnectionsHardLimit)) + fdRequired := ot.Add(cfg.ReservedFDs, cfg.RestConnectionsHardLimit) if ot.Overflowed { return errors.New( - "Initialize() overflowed when adding up ReservedFDs, IncomingConnectionsLimit " + - "RestConnectionsHardLimit; decrease them") + "Initialize() overflowed when adding up ReservedFDs and RestConnectionsHardLimit; decrease them") } err = util.SetFdSoftLimit(fdRequired) if err != nil { return fmt.Errorf("Initialize() err: %w", err) } + if cfg.IsGossipServer() { + var ot basics.OverflowTracker + fdRequired := ot.Add(fdRequired, uint64(cfg.IncomingConnectionsLimit)) + if ot.Overflowed { + return errors.New("Initialize() overflowed when adding up IncomingConnectionsLimit to the existing RLIMIT_NOFILE value; decrease RestConnectionsHardLimit or IncomingConnectionsLimit") + } + _, hard, err := util.GetFdLimits() + if err != nil { + s.log.Errorf("Failed to get RLIMIT_NOFILE values: %s", err.Error()) + } else { + maxFDs := fdRequired + if fdRequired > hard { + // claim as many descriptors are possible + maxFDs = hard + // but try to keep cfg.ReservedFDs untouched by decreasing other limits + if cfg.AdjustConnectionLimits(fdRequired, hard) { + s.log.Warnf( + "Updated connection limits: RestConnectionsSoftLimit=%d, RestConnectionsHardLimit=%d, IncomingConnectionsLimit=%d", + cfg.RestConnectionsSoftLimit, + cfg.RestConnectionsHardLimit, + cfg.IncomingConnectionsLimit, + ) + if cfg.IncomingConnectionsLimit == 0 { + return errors.New("Initialize() failed to adjust connection limits") + } + } + } + err = util.SetFdSoftLimit(maxFDs) + if err != nil { + // do not fail but log the error + s.log.Errorf("Failed to set a new RLIMIT_NOFILE value to %d (max %d): %s", fdRequired, hard, err.Error()) + } + } + } // configure the deadlock detector library switch { diff --git a/network/wsNetwork.go b/network/wsNetwork.go index b617cf63b1..826a7b83f8 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -724,7 +724,7 @@ func (wn *WebsocketNetwork) setup() { wn.server.IdleTimeout = httpServerIdleTimeout wn.server.MaxHeaderBytes = httpServerMaxHeaderBytes wn.ctx, wn.ctxCancel = context.WithCancel(context.Background()) - wn.relayMessages = wn.config.NetAddress != "" || wn.config.ForceRelayMessages + wn.relayMessages = wn.config.IsGossipServer() || wn.config.ForceRelayMessages if wn.relayMessages || wn.config.ForceFetchTransactions { wn.wantTXGossip = wantTXGossipYes } @@ -798,7 +798,7 @@ func (wn *WebsocketNetwork) Start() { wn.messagesOfInterestEnc = MarshallMessageOfInterestMap(wn.messagesOfInterest) } - if wn.config.NetAddress != "" { + if wn.config.IsGossipServer() { listener, err := net.Listen("tcp", wn.config.NetAddress) if err != nil { wn.log.Errorf("network could not listen %v: %s", wn.config.NetAddress, err) diff --git a/util/util.go b/util/util.go index 0ecf357344..f3699188c6 100644 --- a/util/util.go +++ b/util/util.go @@ -26,6 +26,16 @@ import ( /* misc */ +// GetFdLimits returns a current values for file descriptors limits. +func GetFdLimits() (soft uint64, hard uint64, err error) { + var rLimit syscall.Rlimit + err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return 0, 0, fmt.Errorf("GetFdSoftLimit() err: %w", err) + } + return rLimit.Cur, rLimit.Max, nil +} + // SetFdSoftLimit sets a new file descriptors soft limit. func SetFdSoftLimit(newLimit uint64) error { var rLimit syscall.Rlimit diff --git a/util/util_windows.go b/util/util_windows.go index fa91a4f8a8..b485f8e253 100644 --- a/util/util_windows.go +++ b/util/util_windows.go @@ -18,12 +18,18 @@ package util import ( "errors" + "math" "syscall" "time" ) /* misc */ +// GetFdLimits returns a current values for file descriptors limits. +func GetFdLimits() (soft uint64, hard uint64, err error) { + return math.MaxUint64, math.MaxUint64, nil // syscall.RLIM_INFINITY +} + // SetFdSoftLimit sets a new file descriptors soft limit. func SetFdSoftLimit(_ uint64) error { return nil