diff --git a/src/Nethermind/Nethermind.Core.Test/Modules/TestEnvironmentModule.cs b/src/Nethermind/Nethermind.Core.Test/Modules/TestEnvironmentModule.cs index 90b4e668fb9..931a28077f4 100644 --- a/src/Nethermind/Nethermind.Core.Test/Modules/TestEnvironmentModule.cs +++ b/src/Nethermind/Nethermind.Core.Test/Modules/TestEnvironmentModule.cs @@ -8,6 +8,7 @@ using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Consensus; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Db; @@ -117,6 +118,8 @@ protected override void Load(ContainerBuilder builder) pruningConfig.DirtyNodeShardBit = 1; return pruningConfig; }) + + .AddSingleton(new TestHardwareInfo(1.GiB())) ; } } diff --git a/src/Nethermind/Nethermind.Core.Test/TestHardwareInfo.cs b/src/Nethermind/Nethermind.Core.Test/TestHardwareInfo.cs new file mode 100644 index 00000000000..59e15d1f33a --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/TestHardwareInfo.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Core.Test; + +public class TestHardwareInfo(long availableMemory = 10000000) : IHardwareInfo +{ + public long AvailableMemoryBytes => availableMemory; +} diff --git a/src/Nethermind/Nethermind.Core/HardwareInfo.cs b/src/Nethermind/Nethermind.Core/HardwareInfo.cs new file mode 100644 index 00000000000..280c04c5bb6 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/HardwareInfo.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Core; + +public class HardwareInfo : IHardwareInfo +{ + public long AvailableMemoryBytes { get; } + + public HardwareInfo() + { + // Note: Not the same as memory capacity. This take into account current system memory pressure such as + // other process as well as OS level limit such as rlimit. Eh, its good enough. + AvailableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + } +} diff --git a/src/Nethermind/Nethermind.Core/IHardwareInfo.cs b/src/Nethermind/Nethermind.Core/IHardwareInfo.cs new file mode 100644 index 00000000000..0b9e98ff951 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/IHardwareInfo.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Extensions; + +namespace Nethermind.Core; + +public interface IHardwareInfo +{ + public static readonly long StateDbLargerMemoryThreshold = 20.GiB(); + long AvailableMemoryBytes { get; } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs index a3fbd61b6fe..02300a2d7e6 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs @@ -17,8 +17,8 @@ public class ColumnsDb : DbOnTheRocks, IColumnsDb where T : struct, Enum { private readonly IDictionary _columnDbs = new Dictionary(); - public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, ILogManager logManager, IReadOnlyList keys, IntPtr? sharedCache = null) - : base(basePath, settings, dbConfig, logManager, GetEnumKeys(keys).Select(static (key) => key.ToString()).ToList(), sharedCache: sharedCache) + public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IReadOnlyList keys, IntPtr? sharedCache = null) + : base(basePath, settings, dbConfig, rocksDbConfigFactory, logManager, GetEnumKeys(keys).Select(static (key) => key.ToString()).ToList(), sharedCache: sharedCache) { keys = GetEnumKeys(keys); @@ -61,7 +61,7 @@ private static IReadOnlyList GetEnumKeys(IReadOnlyList keys) return keys; } - protected override void BuildOptions(PerTableDbConfig dbConfig, Options options, IntPtr? sharedCache) + protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache) { base.BuildOptions(dbConfig, options, sharedCache); options.SetCreateMissingColumnFamilies(); diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs new file mode 100644 index 00000000000..16ac59bce62 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Db.Rocks.Config; + +public class AdjustedRocksdbConfig( + IRocksDbConfig baseConfig, + string additionalRocksDbOptions, + ulong writeBufferSize +) : IRocksDbConfig +{ + public ulong? WriteBufferSize => writeBufferSize; + + public ulong? WriteBufferNumber => baseConfig.WriteBufferNumber; + + // Note: not AdditionalRocksDbOptions so that user can still override option. + public string RocksDbOptions => baseConfig.RocksDbOptions + additionalRocksDbOptions; + + public string AdditionalRocksDbOptions => baseConfig.AdditionalRocksDbOptions; + + public int? MaxOpenFiles => baseConfig.MaxOpenFiles; + + public bool WriteAheadLogSync => baseConfig.WriteAheadLogSync; + + public ulong? ReadAheadSize => baseConfig.ReadAheadSize; + + public bool EnableDbStatistics => baseConfig.EnableDbStatistics; + + public uint StatsDumpPeriodSec => baseConfig.StatsDumpPeriodSec; + + public bool? VerifyChecksum => baseConfig.VerifyChecksum; + + public ulong? RowCacheSize => baseConfig.RowCacheSize; + + public bool EnableFileWarmer => baseConfig.EnableFileWarmer; + + public double CompressibilityHint => baseConfig.CompressibilityHint; + + public bool FlushOnExit => baseConfig.FlushOnExit; +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs index e3280a9e6e3..b2b506b1d47 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs @@ -158,7 +158,7 @@ public class DbConfig : IDbConfig public ulong? CodeDbRowCacheSize { get; set; } = (ulong)16.MiB(); public string CodeDbRocksDbOptions { get; set; } = - "write_buffer_size=4000000;" + + "write_buffer_size=16000000;" + "block_based_table_factory.block_cache=16000000;" + "optimize_filters_for_hits=false;" + "prefix_extractor=capped:8;" + @@ -234,6 +234,37 @@ public class DbConfig : IDbConfig "max_write_batch_group_size_bytes=4000000;" + ""; + + public string StateDbLargeMemoryRocksDbOptions { get; set; } = + // In large memory, we disable the partitioned block index. This took around additional 2GB of memory, but + // pretty reasonable reduction in latency especially when blocks are already cached. + // Note: Not for archive mode as the index size is proportional to db size. + "block_based_table_factory={index_type=kBinarySearch;partition_filters=0;};"; + + public string StateDbArchiveModeRocksDbOptions { get; set; } = + // For archive mode, we are mainly concerned with write amplification due to very large database. + + // Lowers back the level multiplier as very large database causes very high write amp. + "max_bytes_for_level_multiplier=10;" + + "max_bytes_for_level_base=350000000;" + + + // Change back file size multiplier as we dont want ridiculous file size, making compaction uneven, + // but set high base size. This mean a lot of file, but you are using archive mode, so this should be expected. + "target_file_size_multiplier=1;" + + "target_file_size_base=256000000;" + + + // Change back restart interval for (probably slight) database size reduction + "block_based_table_factory.block_restart_interval=16;" + + + // slight adjustment to util ratio, for db size. + "block_based_table_factory.data_block_hash_table_util_ratio=0.8;" + + + // slight adjustment to block size, for potentially better db size. + "block_based_table_factory.block_size=64000;"; + + public ulong StateDbLargeMemoryWriteBufferSize { get; set; } = (ulong)128.MiB(); + public ulong StateDbArchiveModeWriteBufferSize { get; set; } = (ulong)256.MiB(); + public string? StateDbAdditionalRocksDbOptions { get; set; } public string L1OriginDbRocksDbOptions { get; set; } = ""; diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs index 865a08532d3..bdd18bb3afd 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs @@ -93,6 +93,10 @@ public interface IDbConfig : IConfig double StateDbCompressibilityHint { get; set; } string StateDbRocksDbOptions { get; set; } string? StateDbAdditionalRocksDbOptions { get; set; } + string StateDbLargeMemoryRocksDbOptions { get; set; } + string StateDbArchiveModeRocksDbOptions { get; set; } + ulong StateDbLargeMemoryWriteBufferSize { get; set; } + ulong StateDbArchiveModeWriteBufferSize { get; set; } string L1OriginDbRocksDbOptions { get; set; } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs new file mode 100644 index 00000000000..f70d96006d3 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Db.Rocks.Config; + +public interface IRocksDbConfig +{ + ulong? WriteBufferSize { get; } + ulong? WriteBufferNumber { get; } + string RocksDbOptions { get; } + string AdditionalRocksDbOptions { get; } + int? MaxOpenFiles { get; } + bool WriteAheadLogSync { get; } + ulong? ReadAheadSize { get; } + bool EnableDbStatistics { get; } + uint StatsDumpPeriodSec { get; } + bool? VerifyChecksum { get; } + ulong? RowCacheSize { get; } + bool EnableFileWarmer { get; } + double CompressibilityHint { get; } + bool FlushOnExit { get; } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfigFactory.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfigFactory.cs new file mode 100644 index 00000000000..624c76da4c0 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfigFactory.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Db.Rocks.Config; + +public interface IRocksDbConfigFactory +{ + IRocksDbConfig GetForDatabase(string databaseName, string? columnName); +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index d3393471252..6524901e357 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -10,20 +10,18 @@ namespace Nethermind.Db.Rocks.Config; -public class PerTableDbConfig +public class PerTableDbConfig : IRocksDbConfig { private readonly string _tableName; private readonly string? _columnName; private readonly IDbConfig _dbConfig; - private readonly DbSettings _settings; private readonly string[] _prefixes; private readonly string[] _reversedPrefixes; - public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? columnName = null) + public PerTableDbConfig(IDbConfig dbConfig, string dbName, string? columnName = null) { _dbConfig = dbConfig; - _settings = dbSettings; - _tableName = _settings.DbName; + _tableName = dbName; _columnName = columnName; _prefixes = GetPrefixes(); _reversedPrefixes = _prefixes.Reverse().ToArray(); diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/RocksDbConfigFactory.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/RocksDbConfigFactory.cs new file mode 100644 index 00000000000..aa62daff031 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/RocksDbConfigFactory.cs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Logging; + +namespace Nethermind.Db.Rocks.Config; + +public class RocksDbConfigFactory(IDbConfig dbConfig, IPruningConfig pruningConfig, IHardwareInfo hardwareInfo, ILogManager logManager) : IRocksDbConfigFactory +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + + public IRocksDbConfig GetForDatabase(string databaseName, string? columnName) + { + IRocksDbConfig rocksDbConfig = new PerTableDbConfig(dbConfig, databaseName, columnName); + if (databaseName.StartsWith("State")) + { + if (!pruningConfig.Mode.IsMemory()) + { + if (_logger.IsInfo) _logger.Info($"Using archive mode State Db config."); + + ulong writeBufferSize = rocksDbConfig.WriteBufferSize ?? 0; + if (writeBufferSize < dbConfig.StateDbArchiveModeWriteBufferSize) writeBufferSize = dbConfig.StateDbArchiveModeWriteBufferSize; + + rocksDbConfig = new AdjustedRocksdbConfig( + rocksDbConfig, + dbConfig.StateDbArchiveModeRocksDbOptions!, + writeBufferSize + ); + } + else if (hardwareInfo.AvailableMemoryBytes >= IHardwareInfo.StateDbLargerMemoryThreshold) + { + if (_logger.IsInfo) _logger.Info($"Detected {hardwareInfo.AvailableMemoryBytes / 1.GiB()} GB of available memory. Applying large memory State Db config."); + + ulong writeBufferSize = rocksDbConfig.WriteBufferSize ?? 0; + if (writeBufferSize < dbConfig.StateDbLargeMemoryWriteBufferSize) writeBufferSize = dbConfig.StateDbLargeMemoryWriteBufferSize; + + rocksDbConfig = new AdjustedRocksdbConfig( + rocksDbConfig, + dbConfig.StateDbLargeMemoryRocksDbOptions!, + writeBufferSize + ); + } + + if (pruningConfig.Mode.IsMemory()) + { + ulong totalWriteBufferMb = rocksDbConfig.WriteBufferNumber!.Value * rocksDbConfig.WriteBufferSize!.Value / (ulong)1.MB(); + double minimumWriteBufferMb = 0.2 * pruningConfig.DirtyCacheMb; + if (totalWriteBufferMb < minimumWriteBufferMb) + { + ulong minimumWriteBufferSize = (ulong)Math.Ceiling((minimumWriteBufferMb * 1.MB()) / rocksDbConfig.WriteBufferNumber!.Value); + + if (_logger.IsInfo) _logger.Info($"Adjust state DB write buffer size to {minimumWriteBufferSize / (ulong)1.MB()} MB to account for pruning cache."); + + rocksDbConfig = new AdjustedRocksdbConfig( + rocksDbConfig, + "", + minimumWriteBufferSize + ); + } + } + + } + + return rocksDbConfig; + } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index a2e8a492935..40c90a1f3a4 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -54,6 +54,7 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor internal ReadOptions? _readAheadReadOptions = null; internal DbOptions? DbOptions { get; private set; } + private readonly IRocksDbConfigFactory _rocksDbConfigFactory; public string Name { get; } @@ -65,7 +66,7 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private readonly DbSettings _settings; - private readonly PerTableDbConfig _perTableDbConfig; + private readonly IRocksDbConfig _perTableDbConfig; private ulong _maxBytesForLevelBase; private ulong _targetFileSizeBase; private int _minWriteBufferToMerge; @@ -92,6 +93,7 @@ public DbOnTheRocks( string basePath, DbSettings dbSettings, IDbConfig dbConfig, + IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IList? columnFamilies = null, RocksDbSharp.Native? rocksDbNative = null, @@ -103,7 +105,8 @@ public DbOnTheRocks( Name = _settings.DbName; _fileSystem = fileSystem ?? new FileSystem(); _rocksDbNative = rocksDbNative ?? RocksDbSharp.Native.Instance; - _perTableDbConfig = new PerTableDbConfig(dbConfig, _settings); + _rocksDbConfigFactory = rocksDbConfigFactory; + _perTableDbConfig = rocksDbConfigFactory.GetForDatabase(Name, null); _db = Init(basePath, dbSettings.DbPath, dbConfig, logManager, columnFamilies, dbSettings.DeleteOnStart, sharedCache); _iteratorManager = new IteratorManager(_db, null, _readAheadReadOptions); } @@ -151,7 +154,8 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan string columnFamily = enumColumnName; ColumnFamilyOptions options = new(); - BuildOptions(new PerTableDbConfig(dbConfig, _settings, columnFamily), options, sharedCache); + IRocksDbConfig columnConfig = _rocksDbConfigFactory.GetForDatabase(Name, columnFamily); + BuildOptions(columnConfig, options, sharedCache); // "default" is a special column name with rocksdb, which is what previously not specifying column goes to if (columnFamily == "Default") columnFamily = "default"; @@ -430,7 +434,7 @@ public static IDictionary ExtractOptions(string dbOptions) return asDict; } - protected virtual void BuildOptions(PerTableDbConfig dbConfig, Options options, IntPtr? sharedCache) where T : Options + protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache) where T : Options { // This section is about the table factory.. and block cache apparently. // This effect the format of the SST files and usually require resync to take effect. @@ -563,6 +567,7 @@ protected virtual void BuildOptions(PerTableDbConfig dbConfig, Options opt #endregion #region read-write options + // TODO: These are not applied to column family WriteOptions = CreateWriteOptions(dbConfig); _noWalWrite = CreateWriteOptions(dbConfig); @@ -597,7 +602,7 @@ protected virtual void BuildOptions(PerTableDbConfig dbConfig, Options opt #endregion } - private static WriteOptions CreateWriteOptions(PerTableDbConfig dbConfig) + private static WriteOptions CreateWriteOptions(IRocksDbConfig dbConfig) { WriteOptions options = new(); // potential fix for corruption on hard process termination, may cause performance degradation diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksDbFactory.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksDbFactory.cs index e0c02bbbe04..3883bb2d4cb 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksDbFactory.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/RocksDbFactory.cs @@ -17,15 +17,17 @@ public class RocksDbFactory : IDbFactory private readonly string _basePath; private readonly IntPtr _sharedCache; + private readonly IRocksDbConfigFactory _rocksDbConfigFactory; - public RocksDbFactory(IDbConfig dbConfig, IInitConfig initConfig, ILogManager logManager) - : this(dbConfig, logManager, initConfig.BaseDbPath) + public RocksDbFactory(IRocksDbConfigFactory rocksDbConfigFactory, IDbConfig dbConfig, IInitConfig initConfig, ILogManager logManager) + : this(rocksDbConfigFactory, dbConfig, logManager, initConfig.BaseDbPath) { } - public RocksDbFactory(IDbConfig dbConfig, ILogManager logManager, string basePath) + public RocksDbFactory(IRocksDbConfigFactory rocksDbConfigFactory, IDbConfig dbConfig, ILogManager logManager, string basePath) { + _rocksDbConfigFactory = rocksDbConfigFactory; _dbConfig = dbConfig; _logManager = logManager; _basePath = basePath; @@ -41,10 +43,10 @@ public RocksDbFactory(IDbConfig dbConfig, ILogManager logManager, string basePat } public IDb CreateDb(DbSettings dbSettings) => - new DbOnTheRocks(_basePath, dbSettings, _dbConfig, _logManager, sharedCache: _sharedCache); + new DbOnTheRocks(_basePath, dbSettings, _dbConfig, _rocksDbConfigFactory, _logManager, sharedCache: _sharedCache); public IColumnsDb CreateColumnsDb(DbSettings dbSettings) where T : struct, Enum => - new ColumnsDb(_basePath, dbSettings, _dbConfig, _logManager, Array.Empty(), sharedCache: _sharedCache); + new ColumnsDb(_basePath, dbSettings, _dbConfig, _rocksDbConfigFactory, _logManager, Array.Empty(), sharedCache: _sharedCache); public string GetFullDbPath(DbSettings dbSettings) => DbOnTheRocks.GetFullDbPath(dbSettings.DbPath, _basePath); } diff --git a/src/Nethermind/Nethermind.Db.Test/ColumnsDbTests.cs b/src/Nethermind/Nethermind.Db.Test/ColumnsDbTests.cs index 0260f10790c..fa6e59d0b24 100644 --- a/src/Nethermind/Nethermind.Db.Test/ColumnsDbTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/ColumnsDbTests.cs @@ -6,6 +6,7 @@ using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Extensions; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Db.Rocks; using Nethermind.Db.Rocks.Config; @@ -34,6 +35,7 @@ public void Setup() DeleteOnStart = true, }, new DbConfig(), + new RocksDbConfigFactory(new DbConfig(), new PruningConfig(), new TestHardwareInfo(), LimboLogs.Instance), LimboLogs.Instance, Enum.GetValues() ); diff --git a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs index 5a0976c8cdd..fac5614b55c 100644 --- a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs @@ -31,7 +31,7 @@ public void CanReadAllConfigForAllTable() foreach (string table in tables) { - PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(table, "")); + PerTableDbConfig config = new PerTableDbConfig(dbConfig, table); object _ = config.RocksDbOptions; _ = config.AdditionalRocksDbOptions; @@ -49,7 +49,7 @@ public void When_ColumnDb_UsePerTableConfig() dbConfig.ReceiptsDbRocksDbOptions = "some_option=2;"; dbConfig.ReceiptsBlocksDbRocksDbOptions = "some_option=3;"; - PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, ""), "Blocks"); + PerTableDbConfig config = new PerTableDbConfig(dbConfig, DbNames.Receipts, "Blocks"); config.RocksDbOptions.Should().Be("some_option=1;some_option=2;some_option=3;"); } @@ -61,7 +61,7 @@ public void When_PerTableConfigIsAvailable_UsePerTableConfig() dbConfig.ReceiptsDbRocksDbOptions = "some_option=2;"; dbConfig.ReceiptsBlocksDbRocksDbOptions = "some_option=3;"; - PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, "")); + PerTableDbConfig config = new PerTableDbConfig(dbConfig, DbNames.Receipts); config.RocksDbOptions.Should().Be("some_option=1;some_option=2;"); } @@ -71,7 +71,7 @@ public void When_PerTableConfigIsNotAvailable_UseGeneralConfig() DbConfig dbConfig = new DbConfig(); dbConfig.MaxOpenFiles = 2; - PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, "")); + PerTableDbConfig config = new PerTableDbConfig(dbConfig, DbNames.Receipts); config.MaxOpenFiles.Should().Be(2); } diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 43b33836c2c..db2623d9eca 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -15,6 +15,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; +using Nethermind.Core.Test; using Nethermind.Db.Rocks; using Nethermind.Db.Rocks.Config; using Nethermind.Logging; @@ -29,12 +30,15 @@ namespace Nethermind.Db.Test [Parallelizable(ParallelScope.None)] public class DbOnTheRocksTests { + private RocksDbConfigFactory _rocksdbConfigFactory; string DbPath => "testdb/" + TestContext.CurrentContext.Test.Name; + [SetUp] public void Setup() { Directory.CreateDirectory(DbPath); + _rocksdbConfigFactory = new RocksDbConfigFactory(new DbConfig(), new PruningConfig(), new TestHardwareInfo(1.GiB()), LimboLogs.Instance); } [TearDown] @@ -47,7 +51,7 @@ public void TearDown() public void WriteOptions_is_correct() { IDbConfig config = new DbConfig(); - using DbOnTheRocks db = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, LimboLogs.Instance); + using DbOnTheRocks db = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, _rocksdbConfigFactory, LimboLogs.Instance); WriteOptions? options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority); Native.Instance.rocksdb_writeoptions_get_low_pri(options.Handle).Should().BeTrue(); @@ -66,7 +70,7 @@ public void WriteOptions_is_correct() public async Task Dispose_while_writing_does_not_cause_access_violation_exception() { IDbConfig config = new DbConfig(); - DbOnTheRocks db = new("testDispose1", GetRocksDbSettings("testDispose1", "TestDispose1"), config, LimboLogs.Instance); + DbOnTheRocks db = new("testDispose1", GetRocksDbSettings("testDispose1", "TestDispose1"), config, _rocksdbConfigFactory, LimboLogs.Instance); CancellationTokenSource cancelSource = new(); ManualResetEventSlim firstWriteWait = new(); @@ -108,7 +112,7 @@ public async Task Dispose_while_writing_does_not_cause_access_violation_exceptio public void Dispose_wont_cause_ObjectDisposedException_when_batch_is_still_open() { IDbConfig config = new DbConfig(); - DbOnTheRocks db = new("testDispose2", GetRocksDbSettings("testDispose2", "TestDispose2"), config, LimboLogs.Instance); + DbOnTheRocks db = new("testDispose2", GetRocksDbSettings("testDispose2", "TestDispose2"), config, _rocksdbConfigFactory, LimboLogs.Instance); _ = db.StartWriteBatch(); db.Dispose(); } @@ -119,7 +123,7 @@ public void CanOpenWithFileWarmer() IDbConfig config = new DbConfig(); config.EnableFileWarmer = true; { - using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, _rocksdbConfigFactory, LimboLogs.Instance); for (int i = 0; i < 1000; i++) { db[i.ToBigEndianByteArray()] = i.ToBigEndianByteArray(); @@ -127,7 +131,7 @@ public void CanOpenWithFileWarmer() } { - using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, _rocksdbConfigFactory, LimboLogs.Instance); } } @@ -141,7 +145,8 @@ public void CanOpenWithAdditionalConfig(string opts, bool success) Action act = () => { - using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + var configFactory = new RocksDbConfigFactory(config, new PruningConfig(), new TestHardwareInfo(1.GiB()), LimboLogs.Instance); + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, configFactory, LimboLogs.Instance); }; if (success) @@ -167,6 +172,7 @@ public void Corrupted_exception_on_open_would_create_marker() try { CorruptedDbOnTheRocks db = new("test", GetRocksDbSettings("test", "test"), config, + _rocksdbConfigFactory, LimboLogs.Instance, fileSystem: fileSystem); } @@ -195,7 +201,7 @@ public void If_marker_exists_on_open_then_repair_before_open() try { - DbOnTheRocks db = new(Path.Join(Path.GetTempPath(), "test"), GetRocksDbSettings("test", "test"), config, + DbOnTheRocks db = new(Path.Join(Path.GetTempPath(), "test"), GetRocksDbSettings("test", "test"), config, _rocksdbConfigFactory, LimboLogs.Instance, fileSystem: fileSystem, rocksDbNative: native); @@ -235,6 +241,8 @@ public DbOnTheRocksDbTests(bool useColumnDb) [SetUp] public void Setup() { + RocksDbConfigFactory rocksdbConfigFactory = new RocksDbConfigFactory(new DbConfig(), new PruningConfig(), new TestHardwareInfo(1.GiB()), LimboLogs.Instance); + if (Directory.Exists(DbPath)) { Directory.Delete(DbPath, true); @@ -244,7 +252,7 @@ public void Setup() if (_useColumnDb) { IDbConfig config = new DbConfig(); - ColumnsDb columnsDb = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, + ColumnsDb columnsDb = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, rocksdbConfigFactory, LimboLogs.Instance, new List() { ReceiptsColumns.Blocks }); _dbDisposable = columnsDb; @@ -253,7 +261,7 @@ public void Setup() else { IDbConfig config = new DbConfig(); - _db = new DbOnTheRocks(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, LimboLogs.Instance); + _db = new DbOnTheRocks(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, rocksdbConfigFactory, LimboLogs.Instance); _dbDisposable = _db; } } @@ -397,11 +405,12 @@ public CorruptedDbOnTheRocks( string basePath, DbSettings dbSettings, IDbConfig dbConfig, + IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IList? columnFamilies = null, RocksDbSharp.Native? rocksDbNative = null, IFileSystem? fileSystem = null - ) : base(basePath, dbSettings, dbConfig, logManager, columnFamilies, rocksDbNative, fileSystem) + ) : base(basePath, dbSettings, dbConfig, rocksDbConfigFactory, logManager, columnFamilies, rocksDbNative, fileSystem) { } diff --git a/src/Nethermind/Nethermind.Db.Test/RocksDbConfigFactoryTests.cs b/src/Nethermind/Nethermind.Db.Test/RocksDbConfigFactoryTests.cs new file mode 100644 index 00000000000..61222f9c404 --- /dev/null +++ b/src/Nethermind/Nethermind.Db.Test/RocksDbConfigFactoryTests.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test; +using Nethermind.Db.Rocks.Config; +using Nethermind.Logging; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Db.Test; + +[TestFixture] +[Parallelizable(ParallelScope.Self)] +public class RocksDbConfigFactoryTests +{ + [Test] + public void CanFetchNormally() + { + var dbConfig = new DbConfig(); + var factory = new RocksDbConfigFactory(dbConfig, new PruningConfig(), new TestHardwareInfo(0), LimboLogs.Instance); + IRocksDbConfig config = factory.GetForDatabase("State0", null); + Console.Error.WriteLine(config.RocksDbOptions); + config.RocksDbOptions.Should().Be(dbConfig.RocksDbOptions + dbConfig.StateDbRocksDbOptions); + } + + [Test] + public void WillOverrideStateConfigOnArchiveMode() + { + var dbConfig = new DbConfig(); + var pruningConfig = new PruningConfig(); + pruningConfig.Mode = PruningMode.Full; + var factory = new RocksDbConfigFactory(dbConfig, pruningConfig, new TestHardwareInfo(0), LimboLogs.Instance); + IRocksDbConfig config = factory.GetForDatabase("State0", null); + config.RocksDbOptions.Should().Be(dbConfig.RocksDbOptions + dbConfig.StateDbRocksDbOptions + dbConfig.StateDbArchiveModeRocksDbOptions); + } + + [Test] + public void WillOverrideStateConfigWhenMemoryIsHigh() + { + var dbConfig = new DbConfig(); + var factory = new RocksDbConfigFactory(dbConfig, new PruningConfig(), new TestHardwareInfo(100.GiB()), LimboLogs.Instance); + IRocksDbConfig config = factory.GetForDatabase("State0", null); + config.RocksDbOptions.Should().Be(dbConfig.RocksDbOptions + dbConfig.StateDbRocksDbOptions + dbConfig.StateDbLargeMemoryRocksDbOptions); + } + + [Test] + public void WillOverrideStateConfigWhenDirtyCachesTooHigh() + { + var dbConfig = new DbConfig(); + var pruningConfig = new PruningConfig(); + pruningConfig.CacheMb = 20000; + pruningConfig.DirtyCacheMb = 10000; + var factory = new RocksDbConfigFactory(dbConfig, pruningConfig, new TestHardwareInfo(0), LimboLogs.Instance); + IRocksDbConfig config = factory.GetForDatabase("State0", null); + config.WriteBufferSize.Should().Be((ulong)500.MB()); + } +} diff --git a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs index c47dfff4c42..e61cf1a1078 100644 --- a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs @@ -11,6 +11,7 @@ using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; +using Nethermind.Core.Test; using Nethermind.Core.Test.Db; using Nethermind.Db.FullPruning; using Nethermind.Db.Rocks; @@ -90,10 +91,12 @@ private Task InitializeStandardDb(bool useReceipts, bool useMemDb, DownloadReceiptsInFastSync = useReceipts })) .AddModule(new WorldStateModule(initConfig)) // For the full pruning db + .AddSingleton(new PruningConfig()) .AddSingleton(new DbConfig()) .AddSingleton(initConfig) .AddSingleton(LimboLogs.Instance) .AddSingleton() + .AddSingleton(new TestHardwareInfo(1)) .AddSingleton() .Build(); diff --git a/src/Nethermind/Nethermind.Init/Modules/DbModule.cs b/src/Nethermind/Nethermind.Init/Modules/DbModule.cs index e78d9604220..14b15978c58 100644 --- a/src/Nethermind/Nethermind.Init/Modules/DbModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/DbModule.cs @@ -9,6 +9,7 @@ using Nethermind.Core; using Nethermind.Db; using Nethermind.Db.Rocks; +using Nethermind.Db.Rocks.Config; using Nethermind.Db.Rpc; using Nethermind.JsonRpc.Client; using Nethermind.Logging; @@ -35,6 +36,7 @@ ISyncConfig syncConfig protected override void Load(ContainerBuilder builder) { builder + .AddSingleton() .AddSingleton() .AddSingleton() .AddScoped((dbProvider) => dbProvider.AsReadOnly(false)) diff --git a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs index 5ba9059520d..4c06f958dd5 100644 --- a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs @@ -70,6 +70,8 @@ protected override void Load(ContainerBuilder builder) .AddSingleton() .AddSingleton() .Add() // Not a singleton so that dispose is registered to correct lifetime + + .AddSingleton() ; if (!configProvider.GetConfig().BlobsSupport.IsPersistentStorage()) diff --git a/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs b/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs index 4326995923a..656c1d65101 100644 --- a/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs +++ b/src/Nethermind/Nethermind.Init/PruningTrieStateFactory.cs @@ -34,7 +34,6 @@ public class PruningTrieStateFactory( IInitConfig initConfig, IPruningConfig pruningConfig, IBlocksConfig blockConfig, - IDbConfig dbConfig, IDbProvider dbProvider, IBlockTree blockTree, IFileSystem fileSystem, @@ -54,8 +53,6 @@ ILogManager logManager { CompositePruningTrigger compositePruningTrigger = new CompositePruningTrigger(); - AdviseConfig(); - IPruningTrieStore trieStore = mainPruningTrieStoreFactory.PruningTrieStore; ITrieStore mainWorldTrieStore = trieStore; PreBlockCaches? preBlockCaches = null; @@ -123,36 +120,6 @@ ILogManager logManager return (stateManager, adminRpcModule); } - private void AdviseConfig() - { - // On a 7950x (32 logical coree), assuming write buffer is large enough, the pruning time is about 3 second - // with 8GB of pruning cache. Lets assume that this is a safe estimate as the ssd can be a limitation also. - long maximumCacheMb = Environment.ProcessorCount * 250; - // It must be at least 1GB as on mainnet at least 500MB will remain to support snap sync. So pruning cache only drop to about 500MB after pruning. - maximumCacheMb = Math.Max(1000, maximumCacheMb); - if (pruningConfig.CacheMb > maximumCacheMb) - { - // The user can also change `--Db.StateDbWriteBufferSize`. - // Which may or may not be better as each read will need to go through eacch write buffer. - // So having less of them is probably better.. - if (_logger.IsWarn) _logger.Warn($"Detected {pruningConfig.CacheMb}MB of pruning cache config. Pruning cache more than {maximumCacheMb}MB is not recommended with {Environment.ProcessorCount} logical core as it may cause long memory pruning time which affect attestation."); - } - - var totalWriteBufferMb = dbConfig.StateDbWriteBufferNumber * dbConfig.StateDbWriteBufferSize / (ulong)1.MB(); - var minimumWriteBufferMb = 0.2 * pruningConfig.CacheMb; - if (totalWriteBufferMb < minimumWriteBufferMb) - { - long minimumWriteBufferSize = (int)Math.Ceiling((minimumWriteBufferMb * 1.MB()) / dbConfig.StateDbWriteBufferNumber); - - if (_logger.IsWarn) _logger.Warn($"Detected {totalWriteBufferMb}MB of maximum write buffer size. Write buffer size should be at least 20% of pruning cache MB or memory pruning may slow down. Try setting `--Db.{nameof(dbConfig.StateDbWriteBufferSize)} {minimumWriteBufferSize}`."); - } - - if (pruningConfig.CacheMb <= pruningConfig.DirtyCacheMb) - { - throw new InvalidConfigurationException("Dirty pruning cache size must be less than persisted pruning cache size.", -1); - } - } - private void InitializeFullPruning(IDb stateDb, IStateReader stateReader, INodeStorage mainNodeStorage, @@ -215,13 +182,15 @@ public MainPruningTrieStoreFactory( IPruningConfig pruningConfig, IDbProvider dbProvider, INodeStorageFactory nodeStorageFactory, - IBlockTree blockTree, - IDisposableStack disposeStack, + IDbConfig dbConfig, + IHardwareInfo hardwareInfo, ILogManager logManager - ) + ) { _logger = logManager.GetClassLogger(); + AdviseConfig(pruningConfig, dbConfig, hardwareInfo); + if (syncConfig.SnapServingEnabled == true && pruningConfig.PruningBoundary < 128) { if (_logger.IsInfo) _logger.Info($"Snap serving enabled, but {nameof(pruningConfig.PruningBoundary)} is less than 128. Setting to 128."); @@ -259,9 +228,6 @@ ILogManager logManager if (stateDb is IFullPruningDb fullPruningDb) { - // PruningTriggerPersistenceStrategy triggerPersistenceStrategy = new(fullPruningDb, logManager); - // disposeStack.Push(triggerPersistenceStrategy); - // persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy); pruningStrategy = new PruningTriggerPruningStrategy(fullPruningDb, pruningStrategy); } @@ -275,5 +241,37 @@ ILogManager logManager logManager); } + private void AdviseConfig(IPruningConfig pruningConfig, IDbConfig dbConfig, IHardwareInfo hardwareInfo) + { + if (hardwareInfo.AvailableMemoryBytes >= IHardwareInfo.StateDbLargerMemoryThreshold) + { + // Default is 1280 MB, which translate to 280 MB of persisted cache memory (dirty node cache is 1000 MB). + // So this actually increase it from 280 MB to 1000 MB, reducing dirty node load at DB by 50%. + if (pruningConfig.CacheMb < 2000) + { + if (_logger.IsDebug) _logger.Debug($"Increasing pruning cache to 2 GB due to available additional memory."); + pruningConfig.CacheMb = 2000; + } + } + + // On a 7950x (32 logical coree), assuming write buffer is large enough, the pruning time is about 3 second + // with 8GB of pruning cache. Lets assume that this is a safe estimate as the ssd can be a limitation also. + long maximumDirtyCacheMb = Environment.ProcessorCount * 250; + // It must be at least 1GB as on mainnet at least 500MB will remain to support snap sync. So pruning cache only drop to about 500MB after pruning. + maximumDirtyCacheMb = Math.Max(1000, maximumDirtyCacheMb); + if (pruningConfig.DirtyCacheMb > maximumDirtyCacheMb) + { + // The user can also change `--Db.StateDbWriteBufferSize`. + // Which may or may not be better as each read will need to go through eacch write buffer. + // So having less of them is probably better.. + if (_logger.IsWarn) _logger.Warn($"Detected {pruningConfig.DirtyCacheMb}MB of dirty pruning cache config. Dirty cache more than {maximumDirtyCacheMb}MB is not recommended with {Environment.ProcessorCount} logical core as it may cause long memory pruning time which affect attestation."); + } + + if (pruningConfig.CacheMb <= pruningConfig.DirtyCacheMb) + { + throw new InvalidConfigurationException("Dirty pruning cache size must be less than persisted pruning cache size.", -1); + } + } + public IPruningTrieStore PruningTrieStore { get; } }