Feature/adjust rocksdb config by memory

src/Nethermind/Nethermind.Core.Test/Modules/TestEnvironmentModule.cs

@@ -8,6 +8,7 @@
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Config;
 using Nethermind.Consensus;
+using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
 using Nethermind.Db;
@@ -117,6 +118,8 @@ protected override void Load(ContainerBuilder builder)
                 pruningConfig.DirtyNodeShardBit = 1;
                 return pruningConfig;
             })
+
+            .AddSingleton<IHardwareInfo>(new TestHardwareInfo(1.GiB()))
             ;
     }
 }

src/Nethermind/Nethermind.Core.Test/TestHardwareInfo.cs

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Core.Test;
+
+public class TestHardwareInfo(long availableMemory = 10000000) : IHardwareInfo
+{
+    public long AvailableMemoryBytes => availableMemory;
+}

src/Nethermind/Nethermind.Core/HardwareInfo.cs

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+
+namespace Nethermind.Core;
+
+public class HardwareInfo : IHardwareInfo
+{
+    public long AvailableMemoryBytes { get; }
+
+    public HardwareInfo()
+    {
+        // Note: Not the same as memory capacity. This take into account current system memory pressure such as
+        // other process as well as OS level limit such as rlimit. Eh, its good enough.
+        AvailableMemoryBytes = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
+    }
+}

src/Nethermind/Nethermind.Core/IHardwareInfo.cs

@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Core.Extensions;
+
+namespace Nethermind.Core;
+
+public interface IHardwareInfo
+{
+    public static readonly long StateDbLargerMemoryThreshold = 20.GiB();
+    long AvailableMemoryBytes { get; }
+}

src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs

@@ -17,8 +17,8 @@ public class ColumnsDb<T> : DbOnTheRocks, IColumnsDb<T> where T : struct, Enum
 {
     private readonly IDictionary<T, ColumnDb> _columnDbs = new Dictionary<T, ColumnDb>();
 
-    public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, ILogManager logManager, IReadOnlyList<T> keys, IntPtr? sharedCache = null)
-        : base(basePath, settings, dbConfig, logManager, GetEnumKeys(keys).Select(static (key) => key.ToString()).ToList(), sharedCache: sharedCache)
+    public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IReadOnlyList<T> keys, IntPtr? sharedCache = null)
+        : base(basePath, settings, dbConfig, rocksDbConfigFactory, logManager, GetEnumKeys(keys).Select(static (key) => key.ToString()).ToList(), sharedCache: sharedCache)
     {
         keys = GetEnumKeys(keys);
 
@@ -61,7 +61,7 @@ private static IReadOnlyList<T> GetEnumKeys(IReadOnlyList<T> keys)
         return keys;
     }
 
-    protected override void BuildOptions<O>(PerTableDbConfig dbConfig, Options<O> options, IntPtr? sharedCache)
+    protected override void BuildOptions<O>(IRocksDbConfig dbConfig, Options<O> options, IntPtr? sharedCache)
     {
         base.BuildOptions(dbConfig, options, sharedCache);
         options.SetCreateMissingColumnFamilies();

src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Db.Rocks.Config;
+
+public class AdjustedRocksdbConfig(
+    IRocksDbConfig baseConfig,
+    string additionalRocksDbOptions,
+    ulong writeBufferSize
+) : IRocksDbConfig
+{
+    public ulong? WriteBufferSize => writeBufferSize;
+
+    public ulong? WriteBufferNumber => baseConfig.WriteBufferNumber;
+
+    // Note: not AdditionalRocksDbOptions so that user can still override option.
+    public string RocksDbOptions => baseConfig.RocksDbOptions + additionalRocksDbOptions;
+
+    public string AdditionalRocksDbOptions => baseConfig.AdditionalRocksDbOptions;
+
+    public int? MaxOpenFiles => baseConfig.MaxOpenFiles;
+
+    public bool WriteAheadLogSync => baseConfig.WriteAheadLogSync;
+
+    public ulong? ReadAheadSize => baseConfig.ReadAheadSize;
+
+    public bool EnableDbStatistics => baseConfig.EnableDbStatistics;
+
+    public uint StatsDumpPeriodSec => baseConfig.StatsDumpPeriodSec;
+
+    public bool? VerifyChecksum => baseConfig.VerifyChecksum;
+
+    public ulong? RowCacheSize => baseConfig.RowCacheSize;
+
+    public bool EnableFileWarmer => baseConfig.EnableFileWarmer;
+
+    public double CompressibilityHint => baseConfig.CompressibilityHint;
+
+    public bool FlushOnExit => baseConfig.FlushOnExit;
+}

src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs

@@ -158,7 +158,7 @@ public class DbConfig : IDbConfig
 
     public ulong? CodeDbRowCacheSize { get; set; } = (ulong)16.MiB();
     public string CodeDbRocksDbOptions { get; set; } =
-        "write_buffer_size=4000000;" +
+        "write_buffer_size=16000000;" +
         "block_based_table_factory.block_cache=16000000;" +
         "optimize_filters_for_hits=false;" +
         "prefix_extractor=capped:8;" +
@@ -234,6 +234,37 @@ public class DbConfig : IDbConfig
         "max_write_batch_group_size_bytes=4000000;" +
 
         "";
+
+    public string StateDbLargeMemoryRocksDbOptions { get; set; } =
+        // In large memory, we disable the partitioned block index. This took around additional 2GB of memory, but
+        // pretty reasonable reduction in latency especially when blocks are already cached.
+        // Note: Not for archive mode as the index size is proportional to db size.
+        "block_based_table_factory={index_type=kBinarySearch;partition_filters=0;};";
+
+    public string StateDbArchiveModeRocksDbOptions { get; set; } =
+        // For archive mode, we are mainly concerned with write amplification due to very large database.
+
+        // Lowers back the level multiplier as very large database causes very high write amp.
+        "max_bytes_for_level_multiplier=10;" +
+        "max_bytes_for_level_base=350000000;" +
+
+        // Change back file size multiplier as we dont want ridiculous file size, making compaction uneven,
+        // but set high base size. This mean a lot of file, but you are using archive mode, so this should be expected.
+        "target_file_size_multiplier=1;" +
+        "target_file_size_base=256000000;" +
+
+        // Change back restart interval for (probably slight) database size reduction
+        "block_based_table_factory.block_restart_interval=16;" +
+
+        // slight adjustment to util ratio, for db size.
+        "block_based_table_factory.data_block_hash_table_util_ratio=0.8;" +
+
+        // slight adjustment to block size, for potentially better db size.
+        "block_based_table_factory.block_size=64000;";
+
+    public ulong StateDbLargeMemoryWriteBufferSize { get; set; } = (ulong)128.MiB();
+    public ulong StateDbArchiveModeWriteBufferSize { get; set; } = (ulong)256.MiB();
+
     public string? StateDbAdditionalRocksDbOptions { get; set; }
 
     public string L1OriginDbRocksDbOptions { get; set; } = "";

src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs

@@ -93,6 +93,10 @@ public interface IDbConfig : IConfig
     double StateDbCompressibilityHint { get; set; }
     string StateDbRocksDbOptions { get; set; }
     string? StateDbAdditionalRocksDbOptions { get; set; }
+    string StateDbLargeMemoryRocksDbOptions { get; set; }
+    string StateDbArchiveModeRocksDbOptions { get; set; }
+    ulong StateDbLargeMemoryWriteBufferSize { get; set; }
+    ulong StateDbArchiveModeWriteBufferSize { get; set; }
 
 
     string L1OriginDbRocksDbOptions { get; set; }

src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Db.Rocks.Config;
+
+public interface IRocksDbConfig
+{
+    ulong? WriteBufferSize { get; }
+    ulong? WriteBufferNumber { get; }
+    string RocksDbOptions { get; }
+    string AdditionalRocksDbOptions { get; }
+    int? MaxOpenFiles { get; }
+    bool WriteAheadLogSync { get; }
+    ulong? ReadAheadSize { get; }
+    bool EnableDbStatistics { get; }
+    uint StatsDumpPeriodSec { get; }
+    bool? VerifyChecksum { get; }
+    ulong? RowCacheSize { get; }
+    bool EnableFileWarmer { get; }
+    double CompressibilityHint { get; }
+    bool FlushOnExit { get; }
+}

src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfigFactory.cs

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Db.Rocks.Config;
+
+public interface IRocksDbConfigFactory
+{
+    IRocksDbConfig GetForDatabase(string databaseName, string? columnName);
+}

src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs

@@ -10,20 +10,18 @@
 
 namespace Nethermind.Db.Rocks.Config;
 
-public class PerTableDbConfig
+public class PerTableDbConfig : IRocksDbConfig
 {
     private readonly string _tableName;
     private readonly string? _columnName;
     private readonly IDbConfig _dbConfig;
-    private readonly DbSettings _settings;
     private readonly string[] _prefixes;
     private readonly string[] _reversedPrefixes;
 
-    public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? columnName = null)
+    public PerTableDbConfig(IDbConfig dbConfig, string dbName, string? columnName = null)
     {
         _dbConfig = dbConfig;
-        _settings = dbSettings;
-        _tableName = _settings.DbName;
+        _tableName = dbName;
         _columnName = columnName;
         _prefixes = GetPrefixes();
         _reversedPrefixes = _prefixes.Reverse().ToArray();

src/Nethermind/Nethermind.Db.Rocks/Config/RocksDbConfigFactory.cs

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using Nethermind.Core;
+using Nethermind.Core.Extensions;
+using Nethermind.Logging;
+
+namespace Nethermind.Db.Rocks.Config;
+
+public class RocksDbConfigFactory(IDbConfig dbConfig, IPruningConfig pruningConfig, IHardwareInfo hardwareInfo, ILogManager logManager) : IRocksDbConfigFactory
+{
+    private readonly ILogger _logger = logManager.GetClassLogger<IRocksDbConfigFactory>();
+
+    public IRocksDbConfig GetForDatabase(string databaseName, string? columnName)
+    {
+        IRocksDbConfig rocksDbConfig = new PerTableDbConfig(dbConfig, databaseName, columnName);
+        if (databaseName.StartsWith("State"))
+        {
+            if (!pruningConfig.Mode.IsMemory())
+            {
+                if (_logger.IsInfo) _logger.Info($"Using archive mode State Db config.");
+
+                ulong writeBufferSize = rocksDbConfig.WriteBufferSize ?? 0;
+                if (writeBufferSize < dbConfig.StateDbArchiveModeWriteBufferSize) writeBufferSize = dbConfig.StateDbArchiveModeWriteBufferSize;
+
+                rocksDbConfig = new AdjustedRocksdbConfig(
+                    rocksDbConfig,
+                    dbConfig.StateDbArchiveModeRocksDbOptions!,
+                    writeBufferSize
+                );
+            }
+            else if (hardwareInfo.AvailableMemoryBytes >= IHardwareInfo.StateDbLargerMemoryThreshold)
+            {
+                if (_logger.IsInfo) _logger.Info($"Detected {hardwareInfo.AvailableMemoryBytes / 1.GiB()} GB of available memory. Applying large memory State Db config.");
+
+                ulong writeBufferSize = rocksDbConfig.WriteBufferSize ?? 0;
+                if (writeBufferSize < dbConfig.StateDbLargeMemoryWriteBufferSize) writeBufferSize = dbConfig.StateDbLargeMemoryWriteBufferSize;
+
+                rocksDbConfig = new AdjustedRocksdbConfig(
+                    rocksDbConfig,
+                    dbConfig.StateDbLargeMemoryRocksDbOptions!,
+                    writeBufferSize
+                );
+            }
+
+            if (pruningConfig.Mode.IsMemory())
+            {
+                ulong totalWriteBufferMb = rocksDbConfig.WriteBufferNumber!.Value * rocksDbConfig.WriteBufferSize!.Value / (ulong)1.MB();
+                double minimumWriteBufferMb = 0.2 * pruningConfig.DirtyCacheMb;
+                if (totalWriteBufferMb < minimumWriteBufferMb)
+                {
+                    ulong minimumWriteBufferSize = (ulong)Math.Ceiling((minimumWriteBufferMb * 1.MB()) / rocksDbConfig.WriteBufferNumber!.Value);
+
+                    if (_logger.IsInfo) _logger.Info($"Adjust state DB write buffer size to {minimumWriteBufferSize / (ulong)1.MB()} MB to account for pruning cache.");
+
+                    rocksDbConfig = new AdjustedRocksdbConfig(
+                        rocksDbConfig,
+                        "",
+                        minimumWriteBufferSize
+                    );
+                }
+            }
+
+        }
+
+        return rocksDbConfig;
+    }
+}

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -54,6 +54,7 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor
     internal ReadOptions? _readAheadReadOptions = null;
 
     internal DbOptions? DbOptions { get; private set; }
+    private readonly IRocksDbConfigFactory _rocksDbConfigFactory;
 
     public string Name { get; }
 
@@ -65,7 +66,7 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor
 
     private readonly DbSettings _settings;
 
-    private readonly PerTableDbConfig _perTableDbConfig;
+    private readonly IRocksDbConfig _perTableDbConfig;
     private ulong _maxBytesForLevelBase;
     private ulong _targetFileSizeBase;
     private int _minWriteBufferToMerge;
@@ -92,6 +93,7 @@ public DbOnTheRocks(
         string basePath,
         DbSettings dbSettings,
         IDbConfig dbConfig,
+        IRocksDbConfigFactory rocksDbConfigFactory,
         ILogManager logManager,
         IList<string>? columnFamilies = null,
         RocksDbSharp.Native? rocksDbNative = null,
@@ -103,7 +105,8 @@ public DbOnTheRocks(
         Name = _settings.DbName;
         _fileSystem = fileSystem ?? new FileSystem();
         _rocksDbNative = rocksDbNative ?? RocksDbSharp.Native.Instance;
-        _perTableDbConfig = new PerTableDbConfig(dbConfig, _settings);
+        _rocksDbConfigFactory = rocksDbConfigFactory;
+        _perTableDbConfig = rocksDbConfigFactory.GetForDatabase(Name, null);
         _db = Init(basePath, dbSettings.DbPath, dbConfig, logManager, columnFamilies, dbSettings.DeleteOnStart, sharedCache);
         _iteratorManager = new IteratorManager(_db, null, _readAheadReadOptions);
     }
@@ -151,7 +154,8 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan
                     string columnFamily = enumColumnName;
 
                     ColumnFamilyOptions options = new();
-                    BuildOptions(new PerTableDbConfig(dbConfig, _settings, columnFamily), options, sharedCache);
+                    IRocksDbConfig columnConfig = _rocksDbConfigFactory.GetForDatabase(Name, columnFamily);
+                    BuildOptions(columnConfig, options, sharedCache);
 
                     // "default" is a special column name with rocksdb, which is what previously not specifying column goes to
                     if (columnFamily == "Default") columnFamily = "default";
@@ -430,7 +434,7 @@ public static IDictionary<string, string> ExtractOptions(string dbOptions)
         return asDict;
     }
 
-    protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> options, IntPtr? sharedCache) where T : Options<T>
+    protected virtual void BuildOptions<T>(IRocksDbConfig dbConfig, Options<T> options, IntPtr? sharedCache) where T : Options<T>
     {
         // This section is about the table factory.. and block cache apparently.
         // This effect the format of the SST files and usually require resync to take effect.
@@ -563,6 +567,7 @@ protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> opt
         #endregion
 
         #region read-write options
+        // TODO: These are not applied to column family
         WriteOptions = CreateWriteOptions(dbConfig);
 
         _noWalWrite = CreateWriteOptions(dbConfig);
@@ -597,7 +602,7 @@ protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> opt
         #endregion
     }
 
-    private static WriteOptions CreateWriteOptions(PerTableDbConfig dbConfig)
+    private static WriteOptions CreateWriteOptions(IRocksDbConfig dbConfig)
     {
         WriteOptions options = new();
         // potential fix for corruption on hard process termination, may cause performance degradation