Flat/snap sync

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -165,7 +165,7 @@ public interface ISyncConfig : IConfig
     [ConfigItem(Description = "_Technical._ Memory budget for in memory dependencies of fast headers.", DefaultValue = "0", HiddenFromDocs = true)]
     ulong FastHeadersMemoryBudget { get; set; }
 
-    [ConfigItem(Description = "_Technical._ Enable storage range split.", DefaultValue = "false", HiddenFromDocs = true)]
+    [ConfigItem(Description = "_Technical._ Enable storage range split.", DefaultValue = "true", HiddenFromDocs = true)]
     bool EnableSnapSyncStorageRangeSplit { get; set; }
 
     [ConfigItem(Description = "_Technical._ Estimated size of memory for storing blocks during download.", DefaultValue = "200000000", HiddenFromDocs = true)]

src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs

@@ -84,7 +84,7 @@ public string? PivotHash
         public int HeaderStateDistance { get; set; } = 0;
 
         public ulong FastHeadersMemoryBudget { get; set; } = (ulong)128.MB();
-        public bool EnableSnapSyncStorageRangeSplit { get; set; } = false;
+        public bool EnableSnapSyncStorageRangeSplit { get; set; } = true;
         public long ForwardSyncDownloadBufferMemoryBudget { get; set; } = 200.MiB();
         public long ForwardSyncBlockProcessingQueueMemoryBudget { get; set; } = 200.MiB();
 

src/Nethermind/Nethermind.Core/Crypto/Hash256.cs

@@ -97,6 +97,46 @@ public string ToShortString(bool withZeroX = true)
         public UInt256 ToUInt256(bool isBigEndian = true) => new UInt256(Bytes, isBigEndian: isBigEndian);
         public Hash256 ToHash256() => new Hash256(this);
         private bool IsZero => _bytes == default;
+
+        public ValueHash256 IncrementPath()
+        {
+            ValueHash256 result = this;
+            Span<byte> bytes = result.BytesAsSpan;
+
+            for (int i = 31; i >= 0; i--)
+            {
+                if (bytes[i] < 0xFF)
+                {
+                    bytes[i]++;
+                    return result;
+                }
+                bytes[i] = 0x00;
+            }
+
+            // Overflow - return max (shouldn't happen in practice)
+            result = ValueKeccak.Zero;
+            result.BytesAsSpan.Fill(0xFF);
+            return result;
+        }
+
+        public ValueHash256 DecrementPath()
+        {
+            ValueHash256 result = this;
+            Span<byte> bytes = result.BytesAsSpan;
+
+            for (int i = 31; i >= 0; i--)
+            {
+                if (bytes[i] > 0)
+                {
+                    bytes[i]--;
+                    return result;
+                }
+                bytes[i] = 0xFF;
+            }
+
+            // Underflow - return zero (shouldn't happen in practice)
+            return ValueKeccak.Zero;
+        }
     }
 
     public readonly struct Hash256AsKey(Hash256 key) : IEquatable<Hash256AsKey>, IComparable<Hash256AsKey>

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -85,6 +85,36 @@ public static int Compare(ReadOnlySpan<byte> x, ReadOnlySpan<byte> y)
             {
                 return x.SequenceCompareTo(y);
             }
+
+            public static int CompareWithCorrectLength(ReadOnlySpan<byte> x, ReadOnlySpan<byte> y)
+            {
+                if (Unsafe.AreSame(ref MemoryMarshal.GetReference(x), ref MemoryMarshal.GetReference(y)) &&
+                    x.Length == y.Length)
+                {
+                    return 0;
+                }
+
+                if (x.Length == 0)
+                {
+                    return y.Length == 0 ? 0 : -1;  // empty < non-empty
+                }
+
+                for (int i = 0; i < x.Length; i++)
+                {
+                    if (y.Length <= i)
+                    {
+                        return 1;  // x is longer, so x > y
+                    }
+
+                    int result = x[i].CompareTo(y[i]);
+                    if (result != 0)
+                    {
+                        return result;
+                    }
+                }
+
+                return y.Length > x.Length ? 1 : 0;
+            }
         }
 
         public static readonly byte[] Zero32 = new byte[32];

src/Nethermind/Nethermind.Db/InMemoryColumnBatch.cs

@@ -1,14 +1,14 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using System.Collections.Generic;
+using System.Collections.Concurrent;
 using Nethermind.Core;
 
 namespace Nethermind.Db
 {
     public class InMemoryColumnWriteBatch<TKey> : IColumnsWriteBatch<TKey>
     {
-        private readonly IList<IWriteBatch> _underlyingBatch = new List<IWriteBatch>();
+        private readonly ConcurrentDictionary<TKey, IWriteBatch> _writeBatches = new();
         private readonly IColumnsDb<TKey> _columnsDb;
 
         public InMemoryColumnWriteBatch(IColumnsDb<TKey> columnsDb)
@@ -18,14 +18,12 @@ public InMemoryColumnWriteBatch(IColumnsDb<TKey> columnsDb)
 
         public IWriteBatch GetColumnBatch(TKey key)
         {
-            InMemoryWriteBatch writeBatch = new InMemoryWriteBatch(_columnsDb.GetColumnDb(key));
-            _underlyingBatch.Add(writeBatch);
-            return writeBatch;
+            return _writeBatches.GetOrAdd(key, key => new InMemoryWriteBatch(_columnsDb.GetColumnDb(key)));
         }
 
         public void Dispose()
         {
-            foreach (IWriteBatch batch in _underlyingBatch)
+            foreach (IWriteBatch batch in _writeBatches.Values)
             {
                 batch.Dispose();
             }

src/Nethermind/Nethermind.Db/InMemoryWriteBatch.cs

@@ -2,16 +2,16 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using Nethermind.Core;
+using Nethermind.Core.Collections;
 
 namespace Nethermind.Db
 {
     public class InMemoryWriteBatch : IWriteBatch
     {
         private readonly IKeyValueStore _store;
-        private readonly ConcurrentDictionary<byte[], byte[]?> _currentItems = new();
+        // Note: need to keep order of operation
+        private readonly ArrayPoolList<(byte[] Key, byte[]? Value)> _writes = new(1);
         private WriteFlags _writeFlags = WriteFlags.None;
 
         public InMemoryWriteBatch(IKeyValueStore storeWithNoBatchSupport)
@@ -21,22 +21,23 @@ public InMemoryWriteBatch(IKeyValueStore storeWithNoBatchSupport)
 
         public void Dispose()
         {
-            foreach (KeyValuePair<byte[], byte[]?> keyValuePair in _currentItems)
+            foreach ((byte[] Key, byte[]? Value) item in _writes)
             {
-                _store.Set(keyValuePair.Key, keyValuePair.Value, _writeFlags);
+                _store.Set(item.Key, item.Value, _writeFlags);
             }
 
+            _writes.Dispose();
             GC.SuppressFinalize(this);
         }
 
         public void Clear()
         {
-            _currentItems.Clear();
+            _writes.Clear();
         }
 
         public void Set(ReadOnlySpan<byte> key, byte[]? value, WriteFlags flags = WriteFlags.None)
         {
-            _currentItems[key.ToArray()] = value;
+            _writes.Add((key.ToArray(), value));
             _writeFlags = flags;
         }
 

src/Nethermind/Nethermind.Init/Modules/FlatWorldStateModule.cs

@@ -22,6 +22,10 @@
 using Nethermind.State.Flat;
 using Nethermind.State.Flat.Persistence;
 using Nethermind.State.Flat.ScopeProvider;
+using Nethermind.State.Flat.Sync;
+using Nethermind.Synchronization.FastSync;
+using Nethermind.Synchronization.ParallelSync;
+using Nethermind.Synchronization.SnapSync;
 
 namespace Nethermind.Init.Modules;
 
@@ -66,6 +70,18 @@ protected override void Load(ContainerBuilder builder)
             .AddSingleton<TrieWarmer>()
             .Add<FlatOverridableWorldScope>()
 
+            // Sync components
+            .AddSingleton<ISnapTrieFactory, FlatSnapTrieFactory>()
+            .AddSingleton<IFlatStateRootIndex>((ctx) => new FlatStateRootIndex(
+                ctx.Resolve<IBlockTree>(),
+                ctx.Resolve<ISyncConfig>().SnapServingMaxDepth))
+            .AddSingleton<ITreeSyncStore, FlatTreeSyncStore>()
+            .Intercept<ISyncConfig>((syncConfig) =>
+            {
+                syncConfig.SnapServingEnabled = true;
+            })
+            .AddSingleton<IFullStateFinder, FlatFullStateFinder>()
+
             // Persistences
             .AddColumnDatabase<FlatDbColumns>(DbNames.Flat)
             .AddSingleton<RocksDbPersistence>()
@@ -101,21 +117,6 @@ protected override void Load(ContainerBuilder builder)
                 .AddSingleton<Importer>()
                 .AddStep(typeof(ImportFlatDb));
         }
-        else
-        {
-            builder
-                .AddDecorator<ISyncConfig>((ctx, syncConfig) =>
-                {
-                    ILogger logger = ctx.Resolve<ILogManager>().GetClassLogger<FlatWorldStateModule>();
-                    if (syncConfig.FastSync || syncConfig.SnapSync)
-                    {
-                        if (logger.IsWarn) logger.Warn("Fast sync and snap sync turned off with FlatDB");
-                        syncConfig.FastSync = false;
-                        syncConfig.SnapSync = false;
-                    }
-                    return syncConfig;
-                });
-        }
     }
 
     /// <summary>

src/Nethermind/Nethermind.Runner/packages.lock.json

@@ -1121,11 +1121,13 @@
       "nethermind.state.flat": {
         "type": "Project",
         "dependencies": {
+          "Nethermind.Blockchain": "[1.37.0-unstable, )",
           "Nethermind.Core": "[1.37.0-unstable, )",
           "Nethermind.Db": "[1.37.0-unstable, )",
           "Nethermind.Evm": "[1.37.0-unstable, )",
           "Nethermind.Serialization.Rlp": "[1.37.0-unstable, )",
           "Nethermind.State": "[1.37.0-unstable, )",
+          "Nethermind.Synchronization": "[1.37.0-unstable, )",
           "Nethermind.Trie": "[1.37.0-unstable, )",
           "System.IO.Hashing": "[10.0.2, )"
         }