Fixes to ilevm (ethereum tests)

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -90,7 +90,7 @@ public interface IInitConfig : IConfig
     [ConfigItem(Description = "[TECHNICAL] Exit when block number is reached. Useful for scripting and testing.", DefaultValue = "null", HiddenFromDocs = true)]
     long? ExitOnBlockNumber { get; set; }
 
-    [ConfigItem(Description = "[TECHNICAL] Specify concurrency limit for background task.", DefaultValue = "1", HiddenFromDocs = true)]
+    [ConfigItem(Description = "[TECHNICAL] Specify concurrency limit for background task.", DefaultValue = "2", HiddenFromDocs = true)]
     int BackgroundTaskConcurrency { get; set; }
 
     [ConfigItem(Description = "[TECHNICAL] Specify max number of background task.", DefaultValue = "1024", HiddenFromDocs = true)]

src/Nethermind/Nethermind.Api/IMainProcessingContext.cs

@@ -4,8 +4,8 @@
 using System.Threading.Tasks;
 using Nethermind.Consensus.Processing;
 using Nethermind.Core.ServiceStopper;
+using Nethermind.Evm.State;
 using Nethermind.Evm.TransactionProcessing;
-using Nethermind.State;
 
 namespace Nethermind.Api;
 
@@ -21,7 +21,7 @@ public interface IMainProcessingContext : IStoppableService
     ITransactionProcessor TransactionProcessor { get; }
     IBlockProcessor BlockProcessor { get; }
     IBlockchainProcessor BlockchainProcessor { get; }
-    IVisitingWorldState WorldState { get; }
+    IWorldState WorldState { get; }
 
     Task IStoppableService.StopAsync() => BlockchainProcessor.StopAsync();
     string IStoppableService.Description => "blockchain processor";

src/Nethermind/Nethermind.Api/InitConfig.cs

@@ -36,7 +36,7 @@ public class InitConfig : IInitConfig
         public bool DisableMallocOpts { get; set; } = false;
         public INodeStorage.KeyScheme StateDbKeyScheme { get; set; } = INodeStorage.KeyScheme.Current;
         public long? ExitOnBlockNumber { get; set; } = null;
-        public int BackgroundTaskConcurrency { get; set; } = 1;
+        public int BackgroundTaskConcurrency { get; set; } = 2;
         public int BackgroundTaskMaxNumber { get; set; } = 1024;
         public bool InRunnerTest { get; set; } = false;
 

src/Nethermind/Nethermind.Api/MainProcessingContext.cs

@@ -2,16 +2,16 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using Nethermind.Consensus.Processing;
+using Nethermind.Evm.State;
 using Nethermind.Evm.TransactionProcessing;
-using Nethermind.State;
 
 namespace Nethermind.Api;
 
 public record MainProcessingContext(
     ITransactionProcessor TransactionProcessor,
     IBlockProcessor BlockProcessor,
     IBlockchainProcessor BlockchainProcessor,
-    IVisitingWorldState WorldState
+    IWorldState WorldState
 ) : IMainProcessingContext
 {
 }

src/Nethermind/Nethermind.AuRa.Test/Contract/TestContractBlockchain.cs

@@ -49,8 +49,10 @@ protected override Block GetGenesisBlock(IWorldState worldState) =>
             new GenesisLoader(
                     ChainSpec,
                     SpecProvider,
+                    StateReader,
                     worldState,
-                    TxProcessor)
+                    TxProcessor,
+                    LogManager)
                 .Load();
     }
 }

src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs

@@ -30,14 +30,14 @@ public class ValidatorContractTests
         private readonly Address _contractAddress = Address.FromNumber(long.MaxValue);
         private ITransactionProcessor _transactionProcessor;
         private IReadOnlyTxProcessorSource _readOnlyTxProcessorSource;
-        private IVisitingWorldState _stateProvider;
+        private IWorldState _stateProvider;
 
         [SetUp]
         public void SetUp()
         {
             _block = new Block(Build.A.BlockHeader.WithStateRoot(TestItem.KeccakA).TestObject, new BlockBody());
             _transactionProcessor = Substitute.For<ITransactionProcessor>();
-            _stateProvider = Substitute.For<IVisitingWorldState>();
+            _stateProvider = Substitute.For<IWorldState>();
             _readOnlyTxProcessorSource = Substitute.For<IReadOnlyTxProcessorSource>();
             _readOnlyTxProcessorSource.Build(_block.Header).Returns(new ReadOnlyTxProcessingScope(_transactionProcessor, _stateProvider));
         }

src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs

@@ -38,7 +38,7 @@ namespace Nethermind.AuRa.Test.Validators;
 
 public class ContractBasedValidatorTests
 {
-    private IVisitingWorldState _stateProvider;
+    private IWorldState _stateProvider;
     private IAbiEncoder _abiEncoder;
     private ILogManager _logManager;
     private AuRaParameters.Validator _validator;
@@ -62,7 +62,7 @@ public void SetUp()
     {
         _validatorStore = new ValidatorStore(new MemDb());
         _validSealerStrategy = new ValidSealerStrategy();
-        _stateProvider = Substitute.For<IVisitingWorldState>();
+        _stateProvider = Substitute.For<IWorldState>();
         _abiEncoder = Substitute.For<IAbiEncoder>();
         _logManager = LimboLogs.Instance;
         _blockTree = Substitute.For<IBlockTree>();

src/Nethermind/Nethermind.Blockchain.Test/GenesisLoaderTests.cs

@@ -60,7 +60,7 @@ private Block GetGenesisBlock(string chainspecPath)
         ISpecProvider specProvider = Substitute.For<ISpecProvider>();
         specProvider.GetSpec(Arg.Any<ForkActivation>()).Returns(Berlin.Instance);
         ITransactionProcessor transactionProcessor = Substitute.For<ITransactionProcessor>();
-        GenesisLoader genesisLoader = new(chainSpec, specProvider, stateProvider, transactionProcessor);
+        GenesisLoader genesisLoader = new(chainSpec, specProvider, worldStateManager.GlobalStateReader, stateProvider, transactionProcessor, LimboLogs.Instance);
         return genesisLoader.Load();
     }
 

src/Nethermind/Nethermind.Blockchain/GenesisLoader.cs

@@ -6,67 +6,94 @@
 using System.Linq;
 using Nethermind.Blockchain.Tracing;
 using Nethermind.Core;
+using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
 using Nethermind.Int256;
 using Nethermind.Evm;
 using Nethermind.Evm.State;
 using Nethermind.Evm.TransactionProcessing;
+using Nethermind.Logging;
 using Nethermind.Specs.ChainSpecStyle;
+using Nethermind.State;
+using Nethermind.Trie;
 
 namespace Nethermind.Blockchain
 {
     public class GenesisLoader(
         ChainSpec chainSpec,
         ISpecProvider specProvider,
+        IStateReader stateReader,
         IWorldState stateProvider,
-        ITransactionProcessor transactionProcessor)
+        ITransactionProcessor transactionProcessor,
+        ILogManager logManager,
+        Hash256? expectedGenesisHash = null
+    )
     {
-        private readonly ChainSpec _chainSpec = chainSpec ?? throw new ArgumentNullException(nameof(chainSpec));
-        private readonly ISpecProvider _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider));
-        private readonly IWorldState _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider));
-        private readonly ITransactionProcessor _transactionProcessor = transactionProcessor ?? throw new ArgumentNullException(nameof(transactionProcessor));
+        ILogger _logger = logManager.GetClassLogger<GenesisLoader>();
 
         public Block Load()
         {
-            Block genesis = _chainSpec.Genesis;
+            Block genesis = chainSpec.Genesis;
             Preallocate(genesis);
 
             // we no longer need the allocations - 0.5MB RAM, 9000 objects for mainnet
-            _chainSpec.Allocations = null;
+            chainSpec.Allocations = null;
 
-            if (!_chainSpec.GenesisStateUnavailable)
+            if (!chainSpec.GenesisStateUnavailable)
             {
-                _stateProvider.Commit(_specProvider.GenesisSpec, true);
+                stateProvider.Commit(specProvider.GenesisSpec, true);
 
-                _stateProvider.CommitTree(0);
+                stateProvider.CommitTree(0);
 
-                genesis.Header.StateRoot = _stateProvider.StateRoot;
+                genesis.Header.StateRoot = stateProvider.StateRoot;
             }
 
             genesis.Header.Hash = genesis.Header.CalculateHash();
 
+            ValidateGenesisHash(expectedGenesisHash, genesis.Header);
+
             return genesis;
         }
 
+        /// <summary>
+        /// If <paramref name="expectedGenesisHash"/> is <value>null</value> then it means that we do not care about the genesis hash (e.g. in some quick testing of private chains)/>
+        /// </summary>
+        /// <param name="expectedGenesisHash"></param>
+        private void ValidateGenesisHash(Hash256? expectedGenesisHash, BlockHeader genesis)
+        {
+            if (expectedGenesisHash is not null && genesis.Hash != expectedGenesisHash)
+            {
+                if (_logger.IsTrace) _logger.Trace(stateReader.DumpState(genesis.StateRoot!));
+                if (_logger.IsWarn) _logger.Warn(genesis.ToString(BlockHeader.Format.Full));
+                if (_logger.IsError) _logger.Error($"Unexpected genesis hash, expected {expectedGenesisHash}, but was {genesis.Hash}");
+            }
+            else
+            {
+                if (_logger.IsDebug) _logger.Info($"Genesis hash :  {genesis.Hash}");
+            }
+
+            ThisNodeInfo.AddInfo("Genesis hash :", $"{genesis.Hash}");
+        }
+
         private void Preallocate(Block genesis)
         {
-            _transactionProcessor.SetBlockExecutionContext(new BlockExecutionContext(genesis.Header, specProvider.GetSpec(genesis.Header)));
-            foreach ((Address address, ChainSpecAllocation allocation) in _chainSpec.Allocations.OrderBy(static a => a.Key))
+            transactionProcessor.SetBlockExecutionContext(new BlockExecutionContext(genesis.Header, specProvider.GetSpec(genesis.Header)));
+            foreach ((Address address, ChainSpecAllocation allocation) in chainSpec.Allocations.OrderBy(static a => a.Key))
             {
-                _stateProvider.CreateAccount(address, allocation.Balance, allocation.Nonce);
+                stateProvider.CreateAccount(address, allocation.Balance, allocation.Nonce);
 
                 if (allocation.Code is not null)
                 {
-                    _stateProvider.InsertCode(address, allocation.Code, _specProvider.GenesisSpec, true);
+                    stateProvider.InsertCode(address, allocation.Code, specProvider.GenesisSpec, true);
                 }
 
                 if (allocation.Storage is not null)
                 {
                     foreach (KeyValuePair<UInt256, byte[]> storage in allocation.Storage)
                     {
-                        _stateProvider.Set(new StorageCell(address, storage.Key),
+                        stateProvider.Set(new StorageCell(address, storage.Key),
                             storage.Value.WithoutLeadingZeros().ToArray());
                     }
                 }
@@ -81,7 +108,7 @@ private void Preallocate(Block genesis)
                     };
 
                     CallOutputTracer outputTracer = new();
-                    _transactionProcessor.Execute(constructorTransaction, outputTracer);
+                    transactionProcessor.Execute(constructorTransaction, outputTracer);
 
                     if (outputTracer.StatusCode != StatusCode.Success)
                     {

src/Nethermind/Nethermind.Blockchain/IReadOnlyTxProcessingScope.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using Nethermind.Evm.State;
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.State;
 
@@ -10,6 +11,6 @@ namespace Nethermind.Blockchain;
 public interface IReadOnlyTxProcessingScope : IDisposable
 {
     ITransactionProcessor TransactionProcessor { get; }
-    IVisitingWorldState WorldState { get; }
+    IWorldState WorldState { get; }
     void Reset() => Dispose();
 }

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -171,6 +171,9 @@ public interface ISyncConfig : IConfig
     [ConfigItem(Description = "_Technical._ Enable storage range split.", DefaultValue = "false", HiddenFromDocs = true)]
     bool EnableSnapSyncStorageRangeSplit { get; set; }
 
-    [ConfigItem(Description = "_Technical._ Max tx in forward sync buffer.", DefaultValue = "200000", HiddenFromDocs = true)]
-    int MaxTxInForwardSyncBuffer { get; set; }
+    [ConfigItem(Description = "_Technical._ Estimated size of memory for storing blocks during download.", DefaultValue = "200000000", HiddenFromDocs = true)]
+    long ForwardSyncDownloadBufferMemoryBudget { get; set; }
+
+    [ConfigItem(Description = "_Technical._ Estimated max size of blocks in block processing queue before stop downloading.", DefaultValue = "200000000", HiddenFromDocs = true)]
+    long ForwardSyncBlockProcessingQueueMemoryBudget { get; set; }
 }

src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs

@@ -83,7 +83,8 @@ public string? PivotHash
 
         public ulong FastHeadersMemoryBudget { get; set; } = (ulong)128.MB();
         public bool EnableSnapSyncStorageRangeSplit { get; set; } = false;
-        public int MaxTxInForwardSyncBuffer { get; set; } = 20000;
+        public long ForwardSyncDownloadBufferMemoryBudget { get; set; } = 200.MiB();
+        public long ForwardSyncBlockProcessingQueueMemoryBudget { get; set; } = 200.MiB();
 
         public override string ToString()
         {

src/Nethermind/Nethermind.Consensus.AuRa/Validators/PendingValidatorsDecoder.cs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
-using System.Linq;
 using Nethermind.Core;
 using Nethermind.Serialization.Rlp;
 
@@ -84,6 +83,11 @@ private static (int Total, int Addresses) GetContentLength(PendingValidators ite
             return (contentLength, addressesLength);
         }
 
-        private static int GetAddressesLength(Address[] addresses) => addresses.Sum(Rlp.LengthOf);
+        private static int GetAddressesLength(Address[] addresses)
+        {
+            const int AddressLengthWithRlpLengthPrefix = 1 + 20;
+
+            return addresses.Length * AddressLengthWithRlpLengthPrefix;
+        }
     }
 }

src/Nethermind/Nethermind.Consensus.Test/ReadOnlyTxProcessingScopeTests.cs

@@ -18,7 +18,7 @@ public void Test_WhenDispose_ThenStateRootWillReset()
     {
         ReadOnlyTxProcessingScope env = new ReadOnlyTxProcessingScope(
             Substitute.For<ITransactionProcessor>(),
-            Substitute.For<IVisitingWorldState>());
+            Substitute.For<IWorldState>());
 
         env.Dispose();
 

src/Nethermind/Nethermind.Consensus.Test/Scheduler/BackgroundTaskSchedulerTests.cs

@@ -9,6 +9,7 @@
 using Nethermind.Consensus.Scheduler;
 using Nethermind.Core.Extensions;
 using Nethermind.Logging;
+using Nethermind.TxPool;
 using NSubstitute;
 using NUnit.Framework;
 using TaskCompletionSource = DotNetty.Common.Concurrency.TaskCompletionSource;
@@ -18,18 +19,21 @@ namespace Nethermind.Consensus.Test.Scheduler;
 public class BackgroundTaskSchedulerTests
 {
     private IBlockProcessor _blockProcessor;
+    private IChainHeadInfoProvider _chainHeadInfo;
 
     [SetUp]
     public void Setup()
     {
         _blockProcessor = Substitute.For<IBlockProcessor>();
+        _chainHeadInfo = Substitute.For<IChainHeadInfoProvider>();
+        _chainHeadInfo.IsSyncing.Returns(false);
     }
 
     [Test]
     public async Task Test_task_will_execute()
     {
         TaskCompletionSource tcs = new TaskCompletionSource();
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 1, 65536, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, _chainHeadInfo, 1, 65536, LimboLogs.Instance);
 
         scheduler.ScheduleTask(1, (_, token) =>
         {
@@ -43,7 +47,7 @@ public async Task Test_task_will_execute()
     [Test]
     public async Task Test_task_will_execute_concurrently_when_configured_so()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, _chainHeadInfo, 2, 65536, LimboLogs.Instance);
 
         int counter = 0;
 
@@ -68,7 +72,7 @@ public async Task Test_task_will_execute_concurrently_when_configured_so()
     [Test]
     public async Task Test_task_will_cancel_on_block_processing()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, _chainHeadInfo, 2, 65536, LimboLogs.Instance);
 
         bool wasCancelled = false;
 
@@ -96,7 +100,7 @@ public async Task Test_task_will_cancel_on_block_processing()
     [Test]
     public async Task Test_task_that_is_scheduled_during_block_processing_will_continue_after()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, _chainHeadInfo, 2, 65536, LimboLogs.Instance);
         _blockProcessor.BlocksProcessing += Raise.EventWith(new BlocksProcessingEventArgs(null));
 
         int executionCount = 0;
@@ -119,7 +123,7 @@ public async Task Test_task_that_is_scheduled_during_block_processing_will_conti
     [Test]
     public async Task Test_task_that_is_scheduled_during_block_processing_but_deadlined_will_get_called_and_cancelled()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, _chainHeadInfo, 2, 65536, LimboLogs.Instance);
         _blockProcessor.BlocksProcessing += Raise.EventWith(new BlocksProcessingEventArgs(null));
 
         bool wasCancelled = false;