Fixes for 4884 tests in hive: simulators/eth/pyspec

src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs

@@ -28,7 +28,7 @@ public void When_more_uncles_than_allowed_returns_false()
             bool noiseRemoved = blockValidator.ValidateSuggestedBlock(Build.A.Block.TestObject);
             Assert.True(noiseRemoved);
 
-            bool result = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithUncles(Build.A.BlockHeader.TestObject).TestObject);
+            bool result = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithUncles(Build.A.BlockHeader.TestObject).TestObject, Build.An.EmptyBlockHeader);
             Assert.False(result);
         }
     }

src/Nethermind/Nethermind.Blockchain.Test/Validators/ShardBlobBlockValidatorTests.cs

@@ -24,7 +24,7 @@ public void Not_null_ExcessDataGas_is_invalid_pre_cancun()
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
             .WithWithdrawalsRoot(TestItem.KeccakA)
             .WithWithdrawals(TestItem.WithdrawalA_1Eth)
-            .WithExcessDataGas(1).TestObject);
+            .WithExcessDataGas(1).TestObject, Build.An.EmptyBlockHeader);
         Assert.False(isValid);
     }
 
@@ -36,7 +36,7 @@ public void Null_ExcessDataGas_is_invalid_post_cancun()
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
             .WithWithdrawalsRoot(TestItem.KeccakA)
             .WithWithdrawals(TestItem.WithdrawalA_1Eth)
-            .TestObject);
+            .TestObject, Build.An.EmptyBlockHeader);
         Assert.False(isValid);
     }
 
@@ -54,6 +54,6 @@ public bool Blobs_per_block_count_is_valid(int blobsCount)
                 .WithWithdrawals(TestItem.WithdrawalA_1Eth)
                 .WithExcessDataGas(IntrinsicGasCalculator.CalculateExcessDataGas(0, blobsCount, specProvider.GenesisSpec)!.Value)
                 .WithTransactions(Build.A.Transaction.WithBlobVersionedHashes(blobsCount).TestObject)
-                .TestObject);
+                .TestObject, Build.An.EmptyBlockHeader);
     }
 }

src/Nethermind/Nethermind.Blockchain.Test/Validators/TestBlockValidator.cs

@@ -39,7 +39,7 @@ public bool Validate(BlockHeader header, bool isUncle)
         return _alwaysSameResultForSuggested ?? _suggestedValidationResults.Dequeue();
     }
 
-    public bool ValidateSuggestedBlock(Block block)
+    public bool ValidateSuggestedBlock(Block block, BlockHeader? parentBlockHeader)
     {
         return _alwaysSameResultForSuggested ?? _suggestedValidationResults.Dequeue();
     }

src/Nethermind/Nethermind.Blockchain.Test/Validators/WithdrawalValidatorTests.cs

@@ -42,7 +42,7 @@ public void Withdrawals_with_incorrect_withdrawals_root_are_invalid()
         BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { TestItem.WithdrawalA_1Eth, TestItem.WithdrawalB_2Eth };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
-        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(TestItem.KeccakD).TestObject);
+        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(TestItem.KeccakD).TestObject, Build.An.EmptyBlockHeader);
         Assert.False(isValid);
     }
 
@@ -53,7 +53,7 @@ public void Empty_withdrawals_are_valid_post_shanghai()
         BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
-        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject);
+        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject, Build.An.EmptyBlockHeader);
         Assert.True(isValid);
     }
 
@@ -64,7 +64,7 @@ public void Correct_withdrawals_block_post_shanghai()
         BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { TestItem.WithdrawalA_1Eth, TestItem.WithdrawalB_2Eth };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
-        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject);
+        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject, Build.An.EmptyBlockHeader);
         Assert.True(isValid);
     }
 }

src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs

@@ -432,6 +432,12 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval)
             );
         header.TxRoot = new TxTrie(block.Transactions).RootHash;
         block.Header.Author = _sealer.Address;
+
+        if (spec.IsEip4844Enabled)
+        {
+            header.ExcessDataGas = Evm.IntrinsicGasCalculator.CalculateExcessDataGas(parentBlock.ExcessDataGas,
+                block.Transactions.Sum(x => x.BlobVersionedHashes?.Length ?? 0), spec);
+        }
         return block;
     }
 

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -244,7 +244,7 @@ protected virtual TxReceipt[] ProcessBlock(
         _stateProvider.Commit(spec);
         _stateProvider.RecalculateStateRoot();
 
-        if (_specProvider.GetSpec(block.Header).IsEip4844Enabled)
+        if (_specProvider.GetSpec(block.Header).IsEip4844Enabled && block.Header.Number != 0)
         {
             block.Header.ExcessDataGas = IntrinsicGasCalculator.CalculateExcessDataGas(
                 _blockFinder.FindParentHeader(block.Header)?.ExcessDataGas,

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -557,13 +557,10 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
                 if (_logger.IsTrace)
                     _logger.Trace(
                         $"To be processed (of {suggestedBlock.ToString(Block.Format.Short)}) is {toBeProcessed?.ToString(Block.Format.Short)}");
-                if (toBeProcessed.IsGenesis)
-                {
-                    break;
-                }
 
                 branchingPoint = _blockTree.FindParentHeader(toBeProcessed.Header,
                     BlockTreeLookupOptions.TotalDifficultyNotNeeded);
+
                 if (branchingPoint is null)
                 {
                     // genesis block
@@ -650,7 +647,7 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
             Keccak stateRoot = branchingPoint?.StateRoot;
             if (_logger.IsTrace) _logger.Trace($"State root lookup: {stateRoot}");
             blocksToBeAddedToMain.Reverse();
-            return new ProcessingBranch(stateRoot, blocksToBeAddedToMain);
+            return new ProcessingBranch(stateRoot, blocksToBeAddedToMain, branchingPoint);
         }
 
         [Todo(Improve.Refactor, "This probably can be made conditional (in DEBUG only)")]
@@ -707,16 +704,18 @@ public void Dispose()
         [DebuggerDisplay("Root: {Root}, Length: {BlocksToProcess.Count}")]
         private readonly struct ProcessingBranch
         {
-            public ProcessingBranch(Keccak root, List<Block> blocks)
+            public ProcessingBranch(Keccak root, List<Block> blocks, BlockHeader parentHeader)
             {
                 Root = root;
                 Blocks = blocks;
                 BlocksToProcess = new List<Block>();
+                ParentHeader = parentHeader;
             }
 
             public Keccak Root { get; }
             public List<Block> Blocks { get; }
             public List<Block> BlocksToProcess { get; }
+            public BlockHeader ParentHeader { get; }
         }
 
         public class Options

src/Nethermind/Nethermind.Consensus/Validators/AlwaysValid.cs

@@ -44,7 +44,7 @@ public bool Validate(BlockHeader header, bool isUncle = false)
         return _result;
     }
 
-    public bool ValidateSuggestedBlock(Block block)
+    public bool ValidateSuggestedBlock(Block block, BlockHeader? parent)
     {
         return _result;
     }

src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs

@@ -58,7 +58,7 @@ public bool Validate(BlockHeader header, bool isUncle)
     /// <returns>
     /// <c>true</c> if the <paramref name="block"/> is valid; otherwise, <c>false</c>.
     /// </returns>
-    public bool ValidateSuggestedBlock(Block block)
+    public bool ValidateSuggestedBlock(Block block, BlockHeader? parentBlockHeader)
     {
         Transaction[] txs = block.Transactions;
         IReleaseSpec spec = _specProvider.GetSpec(block.Header);

src/Nethermind/Nethermind.Consensus/Validators/IBlockValidator.cs

@@ -7,7 +7,7 @@ namespace Nethermind.Consensus.Validators;
 
 public interface IBlockValidator : IHeaderValidator, IWithdrawalValidator
 {
-    bool ValidateSuggestedBlock(Block block);
+    bool ValidateSuggestedBlock(Block block, BlockHeader? parentBlockHeader);
 
     bool ValidateProcessedBlock(Block processedBlock, TxReceipt[] receipts, Block suggestedBlock);
 }

src/Nethermind/Nethermind.Consensus/Validators/NeverValidBlockValidator.cs

@@ -22,7 +22,7 @@ public bool Validate(BlockHeader header, bool isUncle)
             return false;
         }
 
-        public bool ValidateSuggestedBlock(Block block)
+        public bool ValidateSuggestedBlock(Block block, BlockHeader? parent)
         {
             return false;
         }

src/Nethermind/Nethermind.Consensus/Validators/TxValidator.cs

@@ -40,7 +40,7 @@ public bool IsWellFormed(Transaction transaction, IReleaseSpec releaseSpec)
                    ValidateChainId(transaction) &&
                    Validate1559GasFields(transaction, releaseSpec) &&
                    Validate3860Rules(transaction, releaseSpec) &&
-                   Validate4844Fields(transaction);
+                   Validate4844Fields(transaction, releaseSpec);
         }
 
         private static bool Validate3860Rules(Transaction transaction, IReleaseSpec releaseSpec) =>
@@ -99,7 +99,7 @@ private bool ValidateSignature(Signature? signature, IReleaseSpec spec)
             return !spec.ValidateChainId || signature.V is 27 or 28;
         }
 
-        private static bool Validate4844Fields(Transaction transaction)
+        private static bool Validate4844Fields(Transaction transaction, IReleaseSpec spec)
         {
             // Execution-payload version verification
             if (!transaction.SupportsBlobs)
@@ -121,8 +121,7 @@ transaction.BlobVersionedHashes is null ||
             {
                 if (transaction.BlobVersionedHashes[i] is null ||
                     transaction.BlobVersionedHashes![i].Length !=
-                    KzgPolynomialCommitments.BytesPerBlobVersionedHash ||
-                    transaction.BlobVersionedHashes![i][0] != KzgPolynomialCommitments.KzgBlobHashVersionV1)
+                    KzgPolynomialCommitments.BytesPerBlobVersionedHash)
                 {
                     return false;
                 }
@@ -145,7 +144,8 @@ transaction.BlobVersionedHashes is null ||
                      i < transaction.BlobVersionedHashes.Length;
                      i++, n += Ckzg.Ckzg.BytesPerCommitment)
                 {
-                    if (!KzgPolynomialCommitments.TryComputeCommitmentHashV1(
+                    if (transaction.BlobVersionedHashes![i][0] != KzgPolynomialCommitments.KzgBlobHashVersionV1 ||
+                        !KzgPolynomialCommitments.TryComputeCommitmentHashV1(
                             commitements[n..(n + Ckzg.Ckzg.BytesPerCommitment)], hash) ||
                         !hash.SequenceEqual(transaction.BlobVersionedHashes[i]))
                     {

src/Nethermind/Nethermind.Core.Test/Builders/BlockValidatorBuilder.cs

@@ -35,7 +35,7 @@ public BlockValidatorBuilder ThatAlwaysReturnsTrue
 
         protected override void BeforeReturn()
         {
-            TestObjectInternal.ValidateSuggestedBlock(Arg.Any<Block>()).Returns(_alwaysTrue);
+            TestObjectInternal.ValidateSuggestedBlock(Arg.Any<Block>(), Arg.Any<BlockHeader>()).Returns(_alwaysTrue);
             TestObjectInternal.ValidateProcessedBlock(Arg.Any<Block>(), Arg.Any<TxReceipt[]>(), Arg.Any<Block>()).Returns(_alwaysTrue);
             base.BeforeReturn();
         }

src/Nethermind/Nethermind.Core.Test/Encoding/ShardBlobTxDecoderTests.cs

@@ -68,6 +68,10 @@ public void NetworkWrapper_is_decoded_correctly(string rlp, Keccak signedHash, R
 
         Transaction? decoded = _txDecoder.Decode(incomingTxRlp, rlpBehaviors);
         Transaction? decodedByValueDecoderContext = _txDecoder.Decode(ref decoderContext, rlpBehaviors);
+        Assert.That(KzgPolynomialCommitments.AreProofsValid(
+            (decoded.NetworkWrapper as ShardBlobNetworkWrapper).Blobs,
+            (decoded.NetworkWrapper as ShardBlobNetworkWrapper).Commitments,
+            (decoded.NetworkWrapper as ShardBlobNetworkWrapper).Proofs));
 
         Assert.That(decoded!.Hash, Is.EqualTo(signedHash));
         Assert.That(decodedByValueDecoderContext!.Hash, Is.EqualTo(signedHash));

src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs

@@ -93,6 +93,9 @@ public void CanCorrectlyCalculateTxHash_when_called_concurrently((Transaction Tx
             Task.WaitAll(tasks.ToArray());
         }
 
+        public static IEnumerable<(Transaction, string)> TestCaseSource()
+            => TestObjectsSource().Select(tos => (tos.Item1.TestObject, tos.Item2));
+
         [TestCaseSource(nameof(TestCaseSource))]
         public void Roundtrip((Transaction Tx, string Description) testCase)
         {

src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs

@@ -9,18 +9,18 @@
 using Nethermind.Core.Specs;
 using Nethermind.Int256;
 
-namespace Nethermind.Evm;
-
-public static class IntrinsicGasCalculator
+namespace Nethermind.Evm
 {
-    public static long Calculate(Transaction transaction, IReleaseSpec releaseSpec)
+    public static class IntrinsicGasCalculator
     {
-        long result = GasCostOf.Transaction;
-        result += DataCost(transaction, releaseSpec);
-        result += CreateCost(transaction, releaseSpec);
-        result += AccessListCost(transaction, releaseSpec);
-        return result;
-    }
+        public static long Calculate(Transaction transaction, IReleaseSpec releaseSpec)
+        {
+            long result = GasCostOf.Transaction;
+            result += DataCost(transaction, releaseSpec);
+            result += CreateCost(transaction, releaseSpec);
+            result += AccessListCost(transaction, releaseSpec);
+            return result;
+        }
 
     private static long CreateCost(Transaction transaction, IReleaseSpec releaseSpec)
     {
@@ -90,54 +90,7 @@ private static long AccessListCost(Transaction transaction, IReleaseSpec release
             }
         }
 
-        return accessListCost;
-    }
-
-    public static UInt256 CalculateDataGasPrice(Transaction transaction, UInt256 parentExcessDataGas) =>
-        CalculateDataGas(transaction.BlobVersionedHashes?.Length ?? 0) * GetDataGasPrice(parentExcessDataGas);
-
-    public static UInt256 CalculateDataGas(int blobCount) =>
-        ((ulong)blobCount * Eip4844Constants.DataGasPerBlob);
-
-    public static UInt256 GetDataGasPrice(UInt256 parentExcessDataGas)
-    {
-        UInt256 dataGasPriceUpdateFraction = 2225652;
-        UInt256 minDataGasPrice = 1L;
-
-        UInt256 FakeExponential(UInt256 factor, UInt256 num, UInt256 denominator)
-        {
-            UInt256 output = UInt256.Zero;
-
-            UInt256 numAccum = factor * denominator;
-
-            for (UInt256 i = 1; numAccum > 0; i++)
-            {
-                output += numAccum;
-                numAccum *= num;
-                numAccum /= i * denominator;
-            }
-
-            return output / denominator;
+            return accessListCost;
         }
-
-        UInt256 scaleDueToParentExcessDataGas =
-            FakeExponential(minDataGasPrice, parentExcessDataGas, dataGasPriceUpdateFraction);
-        return scaleDueToParentExcessDataGas;
-    }
-
-    public static UInt256? CalculateExcessDataGas(UInt256? parentExcessDataGas, int newBlobsCount,
-        IReleaseSpec releaseSpec)
-    {
-        if (!releaseSpec.IsEip4844Enabled)
-        {
-            return null;
-        }
-
-        UInt256 excessDataGas = parentExcessDataGas.GetValueOrDefault();
-        UInt256 consumedGas = Eip4844Constants.DataGasPerBlob * (UInt256)newBlobsCount;
-        excessDataGas += consumedGas;
-        return excessDataGas < Eip4844Constants.TargetDataGasPerBlock
-            ? UInt256.Zero
-            : (excessDataGas - Eip4844Constants.TargetDataGasPerBlock);
     }
 }

src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs

@@ -202,7 +202,7 @@ private void Execute(Transaction transaction, BlockHeader block, ITxTracer txTra
             }
 
             long intrinsicGas = IntrinsicGasCalculator.Calculate(transaction, spec);
-            if (_logger.IsTrace) _logger.Trace($"Intrinsic gas calculated for {transaction.Hash}: " + intrinsicGas);
+            _logger.Warn($"Intrinsic gas calculated for {transaction.Hash}: " + intrinsicGas);
 
             if (notSystemTransaction)
             {

src/Nethermind/Nethermind.Hive/HiveRunner.cs

@@ -130,6 +130,7 @@ private async Task InitializeBlocks(string blocksDir, CancellationToken cancella
             string[] files = Directory.GetFiles(blocksDir).OrderBy(x => x).ToArray();
             if (_logger.IsInfo) _logger.Info($"Loaded {files.Length} files with blocks to process.");
 
+            BlockHeader? parentBlockHeader = null;
             foreach (string file in files)
             {
                 if (cancellationToken.IsCancellationRequested)
@@ -143,7 +144,8 @@ private async Task InitializeBlocks(string blocksDir, CancellationToken cancella
                     if (_logger.IsInfo)
                         _logger.Info(
                             $"HIVE Processing block file: {file} - {block.ToString(Block.Format.Short)}");
-                    await ProcessBlock(block);
+                    await ProcessBlock(block, parentBlockHeader);
+                    parentBlockHeader = block.Header;
                 }
                 catch (RlpException e)
                 {
@@ -174,12 +176,14 @@ private async Task InitializeChain(string chainFile)
                 blocks.Add(block);
             }
 
+            BlockHeader? parentBlockHeader = null;
             for (int i = 0; i < blocks.Count; i++)
             {
                 Block block = blocks[i];
                 if (_logger.IsInfo)
                     _logger.Info($"HIVE Processing a chain.rlp block {block.ToString(Block.Format.Short)}");
-                await ProcessBlock(block);
+                await ProcessBlock(block, parentBlockHeader);
+                parentBlockHeader = blocks[i].Header;
             }
         }
 
@@ -199,14 +203,14 @@ private async Task WaitForBlockProcessing(SemaphoreSlim semaphore)
             }
         }
 
-        private async Task ProcessBlock(Block block)
+        private async Task ProcessBlock(Block block, BlockHeader? parentBlockHeader)
         {
             try
             {
                 // Start of block processing, setting flag BlockSuggested to default value: false
                 BlockSuggested = false;
 
-                if (!_blockValidator.ValidateSuggestedBlock(block))
+                if (!_blockValidator.ValidateSuggestedBlock(block, parentBlockHeader))
                 {
                     if (_logger.IsInfo) _logger.Info($"Invalid block {block}");
                     return;