Wait for background JumpDestinationAnalysis if in progress

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -430,7 +430,9 @@ private void FireProcessingQueueEmpty()
 
         if (!readonlyChain)
         {
-            _stats.UpdateStats(lastProcessed, _blockTree, _recoveryQueue.Count, _blockQueue.Count, _stopwatch.ElapsedMicroseconds());
+            Metrics.RecoveryQueueSize = _recoveryQueue.Count;
+            Metrics.ProcessingQueueSize = _blockQueue.Count;
+            _stats.UpdateStats(lastProcessed, _blockTree, _stopwatch.ElapsedMicroseconds());
         }
 
         return lastProcessed;

src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs

@@ -39,6 +39,8 @@ internal class ProcessingStats
         private long _processingMicroseconds;
         private long _lastTotalCreates;
         private long _lastReportMs;
+        private long _lastContractsAnalysed;
+        private long _lastCachedContractsUsed;
 
         public ProcessingStats(ILogger logger)
         {
@@ -51,7 +53,7 @@ public ProcessingStats(ILogger logger)
 #endif
         }
 
-        public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueueSize, int blockQueueSize, long blockProcessingTimeInMicros)
+        public void UpdateStats(Block? block, IBlockTree blockTreeCtx, long blockProcessingTimeInMicros)
         {
             const string resetColor = "\u001b[37m";
             const string whiteText = "\u001b[97m";
@@ -85,8 +87,6 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue
             Metrics.LastDifficulty = block.Difficulty;
             Metrics.GasUsed = block.GasUsed;
             Metrics.GasLimit = block.GasLimit;
-            Metrics.RecoveryQueueSize = recoveryQueueSize;
-            Metrics.ProcessingQueueSize = blockQueueSize;
 
             Metrics.BlockchainHeight = block.Header.Number;
             Metrics.BestKnownBlockNumber = blockTreeCtx.BestKnownNumber;
@@ -117,6 +117,8 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue
                     long chunkCreates = Evm.Metrics.Creates - _lastTotalCreates;
                     long chunkSload = Evm.Metrics.SloadOpcode - _lastTotalSLoad;
                     long chunkSstore = Evm.Metrics.SstoreOpcode - _lastTotalSStore;
+                    long contractsAnalysed = Evm.Metrics.ContractsAnalysed - _lastContractsAnalysed;
+                    long cachedContractsUsed = Db.Metrics.CodeDbCache - _lastCachedContractsUsed;
                     double chunkMGas = Metrics.Mgas - _lastTotalMGas;
                     double mgasPerSecond = chunkMicroseconds == 0 ? -1 : chunkMGas / chunkMicroseconds * 1_000_000.0;
                     double totalMgasPerSecond = totalMicroseconds == 0 ? -1 : Metrics.Mgas / totalMicroseconds * 1_000_000.0;
@@ -200,8 +202,22 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue
                         > 75 => whiteText,
                         _ => ""
                     };
+
+                    var recoveryQueue = Metrics.RecoveryQueueSize;
+                    var processingQueue = Metrics.ProcessingQueueSize;
+
+                    string extraStats;
+                    if (recoveryQueue > 0 || processingQueue > 0)
+                    {
+                        extraStats = $"recover {recoveryQueue,5:N0} | process {processingQueue,5:N0}";
+                    }
+                    else
+                    {
+                        extraStats = $"exec code {resetColor} from cache {cachedContractsUsed,7:N0} |{resetColor} new {contractsAnalysed,6:N0}";
+                    }
+
                     _logger.Info($"- Block{(chunkBlocks > 1 ? $"s {chunkBlocks,-9:N0}" : "           ")}{(chunkBlocks == 1 ? mgasColor : "")} {chunkMGas,7:F2}{resetColor} MGas   | {chunkTx,6:N0}    txs |  calls {callsColor}{chunkCalls,6:N0}{resetColor} {darkGreyText}({chunkEmptyCalls,3:N0}){resetColor} | sload {chunkSload,7:N0} | sstore {sstoreColor}{chunkSstore,6:N0}{resetColor} | create {createsColor}{chunkCreates,3:N0}{resetColor}{(currentSelfDestructs - _lastSelfDestructs > 0 ? $"{darkGreyText}({-(currentSelfDestructs - _lastSelfDestructs),3:N0}){resetColor}" : "")}");
-                    _logger.Info($"- Block throughput {mgasPerSecondColor}{mgasPerSecond,7:F2}{resetColor} MGas/s | {txps,9:F2} t/s |       {bps,7:F2} Blk/s | recv  {recoveryQueueSize,7:N0} | proc   {blockQueueSize,6:N0}");
+                    _logger.Info($"- Block throughput {mgasPerSecondColor}{mgasPerSecond,7:F2}{resetColor} MGas/s | {txps,9:F2} t/s |       {bps,7:F2} Blk/s | {extraStats}");
                     // Only output the total throughput in debug mode
                     if (_logger.IsDebug)
                     {
@@ -223,6 +239,8 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue
 
                 }
 
+                _lastCachedContractsUsed = Db.Metrics.CodeDbCache;
+                _lastContractsAnalysed = Evm.Metrics.ContractsAnalysed;
                 _lastReportMs = reportMs;
                 _lastBlockNumber = Metrics.Blocks;
                 _lastTotalMGas = Metrics.Mgas;

src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs

@@ -46,5 +46,13 @@ void IThreadPoolWorkItem.Execute()
         {
             _analyzer.Execute();
         }
+
+        public void AnalyseInBackgroundIfRequired()
+        {
+            if (!ReferenceEquals(_analyzer, _emptyAnalyzer) && _analyzer.RequiresAnalysis)
+            {
+                ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false);
+            }
+        }
     }
 }

src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs

@@ -6,6 +6,7 @@
 using System.Runtime.InteropServices;
 using System.Runtime.Intrinsics;
 using System.Runtime.Intrinsics.X86;
+using System.Threading;
 
 namespace Nethermind.Evm.CodeAnalysis
 {
@@ -19,17 +20,62 @@ public sealed class JumpDestinationAnalyzer(ReadOnlyMemory<byte> code)
         private static readonly long[]? _emptyJumpDestinationBitmap = new long[1];
         private long[]? _jumpDestinationBitmap = code.Length == 0 ? _emptyJumpDestinationBitmap : null;
 
+        private object? _analysisComplete;
         private ReadOnlyMemory<byte> MachineCode { get; } = code;
 
         public bool ValidateJump(int destination)
         {
-            ReadOnlySpan<byte> machineCode = MachineCode.Span;
-            _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(machineCode);
+            _jumpDestinationBitmap ??= CreateOrWaitForJumpDestinationBitmap();
 
             // Cast to uint to change negative numbers to very int high numbers
             // Then do length check, this both reduces check by 1 and eliminates the bounds
             // check from accessing the span.
-            return (uint)destination < (uint)machineCode.Length && IsJumpDestination(_jumpDestinationBitmap, destination);
+            return (uint)destination < (uint)MachineCode.Length && IsJumpDestination(_jumpDestinationBitmap, destination);
+        }
+
+        private long[] CreateOrWaitForJumpDestinationBitmap()
+        {
+            object? previous = Volatile.Read(ref _analysisComplete);
+            if (previous is null)
+            {
+                ManualResetEventSlim analysisComplete = new(initialState: false);
+                previous = Interlocked.CompareExchange(ref _analysisComplete, analysisComplete, null);
+                if (previous is null)
+                {
+                    // Not already in progress, so start it.
+                    var bitmap = CreateJumpDestinationBitmap();
+                    _jumpDestinationBitmap = bitmap;
+                    // Release the MRES to be GC'd
+                    _analysisComplete = bitmap;
+                    // Signal complete.
+                    analysisComplete.Set();
+                    return bitmap;
+                }
+            }
+
+            if (previous is ManualResetEventSlim resetEvent)
+            {
+                Thread thread = Thread.CurrentThread;
+                ThreadPriority priority = thread.Priority;
+                try
+                {
+                    // We are waiting, so drop priority to normal (BlockProcessing runs at higher priority).
+                    thread.Priority = ThreadPriority.Normal;
+
+                    // Already in progress, wait for completion.
+                    resetEvent.Wait();
+                }
+                finally
+                {
+                    // Restore the priority of the thread.
+                    thread.Priority = priority;
+                }
+
+                return _jumpDestinationBitmap;
+            }
+
+            // Must be the bitmap, and lost check->create benign data race
+            return (long[])previous;
         }
 
         /// <summary>
@@ -55,8 +101,10 @@ private static int GetInt64ArrayLengthFromBitLength(int n) =>
         /// Collects data locations in code.
         /// An unset bit means the byte is an opcode, a set bit means it's data.
         /// </summary>
-        private static long[] CreateJumpDestinationBitmap(ReadOnlySpan<byte> code)
+        private long[] CreateJumpDestinationBitmap()
         {
+            Metrics.IncrementContractsAnalysed();
+            ReadOnlySpan<byte> code = MachineCode.Span;
             long[] jumpDestinationBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)];
             int programCounter = 0;
             // We accumulate each array segment to a register and then flush to memory when we move to next.
@@ -146,6 +194,7 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan<byte> code)
         /// <summary>
         /// Checks if the position is in a code segment.
         /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static bool IsJumpDestination(long[] bitvec, int pos)
         {
             int vecIndex = pos >> BitShiftPerInt64;
@@ -164,8 +213,34 @@ private static void MarkJumpDestinations(long[] jumpDestinationBitmap, int pos,
 
         public void Execute()
         {
-            // This is to support background thread preparation of the bitmap.
-            _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(MachineCode.Span);
+            if (_jumpDestinationBitmap is null && Volatile.Read(ref _analysisComplete) is null)
+            {
+                ManualResetEventSlim analysisComplete = new(initialState: false);
+                if (Interlocked.CompareExchange(ref _analysisComplete, analysisComplete, null) is null)
+                {
+                    Thread thread = Thread.CurrentThread;
+                    ThreadPriority priority = thread.Priority;
+                    try
+                    {
+                        // Boost the priority of the thread as block processing may be waiting on this.
+                        thread.Priority = ThreadPriority.AboveNormal;
+
+                        _jumpDestinationBitmap ??= CreateJumpDestinationBitmap();
+
+                        // Release the MRES to be GC'd
+                        _analysisComplete = _jumpDestinationBitmap;
+                        // Signal complete.
+                        analysisComplete.Set();
+                    }
+                    finally
+                    {
+                        // Restore the priority of the thread.
+                        thread.Priority = priority;
+                    }
+                }
+            }
         }
+
+        public bool RequiresAnalysis => _jumpDestinationBitmap is null;
     }
 }

src/Nethermind/Nethermind.Evm/Metrics.cs

@@ -3,6 +3,7 @@
 
 using System.ComponentModel;
 using System.Runtime.CompilerServices;
+using System.Threading;
 
 using Nethermind.Core.Attributes;
 
@@ -77,6 +78,11 @@ public class Metrics
     [CounterMetric]
     [Description("Number of contract create calls.")]
     public static long Creates { get; set; }
+    private static long _contractsAnalysed;
+    [Description("Number of contracts' code analysed for jump destinations.")]
+    public static long ContractsAnalysed => _contractsAnalysed;
+    public static void IncrementContractsAnalysed() => Interlocked.Increment(ref _contractsAnalysed);
+
     internal static long Transactions { get; set; }
     internal static float AveGasPrice { get; set; }
     internal static float MinGasPrice { get; set; } = float.MaxValue;

src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs

@@ -411,6 +411,8 @@ protected ExecutionEnvironment BuildExecutionEnvironment(
                 ? new(tx.Data ?? Memory<byte>.Empty)
                 : VirtualMachine.GetCachedCodeInfo(WorldState, recipient, spec);
 
+            codeInfo.AnalyseInBackgroundIfRequired();
+
             byte[] inputData = tx.IsMessageCall ? tx.Data.AsArray() ?? Array.Empty<byte>() : Array.Empty<byte>();
 
             return new ExecutionEnvironment

src/Nethermind/Nethermind.Evm/VirtualMachine.cs

@@ -498,8 +498,7 @@ public TransactionSubstate Run<TTracingActions>(EvmState state, IWorldState worl
     public void InsertCode(ReadOnlyMemory<byte> code, Address callCodeOwner, IReleaseSpec spec)
     {
         var codeInfo = new CodeInfo(code);
-        // Start generating the JumpDestinationBitmap in background.
-        ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false);
+        codeInfo.AnalyseInBackgroundIfRequired();
 
         Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span);
         _state.InsertCode(callCodeOwner, codeHash, code, spec);
@@ -544,6 +543,8 @@ public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IR
             }
 
             cachedCodeInfo = new CodeInfo(code);
+            cachedCodeInfo.AnalyseInBackgroundIfRequired();
+
             CodeCache.Set(codeHash, cachedCodeInfo);
         }
         else
@@ -2216,6 +2217,9 @@ private EvmExceptionType InstructionCall<TTracingInstructions, TTracingRefunds>(
             !UpdateMemoryCost(vmState, ref gasAvailable, in outputOffset, outputLength) ||
             !UpdateGas(gasExtra, ref gasAvailable)) return EvmExceptionType.OutOfGas;
 
+        CodeInfo codeInfo = GetCachedCodeInfo(_worldState, codeSource, spec);
+        codeInfo.AnalyseInBackgroundIfRequired();
+
         if (spec.Use63Over64Rule)
         {
             gasLimit = UInt256.Min((UInt256)(gasAvailable - gasAvailable / 64), gasLimit);
@@ -2269,7 +2273,7 @@ private EvmExceptionType InstructionCall<TTracingInstructions, TTracingRefunds>(
             transferValue: transferValue,
             value: callValue,
             inputData: callData,
-            codeInfo: GetCachedCodeInfo(_worldState, codeSource, spec)
+            codeInfo: codeInfo
         );
         if (typeof(TLogger) == typeof(IsTracing)) _logger.Trace($"Tx call gas {gasLimitUl}");
         if (outputLength == 0)
@@ -2492,6 +2496,7 @@ private EvmExceptionType InstructionSelfDestruct<TTracing>(EvmState vmState, ref
         // pointing to data in this tx and will become invalid
         // for another tx as returned to pool.
         CodeInfo codeInfo = new(initCode);
+        codeInfo.AnalyseInBackgroundIfRequired();
 
         ExecutionEnvironment callEnv = new
         (