FCU Canonical chain corruption after beacon sync (3 fixes + startup healing)

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -98,6 +98,19 @@ public interface IInitConfig : IConfig
 
     [ConfigItem(Description = "[TECHNICAL] True when in runner test. Disable some wait.", DefaultValue = "false", HiddenFromDocs = true)]
     bool InRunnerTest { get; set; }
+
+    [ConfigItem(
+        Description = "On startup, walk backward from the current head for up to `HealCanonicalChainDepth` " +
+                      "blocks via parentHash and repair any incorrect `HasBlockOnMainChain` markers left by " +
+                      "the beacon-sync path. Also clears stale canonical markers above the head. " +
+                      "Use this flag once after observing a canonical-mismatch (wrong `eth_getBlockByNumber` results).",
+        DefaultValue = "false")]
+    bool HealCanonicalChain { get; set; }
+
+    [ConfigItem(
+        Description = "Number of blocks to walk back from the head when `HealCanonicalChain` is enabled.",
+        DefaultValue = "8192")]
+    long HealCanonicalChainDepth { get; set; }
 }
 
 public enum DiagnosticMode

src/Nethermind/Nethermind.Api/InitConfig.cs

@@ -41,6 +41,8 @@ public class InitConfig : IInitConfig
         public int BackgroundTaskMaxNumber { get; set; } = 2048;
         public bool InRunnerTest { get; set; } = false;
         public string? DataDir { get; set; }
+        public bool HealCanonicalChain { get; set; } = false;
+        public long HealCanonicalChainDepth { get; set; } = 8192;
 
         [Obsolete("Use DiagnosticMode with MemDb instead")]
         public bool UseMemDb

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -767,6 +767,14 @@ as it does not require the step of resolving number -> hash */
                 return level.BlockInfos[0].BlockHash;
             }
 
+            // Post-merge: TD never increases, so the best-TD fallback cannot distinguish canonical from orphaned.
+            if (SpecProvider.TerminalTotalDifficulty is not null
+                && (Head?.TotalDifficulty ?? UInt256.Zero) >= SpecProvider.TerminalTotalDifficulty)
+            {
+                return null;
+            }
+
+            // Pre-merge: the block with the highest total difficulty is canonical by PoW rule.
             UInt256 bestDifficultySoFar = UInt256.Zero;
             Hash256 bestHash = null;
             for (int i = 0; i < level.BlockInfos.Length; i++)
@@ -1014,6 +1022,10 @@ public void UpdateMainChain(IReadOnlyList<Block> blocks, bool wereProcessed, boo
                 }
             }
 
+            // Clear stale canonical markers above the new head left by beacon sync.
+            // Covers both same-height FCU (previousHeadNumber <= lastNumber) and ePBS FCU to ancestor.
+            ClearStaleMarkersAbove(Math.Max(previousHeadNumber, lastNumber), batch);
+
             for (int i = 0; i < blocks.Count; i++)
             {
                 Block block = blocks[i];
@@ -1035,6 +1047,84 @@ public void UpdateMainChain(IReadOnlyList<Block> blocks, bool wereProcessed, boo
             OnUpdateMainChain?.Invoke(this, new OnUpdateMainChainArgs(blocks, wereProcessed));
         }
 
+        public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth)
+        {
+            BlockHeader? start = FindHeader(startHash, BlockTreeLookupOptions.None);
+            if (start is null) return;
+
+            using BatchWrite batch = _chainLevelInfoRepository.StartBatch();
+
+            long repairedAbove = ClearStaleMarkersAbove(start.Number, batch);
+            long repairedBelow = RepairMarkersBelow(start, maxBlockDepth, batch);
+
+            if (Logger.IsInfo) Logger.Info($"Canonical chain heal complete: {repairedAbove + repairedBelow} level(s) repaired ({repairedAbove} stale above head cleared, {repairedBelow} incorrect markers fixed).");
+        }
+
+        private long ClearStaleMarkersAbove(long fromExclusive, BatchWrite batch)
+        {
+            long cleared = 0L;
+            for (long levelNumber = fromExclusive + 1; ; levelNumber++)
+            {
+                ChainLevelInfo? level = LoadLevel(levelNumber);
+                if (level is null) break;
+                if (level.HasBlockOnMainChain)
+                {
+                    level.HasBlockOnMainChain = false;
+                    _chainLevelInfoRepository.PersistLevel(levelNumber, level, batch);
+                    cleared++;
+                }
+            }
+            return cleared;
+        }
+
+        private long RepairMarkersBelow(BlockHeader start, long maxBlockDepth, BatchWrite batch)
+        {
+            long repairedCount = 0L;
+            long blocksWalked = 0L;
+            BlockHeader? current = start;
+
+            while (current is not null && blocksWalked < maxBlockDepth)
+            {
+                ChainLevelInfo? level = LoadLevel(current.Number);
+                if (level is not null)
+                {
+                    int? index = level.FindIndex(current.Hash!);
+                    if (index is null)
+                    {
+                        if (Logger.IsWarn) Logger.Warn($"Canonical heal: block {current.Hash} at height {current.Number} not found in any BlockInfo slot — repair halted.");
+                        break;
+                    }
+
+                    bool needsRepair = index.Value != 0 || !level.HasBlockOnMainChain;
+
+                    if (index.Value != 0)
+                        level.SwapToMain(index.Value);
+
+                    level.HasBlockOnMainChain = true;
+
+                    if (needsRepair)
+                    {
+                        _chainLevelInfoRepository.PersistLevel(current.Number, level, batch);
+                        repairedCount++;
+                    }
+                }
+
+                if (current.IsGenesis) break;
+
+                BlockHeader? parent = FindHeader(current.ParentHash!, BlockTreeLookupOptions.None);
+                if (parent is null)
+                {
+                    if (Logger.IsWarn) Logger.Warn($"Canonical heal: parent {current.ParentHash} of block {current.Number} not found — chain may be pruned, repair halted.");
+                    break;
+                }
+
+                current = parent;
+                blocksWalked++;
+            }
+
+            return repairedCount;
+        }
+
         private void TryUpdateSyncPivot()
         {
             BlockHeader? newPivotHeader = null;

src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs

@@ -245,6 +245,7 @@ public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainC
         _overlayTree.UpdateBeaconMainChain(blockInfos, clearBeaconMainChainStartPoint);
 
     public void RecalculateTreeLevels() => _overlayTree.RecalculateTreeLevels();
+    public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth) => _overlayTree.HealCanonicalChain(startHash, maxBlockDepth);
     public (long BlockNumber, Hash256 BlockHash) SyncPivot
     {
         get => _baseTree.SyncPivot;

src/Nethermind/Nethermind.Blockchain/IBlockTree.cs

@@ -191,6 +191,14 @@ AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBlockOption
 
         void RecalculateTreeLevels();
 
+        /// <summary>
+        /// Repairs canonical chain markers starting from <paramref name="startHash"/>, walking backward
+        /// up to <paramref name="maxBlockDepth"/> blocks. Also removes stale canonical markers above
+        /// <paramref name="startHash"/> left by the beacon-sync path. Safe to run at startup after
+        /// observing canonical mismatches (wrong <c>eth_getBlockByNumber</c> results).
+        /// </summary>
+        void HealCanonicalChain(Hash256 startHash, long maxBlockDepth);
+
         /// <summary>
         /// Sync pivot is mainly concerned with old blocks and receipts.
         /// After sync pivot, blocks and headers should be continuous.

src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs

@@ -193,6 +193,7 @@ public int DeleteChainSlice(in long startNumber, long? endNumber = null, bool fo
         public bool IsBetterThanHead(BlockHeader? header) => _wrapped.IsBetterThanHead(header);
         public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainChainStartPoint) => throw new InvalidOperationException($"{nameof(ReadOnlyBlockTree)} does not expect {nameof(UpdateBeaconMainChain)} calls");
         public void RecalculateTreeLevels() => throw new InvalidOperationException($"{nameof(ReadOnlyBlockTree)} does not expect {nameof(RecalculateTreeLevels)} calls");
+        public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth) => throw new InvalidOperationException($"{nameof(ReadOnlyBlockTree)} does not expect {nameof(HealCanonicalChain)} calls");
         public (long BlockNumber, Hash256 BlockHash) SyncPivot
         {
             get => _wrapped.SyncPivot;

src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockTree.cs

@@ -192,6 +192,9 @@ public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainC
     public void RecalculateTreeLevels()
         => throw new NotSupportedException();
 
+    public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth)
+        => throw new NotSupportedException();
+
     public (long BlockNumber, Hash256 BlockHash) SyncPivot
     {
         get => throw new NotSupportedException();

src/Nethermind/Nethermind.Init/Steps/ReviewBlockTree.cs

@@ -10,6 +10,7 @@
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Blockchain.Visitors;
 using Nethermind.Consensus.Processing;
+using Nethermind.Core.Crypto;
 using Nethermind.Logging;
 using Nethermind.State;
 
@@ -29,13 +30,25 @@ ILogManager logManager
 
         public Task Execute(CancellationToken cancellationToken)
         {
-            if (initConfig.ProcessingEnabled)
+            HealCanonicalChainIfEnabled();
+            return initConfig.ProcessingEnabled
+                ? RunBlockTreeInitTasks(cancellationToken)
+                : Task.CompletedTask;
+        }
+
+        private void HealCanonicalChainIfEnabled()
+        {
+            if (!initConfig.HealCanonicalChain) return;
+
+            Hash256? startHash = blockTree.Head?.Hash;
+            if (startHash is not null)
             {
-                return RunBlockTreeInitTasks(cancellationToken);
+                if (_logger.IsInfo) _logger.Info($"Healing canonical chain from head {startHash} (depth {initConfig.HealCanonicalChainDepth})...");
+                blockTree.HealCanonicalChain(startHash, initConfig.HealCanonicalChainDepth);
             }
             else
             {
-                return Task.CompletedTask;
+                if (_logger.IsWarn) _logger.Warn("HealCanonicalChain requested but no head block found — skipping.");
             }
         }
 

src/Nethermind/Nethermind.PerfTest/Program.cs

@@ -219,6 +219,8 @@ public int DeleteChainSlice(in long startNumber, long? endNumber)
             {
                 return _blockTree.DeleteChainSlice(startNumber, endNumber);
             }
+
+            public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth) { }
         }
 
         private const string DbBasePath = @"C:\perf_db";

src/Nethermind/Nethermind.TxPool.Test/TestBlockTree.cs

@@ -105,4 +105,5 @@ public void ForkChoiceUpdated(Hash256? finalizedBlockHash, Hash256? safeBlockBlo
     public bool IsBetterThanHead(BlockHeader? header) => false;
     public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainChainStartPoint) { }
     public void RecalculateTreeLevels() { }
+    public void HealCanonicalChain(Hash256 startHash, long maxBlockDepth) { }
 }