Adds to 22.10.4

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -425,6 +425,8 @@ public BesuController build() {
             ethProtocolManager,
             pivotBlockSelector);
 
+    protocolContext.setSynchronizer(Optional.of(synchronizer));
+
     final MiningCoordinator miningCoordinator =
         createMiningCoordinator(
             protocolSchedule,
@@ -493,7 +495,6 @@ protected Synchronizer createSynchronizer(
             clock,
             metricsSystem,
             getFullSyncTerminationCondition(protocolContext.getBlockchain()),
-            ethProtocolManager,
             pivotBlockSelector);
 
     return toUse;

besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java

@@ -132,6 +132,7 @@ public void setUp() {
         .thenReturn(ValidationResult.valid());
     when(mockTransactionValidator.validateForSender(any(), any(), any()))
         .thenReturn(ValidationResult.valid());
+    when(mockWorldState.copy()).thenReturn(mockWorldState);
     when(mockWorldStateArchive.getMutable(any(), any(), anyBoolean()))
         .thenReturn(Optional.of(mockWorldState));
 

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java

@@ -34,6 +34,7 @@ public enum RpcMethod {
   CLIQUE_GET_SIGNER_METRICS("clique_getSignerMetrics"),
   DEBUG_ACCOUNT_AT("debug_accountAt"),
   DEBUG_METRICS("debug_metrics"),
+  DEBUG_RESYNC_WORLDSTATE("debug_resyncWorldState"),
   DEBUG_SET_HEAD("debug_setHead"),
   DEBUG_REPLAY_BLOCK("debug_replayBlock"),
   DEBUG_STORAGE_RANGE_AT("debug_storageRangeAt"),

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugResyncWorldstate.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods;
+
+import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
+import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.ethereum.core.Synchronizer;
+import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+
+public class DebugResyncWorldstate implements JsonRpcMethod {
+  private final Synchronizer synchronizer;
+  private final ProtocolSchedule protocolSchedule;
+  private final Blockchain blockchain;
+
+  public DebugResyncWorldstate(
+      final ProtocolSchedule protocolSchedule,
+      final Blockchain blockchain,
+      final Synchronizer synchronizer) {
+    this.synchronizer = synchronizer;
+    this.protocolSchedule = protocolSchedule;
+    this.blockchain = blockchain;
+  }
+
+  @Override
+  public String getName() {
+    return RpcMethod.DEBUG_RESYNC_WORLDSTATE.getMethodName();
+  }
+
+  @Override
+  public JsonRpcResponse response(final JsonRpcRequestContext request) {
+    protocolSchedule
+        .getByBlockNumber(blockchain.getChainHeadBlockNumber())
+        .getBadBlocksManager()
+        .reset();
+    return new JsonRpcSuccessResponse(
+        request.getRequest().getId(), synchronizer.resyncWorldState());
+  }
+}

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockReplay.java

@@ -145,15 +145,22 @@ private <T> Optional<T> performActionWithBlock(
     if (previous == null) {
       return Optional.empty();
     }
-    try (final MutableWorldState mutableWorldState =
+    try (final var worldState =
         worldStateArchive
-            .getMutable(previous.getStateRoot(), previous.getHash(), false)
+            .getMutable(previous.getStateRoot(), previous.getBlockHash(), false)
+            .map(
+                ws -> {
+                  if (!ws.isPersistable()) {
+                    return ws.copy();
+                  }
+                  return ws;
+                })
             .orElseThrow(
                 () ->
                     new IllegalArgumentException(
                         "Missing worldstate for stateroot "
                             + previous.getStateRoot().toShortHexString()))) {
-      return action.perform(body, header, blockchain, mutableWorldState, transactionProcessor);
+      return action.perform(body, header, blockchain, worldState, transactionProcessor);
     } catch (Exception ex) {
       return Optional.empty();
     }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/DebugJsonRpcMethods.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugBatchSendRawTransaction;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugGetBadBlocks;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugMetrics;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugResyncWorldstate;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugSetHead;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugStandardTraceBadBlockToFile;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.DebugStandardTraceBlockToFile;
@@ -36,6 +37,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory;
 import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
+import org.hyperledger.besu.ethereum.core.Synchronizer;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
@@ -53,6 +55,7 @@ public class DebugJsonRpcMethods extends ApiGroupJsonRpcMethods {
   private final ProtocolSchedule protocolSchedule;
   private final ObservableMetricsSystem metricsSystem;
   private final TransactionPool transactionPool;
+  private final Synchronizer synchronizer;
   private final Path dataDir;
 
   DebugJsonRpcMethods(
@@ -61,12 +64,14 @@ public class DebugJsonRpcMethods extends ApiGroupJsonRpcMethods {
       final ProtocolSchedule protocolSchedule,
       final ObservableMetricsSystem metricsSystem,
       final TransactionPool transactionPool,
+      final Synchronizer synchronizer,
       final Path dataDir) {
     this.blockchainQueries = blockchainQueries;
     this.protocolContext = protocolContext;
     this.protocolSchedule = protocolSchedule;
     this.metricsSystem = metricsSystem;
     this.transactionPool = transactionPool;
+    this.synchronizer = synchronizer;
     this.dataDir = dataDir;
   }
 
@@ -88,6 +93,7 @@ protected Map<String, JsonRpcMethod> create() {
         new DebugAccountRange(blockchainQueries),
         new DebugStorageRangeAt(blockchainQueries, blockReplay),
         new DebugMetrics(metricsSystem),
+        new DebugResyncWorldstate(protocolSchedule, protocolContext.getBlockchain(), synchronizer),
         new DebugTraceBlock(
             () -> new BlockTracer(blockReplay),
             ScheduleBasedBlockHeaderFunctions.create(protocolSchedule),

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java

@@ -100,6 +100,7 @@ public Map<String, JsonRpcMethod> methods(
                   protocolSchedule,
                   metricsSystem,
                   transactionPool,
+                  synchronizer,
                   dataDir),
               new EeaJsonRpcMethods(
                   blockchainQueries, protocolSchedule, transactionPool, privacyParameters),

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java

@@ -118,6 +118,7 @@ public void setUp() throws Exception {
     when(protocolSpec.getMiningBeneficiaryCalculator()).thenReturn(BlockHeader::getCoinbase);
     when(blockchain.getChainHeadHeader()).thenReturn(blockHeader);
     when(protocolSpec.getBadBlocksManager()).thenReturn(new BadBlockManager());
+    when(mutableWorldState.copy()).thenReturn(mutableWorldState);
   }
 
   @Test

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java

@@ -167,7 +167,16 @@ public BlockProcessingResult validateAndProcessBlock(
         return new BlockProcessingResult(
             Optional.of(new BlockProcessingOutputs(worldState, receipts)));
       }
-    } catch (StorageException | MerkleTrieException ex) {
+    } catch (MerkleTrieException ex) {
+      context
+          .getSynchronizer()
+          .ifPresentOrElse(
+              synchronizer -> synchronizer.healWorldState(ex.getMaybeAddress(), ex.getLocation()),
+              () ->
+                  handleAndLogImportFailure(
+                      block, new BlockProcessingResult(Optional.empty(), ex)));
+      return new BlockProcessingResult(Optional.empty(), ex);
+    } catch (StorageException ex) {
       var retval = new BlockProcessingResult(Optional.empty(), ex);
       handleAndLogImportFailure(block, retval);
       return retval;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.ethereum;
 
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
+import org.hyperledger.besu.ethereum.core.Synchronizer;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 
@@ -30,13 +31,16 @@ public class ProtocolContext {
   private final WorldStateArchive worldStateArchive;
   private final ConsensusContext consensusContext;
 
+  private Optional<Synchronizer> synchronizer;
+
   public ProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final ConsensusContext consensusContext) {
     this.blockchain = blockchain;
     this.worldStateArchive = worldStateArchive;
     this.consensusContext = consensusContext;
+    this.synchronizer = Optional.empty();
   }
 
   public static ProtocolContext init(
@@ -50,6 +54,14 @@ public static ProtocolContext init(
         consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule));
   }
 
+  public Optional<Synchronizer> getSynchronizer() {
+    return synchronizer;
+  }
+
+  public void setSynchronizer(final Optional<Synchronizer> synchronizer) {
+    this.synchronizer = synchronizer;
+  }
+
   public MutableBlockchain getBlockchain() {
     return blockchain;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java

@@ -180,7 +180,7 @@ public void setBalance(final Wei value) {
   @Override
   public Bytes getCode() {
     if (code == null) {
-      code = context.getCode(address).orElse(Bytes.EMPTY);
+      code = context.getCode(address, codeHash).orElse(Bytes.EMPTY);
     }
     return code;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java

@@ -20,9 +20,11 @@
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 
 import org.apache.tuweni.bytes.Bytes;
@@ -84,13 +86,19 @@ protected Hash calculateRootHash(final BonsaiWorldStateUpdater worldStateUpdater
       final Bytes accountKey = accountUpdate.getKey();
       final BonsaiValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
       final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
-      if (updatedAccount == null) {
-        final Hash addressHash = Hash.hash(accountKey);
-        accountTrie.remove(addressHash);
-      } else {
-        final Hash addressHash = updatedAccount.getAddressHash();
-        final Bytes accountValue = updatedAccount.serializeAccount();
-        accountTrie.put(addressHash, accountValue);
+      try {
+        if (updatedAccount == null) {
+          final Hash addressHash = Hash.hash(accountKey);
+          accountTrie.remove(addressHash);
+        } else {
+          final Hash addressHash = updatedAccount.getAddressHash();
+          final Bytes accountValue = updatedAccount.serializeAccount();
+          accountTrie.put(addressHash, accountValue);
+        }
+      } catch (MerkleTrieException e) {
+        // need to throw to trigger the heal
+        throw new MerkleTrieException(
+            e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation());
       }
     }
 
@@ -129,10 +137,19 @@ private void updateAccountStorage(
           storageAccountUpdate.getValue().entrySet()) {
         final Hash keyHash = storageUpdate.getKey();
         final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
-        if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
-          storageTrie.remove(keyHash);
-        } else {
-          storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+        try {
+          if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
+            storageTrie.remove(keyHash);
+          } else {
+            storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+          }
+        } catch (MerkleTrieException e) {
+          // need to throw to trigger the heal
+          throw new MerkleTrieException(
+              e.getMessage(),
+              Optional.of(Address.wrap(updatedAddress)),
+              e.getHash(),
+              e.getLocation());
         }
       }
 
@@ -150,7 +167,12 @@ public void persist(final BlockHeader blockHeader) {
     final Hash newWorldStateRootHash = rootHash(localUpdater);
     archive
         .getTrieLogManager()
-        .saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader, this);
+        .saveTrieLog(
+            archive,
+            localUpdater,
+            newWorldStateRootHash,
+            blockHeader,
+            (BonsaiPersistedWorldState) this.copy());
     worldStateRootHash = newWorldStateRootHash;
     worldStateBlockHash = blockHeader.getBlockHash();
     isPersisted = true;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java

@@ -14,13 +14,10 @@
  */
 package org.hyperledger.besu.ethereum.bonsai;
 
-import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
 
-import java.util.Optional;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,15 +32,8 @@ public BonsaiInMemoryWorldStateKeyValueStorage(
       final KeyValueStorage codeStorage,
       final KeyValueStorage storageStorage,
       final KeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage,
-      final Optional<PeerTrieNodeFinder> fallbackNodeFinder) {
-    super(
-        accountStorage,
-        codeStorage,
-        storageStorage,
-        trieBranchStorage,
-        trieLogStorage,
-        fallbackNodeFinder);
+      final KeyValueStorage trieLogStorage) {
+    super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
   }
 
   @Override