[POC] heal the worldstate

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -632,6 +632,8 @@ public BesuController build() {
             ethProtocolManager,
             pivotBlockSelector);
 
+    protocolContext.setSynchronizer(Optional.of(synchronizer));
+
     final MiningCoordinator miningCoordinator =
         createMiningCoordinator(
             protocolSchedule,
@@ -713,7 +715,6 @@ protected Synchronizer createSynchronizer(
             clock,
             metricsSystem,
             getFullSyncTerminationCondition(protocolContext.getBlockchain()),
-            ethProtocolManager,
             pivotBlockSelector);
 
     return toUse;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java

@@ -167,7 +167,16 @@ public BlockProcessingResult validateAndProcessBlock(
         return new BlockProcessingResult(
             Optional.of(new BlockProcessingOutputs(worldState, receipts)));
       }
-    } catch (StorageException | MerkleTrieException ex) {
+    } catch (MerkleTrieException ex) {
+      context
+          .getSynchronizer()
+          .ifPresentOrElse(
+              synchronizer -> synchronizer.healWorldState(ex.getMaybeAddress(), ex.getLocation()),
+              () ->
+                  handleAndLogImportFailure(
+                      block, new BlockProcessingResult(Optional.empty(), ex)));
+      return new BlockProcessingResult(Optional.empty(), ex);
+    } catch (StorageException ex) {
       var retval = new BlockProcessingResult(Optional.empty(), ex);
       handleAndLogImportFailure(block, retval);
       return retval;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.ethereum;
 
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
+import org.hyperledger.besu.ethereum.core.Synchronizer;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 
@@ -30,13 +31,16 @@ public class ProtocolContext {
   private final WorldStateArchive worldStateArchive;
   private final ConsensusContext consensusContext;
 
+  private Optional<Synchronizer> synchronizer;
+
   public ProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final ConsensusContext consensusContext) {
     this.blockchain = blockchain;
     this.worldStateArchive = worldStateArchive;
     this.consensusContext = consensusContext;
+    this.synchronizer = Optional.empty();
   }
 
   public static ProtocolContext init(
@@ -50,6 +54,14 @@ public static ProtocolContext init(
         consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule));
   }
 
+  public Optional<Synchronizer> getSynchronizer() {
+    return synchronizer;
+  }
+
+  public void setSynchronizer(final Optional<Synchronizer> synchronizer) {
+    this.synchronizer = synchronizer;
+  }
+
   public MutableBlockchain getBlockchain() {
     return blockchain;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java

@@ -180,7 +180,7 @@ public void setBalance(final Wei value) {
   @Override
   public Bytes getCode() {
     if (code == null) {
-      code = context.getCode(address).orElse(Bytes.EMPTY);
+      code = context.getCode(address, codeHash).orElse(Bytes.EMPTY);
     }
     return code;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java

@@ -20,9 +20,11 @@
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 
 import org.apache.tuweni.bytes.Bytes;
@@ -84,13 +86,19 @@ protected Hash calculateRootHash(final BonsaiWorldStateUpdater worldStateUpdater
       final Bytes accountKey = accountUpdate.getKey();
       final BonsaiValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
       final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
-      if (updatedAccount == null) {
-        final Hash addressHash = Hash.hash(accountKey);
-        accountTrie.remove(addressHash);
-      } else {
-        final Hash addressHash = updatedAccount.getAddressHash();
-        final Bytes accountValue = updatedAccount.serializeAccount();
-        accountTrie.put(addressHash, accountValue);
+      try {
+        if (updatedAccount == null) {
+          final Hash addressHash = Hash.hash(accountKey);
+          accountTrie.remove(addressHash);
+        } else {
+          final Hash addressHash = updatedAccount.getAddressHash();
+          final Bytes accountValue = updatedAccount.serializeAccount();
+          accountTrie.put(addressHash, accountValue);
+        }
+      } catch (MerkleTrieException e) {
+        // need to throw to trigger the heal
+        throw new MerkleTrieException(
+            e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation());
       }
     }
 
@@ -129,10 +137,19 @@ private void updateAccountStorage(
           storageAccountUpdate.getValue().entrySet()) {
         final Hash keyHash = storageUpdate.getKey();
         final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
-        if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
-          storageTrie.remove(keyHash);
-        } else {
-          storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+        try {
+          if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
+            storageTrie.remove(keyHash);
+          } else {
+            storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+          }
+        } catch (MerkleTrieException e) {
+          // need to throw to trigger the heal
+          throw new MerkleTrieException(
+              e.getMessage(),
+              Optional.of(Address.wrap(updatedAddress)),
+              e.getHash(),
+              e.getLocation());
         }
       }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java

@@ -14,13 +14,10 @@
  */
 package org.hyperledger.besu.ethereum.bonsai;
 
-import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
 
-import java.util.Optional;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,15 +32,8 @@ public BonsaiInMemoryWorldStateKeyValueStorage(
       final KeyValueStorage codeStorage,
       final KeyValueStorage storageStorage,
       final KeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage,
-      final Optional<PeerTrieNodeFinder> fallbackNodeFinder) {
-    super(
-        accountStorage,
-        codeStorage,
-        storageStorage,
-        trieBranchStorage,
-        trieLogStorage,
-        fallbackNodeFinder);
+      final KeyValueStorage trieLogStorage) {
+    super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState;
+import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.evm.account.Account;
 import org.hyperledger.besu.evm.worldstate.WorldState;
@@ -107,7 +108,7 @@ public long getHeight() {
   }
 
   @Override
-  public Optional<Bytes> getCode(final Address address) {
+  public Optional<Bytes> getCode(final Address address, final Hash codeHash) {
     BonsaiLayeredWorldState currentLayer = this;
     while (currentLayer != null) {
       final Optional<Bytes> maybeCode = currentLayer.trieLog.getCode(address);
@@ -124,7 +125,7 @@ public Optional<Bytes> getCode(final Address address) {
       } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) {
         currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get();
       } else {
-        return currentLayer.getNextWorldView().get().getCode(address);
+        return currentLayer.getNextWorldView().get().getCode(address, codeHash);
       }
     }
     return Optional.empty();
@@ -291,6 +292,8 @@ public MutableWorldState copy() {
                     new StorageException(
                         "Unable to copy Layered Worldstate for " + blockHash().toHexString()))) {
       return new BonsaiInMemoryWorldState(archive, snapshot.getWorldStateStorage());
+    } catch (MerkleTrieException ex) {
+      throw ex; // need to throw to trigger the heal
     } catch (Exception ex) {
       throw new RuntimeException(ex);
     }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java

@@ -25,6 +25,7 @@
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
+import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
 import org.hyperledger.besu.evm.account.Account;
@@ -92,15 +93,14 @@ public MutableWorldState copy() {
             worldStateStorage.codeStorage,
             worldStateStorage.storageStorage,
             worldStateStorage.trieBranchStorage,
-            worldStateStorage.trieLogStorage,
-            getWorldStateStorage().getMaybeFallbackNodeFinder());
+            worldStateStorage.trieLogStorage);
 
     return new BonsaiInMemoryWorldState(archive, bonsaiInMemoryWorldStateKeyValueStorage);
   }
 
   @Override
-  public Optional<Bytes> getCode(@Nonnull final Address address) {
-    return worldStateStorage.getCode(null, Hash.hash(address));
+  public Optional<Bytes> getCode(@Nonnull final Address address, final Hash codeHash) {
+    return worldStateStorage.getCode(codeHash, Hash.hash(address));
   }
 
   public void setArchiveStateUnSafe(final BlockHeader blockHeader) {
@@ -148,7 +148,7 @@ protected Hash calculateRootHash(
     return Hash.wrap(rootHash);
   }
 
-  private static void addTheAccounts(
+  private void addTheAccounts(
       final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater,
       final BonsaiWorldStateUpdater worldStateUpdater,
       final StoredMerklePatriciaTrie<Bytes, Bytes> accountTrie) {
@@ -157,20 +157,26 @@ private static void addTheAccounts(
       final Bytes accountKey = accountUpdate.getKey();
       final BonsaiValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
       final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
-      if (updatedAccount == null) {
-        final Hash addressHash = Hash.hash(accountKey);
-        accountTrie.remove(addressHash);
-        stateUpdater.removeAccountInfoState(addressHash);
-      } else {
-        final Hash addressHash = updatedAccount.getAddressHash();
-        final Bytes accountValue = updatedAccount.serializeAccount();
-        stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue);
-        accountTrie.put(addressHash, accountValue);
+      try {
+        if (updatedAccount == null) {
+          final Hash addressHash = Hash.hash(accountKey);
+          accountTrie.remove(addressHash);
+          stateUpdater.removeAccountInfoState(addressHash);
+        } else {
+          final Hash addressHash = updatedAccount.getAddressHash();
+          final Bytes accountValue = updatedAccount.serializeAccount();
+          stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue);
+          accountTrie.put(addressHash, accountValue);
+        }
+      } catch (MerkleTrieException e) {
+        // need to throw to trigger the heal
+        throw new MerkleTrieException(
+            e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation());
       }
     }
   }
 
-  private static void updateCode(
+  private void updateCode(
       final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater,
       final BonsaiWorldStateUpdater worldStateUpdater) {
     for (final Map.Entry<Address, BonsaiValue<Bytes>> codeUpdate :
@@ -215,12 +221,21 @@ private void updateAccountStorageState(
             storageAccountUpdate.getValue().entrySet()) {
           final Hash keyHash = storageUpdate.getKey();
           final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
-          if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
-            stateUpdater.removeStorageValueBySlotHash(updatedAddressHash, keyHash);
-            storageTrie.remove(keyHash);
-          } else {
-            stateUpdater.putStorageValueBySlotHash(updatedAddressHash, keyHash, updatedStorage);
-            storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+          try {
+            if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
+              stateUpdater.removeStorageValueBySlotHash(updatedAddressHash, keyHash);
+              storageTrie.remove(keyHash);
+            } else {
+              stateUpdater.putStorageValueBySlotHash(updatedAddressHash, keyHash, updatedStorage);
+              storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
+            }
+          } catch (MerkleTrieException e) {
+            // need to throw to trigger the heal
+            throw new MerkleTrieException(
+                e.getMessage(),
+                Optional.of(Address.wrap(updatedAddress)),
+                e.getHash(),
+                e.getLocation());
           }
         }
 
@@ -259,18 +274,24 @@ private void clearStorage(
               oldAccount.getStorageRoot(),
               Function.identity(),
               Function.identity());
-      Map<Bytes32, Bytes> entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256);
-      while (!entriesToDelete.isEmpty()) {
-        entriesToDelete
-            .keySet()
-            .forEach(
-                k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k)));
-        entriesToDelete.keySet().forEach(storageTrie::remove);
-        if (entriesToDelete.size() == 256) {
-          entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256);
-        } else {
-          break;
+      try {
+        Map<Bytes32, Bytes> entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256);
+        while (!entriesToDelete.isEmpty()) {
+          entriesToDelete
+              .keySet()
+              .forEach(
+                  k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k)));
+          entriesToDelete.keySet().forEach(storageTrie::remove);
+          if (entriesToDelete.size() == 256) {
+            entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256);
+          } else {
+            break;
+          }
         }
+      } catch (MerkleTrieException e) {
+        // need to throw to trigger the heal
+        throw new MerkleTrieException(
+            e.getMessage(), Optional.of(Address.wrap(address)), e.getHash(), e.getLocation());
       }
     }
   }