From 636c9a97eabe89454144e66421952ea6d3e69860 Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Sat, 22 Oct 2022 11:46:15 +0200 Subject: [PATCH 1/3] extracting methods out of calculate root hash To properly measure performance and differences between parts of the methods we extract parts of it. Signed-off-by: Jiri Peinlich --- .../bonsai/BonsaiPersistedWorldState.java | 168 ++++++++++-------- 1 file changed, 91 insertions(+), 77 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index aed904c32d5..fd2026f9bf9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -105,43 +105,69 @@ public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { protected Hash calculateRootHash( final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { - // first clear storage - for (final Address address : worldStateUpdater.getStorageToClear()) { - // because we are clearing persisted values we need the account root as persisted - final BonsaiAccount oldAccount = - worldStateStorage - .getAccount(Hash.hash(address)) - .map(bytes -> fromRLP(BonsaiPersistedWorldState.this, address, bytes, true)) - .orElse(null); - if (oldAccount == null) { - // This is when an account is both created and deleted within the scope of the same - // block. A not-uncommon DeFi bot pattern. - continue; + clearStorage(stateUpdater, worldStateUpdater); + + // This must be done before updating the accounts so + // that we can get the storage state hash + updateAccountStorageState(stateUpdater, worldStateUpdater); + + // Third update the code. This has the side effect of ensuring a code hash is calculated. + updateCode(stateUpdater, worldStateUpdater); + + // next walk the account trie + final StoredMerklePatriciaTrie accountTrie = + new StoredMerklePatriciaTrie<>( + this::getAccountStateTrieNode, + worldStateRootHash, + Function.identity(), + Function.identity()); + + // for manicured tries and composting, collect branches here (not implemented) + + addTheAccounts(stateUpdater, worldStateUpdater, accountTrie); + + // TODO write to a cache and then generate a layer update from that and the + // DB tx updates. Right now it is just DB updates. + accountTrie.commit( + (location, hash, value) -> + writeTrieNode(stateUpdater.getTrieBranchStorageTransaction(), location, value)); + final Bytes32 rootHash = accountTrie.getRootHash(); + return Hash.wrap(rootHash); + } + + private static void addTheAccounts(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { + for (final Map.Entry> accountUpdate : + worldStateUpdater.getAccountsToUpdate().entrySet()) { + final Bytes accountKey = accountUpdate.getKey(); + final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); + if (updatedAccount == null) { + final Hash addressHash = Hash.hash(accountKey); + accountTrie.remove(addressHash); + stateUpdater.removeAccountInfoState(addressHash); + } else { + final Hash addressHash = updatedAccount.getAddressHash(); + final Bytes accountValue = updatedAccount.serializeAccount(); + stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue); + accountTrie.put(addressHash, accountValue); } - final Hash addressHash = Hash.hash(address); - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (location, key) -> getStorageTrieNode(addressHash, location, key), - oldAccount.getStorageRoot(), - Function.identity(), - Function.identity()); - Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); - while (!entriesToDelete.isEmpty()) { - entriesToDelete - .keySet() - .forEach( - k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k))); - if (entriesToDelete.size() == 256) { - entriesToDelete.keySet().forEach(storageTrie::remove); - entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); - } else { - break; - } + } + } + + private static void updateCode(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { + for (final Map.Entry> codeUpdate : + worldStateUpdater.getCodeToUpdate().entrySet()) { + final Bytes updatedCode = codeUpdate.getValue().getUpdated(); + final Hash accountHash = Hash.hash(codeUpdate.getKey()); + if (updatedCode == null || updatedCode.size() == 0) { + stateUpdater.removeCode(accountHash); + } else { + stateUpdater.putCode(accountHash, null, updatedCode); } } + } - // second update account storage state. This must be done before updating the accounts so - // that we can get the storage state hash + private void updateAccountStorageState(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { for (final Map.Entry>> storageAccountUpdate : worldStateUpdater.getStorageToUpdate().entrySet()) { final Address updatedAddress = storageAccountUpdate.getKey(); @@ -185,54 +211,42 @@ protected Hash calculateRootHash( } // for manicured tries and composting, trim and compost here } + } - // Third update the code. This has the side effect of ensuring a code hash is calculated. - for (final Map.Entry> codeUpdate : - worldStateUpdater.getCodeToUpdate().entrySet()) { - final Bytes updatedCode = codeUpdate.getValue().getUpdated(); - final Hash accountHash = Hash.hash(codeUpdate.getKey()); - if (updatedCode == null || updatedCode.size() == 0) { - stateUpdater.removeCode(accountHash); - } else { - stateUpdater.putCode(accountHash, null, updatedCode); + private void clearStorage(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { + for (final Address address : worldStateUpdater.getStorageToClear()) { + // because we are clearing persisted values we need the account root as persisted + final BonsaiAccount oldAccount = + worldStateStorage + .getAccount(Hash.hash(address)) + .map(bytes -> fromRLP(BonsaiPersistedWorldState.this, address, bytes, true)) + .orElse(null); + if (oldAccount == null) { + // This is when an account is both created and deleted within the scope of the same + // block. A not-uncommon DeFi bot pattern. + continue; } - } - - // next walk the account trie - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - this::getAccountStateTrieNode, - worldStateRootHash, - Function.identity(), - Function.identity()); - - // for manicured tries and composting, collect branches here (not implemented) - - // now add the accounts - for (final Map.Entry> accountUpdate : - worldStateUpdater.getAccountsToUpdate().entrySet()) { - final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); - final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); - if (updatedAccount == null) { - final Hash addressHash = Hash.hash(accountKey); - accountTrie.remove(addressHash); - stateUpdater.removeAccountInfoState(addressHash); - } else { - final Hash addressHash = updatedAccount.getAddressHash(); - final Bytes accountValue = updatedAccount.serializeAccount(); - stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue); - accountTrie.put(addressHash, accountValue); + final Hash addressHash = Hash.hash(address); + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (location, key) -> getStorageTrieNode(addressHash, location, key), + oldAccount.getStorageRoot(), + Function.identity(), + Function.identity()); + Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + while (!entriesToDelete.isEmpty()) { + entriesToDelete + .keySet() + .forEach( + k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k))); + if (entriesToDelete.size() == 256) { + entriesToDelete.keySet().forEach(storageTrie::remove); + entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + } else { + break; + } } } - - // TODO write to a cache and then generate a layer update from that and the - // DB tx updates. Right now it is just DB updates. - accountTrie.commit( - (location, hash, value) -> - writeTrieNode(stateUpdater.getTrieBranchStorageTransaction(), location, value)); - final Bytes32 rootHash = accountTrie.getRootHash(); - return Hash.wrap(rootHash); } @Override From a3b5d4445f3b72c1220f1f913c516a53e6d67f31 Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Sat, 22 Oct 2022 11:49:25 +0200 Subject: [PATCH 2/3] we should delete the nodes from trie branch storage all the time The algorithm would leave few nodes in the trie branch storage in its last iteration Signed-off-by: Jiri Peinlich --- .../besu/ethereum/bonsai/BonsaiPersistedWorldState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index fd2026f9bf9..aab18da9c6a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -239,8 +239,8 @@ private void clearStorage(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater st .keySet() .forEach( k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k))); + entriesToDelete.keySet().forEach(storageTrie::remove); if (entriesToDelete.size() == 256) { - entriesToDelete.keySet().forEach(storageTrie::remove); entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); } else { break; From cacdaaf8be073900003c16c5bf9d7fe39a2bd4f2 Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Sat, 22 Oct 2022 12:06:15 +0200 Subject: [PATCH 3/3] fixed spottless Signed-off-by: Jiri Peinlich --- .../bonsai/BonsaiPersistedWorldState.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index aab18da9c6a..33b01c28eaf 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -135,7 +135,10 @@ protected Hash calculateRootHash( return Hash.wrap(rootHash); } - private static void addTheAccounts(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { + private static void addTheAccounts( + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, + final BonsaiWorldStateUpdater worldStateUpdater, + final StoredMerklePatriciaTrie accountTrie) { for (final Map.Entry> accountUpdate : worldStateUpdater.getAccountsToUpdate().entrySet()) { final Bytes accountKey = accountUpdate.getKey(); @@ -154,7 +157,9 @@ private static void addTheAccounts(final BonsaiWorldStateKeyValueStorage.BonsaiU } } - private static void updateCode(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { + private static void updateCode( + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, + final BonsaiWorldStateUpdater worldStateUpdater) { for (final Map.Entry> codeUpdate : worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); @@ -167,7 +172,9 @@ private static void updateCode(final BonsaiWorldStateKeyValueStorage.BonsaiUpdat } } - private void updateAccountStorageState(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { + private void updateAccountStorageState( + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, + final BonsaiWorldStateUpdater worldStateUpdater) { for (final Map.Entry>> storageAccountUpdate : worldStateUpdater.getStorageToUpdate().entrySet()) { final Address updatedAddress = storageAccountUpdate.getKey(); @@ -213,7 +220,9 @@ private void updateAccountStorageState(final BonsaiWorldStateKeyValueStorage.Bon } } - private void clearStorage(final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { + private void clearStorage( + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, + final BonsaiWorldStateUpdater worldStateUpdater) { for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount =