Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/rebuild trie from flat #7969

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
GC merkle tries on interim commits, spotless.
Signed-off-by: garyschulte <garyschulte@gmail.com>
garyschulte committed Jan 22, 2025
commit e54addc13520a0d30575d837aef3593afe47fb1a
Original file line number Diff line number Diff line change
@@ -38,6 +38,7 @@
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;

import java.util.function.Consumer;
import java.util.function.Function;

import org.apache.tuweni.bytes.Bytes;
@@ -133,10 +134,9 @@ Hash rebuildTrie(final BonsaiWorldStateKeyValueStorage worldStateStorage) {

// rebuild the trie by inserting everything into a StoredMerklePatriciaTrie
// and incrementally (naively) commit after each account while streaming
// TODO: optimize to incrementally commit tx after a certain threshold

final var wss = worldStateStorage.getComposedWorldStateStorage();
final var accountTrie =
var accountTrie =
new StoredMerklePatriciaTrie<>(
new StoredNodeFactory<>(
// this may be inefficient, and we can read through an incrementally committing tx
@@ -147,9 +147,9 @@ Hash rebuildTrie(final BonsaiWorldStateKeyValueStorage worldStateStorage) {
MerkleTrie.EMPTY_TRIE_NODE_HASH);

final var accountsTx = new WrappedTransaction(worldStateStorage.getComposedWorldStateStorage());
final Runnable accountTrieCommit =
() ->
accountTrie.commit(
final Consumer<StoredMerklePatriciaTrie<Bytes, Bytes>> accountTrieCommit =
(trie) ->
trie.commit(
(loc, hash, value) ->
accountsTx.put(
TRIE_BRANCH_STORAGE, loc.toArrayUnsafe(), value.toArrayUnsafe()));
@@ -181,13 +181,25 @@ Hash rebuildTrie(final BonsaiWorldStateKeyValueStorage worldStateStorage) {
LOG.info("committing account trie at account {}", accountPair.getFirst());

// commit the account trie if we have exceeded the forced commit interval
accountTrieCommit.run();
accountTrieCommit.accept(accountTrie);
accountsTx.commitAndReopen();

// new trie with new root, GC trie nodes
accountTrie =
new StoredMerklePatriciaTrie<>(
new StoredNodeFactory<>(
// this may be inefficient, and we can read through an incrementally committing
// tx
// instead
worldStateStorage::getAccountStateTrieNode,
Function.identity(),
Function.identity()),
accountTrie.getRootHash());
}
}

// final commit
accountTrieCommit.run();
accountTrieCommit.accept(accountTrie);
accountsTx.commit();

// return the new state trie root hash
@@ -209,19 +221,22 @@ Hash rebuildAccountTrie(
var accountStorageTx = new WrappedTransaction(worldStateStorage.getComposedWorldStateStorage());
var wss = worldStateStorage.getComposedWorldStateStorage();

Function<Bytes32, StoredMerklePatriciaTrie<Bytes, Bytes>> newAccountStorageTrie =
(rootHash) ->
new StoredMerklePatriciaTrie<>(
new StoredNodeFactory<>(
(location, hash) ->
worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash),
Function.identity(),
Function.identity()),
rootHash);

// create account storage trie
var accountStorageTrie =
new StoredMerklePatriciaTrie<>(
new StoredNodeFactory<>(
(location, hash) ->
worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash),
Function.identity(),
Function.identity()),
MerkleTrie.EMPTY_TRIE_NODE_HASH);
var accountStorageTrie = newAccountStorageTrie.apply(MerkleTrie.EMPTY_TRIE_NODE_HASH);

Runnable accountStorageCommit =
() ->
accountStorageTrie.commit(
Consumer<StoredMerklePatriciaTrie<Bytes, Bytes>> accountStorageCommit =
(trie) ->
trie.commit(
(location, hash, value) ->
accountStorageTx.put(
TRIE_BRANCH_STORAGE,
@@ -241,11 +256,13 @@ Hash rebuildAccountTrie(

// commit the account storage trie
if (accountStorageCount++ % FORCED_COMMIT_INTERVAL == 0) {
accountStorageCommit.run();
accountStorageCommit.accept(accountStorageTrie);
accountStorageTx.commitAndReopen();
// new trie with new root, GC trie nodes
accountStorageTrie = newAccountStorageTrie.apply(accountStorageTrie.getRootHash());
}
}
accountStorageCommit.run();
accountStorageCommit.accept(accountStorageTrie);
accountStorageTx.commit();

return Hash.wrap(accountStorageTrie.getRootHash());
Original file line number Diff line number Diff line change
@@ -55,9 +55,8 @@ void setupBonsaiBlockchain() {
blockchainSetupUtil =
BlockchainSetupUtil.createForEthashChain(
// Mainnet is a more robust test resource, but it takes upwards of 1 minute to generate
//BlockTestUtil.getMainnetResources(),
BlockTestUtil.getSnapTestChainResources(),
DataStorageFormat.BONSAI);
// BlockTestUtil.getMainnetResources(),
BlockTestUtil.getSnapTestChainResources(), DataStorageFormat.BONSAI);
blockchainSetupUtil.importAllBlocks(HeaderValidationMode.NONE, HeaderValidationMode.NONE);
}