diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 6d6bed2bfcf..32d1501ef08 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -73,6 +73,7 @@ import org.hyperledger.besu.cli.options.unstable.PrivacyPluginOptions; import org.hyperledger.besu.cli.options.unstable.RPCOptions; import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions; +import org.hyperledger.besu.cli.options.unstable.TrieLogPruningOptions; import org.hyperledger.besu.cli.presynctasks.PreSynchronizationTaskRunner; import org.hyperledger.besu.cli.presynctasks.PrivateDatabaseMigrationPreSyncTask; import org.hyperledger.besu.cli.subcommands.PasswordSubCommand; @@ -294,6 +295,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private final EvmOptions unstableEvmOptions = EvmOptions.create(); private final IpcOptions unstableIpcOptions = IpcOptions.create(); private final ChainPruningOptions unstableChainPruningOptions = ChainPruningOptions.create(); + private final TrieLogPruningOptions unstableTrieLogPruningOptions = + TrieLogPruningOptions.create(); // stable CLI options private final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); @@ -1530,6 +1533,7 @@ private void handleUnstableOptions() { .put("EVM Options", unstableEvmOptions) .put("IPC Options", unstableIpcOptions) .put("Chain Data Pruning Options", unstableChainPruningOptions) + .put("Trie Log Pruning Options", unstableTrieLogPruningOptions) .build(); UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions); @@ -2165,6 +2169,7 @@ public BesuControllerBuilder getControllerBuilder() { .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers) .randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority) .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject()) + .trieLogPrunerConfiguration(unstableTrieLogPruningOptions.toDomainObject()) .cacheLastBlocks(numberOfblocksToCache); } @@ -3432,6 +3437,10 @@ private String generateConfigurationOverview() { builder.setHighSpecEnabled(); } + if (unstableTrieLogPruningOptions.getTrieLogPruningEnabled()) { + builder.setTrieLogPruningEnabled(); + } + builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setPluginContext(besuComponent.getBesuPluginContext()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 508b195a8eb..5e178710dee 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -49,6 +49,7 @@ public class ConfigurationOverviewBuilder { private Collection engineApis; private String engineJwtFilePath; private boolean isHighSpec = false; + private boolean isTrieLogPruning = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private Map environment; private BesuPluginContextImpl besuPluginContext; @@ -169,6 +170,16 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() { return this; } + /** + * Sets high spec enabled. + * + * @return the builder + */ + public ConfigurationOverviewBuilder setTrieLogPruningEnabled() { + isTrieLogPruning = true; + return this; + } + /** * Sets the txpool implementation in use. * @@ -258,6 +269,10 @@ public String build() { lines.add("Using " + txPoolImplementation + " transaction pool implementation"); + if (isTrieLogPruning) { + lines.add("Trie log pruning enabled"); + } + lines.add(""); lines.add("Host:"); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TrieLogPruningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TrieLogPruningOptions.java new file mode 100644 index 00000000000..a73272578ab --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TrieLogPruningOptions.java @@ -0,0 +1,64 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.cli.options.unstable; + +import org.hyperledger.besu.cli.options.CLIOptions; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPrunerConfiguration; + +import java.util.Arrays; +import java.util.List; + +import picocli.CommandLine; + +/** The trie log pruning CLI options. */ +public class TrieLogPruningOptions implements CLIOptions { + + private static final String TRIE_LOG_PRUNING_ENABLED_FLAG = "--Xtrie-log-pruning-enabled"; + + @CommandLine.Option( + hidden = true, + names = {TRIE_LOG_PRUNING_ENABLED_FLAG}, + description = "Enable trie log pruning (default: ${DEFAULT-VALUE})") + private final Boolean trieLogPruningEnabled = Boolean.FALSE; + + /** + * Create trie log pruning options. + * + * @return the trie log pruning options + */ + public static TrieLogPruningOptions create() { + return new TrieLogPruningOptions(); + } + + /** + * Gets trie log pruning enabled. + * + * @return the trie log pruning enabled + */ + public Boolean getTrieLogPruningEnabled() { + return trieLogPruningEnabled; + } + + @Override + public TrieLogPrunerConfiguration toDomainObject() { + return new TrieLogPrunerConfiguration(trieLogPruningEnabled); + } + + @Override + public List getCLIOptions() { + return Arrays.asList(TRIE_LOG_PRUNING_ENABLED_FLAG, trieLogPruningEnabled.toString()); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 802be1215b5..b1722b793dc 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -34,6 +34,8 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPrunerConfiguration; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.ChainDataPruner; @@ -179,6 +181,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides private int maxRemotelyInitiatedPeers; /** The Chain pruner configuration. */ protected ChainPrunerConfiguration chainPrunerConfiguration = ChainPrunerConfiguration.DEFAULT; + /** The Trie log pruner configuration. */ + protected TrieLogPrunerConfiguration trieLogPrunerConfiguration = + TrieLogPrunerConfiguration.DEFAULT; private NetworkingConfiguration networkingConfiguration; private Boolean randomPeerPriority; @@ -508,7 +513,19 @@ public BesuControllerBuilder chainPruningConfiguration( } /** - * Chain pruning configuration besu controller builder. + * Trie log pruning configuration besu controller builder. + * + * @param trieLogPrunerConfiguration the trie log pruner configuration + * @return the besu controller builder + */ + public BesuControllerBuilder trieLogPrunerConfiguration( + final TrieLogPrunerConfiguration trieLogPrunerConfiguration) { + this.trieLogPrunerConfiguration = trieLogPrunerConfiguration; + return this; + } + + /** + * Cache last blocks configuration besu controller builder. * * @param numberOfBlocksToCache the number of blocks to cache * @return the besu controller builder @@ -613,8 +630,15 @@ public BesuController build() { .map(BesuComponent::getCachedMerkleTrieLoader) .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); + final TrieLogPruner trieLogPruner = + trieLogPrunerConfiguration.getTrieLogPruningEnabled() + ? new TrieLogPruner((BonsaiWorldStateKeyValueStorage) worldStateStorage, blockchain) + : TrieLogPruner.noOpTrieLogPruner(); + trieLogPruner.initialise(); + final WorldStateArchive worldStateArchive = - createWorldStateArchive(worldStateStorage, blockchain, cachedMerkleTrieLoader); + createWorldStateArchive( + worldStateStorage, blockchain, cachedMerkleTrieLoader, trieLogPruner); if (blockchain.getChainHeadBlockNumber() < 1) { genesisState.writeStateTo(worldStateArchive.getMutable()); @@ -1070,7 +1094,8 @@ private Optional createSnapProtocolManager( WorldStateArchive createWorldStateArchive( final WorldStateStorage worldStateStorage, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final TrieLogPruner trieLogPruner) { switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI: return new BonsaiWorldStateProvider( @@ -1079,7 +1104,8 @@ WorldStateArchive createWorldStateArchive( Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), cachedMerkleTrieLoader, metricsSystem, - besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null)); + besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), + trieLogPruner); case FOREST: default: diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index bccf28c4522..93cf5599ece 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -252,6 +252,7 @@ public void initMocks() throws Exception { when(mockControllerBuilder.randomPeerPriority(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.chainPruningConfiguration(any())).thenReturn(mockControllerBuilder); + when(mockControllerBuilder.trieLogPrunerConfiguration(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.lowerBoundPeers(anyInt())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.maxRemotelyInitiatedPeers(anyInt())) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index a24479e8617..00cf448b924 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; @@ -165,7 +166,10 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { doReturn(worldStateArchive) .when(besuControllerBuilder) .createWorldStateArchive( - any(WorldStateStorage.class), any(Blockchain.class), any(CachedMerkleTrieLoader.class)); + any(WorldStateStorage.class), + any(Blockchain.class), + any(CachedMerkleTrieLoader.class), + any(TrieLogPruner.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index bcb7f9f65fe..d01193e4a8e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -32,10 +33,8 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -68,36 +67,25 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { private final BonsaiWorldStateKeyValueStorage worldStateStorage; private final CachedMerkleTrieLoader cachedMerkleTrieLoader; - public BonsaiWorldStateProvider( - final StorageProvider provider, - final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final ObservableMetricsSystem metricsSystem, - final BesuContext pluginContext) { - this( - (BonsaiWorldStateKeyValueStorage) - provider.createWorldStateStorage(DataStorageFormat.BONSAI), - blockchain, - Optional.empty(), - cachedMerkleTrieLoader, - metricsSystem, - pluginContext); - } - public BonsaiWorldStateProvider( final BonsaiWorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final ObservableMetricsSystem metricsSystem, - final BesuContext pluginContext) { + final BesuContext pluginContext, + final TrieLogPruner trieLogPruner) { this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( - blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext); + blockchain, + worldStateStorage, + maxLayersToLoad.orElse(RETAINED_LAYERS), + pluginContext, + trieLogPruner); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 9756b280e45..452bcdfe601 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -47,6 +47,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -203,6 +204,10 @@ public Optional getTrieLog(final Hash blockHash) { return trieLogStorage.get(blockHash.toArrayUnsafe()); } + public Stream streamTrieLogs(final int limit) { + return trieLogStorage.streamKeys().limit(limit); + } + public Optional getStateTrieNode(final Bytes location) { return composedWorldStateStorage .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) @@ -335,6 +340,10 @@ public long prune(final Predicate inUseCheck) { throw new RuntimeException("Bonsai Tries do not work with pruning."); } + public boolean pruneTrieLog(final byte[] blockHashBytes) { + return trieLogStorage.tryDelete(blockHashBytes); + } + @Override public long addNodeAddedListener(final NodesAddedListener listener) { throw new RuntimeException("addNodeAddedListener not available"); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java index cabd5a2d300..60aff6ec2c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java @@ -47,16 +47,19 @@ public class TrieLogManager { protected final Subscribers trieLogObservers = Subscribers.create(); protected final TrieLogFactory trieLogFactory; + private final TrieLogPruner trieLogPruner; public TrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, final long maxLayersToLoad, - final BesuContext pluginContext) { + final BesuContext pluginContext, + final TrieLogPruner trieLogPruner) { this.blockchain = blockchain; this.rootWorldStateStorage = worldStateStorage; this.maxLayersToLoad = maxLayersToLoad; this.trieLogFactory = setupTrieLogFactory(pluginContext); + this.trieLogPruner = trieLogPruner; } public synchronized void saveTrieLog( @@ -82,6 +85,9 @@ public synchronized void saveTrieLog( } finally { if (success) { stateUpdater.commit(); + trieLogPruner.cacheForLaterPruning( + forBlockHeader.getNumber(), forBlockHeader.getBlockHash().toArrayUnsafe()); + trieLogPruner.pruneFromCache(); } else { stateUpdater.rollback(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPruner.java new file mode 100644 index 00000000000..faca907afcc --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPruner.java @@ -0,0 +1,175 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.bonsai.trielog; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Multimap; +import com.google.common.collect.TreeMultimap; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TrieLogPruner { + + private static final Logger LOG = LoggerFactory.getLogger(TrieLogPruner.class); + + private static final int DEFAULT_PRUNING_LIMIT = 1000; + private final int pruningLimit; + private final int loadingLimit; + private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final Blockchain blockchain; + private final long numBlocksToRetain; + + private static final Multimap knownTrieLogKeysByDescendingBlockNumber = + TreeMultimap.create(Comparator.reverseOrder(), Comparator.comparingInt(Arrays::hashCode)); + + public TrieLogPruner( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain) { + this( + rootWorldStateStorage, + blockchain, + CachedWorldStorageManager.RETAINED_LAYERS, + DEFAULT_PRUNING_LIMIT); + } + + @VisibleForTesting + TrieLogPruner( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final Blockchain blockchain, + final long numBlocksToRetain, + final int pruningLimit) { + this.rootWorldStateStorage = rootWorldStateStorage; + this.blockchain = blockchain; + this.numBlocksToRetain = numBlocksToRetain; + this.pruningLimit = pruningLimit; + this.loadingLimit = pruningLimit; // same as pruningLimit for now + } + + public void initialise() { + preloadCache(); + } + + void preloadCache() { + LOG.atInfo() + .setMessage("Loading first {} trie logs from database...") + .addArgument(loadingLimit) + .log(); + final Stream trieLogs = rootWorldStateStorage.streamTrieLogs(loadingLimit); + final AtomicLong count = new AtomicLong(); + trieLogs.forEach( + hashAsBytes -> { + Hash hash = Hash.wrap(Bytes32.wrap(hashAsBytes)); + final Optional header = blockchain.getBlockHeader(hash); + if (header.isPresent()) { + knownTrieLogKeysByDescendingBlockNumber.put(header.get().getNumber(), hashAsBytes); + count.getAndIncrement(); + } else { + // prune orphaned blocks (sometimes created during block production) + rootWorldStateStorage.pruneTrieLog(hashAsBytes); + } + }); + LOG.atInfo().log("Loaded {} trie logs from database", count); + } + + void cacheForLaterPruning(final long blockNumber, final byte[] trieLogKey) { + LOG.atTrace() + .setMessage("caching trie log for later pruning blockNumber {}; trieLogKey (blockHash) {}") + .addArgument(blockNumber) + .addArgument(Bytes32.wrap(trieLogKey).toHexString()) + .log(); + knownTrieLogKeysByDescendingBlockNumber.put(blockNumber, trieLogKey); + } + + void pruneFromCache() { + final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; + LOG.atDebug() + .setMessage("(chainHeadNumber: {} - numBlocksToRetain: {}) = retainAboveThisBlock: {}") + .addArgument(blockchain.getChainHeadBlockNumber()) + .addArgument(numBlocksToRetain) + .addArgument(retainAboveThisBlock) + .log(); + + final var toPrune = + knownTrieLogKeysByDescendingBlockNumber.asMap().entrySet().stream() + .dropWhile((e) -> e.getKey() > retainAboveThisBlock) + .limit(pruningLimit); + + final List blockNumbersToRemove = new ArrayList<>(); + + final AtomicInteger count = new AtomicInteger(); + toPrune.forEach( + (e) -> { + for (byte[] trieLogKey : e.getValue()) { + rootWorldStateStorage.pruneTrieLog(trieLogKey); + count.getAndIncrement(); + } + blockNumbersToRemove.add(e.getKey()); + }); + + blockNumbersToRemove.forEach(knownTrieLogKeysByDescendingBlockNumber::removeAll); + LOG.atTrace() + .setMessage("pruned {} trie logs for blocks {}") + .addArgument(count) + .addArgument(blockNumbersToRemove) + .log(); + LOG.atDebug() + .setMessage("pruned {} trie logs from {} blocks") + .addArgument(count) + .addArgument(blockNumbersToRemove.size()) + .log(); + } + + public static TrieLogPruner noOpTrieLogPruner() { + return new NoOpTrieLogPruner(null, null); + } + + public static class NoOpTrieLogPruner extends TrieLogPruner { + private NoOpTrieLogPruner( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain) { + super(rootWorldStateStorage, blockchain); + } + + @Override + public void initialise() { + // no-op + } + + @Override + void cacheForLaterPruning(final long blockNumber, final byte[] trieLogKey) { + // no-op + } + + @Override + void pruneFromCache() { + // no-op + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerConfiguration.java new file mode 100644 index 00000000000..2d1cf4c404c --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerConfiguration.java @@ -0,0 +1,28 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.bonsai.trielog; + +public class TrieLogPrunerConfiguration { + public static final TrieLogPrunerConfiguration DEFAULT = new TrieLogPrunerConfiguration(false); + private final boolean enabled; + + public TrieLogPrunerConfiguration(final boolean enabled) { + this.enabled = enabled; + } + + public boolean getTrieLogPruningEnabled() { + return enabled; + } +} diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 3687643c6d9..46e985ea402 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -16,6 +16,8 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -35,6 +37,8 @@ import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; +import java.util.Optional; + public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { public InMemoryKeyValueStorageProvider() { @@ -86,11 +90,14 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); return new BonsaiWorldStateProvider( - inMemoryKeyValueStorageProvider, + (BonsaiWorldStateKeyValueStorage) + inMemoryKeyValueStorageProvider.createWorldStateStorage(DataStorageFormat.BONSAI), blockchain, + Optional.empty(), cachedMerkleTrieLoader, new NoOpMetricsSystem(), - null); + null, + TrieLogPruner.noOpTrieLogPruner()); } public static MutableWorldState createInMemoryWorldState() { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index b936ed28fa3..9f1e30ff3ba 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.BadBlockManager; @@ -82,14 +81,10 @@ public class BlockImportExceptionHandlingTest { private final WorldStateStorage worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); - private CachedMerkleTrieLoader cachedMerkleTrieLoader; - private final WorldStateArchive worldStateArchive = // contains a BonsaiWorldState which we need to spy on. // do we need to also test with a DefaultWorldStateArchive? - spy( - new BonsaiWorldStateProvider( - storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null)); + spy(InMemoryKeyValueStorageProvider.createBonsaiInMemoryWorldStateArchive(blockchain)); private final BonsaiWorldState persisted = spy( @@ -109,7 +104,6 @@ public void setup() { mainnetBlockValidator = new MainnetBlockValidator( blockHeaderValidator, blockBodyValidator, blockProcessor, badBlockManager); - cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java index a6441f970fb..c20fb1b113e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; @@ -148,7 +149,8 @@ public void createStorage() { Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), new NoOpMetricsSystem(), - null); + null, + TrieLogPruner.noOpTrieLogPruner()); var ws = archive.getMutable(); genesisState.writeStateTo(ws); protocolContext = new ProtocolContext(blockchain, archive, null, Optional.empty()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java index 0721522ffb1..48a28e9c212 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java @@ -37,6 +37,7 @@ import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -121,7 +122,8 @@ public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { Optional.of(512L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), new NoOpMetricsSystem(), - null); + null, + TrieLogPruner.noOpTrieLogPruner()); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java index 0e2fb0c0d1a..054e1b88cc7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; @@ -128,11 +127,7 @@ class LogRollingTests { @BeforeEach void createStorage() { provider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); - archive = - new BonsaiWorldStateProvider( - provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null); + archive = InMemoryKeyValueStorageProvider.createBonsaiInMemoryWorldStateArchive(blockchain); accountStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); codeStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); @@ -144,15 +139,8 @@ void createStorage() { provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); secondProvider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader secondOptimizedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); secondArchive = - new BonsaiWorldStateProvider( - secondProvider, - blockchain, - secondOptimizedMerkleTrieLoader, - new NoOpMetricsSystem(), - null); + InMemoryKeyValueStorageProvider.createBonsaiInMemoryWorldStateArchive(blockchain); secondAccountStorage = secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); secondCodeStorage = diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java index 46e5b6af9f8..397ebf1f9ab 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -22,7 +22,6 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; @@ -51,11 +50,8 @@ public static void main(final String[] arg) throws IOException { new RollingFileReader((i, c) -> Path.of(String.format(arg[0] + "-%04d.rdat", i)), false); final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateProvider archive = - new BonsaiWorldStateProvider( - provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null); + InMemoryKeyValueStorageProvider.createBonsaiInMemoryWorldStateArchive(null); final BonsaiWorldState bonsaiState = new BonsaiWorldState( archive, new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem())); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index 0c911d33556..64bf1b1bcc1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -53,7 +53,13 @@ public class TrieLogManagerTests { @BeforeEach public void setup() { - trieLogManager = new TrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); + trieLogManager = + new TrieLogManager( + blockchain, + bonsaiWorldStateKeyValueStorage, + 512, + null, + TrieLogPruner.noOpTrieLogPruner()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerTest.java new file mode 100644 index 00000000000..e16b08db7b5 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogPrunerTest.java @@ -0,0 +1,98 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.bonsai.trielog; + +import static org.mockito.Mockito.times; + +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.chain.Blockchain; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +public class TrieLogPrunerTest { + + private BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private Blockchain blockchain; + + @BeforeEach + public void setup() { + rootWorldStateStorage = Mockito.mock(BonsaiWorldStateKeyValueStorage.class); + blockchain = Mockito.mock(Blockchain.class); + } + + @SuppressWarnings("BannedMethod") + @Test + public void trieLogs_pruned_in_reverse_order_within_pruning_limit() { + Configurator.setLevel(LogManager.getLogger(TrieLogPruner.class).getName(), Level.TRACE); + + // Given + + // pruning window is below numBlocksToRetain and inside the pruningLimit offset. + final long blocksToRetain = 3; + final int pruningLimit = 2; + TrieLogPruner trieLogPruner = + new TrieLogPruner(rootWorldStateStorage, blockchain, blocksToRetain, pruningLimit); + + final byte[] key0 = new byte[] {1, 2, 3}; // older block outside the prune window + final byte[] key1 = new byte[] {1, 2, 3}; // block inside the prune window + final byte[] key2 = new byte[] {4, 5, 6}; // same block (fork) + final byte[] key3 = new byte[] {7, 8, 9}; // different block inside the prune window + final byte[] key4 = new byte[] {10, 11, 12}; // retained block + final byte[] key5 = new byte[] {13, 14, 15}; // different retained block + final byte[] key6 = new byte[] {7, 8, 9}; // another retained block + final long block0 = 1000L; + final long block1 = 1001L; + final long block2 = 1002L; + final long block3 = 1003L; + final long block4 = 1004L; + final long block5 = 1005L; + + trieLogPruner.cacheForLaterPruning(block0, key0); // older block outside prune window + trieLogPruner.cacheForLaterPruning(block1, key1); // block inside the prune window + trieLogPruner.cacheForLaterPruning(block1, key2); // same block number (fork) + trieLogPruner.cacheForLaterPruning(block2, key3); // different block inside prune window + trieLogPruner.cacheForLaterPruning(block3, key4); // retained block + trieLogPruner.cacheForLaterPruning(block4, key5); // different retained block + trieLogPruner.cacheForLaterPruning(block5, key6); // another retained block + + Mockito.when(blockchain.getChainHeadBlockNumber()).thenReturn(block5); + + // When + trieLogPruner.pruneFromCache(); + + // Then + InOrder inOrder = Mockito.inOrder(rootWorldStateStorage); + inOrder.verify(rootWorldStateStorage, times(1)).pruneTrieLog(key3); + inOrder.verify(rootWorldStateStorage, times(1)).pruneTrieLog(key1); + inOrder.verify(rootWorldStateStorage, times(1)).pruneTrieLog(key2); + + // Subsequent run should add one more block, then prune two oldest remaining keys + long block6 = 1006L; + trieLogPruner.cacheForLaterPruning(block6, new byte[] {1, 2, 3}); + Mockito.when(blockchain.getChainHeadBlockNumber()).thenReturn(block6); + + trieLogPruner.pruneFromCache(); + + inOrder.verify(rootWorldStateStorage, times(1)).pruneTrieLog(key4); + inOrder.verify(rootWorldStateStorage, times(1)).pruneTrieLog(key0); + } +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 59eb9329f33..61e9faacaee 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -178,7 +178,7 @@ public void reset() { static class NoOpTrieLogManager extends TrieLogManager { public NoOpTrieLogManager() { - super(null, null, 0, null); + super(null, null, 0, null, null); } @SuppressWarnings("UnsynchronizedOverridesSynchronized")