Add chain data pruning experimental feature

CHANGELOG.md

@@ -21,6 +21,7 @@
 - Remove some log statements that are keeping some objects live in heap for a long time, to reduce the amount of memory required during initial sync [#4705](https://github.com/hyperledger/besu/pull/4705)
 - Add field `type` to Transaction receipt object (eth_getTransactionReceipt) [#4505](https://github.com/hyperledger/besu/issues/4505)
 - Print an overview of configuration and system information at startup [#4451](https://github.com/hyperledger/besu/pull/4451)
+- Add chain data pruning feature with three experimental CLI options: `--Xchain-data-pruning-enabled`, `--Xchain-data-pruning-blocks-retained` and `--Xchain-data-pruning-frequency` [#4686](https://github.com/hyperledger/besu/pull/4686)
 
 ### Bug Fixes
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -56,6 +56,7 @@
 import org.hyperledger.besu.cli.options.stable.LoggingLevelOption;
 import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption;
 import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions;
+import org.hyperledger.besu.cli.options.unstable.ChainDataPruningOptions;
 import org.hyperledger.besu.cli.options.unstable.DnsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.EvmOptions;
@@ -288,6 +289,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   private final PrivacyPluginOptions unstablePrivacyPluginOptions = PrivacyPluginOptions.create();
   private final EvmOptions unstableEvmOptions = EvmOptions.create();
   private final IpcOptions unstableIpcOptions = IpcOptions.create();
+  private final ChainDataPruningOptions unstableChainDataPruningOptions =
+      ChainDataPruningOptions.create();
 
   // stable CLI options
   private final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
@@ -1532,6 +1535,7 @@ private void handleUnstableOptions() {
             .put("Launcher", unstableLauncherOptions)
             .put("EVM Options", unstableEvmOptions)
             .put("IPC Options", unstableIpcOptions)
+            .put("Chain Data Pruning Options", unstableChainDataPruningOptions)
             .build();
 
     UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions);
@@ -1761,6 +1765,7 @@ private void validateOptions() {
     validateDnsOptionsParams();
     ensureValidPeerBoundParams();
     validateRpcOptionsParams();
+    validateChainDataPruningParams();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
@@ -1912,6 +1917,18 @@ public void validateRpcOptionsParams() {
     }
   }
 
+  public void validateChainDataPruningParams() {
+    if (unstableChainDataPruningOptions.getChainDataPruningEnabled()) {
+      if (unstableChainDataPruningOptions.getChainDataPruningBlocksRetained()
+          < ChainDataPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED) {
+        throw new ParameterException(
+            this.commandLine,
+            "--Xchain-data-pruning-blocks-retained must be >= "
+                + ChainDataPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED);
+      }
+    }
+  }
+
   private GenesisConfigOptions readGenesisConfigOptions() {
 
     try {
@@ -2169,7 +2186,12 @@ public BesuControllerBuilder getControllerBuilder() {
         .reorgLoggingThreshold(reorgLoggingThreshold)
         .evmConfiguration(unstableEvmOptions.toDomainObject())
         .dataStorageConfiguration(dataStorageOptions.toDomainObject())
-        .maxPeers(p2PDiscoveryOptionGroup.maxPeers);
+        .maxPeers(p2PDiscoveryOptionGroup.maxPeers)
+        .isChainDataPruningEnabled(unstableChainDataPruningOptions.getChainDataPruningEnabled())
+        .chainDataPruningBlocksRetained(
+            unstableChainDataPruningOptions.getChainDataPruningBlocksRetained())
+        .chainDataPruningFrequency(
+            unstableChainDataPruningOptions.getChainDataPruningBlocksFrequency());
   }
 
   private GraphQLConfiguration graphQLConfiguration() {

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/ChainDataPruningOptions.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+package org.hyperledger.besu.cli.options.unstable;
+
+import org.hyperledger.besu.util.number.PositiveNumber;
+
+import picocli.CommandLine;
+
+public class ChainDataPruningOptions {
+
+  public static final long DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED = 50400;
+  public static final int DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY = 256;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {"--Xchain-data-pruning-enabled"},
+      description =
+          "Enable the chain pruner to actively prune old chain data (default: ${DEFAULT-VALUE})")
+  private final Boolean chainDataPruningEnabled = Boolean.FALSE;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {"--Xchain-data-pruning-blocks-retained"},
+      description =
+          "The number of recent blocks for which to keep the chain data. Must be >= "
+              + DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED
+              + " (default: ${DEFAULT-VALUE})")
+  private final Long chainDataPruningBlocksRetained =
+      DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {"--Xchain-data-pruning-frequency"},
+      description =
+          "The number of blocks added to the chain between two pruning operations. Must be non-negative (default: ${DEFAULT-VALUE})")
+  private final PositiveNumber chainDataPruningBlocksFrequency =
+      PositiveNumber.fromInt(DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY);
+
+  public static ChainDataPruningOptions create() {
+    return new ChainDataPruningOptions();
+  }
+
+  public Boolean getChainDataPruningEnabled() {
+    return chainDataPruningEnabled;
+  }
+
+  public Long getChainDataPruningBlocksRetained() {
+    return chainDataPruningBlocksRetained;
+  }
+
+  public Long getChainDataPruningBlocksFrequency() {
+    return (long) chainDataPruningBlocksFrequency.getValue();
+  }
+}

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -34,6 +34,7 @@
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
+import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
 import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
@@ -139,6 +140,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
       Collections.emptyList();
   protected EvmConfiguration evmConfiguration;
   protected int maxPeers;
+  protected boolean isChainDataPruningEnabled;
+  protected long chainDataPruningBlocksRetained;
+  protected long chainDataPruningFrequency;
 
   public BesuControllerBuilder storageProvider(final StorageProvider storageProvider) {
     this.storageProvider = storageProvider;
@@ -267,6 +271,22 @@ public BesuControllerBuilder maxPeers(final int maxPeers) {
     return this;
   }
 
+  public BesuControllerBuilder isChainDataPruningEnabled(final boolean isChainDataPruningEnabled) {
+    this.isChainDataPruningEnabled = isChainDataPruningEnabled;
+    return this;
+  }
+
+  public BesuControllerBuilder chainDataPruningBlocksRetained(
+      final long chainDataPruningBlocksRetained) {
+    this.chainDataPruningBlocksRetained = chainDataPruningBlocksRetained;
+    return this;
+  }
+
+  public BesuControllerBuilder chainDataPruningFrequency(final long chainDataPruningFrequency) {
+    this.chainDataPruningFrequency = chainDataPruningFrequency;
+    return this;
+  }
+
   public BesuController build() {
     checkNotNull(genesisConfig, "Missing genesis config");
     checkNotNull(syncConfig, "Missing sync config");
@@ -300,6 +320,23 @@ public BesuController build() {
             reorgLoggingThreshold,
             dataDirectory.toString());
 
+    if (isChainDataPruningEnabled) {
+      ChainDataPruner.enablePruning();
+      final ChainDataPruner chainDataPruner =
+          new ChainDataPruner(
+              blockchainStorage,
+              storageProvider.getStorageBySegmentIdentifier(
+                  KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE),
+              chainDataPruningBlocksRetained,
+              chainDataPruningFrequency);
+      blockchain.observeBlockAdded(chainDataPruner);
+      LOG.info(
+          "Chain data pruning enabled with recent blocks retained to be: "
+              + chainDataPruningBlocksRetained
+              + " and frequency to be: "
+              + chainDataPruningFrequency);
+    }
+
     final WorldStateArchive worldStateArchive =
         createWorldStateArchive(worldStateStorage, blockchain);
 

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -225,6 +225,12 @@ public void initMocks() throws Exception {
     when(mockControllerBuilder.dataStorageConfiguration(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.evmConfiguration(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.isChainDataPruningEnabled(anyBoolean()))
+        .thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.chainDataPruningBlocksRetained(anyLong()))
+        .thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.chainDataPruningFrequency(anyLong()))
+        .thenReturn(mockControllerBuilder);
     // doReturn used because of generic BesuController
     doReturn(mockController).when(mockControllerBuilder).build();
     lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java

@@ -35,6 +35,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineUpdateForkchoiceResult;
+import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 
 import java.util.Optional;
@@ -116,7 +117,8 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     }
 
     // TODO: post-merge cleanup, this should be unnecessary after merge
-    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead.get())) {
+    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead.get())
+        && !ChainDataPruner.isPruningEnabled()) {
       logForkchoiceUpdatedCall(INVALID, forkChoice);
       return new JsonRpcSuccessResponse(
           requestId,

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java

@@ -36,6 +36,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EnginePayloadStatusResult;
+import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockBody;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
@@ -194,7 +195,8 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     }
 
     // TODO: post-merge cleanup
-    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newBlockHeader)) {
+    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newBlockHeader)
+        && !ChainDataPruner.isPruningEnabled()) {
       mergeCoordinator.addBadBlock(block, Optional.empty());
       return respondWithInvalid(
           reqId,

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockchainStorage.java

@@ -72,8 +72,16 @@ interface Updater {
 
     void removeBlockHash(long blockNumber);
 
+    void removeBlockHeader(final Hash blockHash);
+
+    void removeBlockBody(final Hash blockHash);
+
+    void removeTransactionReceipts(final Hash blockHash);
+
     void removeTransactionLocation(Hash transactionHash);
 
+    void removeTotalDifficulty(final Hash blockHash);
+
     void commit();
 
     void rollback();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/ChainDataPruner.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+package org.hyperledger.besu.ethereum.chain;
+
+import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
+import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
+
+import java.util.Collection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ChainDataPruner implements BlockAddedObserver {
+  private static final Logger LOG = LoggerFactory.getLogger(ChainDataPruner.class);
+
+  // TODO: cleanup - pruningEnabled will not be required after
+  // https://github.com/hyperledger/besu/pull/4703 is merged.
+  private static boolean pruningEnabled = false;
+  private final BlockchainStorage blockchainStorage;
+  private final ChainDataPrunerStorage prunerStorage;
+  private final long blocksToRetain;
+
+  private final long pruningFrequency;
+
+  public ChainDataPruner(
+      final BlockchainStorage blockchainStorage,
+      final KeyValueStorage storage,
+      final long blocksToRetain,
+      final long pruningFrequency) {
+    this.blockchainStorage = blockchainStorage;
+    this.prunerStorage = new ChainDataPrunerStorage(storage);
+    this.blocksToRetain = blocksToRetain;
+    this.pruningFrequency = pruningFrequency;
+  }
+
+  public static void enablePruning() {
+    pruningEnabled = true;
+  }
+
+  public static boolean isPruningEnabled() {
+    return pruningEnabled;
+  }
+
+  @Override
+  public void onBlockAdded(final BlockAddedEvent event) {
+    LOG.debug("New block added event: " + event);
+    // Get pruning mark
+    final long blockNumber = event.getBlock().getHeader().getNumber();
+    long pruningMark = prunerStorage.getPruningMark().orElse(blockNumber);
+    if (blockNumber < pruningMark) {
+      // Ignore and warn if block number < pruning mark, this normally indicates the blocksToRetain
+      // is too small.
+      LOG.warn(
+          "Block added event: "
+              + event
+              + " has a block number of "
+              + blockNumber
+              + " < pruning mark "
+              + pruningMark);
+      return;
+    }
+    // Append block into fork blocks.
+    final KeyValueStorageTransaction tx = prunerStorage.startTransaction();
+    final Collection<Hash> forkBlocks = prunerStorage.getForkBlocks(blockNumber);
+    forkBlocks.add(event.getBlock().getHash());
+    prunerStorage.setForkBlocks(tx, blockNumber, forkBlocks);
+    // If a block is a new canonical head, start pruning.
+    if (event.isNewCanonicalHead()
+        && blockNumber - blocksToRetain - pruningMark >= pruningFrequency) {
+      while (blockNumber - pruningMark >= blocksToRetain) {
+        LOG.debug("Pruning chain data at pruning mark: " + pruningMark);
+        pruneChainDataAtBlock(tx, pruningMark);
+        pruningMark++;
+      }
+    }
+    // Update pruning mark and commit
+    prunerStorage.setPruningMark(tx, pruningMark);
+    tx.commit();
+  }
+
+  private void pruneChainDataAtBlock(final KeyValueStorageTransaction tx, final long blockNumber) {
+    // Get a collection of old fork blocks that need to be pruned.
+    final Collection<Hash> oldForkBlocks = prunerStorage.getForkBlocks(blockNumber);
+    final BlockchainStorage.Updater updater = blockchainStorage.updater();
+    for (Hash toPrune : oldForkBlocks) {
+      // Prune block header, body, receipts, total difficulty and transaction locations.
+      updater.removeBlockHeader(toPrune);
+      updater.removeBlockBody(toPrune);
+      updater.removeTransactionReceipts(toPrune);
+      updater.removeTotalDifficulty(toPrune);
+      blockchainStorage
+          .getBlockBody(toPrune)
+          .ifPresent(
+              blockBody ->
+                  blockBody
+                      .getTransactions()
+                      .forEach(t -> updater.removeTransactionLocation(t.getHash())));
+    }
+    // Prune canonical chain mapping and commit.
+    updater.removeBlockHash(blockNumber);
+    updater.commit();
+    // Remove old fork blocks.
+    prunerStorage.removeForkBlocks(tx, blockNumber);
+  }
+}