Add chain data pruning experimental feature

CHANGELOG.md

@@ -20,6 +20,7 @@
 - Increase the speed of modexp gas execution and execution. [#4780](https://github.com/hyperledger/besu/pull/4780)
 - Added experimental CLI options `--Xeth-capability-max` and `--Xeth-capability-min` to specify a range of capabilities to be supported by the Eth protocol. [#4752](https://github.com/hyperledger/besu/pull/4752)
 - Set the default curve in the EVMTool, like is done in production operations [#4790](https://github.com/hyperledger/besu/pull/4790)
+- Add chain data pruning feature with three experimental CLI options: `--Xchain-pruning-enabled`, `--Xchain-pruning-blocks-retained` and `--Xchain-pruning-frequency` [#4686](https://github.com/hyperledger/besu/pull/4686)
 
 ### Bug Fixes
 - Fix storage key format for eth_getProof so that it follows the EIP-1474 spec [#4564](https://github.com/hyperledger/besu/pull/4564)

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -56,6 +56,7 @@
 import org.hyperledger.besu.cli.options.stable.LoggingLevelOption;
 import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption;
 import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions;
+import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions;
 import org.hyperledger.besu.cli.options.unstable.DnsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.EvmOptions;
@@ -289,6 +290,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   private final PrivacyPluginOptions unstablePrivacyPluginOptions = PrivacyPluginOptions.create();
   private final EvmOptions unstableEvmOptions = EvmOptions.create();
   private final IpcOptions unstableIpcOptions = IpcOptions.create();
+  private final ChainPruningOptions unstableChainPruningOptions = ChainPruningOptions.create();
 
   // stable CLI options
   private final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
@@ -1539,6 +1541,7 @@ private void handleUnstableOptions() {
             .put("Launcher", unstableLauncherOptions)
             .put("EVM Options", unstableEvmOptions)
             .put("IPC Options", unstableIpcOptions)
+            .put("Chain Data Pruning Options", unstableChainPruningOptions)
             .build();
 
     UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions);
@@ -1776,6 +1779,7 @@ private void validateOptions() {
     validateDnsOptionsParams();
     ensureValidPeerBoundParams();
     validateRpcOptionsParams();
+    validateChainDataPruningParams();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
@@ -1927,6 +1931,17 @@ public void validateRpcOptionsParams() {
     }
   }
 
+  public void validateChainDataPruningParams() {
+    if (unstableChainPruningOptions.getChainDataPruningEnabled()
+        && unstableChainPruningOptions.getChainDataPruningBlocksRetained()
+            < ChainPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED) {
+      throw new ParameterException(
+          this.commandLine,
+          "--Xchain-pruning-blocks-retained must be >= "
+              + ChainPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED);
+    }
+  }
+
   private GenesisConfigOptions readGenesisConfigOptions() {
 
     try {
@@ -2184,7 +2199,8 @@ public BesuControllerBuilder getControllerBuilder() {
         .reorgLoggingThreshold(reorgLoggingThreshold)
         .evmConfiguration(unstableEvmOptions.toDomainObject())
         .dataStorageConfiguration(dataStorageOptions.toDomainObject())
-        .maxPeers(p2PDiscoveryOptionGroup.maxPeers);
+        .maxPeers(p2PDiscoveryOptionGroup.maxPeers)
+        .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject());
   }
 
   private GraphQLConfiguration graphQLConfiguration() {

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/ChainPruningOptions.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+package org.hyperledger.besu.cli.options.unstable;
+
+import org.hyperledger.besu.cli.options.CLIOptions;
+import org.hyperledger.besu.ethereum.chain.ChainPrunerConfiguration;
+import org.hyperledger.besu.util.number.PositiveNumber;
+
+import java.util.Arrays;
+import java.util.List;
+
+import picocli.CommandLine;
+
+public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration> {
+  private static final String CHAIN_PRUNING_ENABLED_FLAG = "--Xchain-pruning-enabled";
+  private static final String CHAIN_PRUNING_BLOCKS_RETAINED_FLAG =
+      "--Xchain-pruning-blocks-retained";
+  private static final String CHAIN_PRUNING_FREQUENCY_FLAG = "--Xchain-pruning-frequency";
+  public static final long DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED = 7200;
+  public static final int DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY = 256;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {CHAIN_PRUNING_ENABLED_FLAG},
+      description =
+          "Enable the chain pruner to actively prune old chain data (default: ${DEFAULT-VALUE})")
+  private final Boolean chainDataPruningEnabled = Boolean.FALSE;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {CHAIN_PRUNING_BLOCKS_RETAINED_FLAG},
+      description =
+          "The number of recent blocks for which to keep the chain data. Must be >= "
+              + DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED
+              + " (default: ${DEFAULT-VALUE})")
+  private final Long chainDataPruningBlocksRetained =
+      DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {CHAIN_PRUNING_FREQUENCY_FLAG},
+      description =
+          "The number of blocks added to the chain between two pruning operations. Must be non-negative (default: ${DEFAULT-VALUE})")
+  private final PositiveNumber chainDataPruningBlocksFrequency =
+      PositiveNumber.fromInt(DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY);
+
+  public static ChainPruningOptions create() {
+    return new ChainPruningOptions();
+  }
+
+  public Boolean getChainDataPruningEnabled() {
+    return chainDataPruningEnabled;
+  }
+
+  public Long getChainDataPruningBlocksRetained() {
+    return chainDataPruningBlocksRetained;
+  }
+
+  @Override
+  public ChainPrunerConfiguration toDomainObject() {
+    return new ChainPrunerConfiguration(
+        chainDataPruningEnabled,
+        chainDataPruningBlocksRetained,
+        chainDataPruningBlocksFrequency.getValue());
+  }
+
+  @Override
+  public List<String> getCLIOptions() {
+    return Arrays.asList(
+        CHAIN_PRUNING_ENABLED_FLAG,
+        chainDataPruningEnabled.toString(),
+        CHAIN_PRUNING_BLOCKS_RETAINED_FLAG,
+        chainDataPruningBlocksRetained.toString(),
+        CHAIN_PRUNING_FREQUENCY_FLAG,
+        chainDataPruningBlocksFrequency.toString());
+  }
+}

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -35,6 +35,9 @@
 import org.hyperledger.besu.ethereum.bonsai.CachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
+import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
+import org.hyperledger.besu.ethereum.chain.ChainDataPrunerStorage;
+import org.hyperledger.besu.ethereum.chain.ChainPrunerConfiguration;
 import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
@@ -51,6 +54,7 @@
 import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
 import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
 import org.hyperledger.besu.ethereum.eth.manager.MergePeerFilter;
+import org.hyperledger.besu.ethereum.eth.manager.MonitoredExecutors;
 import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager;
 import org.hyperledger.besu.ethereum.eth.peervalidation.CheckpointBlocksPeerValidator;
 import org.hyperledger.besu.ethereum.eth.peervalidation.ClassicForkPeerValidator;
@@ -140,6 +144,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
       Collections.emptyList();
   protected EvmConfiguration evmConfiguration;
   protected int maxPeers;
+  protected ChainPrunerConfiguration chainPrunerConfiguration = ChainPrunerConfiguration.DEFAULT;
 
   public BesuControllerBuilder storageProvider(final StorageProvider storageProvider) {
     this.storageProvider = storageProvider;
@@ -268,6 +273,12 @@ public BesuControllerBuilder maxPeers(final int maxPeers) {
     return this;
   }
 
+  public BesuControllerBuilder chainPruningConfiguration(
+      final ChainPrunerConfiguration chainPrunerConfiguration) {
+    this.chainPrunerConfiguration = chainPrunerConfiguration;
+    return this;
+  }
+
   public BesuController build() {
     checkNotNull(genesisConfig, "Missing genesis config");
     checkNotNull(syncConfig, "Missing sync config");
@@ -315,6 +326,22 @@ public BesuController build() {
             blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext);
     validateContext(protocolContext);
 
+    if (chainPrunerConfiguration.getChainPruningEnabled()) {
+      protocolContext
+          .safeConsensusContext(MergeContext.class)
+          .ifPresent(
+              mergeContext -> {
+                mergeContext.setIsChainPruningEnabled(true);
+              });
+      final ChainDataPruner chainDataPruner = createChainPruner(blockchainStorage);
+      blockchain.observeBlockAdded(chainDataPruner);
+      LOG.info(
+          "Chain data pruning enabled with recent blocks retained to be: "
+              + chainPrunerConfiguration.getChainPruningBlocksRetained()
+              + " and frequency to be: "
+              + chainPrunerConfiguration.getChainPruningBlocksFrequency());
+    }
+
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(
         protocolContext.getWorldStateArchive());
 
@@ -651,6 +678,22 @@ private WorldStateArchive createWorldStateArchive(
     }
   }
 
+  private ChainDataPruner createChainPruner(final BlockchainStorage blockchainStorage) {
+    return new ChainDataPruner(
+        blockchainStorage,
+        new ChainDataPrunerStorage(
+            storageProvider.getStorageBySegmentIdentifier(
+                KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE)),
+        chainPrunerConfiguration.getChainPruningBlocksRetained(),
+        chainPrunerConfiguration.getChainPruningBlocksFrequency(),
+        MonitoredExecutors.newBoundedThreadPool(
+            ChainDataPruner.class.getSimpleName(),
+            1,
+            1,
+            ChainDataPruner.MAX_PRUNING_THREAD_QUEUE_SIZE,
+            metricsSystem));
+  }
+
   protected List<PeerValidator> createPeerValidators(final ProtocolSchedule protocolSchedule) {
     final List<PeerValidator> validators = new ArrayList<>();
 

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -226,7 +226,7 @@ public void initMocks() throws Exception {
     when(mockControllerBuilder.dataStorageConfiguration(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.evmConfiguration(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder);
-
+    when(mockControllerBuilder.chainPruningConfiguration(any())).thenReturn(mockControllerBuilder);
     // doReturn used because of generic BesuController
     doReturn(mockController).when(mockControllerBuilder).build();
     lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);

consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java

@@ -65,4 +65,10 @@ void fireNewUnverifiedForkchoiceEvent(
   void putPayloadById(final PayloadIdentifier payloadId, final Block block);
 
   Optional<Block> retrieveBlockById(final PayloadIdentifier payloadId);
+
+  default void setIsChainPruningEnabled(final boolean isChainPruningEnabled) {}
+
+  default boolean isChainPruningEnabled() {
+    return false;
+  }
 }

consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java

@@ -64,6 +64,10 @@ public class PostMergeContext implements MergeContext {
   private final AtomicReference<Optional<BlockHeader>> terminalPoWBlock =
       new AtomicReference<>(Optional.empty());
 
+  // TODO: cleanup - isChainPruningEnabled will not be required after
+  // https://github.com/hyperledger/besu/pull/4703 is merged.
+  private boolean isChainPruningEnabled = false;
+
   @VisibleForTesting
   PostMergeContext() {
     this(Difficulty.ZERO);
@@ -265,4 +269,14 @@ private static class PayloadTuple {
       this.block = block;
     }
   }
+
+  @Override
+  public void setIsChainPruningEnabled(final boolean isChainPruningEnabled) {
+    this.isChainPruningEnabled = isChainPruningEnabled;
+  }
+
+  @Override
+  public boolean isChainPruningEnabled() {
+    return isChainPruningEnabled;
+  }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java

@@ -119,7 +119,8 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     }
 
     // TODO: post-merge cleanup, this should be unnecessary after merge
-    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead)) {
+    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead)
+        && !mergeContext.get().isChainPruningEnabled()) {
       logForkchoiceUpdatedCall(INVALID, forkChoice);
       return new JsonRpcSuccessResponse(
           requestId,

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java

@@ -196,7 +196,8 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     }
 
     // TODO: post-merge cleanup
-    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newBlockHeader)) {
+    if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newBlockHeader)
+        && !mergeContext.get().isChainPruningEnabled()) {
       mergeCoordinator.addBadBlock(block, Optional.empty());
       return respondWithInvalid(
           reqId,

ethereum/core/build.gradle

@@ -81,6 +81,7 @@ dependencies {
   testImplementation 'org.junit.jupiter:junit-jupiter'
   testImplementation 'org.junit.jupiter:junit-jupiter-params'
   testImplementation 'org.mockito:mockito-core'
+  testImplementation 'org.awaitility:awaitility'
 
   testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockchainStorage.java

@@ -72,8 +72,16 @@ interface Updater {
 
     void removeBlockHash(long blockNumber);
 
+    void removeBlockHeader(final Hash blockHash);
+
+    void removeBlockBody(final Hash blockHash);
+
+    void removeTransactionReceipts(final Hash blockHash);
+
     void removeTransactionLocation(Hash transactionHash);
 
+    void removeTotalDifficulty(final Hash blockHash);
+
     void commit();
 
     void rollback();