Post Merge checkpoint sync

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 ## 23.1
 
+### Additions and Improvements
+- Add access list to Transaction Call Object [#4802](https://github.com/hyperledger/besu/issues/4801)
+
 ### Breaking Changes
 - GoQuorum-compatible privacy is deprecated and will be removed in 23.4
 - IBFT 1.0 is deprecated and will be removed in 23.4

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -58,6 +58,7 @@
 import java.io.File;
 import java.nio.file.Path;
 import java.time.Clock;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -147,8 +148,12 @@ public void startNode(final BesuNode node) {
             .setBootNodes(bootnodes);
     node.getConfiguration().getGenesisConfig().ifPresent(networkConfigBuilder::setGenesisConfig);
     final EthNetworkConfig ethNetworkConfig = networkConfigBuilder.build();
+    final SynchronizerConfiguration synchronizerConfiguration =
+        new SynchronizerConfiguration.Builder().build();
     final BesuControllerBuilder builder =
-        new BesuController.Builder().fromEthNetworkConfig(ethNetworkConfig);
+        new BesuController.Builder()
+            .fromEthNetworkConfig(
+                ethNetworkConfig, Collections.emptyMap(), synchronizerConfiguration.getSyncMode());
 
     final KeyValueStorageProvider storageProvider =
         new KeyValueStorageProviderBuilder()

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -85,6 +85,7 @@
 import org.hyperledger.besu.cli.util.CommandLineUtils;
 import org.hyperledger.besu.cli.util.ConfigOptionSearchAndRunHandler;
 import org.hyperledger.besu.cli.util.VersionProvider;
+import org.hyperledger.besu.config.CheckpointConfigOptions;
 import org.hyperledger.besu.config.GenesisConfigFile;
 import org.hyperledger.besu.config.GenesisConfigOptions;
 import org.hyperledger.besu.config.GoQuorumOptions;
@@ -1779,6 +1780,7 @@ private void validateOptions() {
     validateDnsOptionsParams();
     ensureValidPeerBoundParams();
     validateRpcOptionsParams();
+    validatePostMergeCheckpointBlockRequirements();
     validateChainDataPruningParams();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
@@ -2003,6 +2005,12 @@ private void issueOptionWarnings() {
         !SyncMode.isFullSync(getDefaultSyncModeIfNotSet(syncMode)),
         singletonList("--fast-sync-min-peers"));
 
+    CommandLineUtils.failIfOptionDoesntMeetRequirement(
+        commandLine,
+        "--Xcheckpoint-post-merge-enabled can only be used with X_CHECKPOINT sync-mode",
+        SyncMode.X_CHECKPOINT.equals(getDefaultSyncModeIfNotSet(syncMode)),
+        singletonList("--Xcheckpoint-post-merge-enabled"));
+
     if (!securityModuleName.equals(DEFAULT_SECURITY_MODULE)
         && nodePrivateKeyFileOption.getNodePrivateKeyFile() != null) {
       logger.warn(
@@ -2156,7 +2164,7 @@ public BesuController buildController() {
   public BesuControllerBuilder getControllerBuilder() {
     final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName);
     return controllerBuilderFactory
-        .fromEthNetworkConfig(updateNetworkConfig(network), genesisConfigOverrides)
+        .fromEthNetworkConfig(updateNetworkConfig(network), genesisConfigOverrides, syncMode)
         .synchronizerConfiguration(buildSyncConfig())
         .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject())
         .dataDirectory(dataDir())
@@ -3344,6 +3352,39 @@ private void setMergeConfigOptions() {
             .isPresent());
   }
 
+  private void validatePostMergeCheckpointBlockRequirements() {
+    final GenesisConfigOptions genesisOptions =
+        Optional.ofNullable(genesisConfigOptions)
+            .orElseGet(
+                () ->
+                    GenesisConfigFile.fromConfig(
+                            genesisConfig(Optional.ofNullable(network).orElse(MAINNET)))
+                        .getConfigOptions(genesisConfigOverrides));
+    final SynchronizerConfiguration synchronizerConfiguration =
+        unstableSynchronizerOptions.toDomainObject().build();
+    final Optional<UInt256> terminalTotalDifficulty = genesisOptions.getTerminalTotalDifficulty();
+    final CheckpointConfigOptions checkpointConfigOptions = genesisOptions.getCheckpointOptions();
+    if (synchronizerConfiguration.isCheckpointPostMergeEnabled()) {
+      if (!checkpointConfigOptions.isValid()) {
+        throw new InvalidConfigurationException(
+            "Near head checkpoint sync requires a checkpoint block configured in the genesis file");
+      }
+      terminalTotalDifficulty.ifPresentOrElse(
+          value -> {
+            if (UInt256.fromHexString(
+                    genesisOptions.getCheckpointOptions().getTotalDifficulty().orElse("0x0"))
+                .lessOrEqualThan(value)) {
+              throw new InvalidConfigurationException(
+                  "Near head checkpoint sync requires a block with total difficulty greater than the TTD");
+            }
+          },
+          () -> {
+            throw new InvalidConfigurationException(
+                "Near head checkpoint sync requires TTD in the genesis file");
+          });
+    }
+  }
+
   private boolean isMergeEnabled() {
     return MergeConfigOptions.isMergeEnabled();
   }

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java

@@ -72,6 +72,8 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
   private static final String SNAP_TRIENODE_COUNT_PER_REQUEST_FLAG =
       "--Xsnapsync-synchronizer-trienode-count-per-request";
 
+  private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
+
   @CommandLine.Option(
       names = BLOCK_PROPAGATION_RANGE_FLAG,
       hidden = true,
@@ -272,6 +274,13 @@ public void parseBlockPropagationRange(final String arg) {
   private int snapsyncTrieNodeCountPerRequest =
       SnapSyncConfiguration.DEFAULT_TRIENODE_COUNT_PER_REQUEST;
 
+  @CommandLine.Option(
+      names = {CHECKPOINT_POST_MERGE_FLAG},
+      hidden = true,
+      description = "Enable the sync to start from a post-merge block.")
+  private Boolean checkpointPostMergeSyncEnabled =
+      SynchronizerConfiguration.DEFAULT_CHECKPOINT_POST_MERGE_ENABLED;
+
   private SynchronizerOptions() {}
 
   public static SynchronizerOptions create() {
@@ -308,6 +317,7 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
         config.getSnapSyncConfiguration().getBytecodeCountPerRequest();
     options.snapsyncTrieNodeCountPerRequest =
         config.getSnapSyncConfiguration().getTrienodeCountPerRequest();
+    options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
     return options;
   }
 
@@ -338,6 +348,7 @@ public SynchronizerConfiguration.Builder toDomainObject() {
             .bytecodeCountPerRequest(snapsyncBytecodeCountPerRequest)
             .trienodeCountPerRequest(snapsyncTrieNodeCountPerRequest)
             .build());
+    builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
 
     return builder;
   }

besu/src/main/java/org/hyperledger/besu/controller/BesuController.java

@@ -14,6 +14,8 @@
  */
 package org.hyperledger.besu.controller;
 
+import static org.hyperledger.besu.ethereum.eth.sync.SyncMode.isCheckpointSync;
+
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.config.GenesisConfigFile;
 import org.hyperledger.besu.config.GenesisConfigOptions;
@@ -28,10 +30,12 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Synchronizer;
 import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration;
+import org.hyperledger.besu.util.InvalidConfigurationException;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -41,6 +45,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.tuweni.units.bigints.UInt256;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -171,24 +176,26 @@ public PluginServiceFactory getAdditionalPluginServices() {
 
   public static class Builder {
 
-    public BesuControllerBuilder fromEthNetworkConfig(final EthNetworkConfig ethNetworkConfig) {
-      return fromEthNetworkConfig(ethNetworkConfig, Collections.emptyMap());
-    }
-
     public BesuControllerBuilder fromEthNetworkConfig(
-        final EthNetworkConfig ethNetworkConfig, final Map<String, String> genesisConfigOverrides) {
+        final EthNetworkConfig ethNetworkConfig,
+        final Map<String, String> genesisConfigOverrides,
+        final SyncMode syncMode) {
       return fromGenesisConfig(
               GenesisConfigFile.fromConfig(ethNetworkConfig.getGenesisConfig()),
-              genesisConfigOverrides)
+              genesisConfigOverrides,
+              syncMode)
           .networkId(ethNetworkConfig.getNetworkId());
     }
 
-    public BesuControllerBuilder fromGenesisConfig(final GenesisConfigFile genesisConfig) {
-      return fromGenesisConfig(genesisConfig, Collections.emptyMap());
+    public BesuControllerBuilder fromGenesisConfig(
+        final GenesisConfigFile genesisConfig, final SyncMode syncMode) {
+      return fromGenesisConfig(genesisConfig, Collections.emptyMap(), syncMode);
     }
 
     BesuControllerBuilder fromGenesisConfig(
-        final GenesisConfigFile genesisConfig, final Map<String, String> genesisConfigOverrides) {
+        final GenesisConfigFile genesisConfig,
+        final Map<String, String> genesisConfigOverrides,
+        final SyncMode syncMode) {
       final GenesisConfigOptions configOptions =
           genesisConfig.getConfigOptions(genesisConfigOverrides);
       final BesuControllerBuilder builder;
@@ -213,11 +220,17 @@ BesuControllerBuilder fromGenesisConfig(
 
       // wrap with TransitionBesuControllerBuilder if we have a terminal total difficulty:
       if (configOptions.getTerminalTotalDifficulty().isPresent()) {
-        // TODO this should be changed to vanilla MergeBesuControllerBuilder and the Transition*
-        // series of classes removed after we successfully transition to PoS
-        // https://github.com/hyperledger/besu/issues/2897
-        return new TransitionBesuControllerBuilder(builder, new MergeBesuControllerBuilder())
-            .genesisConfigFile(genesisConfig);
+        // Enable start with vanilla MergeBesuControllerBuilder for PoS checkpoint block
+        if (isCheckpointSync(syncMode) && isCheckpointPoSBlock(configOptions)) {
+          return new MergeBesuControllerBuilder().genesisConfigFile(genesisConfig);
+        } else {
+          // TODO this should be changed to vanilla MergeBesuControllerBuilder and the Transition*
+          // series of classes removed after we successfully transition to PoS
+          // https://github.com/hyperledger/besu/issues/2897
+          return new TransitionBesuControllerBuilder(builder, new MergeBesuControllerBuilder())
+              .genesisConfigFile(genesisConfig);
+        }
+
       } else return builder.genesisConfigFile(genesisConfig);
     }
 
@@ -258,5 +271,19 @@ private Long readQbftStartBlockConfig(final QbftConfigOptions qbftConfigOptions)
 
       return startBlock;
     }
+
+    private boolean isCheckpointPoSBlock(final GenesisConfigOptions configOptions) {
+      UInt256 terminalTotalDifficulty = configOptions.getTerminalTotalDifficulty().get();
+      // this is currently a limitation that besu has in its codebase
+      if (UInt256.fromHexString(configOptions.getCheckpointOptions().getTotalDifficulty().get())
+              .equals(UInt256.ZERO)
+          && terminalTotalDifficulty.equals(UInt256.ZERO)) {
+        throw new InvalidConfigurationException(
+            "Post Merge checkpoint sync can't be used with TTD = 0 and checkpoint totalDifficulty = 0");
+      }
+      return configOptions.getCheckpointOptions().isValid()
+          && (UInt256.fromHexString(configOptions.getCheckpointOptions().getTotalDifficulty().get())
+              .greaterThan(terminalTotalDifficulty));
+    }
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java

@@ -184,7 +184,8 @@ protected MergeContext createConsensusContext(
                     .get()
                     .getTerminalTotalDifficulty()
                     .map(Difficulty::of)
-                    .orElse(Difficulty.ZERO));
+                    .orElse(Difficulty.ZERO))
+            .setIsNearHeadCheckpointSync(syncConfig.isCheckpointPostMergeEnabled());
 
     blockchain
         .getFinalized()
@@ -235,6 +236,13 @@ protected List<PeerValidator> createPeerValidators(final ProtocolSchedule protoc
     return retval;
   }
 
+  @Override
+  public BesuController build() {
+    BesuController controller = super.build();
+    PostMergeContext.get().setSyncState(controller.getSyncState());
+    return controller;
+  }
+
   public TimestampSchedule createTimestampProtocolSchedule() {
     return MergeProtocolSchedule.createTimestamp(
         configOptionsSupplier.get(), privacyParameters, isRevertReasonEnabled);

besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java

@@ -45,6 +45,7 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode;
@@ -188,7 +189,7 @@ public void setUp() throws IOException {
 
     besuController =
         new BesuController.Builder()
-            .fromGenesisConfig(GenesisConfigFile.development())
+            .fromGenesisConfig(GenesisConfigFile.development(), SyncMode.FULL)
             .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
             .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
             .storageProvider(new InMemoryKeyValueStorageProvider())

besu/src/test/java/org/hyperledger/besu/PrivacyTest.java

@@ -33,6 +33,7 @@
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider;
@@ -107,7 +108,7 @@ private BesuController setUpControllerWithPrivacyEnabled(final boolean flexibleE
             .setFlexiblePrivacyGroupsEnabled(flexibleEnabled)
             .build();
     return new BesuController.Builder()
-        .fromGenesisConfig(GenesisConfigFile.mainnet())
+        .fromGenesisConfig(GenesisConfigFile.mainnet(), SyncMode.FAST)
         .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
         .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
         .storageProvider(new InMemoryKeyValueStorageProvider())

besu/src/test/java/org/hyperledger/besu/chainexport/RlpBlockExporterTest.java

@@ -30,6 +30,7 @@
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
@@ -83,7 +84,7 @@ private static Blockchain importBlocks(final BesuController controller, final Pa
   private static BesuController createController() throws IOException {
     final Path dataDir = folder.newFolder().toPath();
     return new BesuController.Builder()
-        .fromGenesisConfig(GenesisConfigFile.mainnet())
+        .fromGenesisConfig(GenesisConfigFile.mainnet(), SyncMode.FAST)
         .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
         .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
         .storageProvider(new InMemoryKeyValueStorageProvider())

besu/src/test/java/org/hyperledger/besu/chainimport/JsonBlockImporterTest.java

@@ -34,6 +34,7 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
@@ -413,7 +414,7 @@ protected BesuController createController(final GenesisConfigFile genesisConfigF
       throws IOException {
     final Path dataDir = folder.newFolder().toPath();
     return new BesuController.Builder()
-        .fromGenesisConfig(genesisConfigFile)
+        .fromGenesisConfig(genesisConfigFile, SyncMode.FAST)
         .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
         .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
         .storageProvider(new InMemoryKeyValueStorageProvider())

besu/src/test/java/org/hyperledger/besu/chainimport/RlpBlockImporterTest.java

@@ -27,6 +27,7 @@
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
@@ -64,7 +65,7 @@ public void blockImport() throws IOException {
     BlockTestUtil.write1000Blocks(source);
     final BesuController targetController =
         new BesuController.Builder()
-            .fromGenesisConfig(GenesisConfigFile.mainnet())
+            .fromGenesisConfig(GenesisConfigFile.mainnet(), SyncMode.FAST)
             .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
             .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
             .storageProvider(new InMemoryKeyValueStorageProvider())
@@ -96,7 +97,7 @@ public void blockImportRejectsBadPow() throws IOException {
     BlockTestUtil.writeBadPowBlocks(source);
     final BesuController targetController =
         new BesuController.Builder()
-            .fromGenesisConfig(GenesisConfigFile.mainnet())
+            .fromGenesisConfig(GenesisConfigFile.mainnet(), SyncMode.FAST)
             .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
             .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
             .storageProvider(new InMemoryKeyValueStorageProvider())
@@ -125,7 +126,7 @@ public void blockImportCanSkipPow() throws IOException {
     BlockTestUtil.writeBadPowBlocks(source);
     final BesuController targetController =
         new BesuController.Builder()
-            .fromGenesisConfig(GenesisConfigFile.mainnet())
+            .fromGenesisConfig(GenesisConfigFile.mainnet(), SyncMode.FAST)
             .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
             .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
             .storageProvider(new InMemoryKeyValueStorageProvider())
@@ -166,7 +167,7 @@ public void ibftImport() throws IOException {
 
     final BesuController controller =
         new BesuController.Builder()
-            .fromGenesisConfig(GenesisConfigFile.fromConfig(config))
+            .fromGenesisConfig(GenesisConfigFile.fromConfig(config), SyncMode.FULL)
             .synchronizerConfiguration(SynchronizerConfiguration.builder().build())
             .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
             .storageProvider(new InMemoryKeyValueStorageProvider())