merge Parallelize calculate root hash into RC3

CHANGELOG.md

@@ -5,12 +5,14 @@
 
 ### Additions and Improvements
 - Updated jackson-databind library to version 2.13.4.2 addressing [CVE-2022-42003](https://nvd.nist.gov/vuln/detail/CVE-2022-42003)
-- Improve calculateRootHash method performance during Block processing [#4568](https://github.com/hyperledger/besu/pull/4568)
+- Gradle task allows custom docker image configs e.g. `./gradlew distDocker -PdockerImageName=my/besu -PdockerVariants=openjdk-17,openjdk-19`
+- Update snapsync feature to avoid restarting the download of the world state from scratch when restarting Besu [#4381](https://github.com/hyperledger/besu/pull/4381)
+- Added worldstate snapshot isolation to improve the stability of bonsai (`--Xbonsai-use-snapshots=true`) [#4351](https://github.com/hyperledger/besu/pull/4531)
 
 ### Bug Fixes
-- Fixed default fromBlock value and improved parameter interpetation in eth_getLogs RPC handler [#4513](https://github.com/hyperledger/besu/pull/4513)
-- Corrects treating a block as bad on internal error during either validation or processing [#4512](https://github.com/hyperledger/besu/issues/4512)
+- Fixed default fromBlock value and improved parameter interpretation in eth_getLogs RPC handler [#4513](https://github.com/hyperledger/besu/pull/4513)
 - Fix for NoSuchElementException for missing invalid reason when rejecting a local sent transaction [#4569](https://github.com/hyperledger/besu/pull/4569) 
+- Corrects treating a block as bad on internal error during either validation or processing [#4512](https://github.com/hyperledger/besu/issues/4512)
 
 ## 22.10.0-RC2
 
@@ -35,6 +37,7 @@
 - Upgrade RocksDB database version from 6.29.5 to 7.6.0 [#4517](https://github.com/hyperledger/besu/pull/4517)
 - Avoid connecting to self when using static-nodes [#4521](https://github.com/hyperledger/besu/pull/4521)
 - EVM performance has increased 20%-100% depending on the particulars of the contract. [#4540](https://github.com/hyperledger/besu/pull/4540)
+- Improve calculateRootHash method performance during Block processing [#4568](https://github.com/hyperledger/besu/pull/4568)
 
 ### Bug Fixes
 - Corrects emission of blockadded events when rewinding during a re-org. Fix for [#4495](https://github.com/hyperledger/besu/issues/4495)

build.gradle

@@ -604,7 +604,9 @@ artifactoryPublish {
   dependsOn distZip
 }
 
-def dockerVariants = [
+def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
+def dockerImageName = project.hasProperty("dockerImageName") ? project.getProperty("dockerImageName") : "hyperledger/besu"
+def dockerVariants = project.hasProperty("dockerVariants") ? project.getProperty("dockerVariants").split(",") : [
   "openjdk-11",
   "openjdk-11-debug",
   "graalvm",
@@ -634,9 +636,7 @@ tasks.register("dockerDistUntar") {
 task distDocker {
   dependsOn dockerDistUntar
   inputs.dir("build/docker-besu/")
-  def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
   def dockerBuildDir = "build/docker-besu/"
-  def imageName = "hyperledger/besu"
 
   doLast {
     for (def variant in dockerVariants) {
@@ -645,7 +645,7 @@ task distDocker {
         into(dockerBuildDir)
       }
       exec {
-        def image = "${imageName}:${dockerBuildVersion}-${variant}"
+        def image = "${dockerImageName}:${dockerBuildVersion}-${variant}"
         executable "sh"
         workingDir dockerBuildDir
         args "-c", "docker build --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} -t ${image} ."
@@ -654,12 +654,12 @@ task distDocker {
     // tag the "default" (which is the variant in the zero position)
     exec {
       executable "sh"
-      args "-c", "docker tag '${imageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${imageName}:${dockerBuildVersion}'"
+      args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:${dockerBuildVersion}'"
     }
     // create a static tag for the benchmark target
     exec {
       executable "sh"
-      args "-c", "docker tag '${imageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${imageName}:benchmark'"
+      args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:benchmark'"
     }
   }
 }
@@ -675,7 +675,7 @@ task testDocker {
   doLast {
     for (def variant in dockerVariants) {
       exec {
-        def image = project.hasProperty('release.releaseVersion') ? "hyperledger/besu:" + project.property('release.releaseVersion') : "hyperledger/besu:${project.version}"
+        def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}"
         workingDir "${projectDir}/docker/${variant}"
         executable "sh"
         args "-c", "bash ../test.sh ${image}-${variant}"
@@ -710,7 +710,7 @@ task acceptanceTestsQuorum {
       new File(dataDirectory).mkdirs()
       new File(reportsDirectory).mkdirs()
 
-      def image = project.hasProperty('release.releaseVersion') ? "hyperledger/besu:" + project.property('release.releaseVersion') : "hyperledger/besu:${project.version}"
+      def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}"
       def dockerEnv = "--env LOGGING_LEVEL_COM_QUORUM_GAUGE=DEBUG --env TF_VAR_besu_docker_image='{name=\"${image}-${variant}\",local=true}'"
       def dockerVolumes = "-v ${reportsDirectory}:/workspace/target/gauge/reports/ -v /var/run/docker.sock:/var/run/docker.sock -v ${dataDirectory}:${dataDirectory}"
       executable "sh"
@@ -719,13 +719,10 @@ task acceptanceTestsQuorum {
   }
 }
 
-def dockerImage = "hyperledger/besu"
-
 task dockerUpload {
   dependsOn distDocker
-  def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
   def architecture = System.getenv('architecture')
-  def image = "${dockerImage}:${dockerBuildVersion}"
+  def image = "${dockerImageName}:${dockerBuildVersion}"
   def additionalTags = []
 
   if (project.hasProperty('branch') && project.property('branch') == 'main') {
@@ -743,7 +740,7 @@ task dockerUpload {
       exec {
         def archVariantImage = "${variantImage}-${architecture}"
         def cmd = "docker tag '${variantImage}' '${archVariantImage}' && docker push '${archVariantImage}'"
-        additionalTags.each { tag -> cmd += " && docker tag '${variantImage}' '${dockerImage}:${tag.trim()}-${variant}-${architecture}' && docker push '${dockerImage}:${tag.trim()}-${variant}-${architecture}'" }
+        additionalTags.each { tag -> cmd += " && docker tag '${variantImage}' '${dockerImageName}:${tag.trim()}-${variant}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${variant}-${architecture}'" }
         executable "sh"
         args "-c", cmd
       }
@@ -752,7 +749,7 @@ task dockerUpload {
     exec {
       def archImage = "${image}-${architecture}"
       def cmd = "docker tag ${image} ${archImage} && docker push '${archImage}'"
-      additionalTags.each { tag -> cmd += " && docker tag '${image}' '${dockerImage}:${tag.trim()}-${architecture}' && docker push '${dockerImage}:${tag.trim()}-${architecture}'" }
+      additionalTags.each { tag -> cmd += " && docker tag '${image}' '${dockerImageName}:${tag.trim()}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${architecture}'" }
       executable "sh"
       args "-c", cmd
     }
@@ -761,18 +758,17 @@ task dockerUpload {
 
 task manifestDocker {
 
-  def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}"
-  def image = "${dockerImage}:${dockerBuildVersion}"
+  def image = "${dockerImageName}:${dockerBuildVersion}"
   def archs = ["arm64", "amd64"]
   def tags = ["${image}"]
 
   if (project.hasProperty('branch') && project.property('branch') == 'main') {
-    tags.add("${dockerImage}:develop")
+    tags.add("${dockerImageName}:develop")
   }
 
   if (!(dockerBuildVersion ==~ /.*-SNAPSHOT/)) {
-    tags.add("${dockerImage}:latest")
-    tags.add("${dockerImage}:" + dockerBuildVersion.split(/\./)[0..1].join('.'))
+    tags.add("${dockerImageName}:latest")
+    tags.add("${dockerImageName}:" + dockerBuildVersion.split(/\./)[0..1].join('.'))
   }
 
   doLast {

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java

@@ -46,7 +46,6 @@
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
 import org.hyperledger.besu.evm.account.Account;
 import org.hyperledger.besu.evm.log.LogTopic;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.plugin.data.SyncStatus;
 
 import java.math.BigInteger;
@@ -219,31 +218,37 @@ DataFetcher<Optional<AccountAdapter>> getAccountDataFetcher() {
       final Address addr = dataFetchingEnvironment.getArgument("address");
       final Long bn = dataFetchingEnvironment.getArgument("blockNumber");
       if (bn != null) {
-        final Optional<WorldState> ws = blockchainQuery.getWorldState(bn);
-        if (ws.isPresent()) {
-          final Account account = ws.get().get(addr);
-          if (account == null) {
-            return Optional.of(new EmptyAccountAdapter(addr));
-          }
-          return Optional.of(new AccountAdapter(account));
-        } else if (bn > blockchainQuery.getBlockchain().getChainHeadBlockNumber()) {
-          // block is past chainhead
-          throw new GraphQLException(GraphQLError.INVALID_PARAMS);
-        } else {
-          // we don't have that block
-          throw new GraphQLException(GraphQLError.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE);
-        }
+        return blockchainQuery
+            .getAndMapWorldState(
+                bn,
+                ws -> {
+                  final Account account = ws.get(addr);
+                  if (account == null) {
+                    return new EmptyAccountAdapter(addr);
+                  }
+                  return new AccountAdapter(account);
+                })
+            .or(
+                () -> {
+                  if (bn > blockchainQuery.getBlockchain().getChainHeadBlockNumber()) {
+                    // block is past chainhead
+                    throw new GraphQLException(GraphQLError.INVALID_PARAMS);
+                  } else {
+                    // we don't have that block
+                    throw new GraphQLException(GraphQLError.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE);
+                  }
+                });
       } else {
         // return account on latest block
         final long latestBn = blockchainQuery.latestBlock().get().getHeader().getNumber();
-        final Optional<WorldState> ows = blockchainQuery.getWorldState(latestBn);
-        return ows.flatMap(
+        return blockchainQuery.getAndMapWorldState(
+            latestBn,
             ws -> {
               final Account account = ws.get(addr);
               if (account == null) {
-                return Optional.of(new EmptyAccountAdapter(addr));
+                return new EmptyAccountAdapter(addr);
               }
-              return Optional.of(new AccountAdapter(account));
+              return new AccountAdapter(account);
             });
       }
     };

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java

@@ -96,8 +96,7 @@ public Optional<AdapterBase> getMiner(final DataFetchingEnvironment environment)
     }
 
     return query
-        .getWorldState(blockNumber)
-        .map(ws -> ws.get(header.getCoinbase()))
+        .getAndMapWorldState(blockNumber, ws -> ws.get(header.getCoinbase()))
         .map(account -> (AdapterBase) new AccountAdapter(account))
         .or(() -> Optional.of(new EmptyAccountAdapter(header.getCoinbase())));
   }
@@ -147,13 +146,12 @@ public Optional<AccountAdapter> getAccount(final DataFetchingEnvironment environ
 
     final BlockchainQueries query = getBlockchainQueries(environment);
     final long bn = header.getNumber();
-    return query
-        .getWorldState(bn)
-        .map(
-            ws -> {
-              final Address address = environment.getArgument("address");
-              return new AccountAdapter(ws.get(address));
-            });
+    return query.getAndMapWorldState(
+        bn,
+        ws -> {
+          final Address address = environment.getArgument("address");
+          return new AccountAdapter(ws.get(address));
+        });
   }
 
   public List<LogAdapter> getLogs(final DataFetchingEnvironment environment) {

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java

@@ -63,8 +63,7 @@ public Optional<AccountAdapter> getAccount(final DataFetchingEnvironment environ
       blockNumber = bn;
     }
 
-    return query
-        .getWorldState(blockNumber)
-        .map(ws -> new AccountAdapter(ws.get(logWithMetadata.getLogger())));
+    return query.getAndMapWorldState(
+        blockNumber, ws -> new AccountAdapter(ws.get(logWithMetadata.getLogger())));
   }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java

@@ -24,7 +24,6 @@
 import org.hyperledger.besu.ethereum.transaction.CallParameter;
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 
 import java.util.List;
 import java.util.Map;
@@ -65,10 +64,8 @@ public Optional<AccountAdapter> getAccount(
     final Address addr = dataFetchingEnvironment.getArgument("address");
     final Long blockNumber = dataFetchingEnvironment.getArgument("blockNumber");
     final long latestBlockNumber = blockchainQuery.latestBlock().get().getHeader().getNumber();
-    final Optional<WorldState> optionalWorldState =
-        blockchainQuery.getWorldState(latestBlockNumber);
-    return optionalWorldState
-        .flatMap(worldState -> Optional.ofNullable(worldState.get(addr)))
+    return blockchainQuery
+        .getAndMapWorldState(latestBlockNumber, ws -> ws.get(addr))
         .map(AccountAdapter::new);
   }
 

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java

@@ -25,7 +25,6 @@
 import org.hyperledger.besu.ethereum.core.LogWithMetadata;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -83,12 +82,11 @@ public Optional<AccountAdapter> getFrom(final DataFetchingEnvironment environmen
     if (blockNumber == null) {
       blockNumber = transactionWithMetadata.getBlockNumber().orElseGet(query::headBlockNumber);
     }
-    return query
-        .getWorldState(blockNumber)
-        .map(
-            mutableWorldState ->
-                new AccountAdapter(
-                    mutableWorldState.get(transactionWithMetadata.getTransaction().getSender())));
+    return query.getAndMapWorldState(
+        blockNumber,
+        mutableWorldState ->
+            new AccountAdapter(
+                mutableWorldState.get(transactionWithMetadata.getTransaction().getSender())));
   }
 
   public Optional<AccountAdapter> getTo(final DataFetchingEnvironment environment) {
@@ -98,15 +96,15 @@ public Optional<AccountAdapter> getTo(final DataFetchingEnvironment environment)
       blockNumber = transactionWithMetadata.getBlockNumber().orElseGet(query::headBlockNumber);
     }
 
-    return query
-        .getWorldState(blockNumber)
-        .flatMap(
-            ws -> {
-              return transactionWithMetadata
-                  .getTransaction()
-                  .getTo()
-                  .map(address -> new AccountAdapter(address, ws.get(address)));
-            });
+    return query.getAndMapWorldState(
+        blockNumber,
+        ws ->
+            transactionWithMetadata
+                .getTransaction()
+                .getTo()
+                .map(address -> new AccountAdapter(address, ws.get(address)))
+                // safe because mapWorldState returns Optional.ofNullable
+                .orElse(null));
   }
 
   public Optional<Wei> getValue() {
@@ -176,11 +174,7 @@ public Optional<AccountAdapter> getCreatedContract(final DataFetchingEnvironment
           return Optional.empty();
         }
         final long blockNumber = bn.orElseGet(txBlockNumber::get);
-
-        final Optional<WorldState> ws = query.getWorldState(blockNumber);
-        if (ws.isPresent()) {
-          return Optional.of(new AccountAdapter(ws.get().get(addr.get())));
-        }
+        return query.getAndMapWorldState(blockNumber, ws -> new AccountAdapter(ws.get(addr.get())));
       }
     }
     return Optional.empty();

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java

@@ -49,7 +49,6 @@ public enum RpcMethod {
   ENGINE_NEW_PAYLOAD("engine_newPayloadV1"),
   ENGINE_FORKCHOICE_UPDATED("engine_forkchoiceUpdatedV1"),
   ENGINE_EXCHANGE_TRANSITION_CONFIGURATION("engine_exchangeTransitionConfigurationV1"),
-  ENGINE_PREPARE_PAYLOAD("engine_preparePayload_debug"),
 
   GOQUORUM_ETH_GET_QUORUM_PAYLOAD("eth_getQuorumPayload"),
   GOQUORUM_STORE_RAW("goquorum_storeRaw"),