Cherry picks from master for RC2

.circleci/config.yml

@@ -230,7 +230,7 @@ jobs:
       - run:
           name: Publish
           command: |
-            ./gradlew --no-daemon bintrayUpload
+            ./gradlew --no-daemon artifactoryPublish bintrayUpload
 
   publishDocker:
     executor: besu_executor_med

CHANGELOG.md

@@ -1,8 +1,23 @@
 # Changelog
 
-## 21.2.0-RC1
+## 21.1.0-RC2
 
-### 21.2.0 Breaking Changes
+### Additions and Improvements
+* Distributions and maven artifacts have been moved off of bintray [\#1886](https://github.com/hyperledger/besu/pull/1886)
+* Support "eth" field in ENR records [\#1893](https://github.com/hyperledger/besu/pull/1893)
+
+### Bug Fixes
+
+### Early Access Features
+
+#### Previously identified known issues
+
+- [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)
+- [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
+
+## 21.1.0-RC1
+
+### 21.1.0 Breaking Changes
 * `--skip-pow-validation-enabled` is now an error with `block import --format JSON`. This is because the JSON format doesn't include the nonce so the proof of work must be calculated.
 * `eth_call` will not return a JSON-RPC result if the call fails, but will return an error instead. If it was for a revert the revert reason will be included.
 * `eth_call` will not fail for account balance issues by default. An parameter `"strict": true` can be added to the call parameters (with `to` and `from`) to enforce balance checks.
@@ -25,6 +40,11 @@
 - [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)
 - [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
 
+
+### Download link
+https://dl.bintray.com/hyperledger-org/besu-repo/besu-21.1.0-RC1.zip
+sha256: `b0fe3942052b8fd43fc3025a298a6c701f9edae2e100f0c563a1c5a4ceef71f1`
+
 ## 20.10.4
 
 ### Additions and Improvements

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -205,6 +205,7 @@ public void startNode(final BesuNode node) {
             .besuPluginContext(new BesuPluginContextImpl())
             .autoLogBloomCaching(false)
             .storageProvider(storageProvider)
+            .forkIdSupplier(() -> besuController.getProtocolManager().getForkIdAsBytesList())
             .build();
 
     runner.start();

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -119,6 +119,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -170,6 +171,7 @@ public class RunnerBuilder {
   private boolean autoLogBloomCaching = true;
   private boolean randomPeerPriority;
   private StorageProvider storageProvider;
+  private Supplier<List<Bytes>> forkIdSupplier;
 
   public RunnerBuilder vertx(final Vertx vertx) {
     this.vertx = vertx;
@@ -341,6 +343,11 @@ public RunnerBuilder storageProvider(final StorageProvider storageProvider) {
     return this;
   }
 
+  public RunnerBuilder forkIdSupplier(final Supplier<List<Bytes>> forkIdSupplier) {
+    this.forkIdSupplier = forkIdSupplier;
+    return this;
+  }
+
   public Runner build() {
 
     Preconditions.checkNotNull(besuController);
@@ -414,9 +421,9 @@ public Runner build() {
     LOG.info("Detecting NAT service.");
     final boolean fallbackEnabled = natMethod == NatMethod.AUTO || natMethodFallbackEnabled;
     final NatService natService = new NatService(buildNatManager(natMethod), fallbackEnabled);
-    final NetworkBuilder inactiveNetwork = (caps) -> new NoopP2PNetwork();
+    final NetworkBuilder inactiveNetwork = caps -> new NoopP2PNetwork();
     final NetworkBuilder activeNetwork =
-        (caps) ->
+        caps ->
             DefaultP2PNetwork.builder()
                 .vertx(vertx)
                 .nodeKey(nodeKey)
@@ -427,6 +434,7 @@ public Runner build() {
                 .natService(natService)
                 .randomPeerPriority(randomPeerPriority)
                 .storageProvider(storageProvider)
+                .forkIdSupplier(forkIdSupplier)
                 .build();
 
     final NetworkRunner networkRunner =

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2227,6 +2227,7 @@ private void synchronize(
             .ethstatsUrl(unstableEthstatsOptions.getEthstatsUrl())
             .ethstatsContact(unstableEthstatsOptions.getEthstatsContact())
             .storageProvider(keyStorageProvider(keyValueStorageName))
+            .forkIdSupplier(() -> besuController.getProtocolManager().getForkIdAsBytesList())
             .build();
 
     addShutdownHook(runner);

besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java

@@ -26,7 +26,7 @@
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_DISCOVERY_URL;
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_BOOTSTRAP_NODES;
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_DISCOVERY_URL;
-import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.YOLO_V2_BOOTSTRAP_NODES;
+import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.YOLO_V3_BOOTSTRAP_NODES;
 
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
 
@@ -50,7 +50,7 @@ public class EthNetworkConfig {
   public static final BigInteger CLASSIC_NETWORK_ID = BigInteger.valueOf(1);
   public static final BigInteger KOTTI_NETWORK_ID = BigInteger.valueOf(6);
   public static final BigInteger MORDOR_NETWORK_ID = BigInteger.valueOf(7);
-  private static final BigInteger YOLO_V2_NETWORK_ID = BigInteger.valueOf(133519467574834L);
+  private static final BigInteger YOLO_V3_NETWORK_ID = BigInteger.valueOf(133519467574835L);
   private static final String MAINNET_GENESIS = "/mainnet.json";
   private static final String ROPSTEN_GENESIS = "/ropsten.json";
   private static final String RINKEBY_GENESIS = "/rinkeby.json";
@@ -160,9 +160,9 @@ public static EthNetworkConfig getNetworkConfig(final NetworkName networkName) {
       case MORDOR:
         return new EthNetworkConfig(
             jsonConfig(MORDOR_GENESIS), MORDOR_NETWORK_ID, MORDOR_BOOTSTRAP_NODES, null);
-      case YOLO_V2:
+      case YOLO_V3:
         return new EthNetworkConfig(
-            jsonConfig(YOLO_GENESIS), YOLO_V2_NETWORK_ID, YOLO_V2_BOOTSTRAP_NODES, null);
+            jsonConfig(YOLO_GENESIS), YOLO_V3_NETWORK_ID, YOLO_V3_BOOTSTRAP_NODES, null);
       case MAINNET:
       default:
         return new EthNetworkConfig(
@@ -200,7 +200,7 @@ public static String jsonConfig(final NetworkName network) {
         return jsonConfig(KOTTI_GENESIS);
       case MORDOR:
         return jsonConfig(MORDOR_GENESIS);
-      case YOLO_V2:
+      case YOLO_V3:
         return jsonConfig(YOLO_GENESIS);
       default:
         throw new IllegalArgumentException("Unknown network:" + network);

besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java

@@ -23,5 +23,5 @@ public enum NetworkName {
   CLASSIC,
   KOTTI,
   MORDOR,
-  YOLO_V2
+  YOLO_V3
 }

besu/src/main/java/org/hyperledger/besu/controller/BftQueryPluginServiceFactory.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.controller;
+
+import org.hyperledger.besu.consensus.common.BlockInterface;
+import org.hyperledger.besu.consensus.common.bft.BftBlockInterface;
+import org.hyperledger.besu.consensus.common.bft.queries.BftQueryServiceImpl;
+import org.hyperledger.besu.crypto.NodeKey;
+import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.plugin.services.metrics.PoAMetricsService;
+import org.hyperledger.besu.plugin.services.query.BftQueryService;
+import org.hyperledger.besu.plugin.services.query.PoaQueryService;
+import org.hyperledger.besu.services.BesuPluginContextImpl;
+
+public class BftQueryPluginServiceFactory implements PluginServiceFactory {
+
+  private final Blockchain blockchain;
+  private final NodeKey nodeKey;
+  private final String consensusMechanismName;
+
+  public BftQueryPluginServiceFactory(
+      final Blockchain blockchain, final NodeKey nodeKey, final String consensusMechanismName) {
+    this.blockchain = blockchain;
+    this.nodeKey = nodeKey;
+    this.consensusMechanismName = consensusMechanismName;
+  }
+
+  @Override
+  public void appendPluginServices(final BesuPluginContextImpl besuContext) {
+    final BlockInterface blockInterface = new BftBlockInterface();
+
+    final BftQueryServiceImpl service =
+        new BftQueryServiceImpl(blockInterface, blockchain, nodeKey, consensusMechanismName);
+    besuContext.addService(BftQueryService.class, service);
+    besuContext.addService(PoaQueryService.class, service);
+    besuContext.addService(PoAMetricsService.class, service);
+  }
+}

besu/src/main/java/org/hyperledger/besu/controller/IbftQueryPluginServiceFactory.java

@@ -16,10 +16,12 @@
 
 import org.hyperledger.besu.consensus.common.BlockInterface;
 import org.hyperledger.besu.consensus.common.bft.BftBlockInterface;
+import org.hyperledger.besu.consensus.common.bft.queries.BftQueryServiceImpl;
 import org.hyperledger.besu.consensus.ibft.queries.IbftQueryServiceImpl;
 import org.hyperledger.besu.crypto.NodeKey;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.plugin.services.metrics.PoAMetricsService;
+import org.hyperledger.besu.plugin.services.query.BftQueryService;
 import org.hyperledger.besu.plugin.services.query.IbftQueryService;
 import org.hyperledger.besu.plugin.services.query.PoaQueryService;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -43,5 +45,9 @@ public void appendPluginServices(final BesuPluginContextImpl besuContext) {
     besuContext.addService(IbftQueryService.class, service);
     besuContext.addService(PoaQueryService.class, service);
     besuContext.addService(PoAMetricsService.class, service);
+
+    final BftQueryServiceImpl bftService =
+        new BftQueryServiceImpl(blockInterface, blockchain, nodeKey, "ibft");
+    besuContext.addService(BftQueryService.class, bftService);
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java

@@ -214,7 +214,7 @@ protected MiningCoordinator createMiningCoordinator(
 
   @Override
   protected PluginServiceFactory createAdditionalPluginServices(final Blockchain blockchain) {
-    return new NoopPluginServiceFactory();
+    return new BftQueryPluginServiceFactory(blockchain, nodeKey, "qbft");
   }
 
   @Override

besu/src/main/resources/org/hyperledger/besu/cli/launcher.json

@@ -7,7 +7,7 @@
       "config-key": "network",
       "available-options": "org.hyperledger.besu.cli.config.NetworkName",
       "additional-flag": {
-        "yolo_v2": "Xberlin-enabled"
+        "yolo_v3": "Xberlin-enabled"
       }
     },
     {
@@ -184,4 +184,4 @@
       ]
     }
   ]
-}
+}

besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java

@@ -80,7 +80,6 @@
 import org.junit.rules.TemporaryFolder;
 
 @SuppressWarnings("rawtypes")
-// todo request lucas look at this pr
 public class PrivacyReorgTest {
   @Rule public final TemporaryFolder folder = new TemporaryFolder();
 
@@ -93,9 +92,9 @@ public class PrivacyReorgTest {
       Bytes.fromBase64String("A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo=");
 
   private static final String FIRST_BLOCK_WITH_NO_TRANSACTIONS_STATE_ROOT =
-      "0xc9dcaffbebc7edc20839c80e706430a8fa885e8f703bb89f6e95633cc2b05d4d";
+      "0xe368938a01d983e331eb0e4ea61224726d06075c1ad525569b369f664067ff26";
   private static final String FIRST_BLOCK_WITH_SINGLE_TRANSACTION_STATE_ROOT =
-      "0x66fbc6ad12ef1da78740093ea2b3362e773e510e27d8f88b68c27bfc1f4d58c8";
+      "0x9c88988f9602184efc538cf1c2f482a6b8757ff918d234602884dc8e3b983edd";
   private static final String BLOCK_WITH_SINGLE_TRANSACTION_RECEIPTS_ROOT =
       "0xc8267b3f9ed36df3ff8adb51a6d030716f23eeb50270e7fce8d9822ffa7f0461";
   private static final String STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMPTY_STATE =
@@ -338,7 +337,7 @@ public void reorgToLongerChain() {
         privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMPTY_STATE);
 
     final String secondForkBlockStateRoot =
-        "0x57ccc80f4e50d2e669d82aefa7d3bbe763cf47df27665af14c90b2f8641953f5";
+        "0x2c37a360a700c614b10c980138f64be9ad66fc4a14cd5145199cd0d8ec43d51d";
     final Block secondForkBlock =
         gen.block(
             getBlockOptionsNoTransactionWithDifficulty(
@@ -368,8 +367,7 @@ public void reorgToLongerChain() {
     appendBlock(besuController, blockchain, protocolContext, thirdForkBlock);
 
     // Check that the private state did change after reorg
-    assertPrivateStateRoot(
-        privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMPTY_STATE);
+    assertPrivateStateRoot(privateStateRootResolver, blockchain, EMPTY_ROOT_HASH);
   }
 
   @SuppressWarnings("unchecked")

besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java

@@ -124,6 +124,7 @@ public void enodeUrlShouldHaveAdvertisedHostWhenDiscoveryDisabled() {
             .vertx(vertx)
             .dataDir(dataDir.getRoot().toPath())
             .storageProvider(mock(KeyValueStorageProvider.class))
+            .forkIdSupplier(() -> Collections.singletonList(Bytes.EMPTY))
             .build();
     runner.start();
 

besu/src/test/java/org/hyperledger/besu/RunnerTest.java

@@ -82,6 +82,7 @@
 import okhttp3.Request;
 import okhttp3.RequestBody;
 import okhttp3.Response;
+import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.units.bigints.UInt256;
 import org.awaitility.Awaitility;
 import org.junit.After;
@@ -203,7 +204,8 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi
             .maxPeers(3)
             .metricsSystem(noOpMetricsSystem)
             .staticNodes(emptySet())
-            .storageProvider(new InMemoryStorageProvider());
+            .storageProvider(new InMemoryStorageProvider())
+            .forkIdSupplier(() -> Collections.singletonList(Bytes.EMPTY));
 
     Runner runnerBehind = null;
     final Runner runnerAhead =
@@ -217,6 +219,7 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi
             .dataDir(dbAhead)
             .pidPath(pidPath)
             .besuPluginContext(new BesuPluginContextImpl())
+            .forkIdSupplier(() -> controllerAhead.getProtocolManager().getForkIdAsBytesList())
             .build();
     try {
 
@@ -269,6 +272,7 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi
               .metricsConfiguration(behindMetricsConfiguration)
               .dataDir(temp.newFolder().toPath())
               .metricsSystem(noOpMetricsSystem)
+              .forkIdSupplier(() -> controllerBehind.getProtocolManager().getForkIdAsBytesList())
               .build();
 
       runnerBehind.start();

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -238,6 +238,7 @@ public void initMocks() throws Exception {
     when(mockRunnerBuilder.ethstatsUrl(anyString())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.ethstatsContact(anyString())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.storageProvider(any())).thenReturn(mockRunnerBuilder);
+    when(mockRunnerBuilder.forkIdSupplier(any())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.build()).thenReturn(mockRunner);
 
     final Bytes32 keyPairPrvKey =

besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java

@@ -299,11 +299,22 @@ public void reorgedBlockEventFiresAfterSubscribe() {
     blockchain.appendBlock(block, gen.receipts(block));
     assertThat(result.get()).isNull();
 
-    final var reorgBlock =
+    final var forkBlock =
         gen.block(
             new BlockDataGenerator.BlockOptions()
                 .setParentHash(blockchain.getGenesisBlock().getHash())
-                .setBlockNumber(blockchain.getGenesisBlock().getHeader().getNumber() + 2));
+                .setDifficulty(block.getHeader().getDifficulty().subtract(1))
+                .setBlockNumber(blockchain.getGenesisBlock().getHeader().getNumber() + 1));
+    blockchain.appendBlock(forkBlock, gen.receipts(forkBlock));
+    assertThat(result.get()).isNull();
+
+    final var reorgBlock =
+        gen.block(
+            new BlockDataGenerator.BlockOptions()
+                .setParentHash(forkBlock.getHash())
+                .setDifficulty(Difficulty.of(10000000))
+                .setBlockNumber(forkBlock.getHeader().getNumber() + 1));
+
     List<TransactionReceipt> transactionReceipts = gen.receipts(reorgBlock);
     blockchain.appendBlock(reorgBlock, transactionReceipts);
     assertThat(result.get()).isNotNull();