Private state update metadata and migration (release 1.4)

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 1.4.0 RC-2 
+
+### Private State Migration 
+Hyperledger Besu v1.4 implements a new data structure for private state storage that is not backwards compatible. 
+A migration will be performed when starting v1.4 for the first time to reprocess existing private transactions 
+and re-create the private state data in the v1.4 format. 
+If you have existing private transactions, see [migration details](docs/Private-Txns-Migration.md).
+
 ## 1.4.0 RC-1 
 
 ### Additions and Improvements 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/ethsigner/EthSignerClientTest.java

@@ -38,8 +38,7 @@ public class EthSignerClientTest {
   @ClassRule public static final TemporaryFolder folder = new TemporaryFolder();
 
   @ClassRule
-  public static final WireMockRule wireMockRule =
-      new WireMockRule(wireMockConfig().dynamicPort().dynamicPort());
+  public static final WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
 
   private static final String MOCK_RESPONSE = "mock_transaction_hash";
   private static final String MOCK_SEND_TRANSACTION_RESPONSE =

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyAcceptanceTest.java

@@ -57,7 +57,7 @@ public class MultiTenancyAcceptanceTest extends AcceptanceTestBase {
   private Cluster multiTenancyCluster;
 
   private static final int ENCLAVE_PORT = 1080;
-  private static final String PRIVACY_GROUP_ID = "Z3JvdXBJZA==";
+  private static final String PRIVACY_GROUP_ID = "B1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo=";
   private static final String ENCLAVE_KEY = "A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo=";
   private static final String KEY1 = "sgFkVOyFndZe/5SAZJO5UYbrl7pezHetveriBBWWnE8=";
   private static final String KEY2 = "R1kW75NQC9XX3kwNpyPjCBFflM29+XvnKKS9VLrUkzo=";

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/EthSignerAcceptanceTest.java

@@ -57,12 +57,11 @@ public void setUp() throws Exception {
   }
 
   @Test
-  @Ignore
   public void privateSmartContractMustDeploy() throws IOException {
     final String transactionHash =
         ethSignerClient.eeaSendTransaction(
             null,
-            BigInteger.valueOf(63992),
+            BigInteger.valueOf(23176),
             BigInteger.valueOf(1000),
             EventEmitter.BINARY,
             BigInteger.valueOf(0),
@@ -77,13 +76,15 @@ public void privateSmartContractMustDeploy() throws IOException {
         privateTransactionVerifier.validPrivateTransactionReceipt(transactionHash, receipt));
   }
 
+  // requires ethsigner jar > 0.3.0
+  // https://bintray.com/consensys/pegasys-repo/ethsigner
   @Test
   @Ignore
   public void privateSmartContractMustDeployNoNonce() throws IOException {
     final String transactionHash =
         ethSignerClient.eeaSendTransaction(
             null,
-            BigInteger.valueOf(63992),
+            BigInteger.valueOf(23176),
             BigInteger.valueOf(1000),
             EventEmitter.BINARY,
             minerNode.getEnclaveKey(),
@@ -114,7 +115,7 @@ public void privateSmartContractMustDeployWithPrivacyGroup() throws IOException
     final String transactionHash =
         ethSignerClient.eeaSendTransaction(
             null,
-            BigInteger.valueOf(63992),
+            BigInteger.valueOf(23176),
             BigInteger.valueOf(1000),
             EventEmitter.BINARY,
             BigInteger.valueOf(0),
@@ -130,7 +131,6 @@ public void privateSmartContractMustDeployWithPrivacyGroup() throws IOException
   }
 
   @Test
-  @Ignore
   public void privateSmartContractMustDeployWithPrivacyGroupNoNonce() throws IOException {
     final String privacyGroupId =
         minerNode.execute(privacyTransactions.createPrivacyGroup(null, null, minerNode));
@@ -140,14 +140,14 @@ public void privateSmartContractMustDeployWithPrivacyGroupNoNonce() throws IOExc
             new PrivacyGroup(
                 privacyGroupId,
                 PrivacyGroup.Type.PANTHEON,
-                "Default Name",
-                "Default Description",
+                "",
+                "",
                 Base64String.wrapList(minerNode.getEnclaveKey()))));
 
     final String transactionHash =
         ethSignerClient.eeaSendTransaction(
             null,
-            BigInteger.valueOf(63992),
+            BigInteger.valueOf(23176),
             BigInteger.valueOf(1000),
             EventEmitter.BINARY,
             minerNode.getEnclaveKey(),

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -48,6 +48,8 @@
 import org.hyperledger.besu.cli.options.PrunerOptions;
 import org.hyperledger.besu.cli.options.SynchronizerOptions;
 import org.hyperledger.besu.cli.options.TransactionPoolOptions;
+import org.hyperledger.besu.cli.presynctasks.PreSynchronizationTaskRunner;
+import org.hyperledger.besu.cli.presynctasks.PrivateDatabaseMigrationPreSyncTask;
 import org.hyperledger.besu.cli.subcommands.PasswordSubCommand;
 import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand;
 import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand.KeyLoader;
@@ -212,6 +214,9 @@ protected KeyLoader getKeyLoader() {
   // Property to indicate whether Besu has been launched via docker
   private final boolean isDocker = Boolean.getBoolean("besu.docker");
 
+  private final PreSynchronizationTaskRunner preSynchronizationTaskRunner =
+      new PreSynchronizationTaskRunner();
+
   // CLI options defined by user at runtime.
   // Options parsing is done with CLI library Picocli https://picocli.info/
 
@@ -760,6 +765,11 @@ void setBannedNodeIds(final List<String> values) {
           "The name of a file containing the private key used to sign privacy marker transactions. If unset, each will be signed with a random key.")
   private final Path privacyMarkerTransactionSigningKeyPath = null;
 
+  @Option(
+      names = {"--privacy-enable-database-migration"},
+      description = "Enable private database metadata migration (default: ${DEFAULT-VALUE})")
+  private final Boolean migratePrivateDatabase = false;
+
   @Option(
       names = {"--target-gas-limit"},
       description =
@@ -882,7 +892,11 @@ public void run() {
       // Need to create vertx after cmdline has been parsed, such that metricSystem is configurable
       vertx = createVertx(createVertxOptions(metricsSystem.get()));
 
-      validateOptions().configure().controller().startPlugins().startSynchronization();
+      final BesuCommand controller = validateOptions().configure().controller();
+
+      preSynchronizationTaskRunner.runTasks(controller.besuController);
+
+      controller.startPlugins().startSynchronization();
     } catch (final Exception e) {
       throw new ParameterException(this.commandLine, e.getMessage(), e);
     }
@@ -1590,7 +1604,14 @@ private PrivacyParameters privacyParameters() {
       }
     }
 
-    return privacyParametersBuilder.build();
+    final PrivacyParameters privacyParameters = privacyParametersBuilder.build();
+
+    if (isPrivacyEnabled) {
+      preSynchronizationTaskRunner.addTask(
+          new PrivateDatabaseMigrationPreSyncTask(privacyParameters, migratePrivateDatabase));
+    }
+
+    return privacyParameters;
   }
 
   private boolean anyPrivacyApiEnabled() {
@@ -1602,19 +1623,20 @@ private boolean anyPrivacyApiEnabled() {
 
   private PrivacyKeyValueStorageProvider privacyKeyStorageProvider(final String name) {
     return new PrivacyKeyValueStorageProviderBuilder()
-        .withStorageFactory(
-            (PrivacyKeyValueStorageFactory)
-                storageService
-                    .getByName(name)
-                    .orElseThrow(
-                        () ->
-                            new StorageException(
-                                "No KeyValueStorageFactory found for key: " + name)))
+        .withStorageFactory(privacyKeyValueStorageFactory(name))
         .withCommonConfiguration(pluginCommonConfiguration)
         .withMetricsSystem(getMetricsSystem())
         .build();
   }
 
+  private PrivacyKeyValueStorageFactory privacyKeyValueStorageFactory(final String name) {
+    return (PrivacyKeyValueStorageFactory)
+        storageService
+            .getByName(name)
+            .orElseThrow(
+                () -> new StorageException("No KeyValueStorageFactory found for key: " + name));
+  }
+
   private KeyValueStorageProvider keyStorageProvider(final String name) {
     return new KeyValueStorageProviderBuilder()
         .withStorageFactory(

besu/src/main/java/org/hyperledger/besu/cli/error/BesuExceptionHandler.java

@@ -14,11 +14,14 @@
  */
 package org.hyperledger.besu.cli.error;
 
+import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationException;
+
 import java.util.List;
 import java.util.function.Supplier;
 
 import org.apache.logging.log4j.Level;
 import picocli.CommandLine;
+import picocli.CommandLine.ParameterException;
 
 public class BesuExceptionHandler
     extends CommandLine.AbstractHandler<List<Object>, BesuExceptionHandler>
@@ -39,12 +42,17 @@ public List<Object> handleParseException(
     } else {
       err().println(ex.getMessage());
     }
-    if (!CommandLine.UnmatchedArgumentException.printSuggestions(ex, err())) {
+    if (shouldPrintUsage(ex)) {
       ex.getCommandLine().usage(err(), ansi());
     }
     return returnResultOrExit(null);
   }
 
+  private boolean shouldPrintUsage(final ParameterException ex) {
+    return !CommandLine.UnmatchedArgumentException.printSuggestions(ex, err())
+        && !(ex.getCause() instanceof PrivateStorageMigrationException);
+  }
+
   @Override
   public List<Object> handleExecutionException(
       final CommandLine.ExecutionException ex, final CommandLine.ParseResult parseResult) {

besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.presynctasks;
+
+import org.hyperledger.besu.cli.BesuCommand;
+import org.hyperledger.besu.controller.BesuController;
+
+/**
+ * All PreSynchronizationTask instances execute after the {@link BesuController} instance in {@link
+ * BesuCommand} is ready and before {@link BesuCommand#startSynchronization()} is called
+ */
+public interface PreSynchronizationTask {
+
+  void run(final BesuController<?> besuController);
+}

besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTaskRunner.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.presynctasks;
+
+import org.hyperledger.besu.controller.BesuController;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PreSynchronizationTaskRunner {
+
+  private final List<PreSynchronizationTask> tasks = new ArrayList<>();
+
+  public void addTask(final PreSynchronizationTask task) {
+    tasks.add(task);
+  }
+
+  public void runTasks(final BesuController<?> besuController) {
+    tasks.forEach(t -> t.run(besuController));
+  }
+}

besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PrivateDatabaseMigrationPreSyncTask.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.presynctasks;
+
+import org.hyperledger.besu.controller.BesuController;
+import org.hyperledger.besu.ethereum.core.PrivacyParameters;
+import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationService;
+import org.hyperledger.besu.util.PrivateStorageMigrationBuilder;
+
+public class PrivateDatabaseMigrationPreSyncTask implements PreSynchronizationTask {
+
+  private final PrivacyParameters privacyParameters;
+  private final boolean migratePrivateDatabaseFlag;
+
+  public PrivateDatabaseMigrationPreSyncTask(
+      final PrivacyParameters privacyParameters, final boolean migratePrivateDatabaseFlag) {
+    this.privacyParameters = privacyParameters;
+    this.migratePrivateDatabaseFlag = migratePrivateDatabaseFlag;
+  }
+
+  @Override
+  public void run(final BesuController<?> besuController) {
+    final PrivateStorageMigrationBuilder privateStorageMigrationBuilder =
+        new PrivateStorageMigrationBuilder(besuController, privacyParameters);
+    final PrivateStorageMigrationService privateStorageMigrationService =
+        new PrivateStorageMigrationService(
+            privacyParameters.getPrivateStateStorage(),
+            migratePrivateDatabaseFlag,
+            privateStorageMigrationBuilder::build);
+
+    privateStorageMigrationService.runMigrationIfRequired();
+  }
+}

besu/src/main/java/org/hyperledger/besu/util/PrivateStorageMigrationBuilder.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.util;
+
+import org.hyperledger.besu.controller.BesuController;
+import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.ethereum.core.Address;
+import org.hyperledger.besu.ethereum.core.PrivacyParameters;
+import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver;
+import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage;
+import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage;
+import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateMigrationBlockProcessor;
+import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigration;
+import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
+
+public class PrivateStorageMigrationBuilder {
+
+  private final BesuController<?> besuController;
+  private final PrivacyParameters privacyParameters;
+
+  public PrivateStorageMigrationBuilder(
+      final BesuController<?> besuController, final PrivacyParameters privacyParameters) {
+    this.besuController = besuController;
+    this.privacyParameters = privacyParameters;
+  }
+
+  public PrivateStorageMigration build() {
+    final Blockchain blockchain = besuController.getProtocolContext().getBlockchain();
+    final Address privacyPrecompileAddress =
+        Address.privacyPrecompiled(privacyParameters.getPrivacyAddress());
+    final ProtocolSchedule<?> protocolSchedule = besuController.getProtocolSchedule();
+    final WorldStateArchive publicWorldStateArchive =
+        besuController.getProtocolContext().getWorldStateArchive();
+    final PrivateStateStorage privateStateStorage = privacyParameters.getPrivateStateStorage();
+    final PrivateStateRootResolver privateStateRootResolver =
+        new PrivateStateRootResolver(privateStateStorage);
+    final LegacyPrivateStateStorage legacyPrivateStateStorage =
+        privacyParameters.getPrivateStorageProvider().createLegacyPrivateStateStorage();
+
+    return new PrivateStorageMigration(
+        blockchain,
+        privacyPrecompileAddress,
+        protocolSchedule,
+        publicWorldStateArchive,
+        privateStateStorage,
+        privateStateRootResolver,
+        legacyPrivateStateStorage,
+        PrivateMigrationBlockProcessor::new);
+  }
+}