From 6164025aa86b6ad87a5d9a7b104f99c68ad8687d Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Mon, 17 Feb 2020 13:12:44 +1300 Subject: [PATCH] Private state update metadata and migration (release 1.4) (#402) (backport from release-1.4) Private state update metadata and migration Signed-off-by: Lucas Saldanha --- CHANGELOG.md | 7 + .../dsl/ethsigner/EthSignerClientTest.java | 3 +- .../MultiTenancyAcceptanceTest.java | 2 +- ...tiTenancyValidationFailAcceptanceTest.java | 2 +- .../privacy/EthSignerAcceptanceTest.java | 16 +- .../org/hyperledger/besu/cli/BesuCommand.java | 42 +- .../besu/cli/error/BesuExceptionHandler.java | 10 +- .../presynctasks/PreSynchronizationTask.java | 27 + .../PreSynchronizationTaskRunner.java | 33 ++ .../PrivateDatabaseMigrationPreSyncTask.java | 45 ++ .../util/PrivateStorageMigrationBuilder.java | 63 ++ .../hyperledger/besu/PrivacyReorgTest.java | 538 ++++++++++++++++++ .../org/hyperledger/besu/PrivacyTest.java | 5 +- .../hyperledger/besu/cli/BesuCommandTest.java | 18 +- .../besu/cli/CommandTestAbstract.java | 19 +- .../PrivateStorageMigrationServiceTest.java | 105 ++++ besu/src/test/resources/enclavePrivateKey | 1 + besu/src/test/resources/enclavePublicKey | 1 + .../src/test/resources/everything_config.toml | 1 + build.gradle | 2 +- docs/Private-Txns-Migration.md | 36 ++ ...vGetPrivateTransactionIntegrationTest.java | 2 +- .../privacy/methods/priv/PrivCall.java | 11 - .../priv/PrivGetTransactionReceipt.java | 62 +- .../privacy/PrivateTransactionResult.java | 2 +- .../PrivacyApiGroupJsonRpcMethods.java | 15 +- .../privacy/methods/priv/PrivCallTest.java | 1 - .../priv/PrivGetTransactionCountTest.java | 19 +- .../priv/PrivGetTransactionReceiptTest.java | 21 +- ...acyPrecompiledContractIntegrationTest.java | 25 +- .../mainnet/AbstractBlockProcessor.java | 9 + .../mainnet/MainnetTransactionProcessor.java | 2 + .../mainnet/PrivacyBlockProcessor.java | 54 ++ .../ethereum/mainnet/ProtocolSpecBuilder.java | 29 +- .../privacy/PrivacyPrecompiledContract.java | 101 +++- .../ChainHeadPrivateNonceProvider.java | 54 ++ .../privacy/DefaultPrivacyController.java | 46 +- .../privacy/PrivateNonceProvider.java | 23 + .../privacy/PrivateStateRootResolver.java | 70 +++ ...geMigrationTransactionProcessorResult.java | 76 +++ .../ethereum/privacy/PrivateTransaction.java | 2 +- .../privacy/PrivateTransactionProcessor.java | 2 + .../privacy/PrivateTransactionReceipt.java | 184 ++++++ .../privacy/PrivateTransactionSimulator.java | 14 +- .../privacy/PrivateTransactionValidator.java | 8 +- .../LegacyPrivateStateKeyValueStorage.java | 178 ++++++ .../storage/LegacyPrivateStateStorage.java | 68 +++ .../storage/PrivacyGroupHeadBlockMap.java | 134 +++++ .../storage/PrivacyStorageProvider.java | 3 + .../privacy/storage/PrivateBlockMetadata.java | 68 +++ .../storage/PrivateStateKeyValueStorage.java | 133 ++--- .../privacy/storage/PrivateStateStorage.java | 40 +- .../storage/PrivateTransactionMetadata.java | 12 +- .../ethereum/privacy/storage/RLPMapEntry.java | 71 +++ .../PrivacyKeyValueStorageProvider.java | 39 +- .../PrivateMigrationBlockProcessor.java | 160 ++++++ .../migration/PrivateStorageMigration.java | 176 ++++++ .../PrivateStorageMigrationException.java | 33 ++ .../PrivateStorageMigrationService.java | 78 +++ .../besu/ethereum/vm/MessageFrame.java | 22 + .../ethereum/core/BlockDataGenerator.java | 121 +++- .../core/InMemoryPrivacyStorageProvider.java | 7 + .../core/InMemoryStorageProvider.java | 6 + .../mainnet/PrivacyBlockProcessorTest.java | 77 +++ .../PrivacyPrecompiledContractTest.java | 45 +- .../ChainHeadPrivateNonceProviderTest.java | 102 ++++ .../privacy/DefaultPrivacyControllerTest.java | 107 +--- .../privacy/PrivateStateRootResolverTest.java | 180 ++++++ .../PrivateStateKeyValueStorageTest.java | 55 ++ .../PrivateStorageMigrationTest.java | 315 ++++++++++ .../PrivateTransactionDataFixture.java | 85 +++ 71 files changed, 3677 insertions(+), 446 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTaskRunner.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PrivateDatabaseMigrationPreSyncTask.java create mode 100644 besu/src/main/java/org/hyperledger/besu/util/PrivateStorageMigrationBuilder.java create mode 100644 besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java create mode 100644 besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java create mode 100644 besu/src/test/resources/enclavePrivateKey create mode 100644 besu/src/test/resources/enclavePublicKey create mode 100644 docs/Private-Txns-Migration.md create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProvider.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateNonceProvider.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolver.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStorageMigrationTransactionProcessorResult.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyGroupHeadBlockMap.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateBlockMetadata.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/RLPMapEntry.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigration.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationException.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationService.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProviderTest.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolverTest.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateTransactionDataFixture.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 096b4826311..72837156174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ - [BESU-25](https://jira.hyperledger.org/browse/BESU-25) Use v5 Devp2p when pinging [\#392](https://github.com/hyperledger/besu/pull/392) +## 1.4.0 RC-2 + +### Private State Migration +Hyperledger Besu v1.4 implements a new data structure for private state storage that is not backwards compatible. +A migration will be performed when starting v1.4 for the first time to reprocess existing private transactions +and re-create the private state data in the v1.4 format. +If you have existing private transactions, see [migration details](docs/Private-Txns-Migration.md). ## 1.4.0 RC-1 diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/ethsigner/EthSignerClientTest.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/ethsigner/EthSignerClientTest.java index 9648971f9d0..06b33f2b6b1 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/ethsigner/EthSignerClientTest.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/ethsigner/EthSignerClientTest.java @@ -38,8 +38,7 @@ public class EthSignerClientTest { @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); @ClassRule - public static final WireMockRule wireMockRule = - new WireMockRule(wireMockConfig().dynamicPort().dynamicPort()); + public static final WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); private static final String MOCK_RESPONSE = "mock_transaction_hash"; private static final String MOCK_SEND_TRANSACTION_RESPONSE = diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyAcceptanceTest.java index 83aed561c28..1971fbab6f2 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyAcceptanceTest.java @@ -56,7 +56,7 @@ public class MultiTenancyAcceptanceTest extends AcceptanceTestBase { private final ObjectMapper mapper = new ObjectMapper(); private Cluster multiTenancyCluster; - private static final String PRIVACY_GROUP_ID = "Z3JvdXBJZA=="; + private static final String PRIVACY_GROUP_ID = "B1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; private static final String ENCLAVE_KEY = "A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; private static final String KEY1 = "sgFkVOyFndZe/5SAZJO5UYbrl7pezHetveriBBWWnE8="; private static final String KEY2 = "R1kW75NQC9XX3kwNpyPjCBFflM29+XvnKKS9VLrUkzo="; diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyValidationFailAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyValidationFailAcceptanceTest.java index 88df1930a6e..fddfe2f28ca 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyValidationFailAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/multitenancy/MultiTenancyValidationFailAcceptanceTest.java @@ -55,7 +55,7 @@ public class MultiTenancyValidationFailAcceptanceTest extends AcceptanceTestBase private final ObjectMapper mapper = new ObjectMapper(); private Cluster multiTenancyCluster; - private static final String PRIVACY_GROUP_ID = "Z3JvdXBJZA=="; + private static final String PRIVACY_GROUP_ID = "B1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; private static final String ENCLAVE_PUBLIC_KEY = "B1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; private static final String OTHER_ENCLAVE_PUBLIC_KEY = "A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/EthSignerAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/EthSignerAcceptanceTest.java index e334e42f3d0..0cbd711e3db 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/EthSignerAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/EthSignerAcceptanceTest.java @@ -57,12 +57,11 @@ public void setUp() throws Exception { } @Test - @Ignore public void privateSmartContractMustDeploy() throws IOException { final String transactionHash = ethSignerClient.eeaSendTransaction( null, - BigInteger.valueOf(63992), + BigInteger.valueOf(23176), BigInteger.valueOf(1000), EventEmitter.BINARY, BigInteger.valueOf(0), @@ -77,13 +76,15 @@ public void privateSmartContractMustDeploy() throws IOException { privateTransactionVerifier.validPrivateTransactionReceipt(transactionHash, receipt)); } + // requires ethsigner jar > 0.3.0 + // https://bintray.com/consensys/pegasys-repo/ethsigner @Test @Ignore public void privateSmartContractMustDeployNoNonce() throws IOException { final String transactionHash = ethSignerClient.eeaSendTransaction( null, - BigInteger.valueOf(63992), + BigInteger.valueOf(23176), BigInteger.valueOf(1000), EventEmitter.BINARY, minerNode.getEnclaveKey(), @@ -114,7 +115,7 @@ public void privateSmartContractMustDeployWithPrivacyGroup() throws IOException final String transactionHash = ethSignerClient.eeaSendTransaction( null, - BigInteger.valueOf(63992), + BigInteger.valueOf(23176), BigInteger.valueOf(1000), EventEmitter.BINARY, BigInteger.valueOf(0), @@ -130,7 +131,6 @@ public void privateSmartContractMustDeployWithPrivacyGroup() throws IOException } @Test - @Ignore public void privateSmartContractMustDeployWithPrivacyGroupNoNonce() throws IOException { final String privacyGroupId = minerNode.execute(privacyTransactions.createPrivacyGroup(null, null, minerNode)); @@ -140,14 +140,14 @@ public void privateSmartContractMustDeployWithPrivacyGroupNoNonce() throws IOExc new PrivacyGroup( privacyGroupId, PrivacyGroup.Type.PANTHEON, - "Default Name", - "Default Description", + "", + "", Base64String.wrapList(minerNode.getEnclaveKey())))); final String transactionHash = ethSignerClient.eeaSendTransaction( null, - BigInteger.valueOf(63992), + BigInteger.valueOf(23176), BigInteger.valueOf(1000), EventEmitter.BINARY, minerNode.getEnclaveKey(), diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index ec969df7279..784d4b90d35 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -48,6 +48,8 @@ import org.hyperledger.besu.cli.options.PrunerOptions; import org.hyperledger.besu.cli.options.SynchronizerOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; +import org.hyperledger.besu.cli.presynctasks.PreSynchronizationTaskRunner; +import org.hyperledger.besu.cli.presynctasks.PrivateDatabaseMigrationPreSyncTask; import org.hyperledger.besu.cli.subcommands.PasswordSubCommand; import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand; import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand.KeyLoader; @@ -212,6 +214,9 @@ protected KeyLoader getKeyLoader() { // Property to indicate whether Besu has been launched via docker private final boolean isDocker = Boolean.getBoolean("besu.docker"); + private final PreSynchronizationTaskRunner preSynchronizationTaskRunner = + new PreSynchronizationTaskRunner(); + // CLI options defined by user at runtime. // Options parsing is done with CLI library Picocli https://picocli.info/ @@ -760,6 +765,11 @@ void setBannedNodeIds(final List values) { "The name of a file containing the private key used to sign privacy marker transactions. If unset, each will be signed with a random key.") private final Path privacyMarkerTransactionSigningKeyPath = null; + @Option( + names = {"--privacy-enable-database-migration"}, + description = "Enable private database metadata migration (default: ${DEFAULT-VALUE})") + private final Boolean migratePrivateDatabase = false; + @Option( names = {"--target-gas-limit"}, description = @@ -888,7 +898,11 @@ public void run() { // Need to create vertx after cmdline has been parsed, such that metricSystem is configurable vertx = createVertx(createVertxOptions(metricsSystem.get())); - validateOptions().configure().controller().startPlugins().startSynchronization(); + final BesuCommand controller = validateOptions().configure().controller(); + + preSynchronizationTaskRunner.runTasks(controller.besuController); + + controller.startPlugins().startSynchronization(); } catch (final Exception e) { throw new ParameterException(this.commandLine, e.getMessage(), e); } @@ -1596,7 +1610,14 @@ private PrivacyParameters privacyParameters() { } } - return privacyParametersBuilder.build(); + final PrivacyParameters privacyParameters = privacyParametersBuilder.build(); + + if (isPrivacyEnabled) { + preSynchronizationTaskRunner.addTask( + new PrivateDatabaseMigrationPreSyncTask(privacyParameters, migratePrivateDatabase)); + } + + return privacyParameters; } private boolean anyPrivacyApiEnabled() { @@ -1608,19 +1629,20 @@ private boolean anyPrivacyApiEnabled() { private PrivacyKeyValueStorageProvider privacyKeyStorageProvider(final String name) { return new PrivacyKeyValueStorageProviderBuilder() - .withStorageFactory( - (PrivacyKeyValueStorageFactory) - storageService - .getByName(name) - .orElseThrow( - () -> - new StorageException( - "No KeyValueStorageFactory found for key: " + name))) + .withStorageFactory(privacyKeyValueStorageFactory(name)) .withCommonConfiguration(pluginCommonConfiguration) .withMetricsSystem(getMetricsSystem()) .build(); } + private PrivacyKeyValueStorageFactory privacyKeyValueStorageFactory(final String name) { + return (PrivacyKeyValueStorageFactory) + storageService + .getByName(name) + .orElseThrow( + () -> new StorageException("No KeyValueStorageFactory found for key: " + name)); + } + private KeyValueStorageProvider keyStorageProvider(final String name) { return new KeyValueStorageProviderBuilder() .withStorageFactory( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/error/BesuExceptionHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/error/BesuExceptionHandler.java index 664f70c462c..59ccedefcbb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/error/BesuExceptionHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/error/BesuExceptionHandler.java @@ -14,11 +14,14 @@ */ package org.hyperledger.besu.cli.error; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationException; + import java.util.List; import java.util.function.Supplier; import org.apache.logging.log4j.Level; import picocli.CommandLine; +import picocli.CommandLine.ParameterException; public class BesuExceptionHandler extends CommandLine.AbstractHandler, BesuExceptionHandler> @@ -39,12 +42,17 @@ public List handleParseException( } else { err().println(ex.getMessage()); } - if (!CommandLine.UnmatchedArgumentException.printSuggestions(ex, err())) { + if (shouldPrintUsage(ex)) { ex.getCommandLine().usage(err(), ansi()); } return returnResultOrExit(null); } + private boolean shouldPrintUsage(final ParameterException ex) { + return !CommandLine.UnmatchedArgumentException.printSuggestions(ex, err()) + && !(ex.getCause() instanceof PrivateStorageMigrationException); + } + @Override public List handleExecutionException( final CommandLine.ExecutionException ex, final CommandLine.ParseResult parseResult) { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java new file mode 100644 index 00000000000..28238ca241c --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTask.java @@ -0,0 +1,27 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.presynctasks; + +import org.hyperledger.besu.cli.BesuCommand; +import org.hyperledger.besu.controller.BesuController; + +/** + * All PreSynchronizationTask instances execute after the {@link BesuController} instance in {@link + * BesuCommand} is ready and before {@link BesuCommand#startSynchronization()} is called + */ +public interface PreSynchronizationTask { + + void run(final BesuController besuController); +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTaskRunner.java b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTaskRunner.java new file mode 100644 index 00000000000..55cf9850d23 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PreSynchronizationTaskRunner.java @@ -0,0 +1,33 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.presynctasks; + +import org.hyperledger.besu.controller.BesuController; + +import java.util.ArrayList; +import java.util.List; + +public class PreSynchronizationTaskRunner { + + private final List tasks = new ArrayList<>(); + + public void addTask(final PreSynchronizationTask task) { + tasks.add(task); + } + + public void runTasks(final BesuController besuController) { + tasks.forEach(t -> t.run(besuController)); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PrivateDatabaseMigrationPreSyncTask.java b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PrivateDatabaseMigrationPreSyncTask.java new file mode 100644 index 00000000000..a6ef39dc8a0 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/presynctasks/PrivateDatabaseMigrationPreSyncTask.java @@ -0,0 +1,45 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.presynctasks; + +import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationService; +import org.hyperledger.besu.util.PrivateStorageMigrationBuilder; + +public class PrivateDatabaseMigrationPreSyncTask implements PreSynchronizationTask { + + private final PrivacyParameters privacyParameters; + private final boolean migratePrivateDatabaseFlag; + + public PrivateDatabaseMigrationPreSyncTask( + final PrivacyParameters privacyParameters, final boolean migratePrivateDatabaseFlag) { + this.privacyParameters = privacyParameters; + this.migratePrivateDatabaseFlag = migratePrivateDatabaseFlag; + } + + @Override + public void run(final BesuController besuController) { + final PrivateStorageMigrationBuilder privateStorageMigrationBuilder = + new PrivateStorageMigrationBuilder(besuController, privacyParameters); + final PrivateStorageMigrationService privateStorageMigrationService = + new PrivateStorageMigrationService( + privacyParameters.getPrivateStateStorage(), + migratePrivateDatabaseFlag, + privateStorageMigrationBuilder::build); + + privateStorageMigrationService.runMigrationIfRequired(); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/util/PrivateStorageMigrationBuilder.java b/besu/src/main/java/org/hyperledger/besu/util/PrivateStorageMigrationBuilder.java new file mode 100644 index 00000000000..f8ffb9c39de --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/util/PrivateStorageMigrationBuilder.java @@ -0,0 +1,63 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.util; + +import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateMigrationBlockProcessor; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigration; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; + +public class PrivateStorageMigrationBuilder { + + private final BesuController besuController; + private final PrivacyParameters privacyParameters; + + public PrivateStorageMigrationBuilder( + final BesuController besuController, final PrivacyParameters privacyParameters) { + this.besuController = besuController; + this.privacyParameters = privacyParameters; + } + + public PrivateStorageMigration build() { + final Blockchain blockchain = besuController.getProtocolContext().getBlockchain(); + final Address privacyPrecompileAddress = + Address.privacyPrecompiled(privacyParameters.getPrivacyAddress()); + final ProtocolSchedule protocolSchedule = besuController.getProtocolSchedule(); + final WorldStateArchive publicWorldStateArchive = + besuController.getProtocolContext().getWorldStateArchive(); + final PrivateStateStorage privateStateStorage = privacyParameters.getPrivateStateStorage(); + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + final LegacyPrivateStateStorage legacyPrivateStateStorage = + privacyParameters.getPrivateStorageProvider().createLegacyPrivateStateStorage(); + + return new PrivateStorageMigration( + blockchain, + privacyPrecompileAddress, + protocolSchedule, + publicWorldStateArchive, + privateStateStorage, + privateStateRootResolver, + legacyPrivateStateStorage, + PrivateMigrationBlockProcessor::new); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java b/besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java new file mode 100644 index 00000000000..704eaeaf423 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/PrivacyReorgTest.java @@ -0,0 +1,538 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver.EMPTY_ROOT_HASH; + +import org.hyperledger.besu.config.GenesisConfigFile; +import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.controller.GasLimitCalculator; +import org.hyperledger.besu.crypto.SECP256K1; +import org.hyperledger.besu.enclave.Enclave; +import org.hyperledger.besu.enclave.EnclaveFactory; +import org.hyperledger.besu.enclave.types.SendResponse; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.InMemoryStorageProvider; +import org.hyperledger.besu.ethereum.core.LogsBloomFilter; +import org.hyperledger.besu.ethereum.core.MiningParametersTestBuilder; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; +import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; +import org.hyperledger.besu.ethereum.privacy.Restriction; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.keyvalue.PrivacyKeyValueStorageProviderBuilder; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValuePrivacyStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; +import org.hyperledger.besu.services.BesuConfigurationImpl; +import org.hyperledger.besu.testutil.TestClock; +import org.hyperledger.orion.testutil.OrionKeyConfiguration; +import org.hyperledger.orion.testutil.OrionTestHarness; +import org.hyperledger.orion.testutil.OrionTestHarnessFactory; + +import java.io.IOException; +import java.math.BigInteger; +import java.net.URI; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import io.vertx.core.Vertx; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +@SuppressWarnings("rawtypes") +public class PrivacyReorgTest { + private static final int MAX_OPEN_FILES = 1024; + private static final long CACHE_CAPACITY = 8388608; + private static final int MAX_BACKGROUND_COMPACTIONS = 4; + private static final int BACKGROUND_THREAD_COUNT = 4; + + @Rule public final TemporaryFolder folder = new TemporaryFolder(); + + private static final SECP256K1.KeyPair KEY_PAIR = + SECP256K1.KeyPair.create( + SECP256K1.PrivateKey.create( + new BigInteger( + "8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63", 16))); + private static final Bytes ENCLAVE_PUBLIC_KEY = + Bytes.fromBase64String("A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="); + + private static final String FIRST_BLOCK_WITH_NO_TRANSACTIONS_STATE_ROOT = + "0x1bdf13f6d14c7322d6e695498aab258949e55574bef7eac366eb777f43d7dd2b"; + private static final String FIRST_BLOCK_WITH_SINGLE_TRANSACTION_STATE_ROOT = + "0x16979b290f429e06d86a43584c7d8689d4292ade9a602e5c78e2867c6ebd904e"; + private static final String BLOCK_WITH_SINGLE_TRANSACTION_RECEIPTS_ROOT = + "0xc8267b3f9ed36df3ff8adb51a6d030716f23eeb50270e7fce8d9822ffa7f0461"; + private static final String STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE = + "0x2121b68f1333e93bae8cd717a3ca68c9d7e7003f6b288c36dfc59b0f87be9590"; + private static final Bytes32 PRIVACY_GROUP_BYTES32 = + Bytes32.fromHexString("0xf250d523ae9164722b06ca25cfa2a7f3c45df96b09e215236f886c876f715bfa"); + + // EventEmitter contract binary + private static final Bytes MOCK_PAYLOAD = + Bytes.fromHexString( + "0x608060405234801561001057600080fd5b5060008054600160a060020a03191633179055610199806100326000396000f3fe6080604052600436106100565763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416633fa4f245811461005b5780636057361d1461008257806367e404ce146100ae575b600080fd5b34801561006757600080fd5b506100706100ec565b60408051918252519081900360200190f35b34801561008e57600080fd5b506100ac600480360360208110156100a557600080fd5b50356100f2565b005b3480156100ba57600080fd5b506100c3610151565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b60025490565b604080513381526020810183905281517fc9db20adedc6cf2b5d25252b101ab03e124902a73fcb12b753f3d1aaa2d8f9f5929181900390910190a16002556001805473ffffffffffffffffffffffffffffffffffffffff191633179055565b60015473ffffffffffffffffffffffffffffffffffffffff169056fea165627a7a72305820c7f729cb24e05c221f5aa913700793994656f233fe2ce3b9fd9a505ea17e8d8a0029"); + private static final PrivateTransaction PRIVATE_TRANSACTION = + PrivateTransaction.builder() + .chainId(BigInteger.valueOf(2018)) + .gasLimit(1000) + .gasPrice(Wei.ZERO) + .nonce(0) + .payload(MOCK_PAYLOAD) + .to(null) + .privateFrom(ENCLAVE_PUBLIC_KEY) + .privateFor(Collections.singletonList(ENCLAVE_PUBLIC_KEY)) + .restriction(Restriction.RESTRICTED) + .value(Wei.ZERO) + .signAndBuild(KEY_PAIR); + + private final BlockDataGenerator gen = new BlockDataGenerator(); + private BesuController besuController; + private OrionTestHarness enclave; + private PrivateStateRootResolver privateStateRootResolver; + private PrivacyParameters privacyParameters; + + @Before + public void setUp() throws IOException { + // Start Enclave + enclave = + OrionTestHarnessFactory.create( + folder.newFolder().toPath(), + new OrionKeyConfiguration("enclavePublicKey", "enclavePrivateKey")); + enclave.start(); + + // Create Storage + final Path dataDir = folder.newFolder().toPath(); + final Path dbDir = dataDir.resolve("database"); + + // Configure Privacy + privacyParameters = + new PrivacyParameters.Builder() + .setEnabled(true) + .setStorageProvider(createKeyValueStorageProvider(dataDir, dbDir)) + .setEnclaveUrl(enclave.clientUrl()) + .setEnclaveFactory(new EnclaveFactory(Vertx.vertx())) + .build(); + privacyParameters.setEnclavePublicKey(ENCLAVE_PUBLIC_KEY.toBase64String()); + + privateStateRootResolver = + new PrivateStateRootResolver(privacyParameters.getPrivateStateStorage()); + + besuController = + new BesuController.Builder() + .fromGenesisConfig(GenesisConfigFile.development()) + .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) + .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) + .storageProvider(new InMemoryStorageProvider()) + .networkId(BigInteger.ONE) + .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) + .nodeKeys(SECP256K1.KeyPair.generate()) + .metricsSystem(new NoOpMetricsSystem()) + .dataDirectory(dataDir) + .clock(TestClock.fixed()) + .privacyParameters(privacyParameters) + .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) + .targetGasLimit(GasLimitCalculator.DEFAULT) + .build(); + } + + @Test + public void privacyGroupHeadIsTracked() { + // Setup an initial blockchain with one private transaction + final ProtocolContext protocolContext = besuController.getProtocolContext(); + final DefaultBlockchain blockchain = (DefaultBlockchain) protocolContext.getBlockchain(); + final PrivateStateStorage privateStateStorage = privacyParameters.getPrivateStateStorage(); + + final Transaction privacyMarkerTransaction = + buildMarkerTransaction(getEnclaveKey(enclave.clientUrl())); + final Block firstBlock = + gen.block( + getBlockOptionsWithTransaction( + blockchain.getGenesisBlock(), + privacyMarkerTransaction, + FIRST_BLOCK_WITH_SINGLE_TRANSACTION_STATE_ROOT)); + + appendBlock(besuController, blockchain, protocolContext, firstBlock); + + final PrivacyGroupHeadBlockMap expected = + new PrivacyGroupHeadBlockMap( + Collections.singletonMap(PRIVACY_GROUP_BYTES32, firstBlock.getHash())); + + assertThat( + privateStateStorage.getPrivacyGroupHeadBlockMap(blockchain.getGenesisBlock().getHash())) + .isEmpty(); + assertThat(privateStateStorage.getPrivacyGroupHeadBlockMap(firstBlock.getHash())).isNotEmpty(); + assertThat(privateStateStorage.getPrivacyGroupHeadBlockMap(firstBlock.getHash())) + .contains(expected); + + final String secondBlockStateRoot = + "0xd86a520e49caf215e7e4028262924db50540a5b26e415ab7c944e46a0c01d704"; + final Block secondBlock = + gen.block(getBlockOptionsNoTransaction(firstBlock, secondBlockStateRoot)); + + appendBlock(besuController, blockchain, protocolContext, secondBlock); + + assertThat(privateStateStorage.getPrivacyGroupHeadBlockMap(secondBlock.getHash())).isNotEmpty(); + + assertThat(privateStateStorage.getPrivacyGroupHeadBlockMap(secondBlock.getHash())) + .contains(expected); + } + + @Test + public void reorgToChainAtEqualHeight() { + // Setup an initial blockchain with one private transaction + final ProtocolContext protocolContext = besuController.getProtocolContext(); + final DefaultBlockchain blockchain = (DefaultBlockchain) protocolContext.getBlockchain(); + + final Block firstBlock = + gen.block( + getBlockOptionsWithTransaction( + blockchain.getGenesisBlock(), + buildMarkerTransaction(getEnclaveKey(enclave.clientUrl())), + FIRST_BLOCK_WITH_SINGLE_TRANSACTION_STATE_ROOT)); + + appendBlock(besuController, blockchain, protocolContext, firstBlock); + + // Check that the private state root is not the empty state + assertPrivateStateRoot( + privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE); + + // Create parallel fork of length 1 which removes privacy marker transaction + final Block forkBlock = + gen.block( + getBlockOptionsNoTransactionWithDifficulty( + blockchain.getGenesisBlock(), + blockchain.getChainHeadHeader().getDifficulty().plus(10L), + FIRST_BLOCK_WITH_NO_TRANSACTIONS_STATE_ROOT)); + + appendBlock(besuController, blockchain, protocolContext, forkBlock); + + // Check that the private state root is the empty state + assertPrivateStateRoot(privateStateRootResolver, blockchain, EMPTY_ROOT_HASH); + } + + @Test + public void reorgToShorterChain() { + // Setup an initial blockchain with one private transaction + final ProtocolContext protocolContext = besuController.getProtocolContext(); + final DefaultBlockchain blockchain = (DefaultBlockchain) protocolContext.getBlockchain(); + + final String firstBlockStateRoot = + "0xbca927086c294984d6c2add82731c386cf2df3cd75509907dac928de12b7c472"; + final Block firstBlock = + gen.block(getBlockOptionsNoTransaction(blockchain.getGenesisBlock(), firstBlockStateRoot)); + + final String secondBlockStateRoot = + "0x35c315ee7d272e5b612d454ee87c948657310ab33208b57122f8d0525e91f35e"; + final Block secondBlock = + gen.block( + getBlockOptionsWithTransaction( + firstBlock, + buildMarkerTransaction(getEnclaveKey(enclave.clientUrl())), + secondBlockStateRoot)); + + appendBlock(besuController, blockchain, protocolContext, firstBlock); + appendBlock(besuController, blockchain, protocolContext, secondBlock); + + assertThat(blockchain.getChainHeadBlockNumber()).isEqualTo(2); + + // Check that the private state root is not the empty state + assertPrivateStateRoot( + privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE); + + // Create parallel fork of length 1 which removes privacy marker transaction + final Difficulty remainingDifficultyToOutpace = + blockchain + .getBlockByNumber(1) + .get() + .getHeader() + .getDifficulty() + .plus(blockchain.getBlockByNumber(2).get().getHeader().getDifficulty()); + + final String forkBlockStateRoot = + "0x4a33bdf9d16e6dd4f4c67f1638971f663f132ebceac0c7c65c9a3f35172af4de"; + final Block forkBlock = + gen.block( + getBlockOptionsNoTransactionWithDifficulty( + blockchain.getGenesisBlock(), + remainingDifficultyToOutpace.plus(10L), + forkBlockStateRoot)); + + appendBlock(besuController, blockchain, protocolContext, forkBlock); + + assertThat(blockchain.getChainHeadBlockNumber()).isEqualTo(1); + + // Check that the private state root is the empty state + assertPrivateStateRoot(privateStateRootResolver, blockchain, EMPTY_ROOT_HASH); + } + + @Test + public void reorgToLongerChain() { + // Setup an initial blockchain with one private transaction + final ProtocolContext protocolContext = besuController.getProtocolContext(); + final DefaultBlockchain blockchain = (DefaultBlockchain) protocolContext.getBlockchain(); + + final Block firstBlock = + gen.block( + getBlockOptionsWithTransaction( + blockchain.getGenesisBlock(), + buildMarkerTransaction(getEnclaveKey(enclave.clientUrl())), + FIRST_BLOCK_WITH_SINGLE_TRANSACTION_STATE_ROOT)); + + appendBlock(besuController, blockchain, protocolContext, firstBlock); + + assertThat(blockchain.getChainHeadBlockNumber()).isEqualTo(1); + + // Check that the private state root is not the empty state + assertPrivateStateRoot( + privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE); + + // Create parallel fork of length 1 which removes privacy marker transaction + final Block forkBlock = + gen.block( + getBlockOptionsNoTransactionWithDifficulty( + blockchain.getGenesisBlock(), + firstBlock.getHeader().getDifficulty().plus(10L), + FIRST_BLOCK_WITH_NO_TRANSACTIONS_STATE_ROOT)); + + // Check that the private state root did not change + assertPrivateStateRoot( + privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE); + + final String secondForkBlockStateRoot = + "0xd35eea814b8b5a0b12e690ab320785f3a33d9685bbf6875637c40a64203915da"; + final Block secondForkBlock = + gen.block( + getBlockOptionsNoTransactionWithDifficulty( + forkBlock, + forkBlock.getHeader().getDifficulty().plus(10L), + secondForkBlockStateRoot)); + + appendBlock(besuController, blockchain, protocolContext, forkBlock); + appendBlock(besuController, blockchain, protocolContext, secondForkBlock); + + assertThat(blockchain.getChainHeadBlockNumber()).isEqualTo(2); + + // Check that the private state root is the empty state + assertPrivateStateRoot(privateStateRootResolver, blockchain, EMPTY_ROOT_HASH); + + // Add another private transaction + final String thirdForkBlockStateRoot = + "0xe22344ade05260177b79dcc6c4fed8f87ab95a506c2a6147631ac6547cf44846"; + final Block thirdForkBlock = + gen.block( + getBlockOptionsWithTransactionAndDifficulty( + secondForkBlock, + buildMarkerTransaction(getEnclaveKey(enclave.clientUrl())), + secondForkBlock.getHeader().getDifficulty().plus(10L), + thirdForkBlockStateRoot)); + + appendBlock(besuController, blockchain, protocolContext, thirdForkBlock); + + // Check that the private state did change after reorg + assertPrivateStateRoot( + privateStateRootResolver, blockchain, STATE_ROOT_AFTER_TRANSACTION_APPENDED_TO_EMTPY_STATE); + } + + @SuppressWarnings("unchecked") + private void appendBlock( + final BesuController besuController, + final DefaultBlockchain blockchain, + final ProtocolContext protocolContext, + final Block block) { + besuController + .getProtocolSchedule() + .getByBlockNumber(blockchain.getChainHeadBlockNumber()) + .getBlockImporter() + .importBlock(protocolContext, block, HeaderValidationMode.NONE); + } + + private PrivacyStorageProvider createKeyValueStorageProvider( + final Path dataLocation, final Path dbLocation) { + return new PrivacyKeyValueStorageProviderBuilder() + .withStorageFactory( + new RocksDBKeyValuePrivacyStorageFactory( + new RocksDBKeyValueStorageFactory( + () -> + new RocksDBFactoryConfiguration( + MAX_OPEN_FILES, + MAX_BACKGROUND_COMPACTIONS, + BACKGROUND_THREAD_COUNT, + CACHE_CAPACITY), + Arrays.asList(KeyValueSegmentIdentifier.values()), + RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS))) + .withCommonConfiguration(new BesuConfigurationImpl(dataLocation, dbLocation)) + .withMetricsSystem(new NoOpMetricsSystem()) + .build(); + } + + private Bytes getEnclaveKey(final URI enclaveURI) { + final Enclave enclave = new EnclaveFactory(Vertx.vertx()).createVertxEnclave(enclaveURI); + final SendResponse sendResponse = + sendRequest(enclave, PRIVATE_TRANSACTION, ENCLAVE_PUBLIC_KEY.toBase64String()); + final Bytes payload = Bytes.fromBase64String(sendResponse.getKey()); + + // If the key has 0 bytes generate a new key. + // This is to keep the gasUsed constant allowing + // hard-coded receipt roots in the block headers + for (int i = 0; i < payload.size(); i++) { + if (payload.get(i) == 0) { + return getEnclaveKey(enclaveURI); + } + } + + return payload; + } + + private SendResponse sendRequest( + final Enclave enclave, + final PrivateTransaction privateTransaction, + final String enclavePublicKey) { + final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); + privateTransaction.writeTo(rlpOutput); + final String payload = rlpOutput.encoded().toBase64String(); + + if (privateTransaction.getPrivacyGroupId().isPresent()) { + return enclave.send( + payload, enclavePublicKey, privateTransaction.getPrivacyGroupId().get().toBase64String()); + } else { + final List privateFor = + privateTransaction.getPrivateFor().get().stream() + .map(Bytes::toBase64String) + .collect(Collectors.toList()); + + if (privateFor.isEmpty()) { + privateFor.add(privateTransaction.getPrivateFrom().toBase64String()); + } + return enclave.send( + payload, privateTransaction.getPrivateFrom().toBase64String(), privateFor); + } + } + + private Transaction buildMarkerTransaction(final Bytes payload) { + return Transaction.builder() + .chainId(BigInteger.valueOf(2018)) + .gasLimit(60000) + .gasPrice(Wei.of(1000)) + .nonce(0) + .payload(payload) + .to(Address.DEFAULT_PRIVACY) + .value(Wei.ZERO) + .signAndBuild(KEY_PAIR); + } + + private void assertPrivateStateRoot( + final PrivateStateRootResolver privateStateRootResolver, + final DefaultBlockchain blockchain, + final String expected) { + assertPrivateStateRoot(privateStateRootResolver, blockchain, Hash.fromHexString(expected)); + } + + private void assertPrivateStateRoot( + final PrivateStateRootResolver privateStateRootResolver, + final DefaultBlockchain blockchain, + final Hash expected) { + assertThat( + privateStateRootResolver.resolveLastStateRoot( + Bytes32.wrap( + Bytes.fromBase64String("8lDVI66RZHIrBsolz6Kn88Rd+WsJ4hUjb4hsh29xW/o=")), + blockchain.getChainHeadHash())) + .isEqualTo(expected); + } + + private BlockDataGenerator.BlockOptions getBlockOptionsNoTransaction( + final Block parentBlock, final String stateRoot) { + return getBlockOptions( + new BlockDataGenerator.BlockOptions() + .hasTransactions(false) + .setReceiptsRoot(PrivateStateRootResolver.EMPTY_ROOT_HASH) + .setGasUsed(0) + .setStateRoot(Hash.fromHexString(stateRoot)), + parentBlock); + } + + private BlockDataGenerator.BlockOptions getBlockOptionsWithTransaction( + final Block parentBlock, final Transaction transaction, final String stateRoot) { + return getBlockOptions( + new BlockDataGenerator.BlockOptions() + .addTransaction(transaction) + .setReceiptsRoot(Hash.fromHexString(BLOCK_WITH_SINGLE_TRANSACTION_RECEIPTS_ROOT)) + .setGasUsed(23176) + .setStateRoot(Hash.fromHexString(stateRoot)), + parentBlock); + } + + private BlockDataGenerator.BlockOptions getBlockOptionsNoTransactionWithDifficulty( + final Block parentBlock, final Difficulty difficulty, final String stateRoot) { + return getBlockOptions( + new BlockDataGenerator.BlockOptions() + .hasTransactions(false) + .setDifficulty(difficulty) + .setReceiptsRoot(PrivateStateRootResolver.EMPTY_ROOT_HASH) + .setGasUsed(0) + .setStateRoot(Hash.fromHexString(stateRoot)), + parentBlock); + } + + private BlockDataGenerator.BlockOptions getBlockOptionsWithTransactionAndDifficulty( + final Block parentBlock, + final Transaction transaction, + final Difficulty difficulty, + final String stateRoot) { + return getBlockOptions( + new BlockDataGenerator.BlockOptions() + .addTransaction(transaction) + .setDifficulty(difficulty) + .setReceiptsRoot(Hash.fromHexString(BLOCK_WITH_SINGLE_TRANSACTION_RECEIPTS_ROOT)) + .setGasUsed(23176) + .setStateRoot(Hash.fromHexString(stateRoot)), + parentBlock); + } + + private BlockDataGenerator.BlockOptions getBlockOptions( + final BlockDataGenerator.BlockOptions blockOptions, final Block parentBlock) { + return blockOptions + .setBlockNumber(parentBlock.getHeader().getNumber() + 1) + .setParentHash(parentBlock.getHash()) + .hasOmmers(false) + .setLogsBloom(LogsBloomFilter.empty()); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java b/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java index 39aa32c74ff..bdb6a9728eb 100644 --- a/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java +++ b/besu/src/test/java/org/hyperledger/besu/PrivacyTest.java @@ -62,7 +62,6 @@ public class PrivacyTest { private static final int BACKGROUND_THREAD_COUNT = 4; private final Vertx vertx = Vertx.vertx(); - private static final Integer ADDRESS = 9; @Rule public final TemporaryFolder folder = new TemporaryFolder(); @After @@ -76,7 +75,7 @@ public void privacyPrecompiled() throws IOException, URISyntaxException { final Path dbDir = dataDir.resolve("database"); final PrivacyParameters privacyParameters = new PrivacyParameters.Builder() - .setPrivacyAddress(ADDRESS) + .setPrivacyAddress(Address.PRIVACY) .setEnabled(true) .setEnclaveUrl(new URI("http://127.0.0.1:8000")) .setStorageProvider(createKeyValueStorageProvider(dataDir, dbDir)) @@ -99,7 +98,7 @@ public void privacyPrecompiled() throws IOException, URISyntaxException { .targetGasLimit(GasLimitCalculator.DEFAULT) .build(); - final Address privacyContractAddress = Address.privacyPrecompiled(ADDRESS); + final Address privacyContractAddress = Address.DEFAULT_PRIVACY; final PrecompiledContract precompiledContract = besuController .getProtocolSchedule() diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 985b7bd8884..b65bcfd4185 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -807,7 +807,7 @@ public void configOptionDisabledUnderDocker() { @Test public void nodekeyOptionMustBeUsed() throws Exception { - final File file = new File("./specific/key"); + final File file = new File("./specific/enclavePrivateKey"); file.deleteOnExit(); parseCommand("--node-private-key-file", file.getPath()); @@ -828,13 +828,12 @@ public void nodekeyOptionDisabledUnderDocker() { assumeFalse(isFullInstantiation()); - final File file = new File("./specific/key"); + final File file = new File("./specific/enclavePrivateKey"); file.deleteOnExit(); parseCommand("--node-private-key-file", file.getPath()); - assertThat(commandErrorOutput.toString()) - .startsWith("Unknown options: '--node-private-key-file', './specific/key'"); + .startsWith("Unknown options: '--node-private-key-file', './specific/enclavePrivateKey'"); assertThat(commandOutput.toString()).isEmpty(); } @@ -2877,8 +2876,6 @@ public void fullCLIOptionsShownWhenNotInDockerContainer() { @Test public void mustUseEnclaveUriAndOptions() { - when(storageService.getByName("rocksdb-privacy")) - .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); final URL configFile = this.getClass().getResource("/orion_publickey.pub"); parseCommand( @@ -2905,7 +2902,7 @@ public void mustUseEnclaveUriAndOptions() { @Test public void privacyOptionsRequiresServiceToBeEnabled() { - final File file = new File("./specific/public_key"); + final File file = new File("./specific/enclavePublicKey"); file.deleteOnExit(); parseCommand( @@ -2951,9 +2948,6 @@ public void mustVerifyPrivacyIsDisabled() { @Test public void privacyMultiTenancyIsConfiguredWhenConfiguredWithNecessaryOptions() { - when(storageService.getByName("rocksdb-privacy")) - .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); - parseCommand( "--privacy-enabled", "--rpc-http-authentication-enabled", @@ -3064,9 +3058,9 @@ public void privacyPublicKeyFileOptionDisabledUnderDocker() { assumeFalse(isFullInstantiation()); final Path path = Paths.get("."); - parseCommand("--privacy-public-key-file", path.toString()); + parseCommand("--privacy-public-enclavePrivateKey-file", path.toString()); assertThat(commandErrorOutput.toString()) - .startsWith("Unknown options: '--privacy-public-key-file', '.'"); + .startsWith("Unknown options: '--privacy-public-enclavePrivateKey-file', '.'"); assertThat(commandOutput.toString()).isEmpty(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index e689fffd7ee..36678d287f7 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -19,6 +19,7 @@ import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -55,10 +56,12 @@ import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.PicoCLIOptions; +import org.hyperledger.besu.plugin.services.StorageService; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory; import org.hyperledger.besu.services.BesuPluginContextImpl; import org.hyperledger.besu.services.StorageServiceImpl; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.io.ByteArrayOutputStream; import java.io.File; @@ -223,10 +226,22 @@ public void initMocks() throws Exception { when(mockRunnerBuilder.autoLogBloomCaching(anyBoolean())).thenReturn(mockRunnerBuilder); when(mockRunnerBuilder.build()).thenReturn(mockRunner); - when(storageService.getByName("rocksdb")).thenReturn(Optional.of(rocksDBStorageFactory)); + lenient() + .when(storageService.getByName(eq("rocksdb"))) + .thenReturn(Optional.of(rocksDBStorageFactory)); + lenient() + .when(storageService.getByName(eq("rocksdb-privacy"))) + .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); + lenient() + .when(rocksDBSPrivacyStorageFactory.create(any(), any(), any())) + .thenReturn(new InMemoryKeyValueStorage()); - when(mockBesuPluginContext.getService(PicoCLIOptions.class)) + lenient() + .when(mockBesuPluginContext.getService(PicoCLIOptions.class)) .thenReturn(Optional.of(cliOptions)); + lenient() + .when(mockBesuPluginContext.getService(StorageService.class)) + .thenReturn(Optional.of(storageService)); } // Display outputs for debug purpose diff --git a/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java b/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java new file mode 100644 index 00000000000..fdccf5b8740 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java @@ -0,0 +1,105 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_0_0; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage.Updater; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigration; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationException; +import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationService; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class PrivateStorageMigrationServiceTest { + + @Mock private PrivateStateStorage privateStateStorage; + @Mock private PrivateStorageMigration migration; + + private PrivateStorageMigrationService migrationService; + + @Test + public void migrationShouldNotRunIfDatabaseIsFreshAndVersionShouldBeSet() { + when(privateStateStorage.getSchemaVersion()).thenReturn(SCHEMA_VERSION_1_0_0); + when(privateStateStorage.isEmpty()).thenReturn(true); + final Updater privateStateStorageUpdater = mock(Updater.class); + when(privateStateStorage.updater()).thenReturn(privateStateStorageUpdater); + when(privateStateStorageUpdater.putDatabaseVersion(anyInt())) + .thenReturn(privateStateStorageUpdater); + + migrationService = + new PrivateStorageMigrationService(privateStateStorage, true, () -> migration); + + migrationService.runMigrationIfRequired(); + + verify(privateStateStorageUpdater).putDatabaseVersion(eq(SCHEMA_VERSION_1_4_0)); + verifyNoInteractions(migration); + } + + @Test + public void migrationShouldNotRunIfSchemaVersionIsGreaterThanOne() { + when(privateStateStorage.getSchemaVersion()).thenReturn(SCHEMA_VERSION_1_4_0); + + migrationService = + new PrivateStorageMigrationService(privateStateStorage, true, () -> migration); + + migrationService.runMigrationIfRequired(); + + verifyNoInteractions(migration); + } + + @Test + public void migrationShouldNotRunIfFlagIsNotSetEvenIfVersionRequiresMigration() { + when(privateStateStorage.getSchemaVersion()).thenReturn(SCHEMA_VERSION_1_0_0); + when(privateStateStorage.isEmpty()).thenReturn(false); + + migrationService = + new PrivateStorageMigrationService(privateStateStorage, false, () -> migration); + + final Throwable thrown = catchThrowable(() -> migrationService.runMigrationIfRequired()); + assertThat(thrown) + .isInstanceOf(PrivateStorageMigrationException.class) + .hasMessageContaining("Private database metadata requires migration"); + + verifyNoInteractions(migration); + } + + @Test + public void migrationShouldRunIfVersionIsOneAndFlagIsSet() { + when(privateStateStorage.getSchemaVersion()).thenReturn(SCHEMA_VERSION_1_0_0); + when(privateStateStorage.isEmpty()).thenReturn(false); + + migrationService = + new PrivateStorageMigrationService(privateStateStorage, true, () -> migration); + + migrationService.runMigrationIfRequired(); + + verify(migration).migratePrivateStorage(); + } +} diff --git a/besu/src/test/resources/enclavePrivateKey b/besu/src/test/resources/enclavePrivateKey new file mode 100644 index 00000000000..eaae9b0867c --- /dev/null +++ b/besu/src/test/resources/enclavePrivateKey @@ -0,0 +1 @@ +{"data":{"bytes":"hBsuQsGJzx4QHmFmBkNoI7YGnTmaZP4P+wBOdu56ljk="},"type":"unlocked"} \ No newline at end of file diff --git a/besu/src/test/resources/enclavePublicKey b/besu/src/test/resources/enclavePublicKey new file mode 100644 index 00000000000..3c8b7f3bfb8 --- /dev/null +++ b/besu/src/test/resources/enclavePublicKey @@ -0,0 +1 @@ +A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo= \ No newline at end of file diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 5fbc6f48470..f776ca43a80 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -120,6 +120,7 @@ privacy-enabled=false privacy-multi-tenancy-enabled=true privacy-precompiled-address=9 privacy-marker-transaction-signing-key-file="./signerKey" +privacy-enable-database-migration=false # Transaction Pool tx-pool-retention-hours=999 diff --git a/build.gradle b/build.gradle index 840728d0537..ec056cb081b 100644 --- a/build.gradle +++ b/build.gradle @@ -601,7 +601,7 @@ task checkSpdxHeader(type: CheckSpdxHeader) { rootPath = "${projectDir}" spdxHeader = "* SPDX-License-Identifier: Apache-2.0" filesRegex = "(.*.java)|(.*.groovy)" - excludeRegex = "(.*generalstate/GeneralStateRegressionReferenceTest.*)|(.*generalstate/GeneralStateReferenceTest.*)|(.*blockchain/BlockchainReferenceTest.*)|(.*.gradle/.*)" + excludeRegex = "(.*generalstate/GeneralStateRegressionReferenceTest.*)|(.*generalstate/GeneralStateReferenceTest.*)|(.*blockchain/BlockchainReferenceTest.*)|(.*.gradle/.*)|(.*.idea/.*)" } task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { diff --git a/docs/Private-Txns-Migration.md b/docs/Private-Txns-Migration.md new file mode 100644 index 00000000000..9e828008dc9 --- /dev/null +++ b/docs/Private-Txns-Migration.md @@ -0,0 +1,36 @@ +# Private Transactions Migration + +Hyperledger Besu v1.4 implements a new data structure for private state storage that is not backwards compatible. +A migration will be performed when starting v1.4 for the first time to reprocess existing private transactions +and re-create the private state data in the v1.4 format. + +**Important** + +All nodes with existing private transactions will be migrated to the new private state storage +when upgrading to v1.4. It is not possible to upgrade to v1.4 without migrating. + +## How to migrate + +**Important** + +As a precaution (that is, resyncing should not be required), ensure your Hyperledger Besu database is backed-up +or other Besu nodes in your network are available to resync from if the migration does not complete as expected. + +We recommend that all nodes in a network do not upgrade and migrate at once. + +To migrate, add the `--privacy-enable-database-migration` flag to your Besu command line options. The migration starts +automatically when this flag is specified. If you have existing private transactions and do not specify this flag, +v1.4 will not start. + +After the migration starts, logs display the progress. When the migration is complete, Besu continues +starting up as usual. + +## During migration + +Do not stop the migration running once it has started. If the migration is stopped, you will need to restore +your Besu database from backup and restart the migration process. + +## Migration support + +If you have a long running network with a large volume of private transactions and/or would like to discuss +the migration process with us before upgrading, contact us at support@pegasys.tech \ No newline at end of file diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivGetPrivateTransactionIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivGetPrivateTransactionIntegrationTest.java index 37289c7fb68..cc55a245466 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivGetPrivateTransactionIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivGetPrivateTransactionIntegrationTest.java @@ -92,7 +92,7 @@ public static void setUpOnce() throws Exception { final EnclaveFactory factory = new EnclaveFactory(vertx); enclave = factory.createVertxEnclave(testHarness.clientUrl()); - privacyController = new DefaultPrivacyController(enclave, null, null, null, null, null); + privacyController = new DefaultPrivacyController(enclave, null, null, null, null); } @AfterClass diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java index 58bb3966733..97210f68663 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.privacy.methods.EnclavePublicKeyProvider; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; @@ -60,16 +59,6 @@ protected Object resultByBlockNumber( final CallParameter callParams = validateAndGetCallParams(request); final String privacyGroupId = request.getRequiredParameter(0, String.class); - // For now we do only support privCall on the head of the chain. - // TODO: Once we support privacy on PoW chains (mainnet) this can be removed and the - // blockchainQueries field can be made private again - if (blockNumber != blockchainQueries.get().headBlockNumber()) { - // TODO: Remove PRIV_CALL_ONLY_SUPPORTED_ON_CHAIN_HEAD in JsonRpcError when removing this - // code. - return new JsonRpcErrorResponse( - request.getRequest().getId(), JsonRpcError.PRIV_CALL_ONLY_SUPPORTED_ON_CHAIN_HEAD); - } - final String enclavePublicKey = enclavePublicKeyProvider.getEnclaveKey(request.getUser()); return privacyController diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceipt.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceipt.java index 03aad796a0b..28c1fb9eb04 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceipt.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceipt.java @@ -34,16 +34,14 @@ import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.Hash; -import org.hyperledger.besu.ethereum.core.Log; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.privacy.PrivacyController; import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionReceipt; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import java.util.Collections; -import java.util.List; import java.util.Optional; import org.apache.logging.log4j.Logger; @@ -78,18 +76,19 @@ public String getName() { @Override public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { LOG.trace("Executing {}", RpcMethod.PRIV_GET_TRANSACTION_RECEIPT.getMethodName()); - final Hash transactionHash = requestContext.getRequiredParameter(0, Hash.class); + final Hash pmtTransactionHash = requestContext.getRequiredParameter(0, Hash.class); final Optional maybeLocation = - blockchain.getBlockchain().getTransactionLocation(transactionHash); + blockchain.getBlockchain().getTransactionLocation(pmtTransactionHash); if (maybeLocation.isEmpty()) { return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), null); } - final TransactionLocation location = maybeLocation.get(); + final TransactionLocation pmtLocation = maybeLocation.get(); final BlockBody blockBody = - blockchain.getBlockchain().getBlockBody(location.getBlockHash()).get(); - final Transaction transaction = blockBody.getTransactions().get(location.getTransactionIndex()); + blockchain.getBlockchain().getBlockBody(pmtLocation.getBlockHash()).get(); + final Transaction pmtTransaction = + blockBody.getTransactions().get(pmtLocation.getTransactionIndex()); - final Hash blockhash = location.getBlockHash(); + final Hash blockhash = pmtLocation.getBlockHash(); final long blockNumber = blockchain.getBlockchain().getBlockHeader(blockhash).get().getNumber(); final PrivateTransaction privateTransaction; @@ -97,9 +96,9 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { try { final ReceiveResponse receiveResponse = privacyController.retrieveTransaction( - transaction.getPayload().toBase64String(), + pmtTransaction.getPayload().toBase64String(), enclavePublicKeyProvider.getEnclaveKey(requestContext.getUser())); - LOG.trace("Received transaction information"); + LOG.trace("Received private transaction information"); final BytesValueRLPInput input = new BytesValueRLPInput( @@ -124,52 +123,35 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { final Bytes rlpEncoded = RLP.encode(privateTransaction::writeTo); final Bytes32 txHash = org.hyperledger.besu.crypto.Hash.keccak256(rlpEncoded); - LOG.trace("Calculated private transaction hash: {}", txHash); - final List transactionLogs = + final PrivateTransactionReceipt privateTransactioReceipt = privacyParameters .getPrivateStateStorage() - .getTransactionLogs(txHash) - .orElse(Collections.emptyList()); - - LOG.trace("Processed private transaction events"); - - final Bytes transactionOutput = - privacyParameters.getPrivateStateStorage().getTransactionOutput(txHash).orElse(Bytes.EMPTY); - - final Bytes revertReason = - privacyParameters.getPrivateStateStorage().getRevertReason(txHash).orElse(null); - - final String transactionStatus = - Quantity.create( - privacyParameters - .getPrivateStateStorage() - .getStatus(txHash) - .orElse(Bytes.EMPTY) - .toUnsignedBigInteger()); + .getTransactionReceipt(blockhash, txHash) + .orElse(PrivateTransactionReceipt.EMPTY); - LOG.trace("Processed private transaction output"); + LOG.trace("Processed private transaction receipt"); final PrivateTransactionReceiptResult result = new PrivateTransactionReceiptResult( contractAddress, privateTransaction.getSender().toString(), privateTransaction.getTo().map(Address::toString).orElse(null), - transactionLogs, - transactionOutput, + privateTransactioReceipt.getLogs(), + privateTransactioReceipt.getOutput(), blockhash, blockNumber, - location.getTransactionIndex(), - transaction.getHash(), - privateTransaction.hash(), + pmtLocation.getTransactionIndex(), + pmtTransaction.getHash(), + privateTransaction.getHash(), privateTransaction.getPrivateFrom(), privateTransaction.getPrivateFor().orElse(null), privateTransaction.getPrivacyGroupId().orElse(null), - revertReason, - transactionStatus); + privateTransactioReceipt.getRevertReason().orElse(null), + Quantity.create(privateTransactioReceipt.getStatus())); - LOG.trace("Created Private Transaction from given Transaction Hash"); + LOG.trace("Created Private Transaction Receipt Result from given Transaction Hash"); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), result); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/privacy/PrivateTransactionResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/privacy/PrivateTransactionResult.java index ab50299b7b6..9808446d661 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/privacy/PrivateTransactionResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/privacy/PrivateTransactionResult.java @@ -41,7 +41,7 @@ public PrivateTransactionResult(final PrivateTransaction tx) { this.from = tx.getSender().toString(); this.gas = Quantity.create(tx.getGasLimit()); this.gasPrice = Quantity.create(tx.getGasPrice()); - this.hash = tx.hash().toString(); + this.hash = tx.getHash().toString(); this.input = tx.getPayload().toString(); this.nonce = Quantity.create(tx.getNonce()); this.to = tx.getTo().map(Address::toHexString).orElse(null); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivacyApiGroupJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivacyApiGroupJsonRpcMethods.java index 4b8f56ffb6a..bd17c4fe5d9 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivacyApiGroupJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivacyApiGroupJsonRpcMethods.java @@ -27,9 +27,12 @@ import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.privacy.ChainHeadPrivateNonceProvider; import org.hyperledger.besu.ethereum.privacy.DefaultPrivacyController; import org.hyperledger.besu.ethereum.privacy.MultiTenancyPrivacyController; import org.hyperledger.besu.ethereum.privacy.PrivacyController; +import org.hyperledger.besu.ethereum.privacy.PrivateNonceProvider; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; import org.hyperledger.besu.ethereum.privacy.PrivateTransactionSimulator; import org.hyperledger.besu.ethereum.privacy.markertransaction.FixedKeySigningPrivateMarkerTransactionFactory; import org.hyperledger.besu.ethereum.privacy.markertransaction.PrivateMarkerTransactionFactory; @@ -45,6 +48,7 @@ public abstract class PrivacyApiGroupJsonRpcMethods extends ApiGroupJsonRpcMetho private final ProtocolSchedule protocolSchedule; private final TransactionPool transactionPool; private final PrivacyParameters privacyParameters; + private final PrivateNonceProvider privateNonceProvider; public PrivacyApiGroupJsonRpcMethods( final BlockchainQueries blockchainQueries, @@ -55,6 +59,14 @@ public PrivacyApiGroupJsonRpcMethods( this.protocolSchedule = protocolSchedule; this.transactionPool = transactionPool; this.privacyParameters = privacyParameters; + + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privacyParameters.getPrivateStateStorage()); + this.privateNonceProvider = + new ChainHeadPrivateNonceProvider( + blockchainQueries.getBlockchain(), + privateStateRootResolver, + privacyParameters.getPrivateWorldStateArchive()); } public BlockchainQueries getBlockchainQueries() { @@ -131,7 +143,8 @@ private PrivacyController createPrivacyController( privacyParameters, protocolSchedule.getChainId(), markerTransactionFactory, - createPrivateTransactionSimulator()); + createPrivateTransactionSimulator(), + privateNonceProvider); return privacyParameters.isMultiTenancyEnabled() ? new MultiTenancyPrivacyController( defaultPrivacyController, privacyParameters.getEnclave()) diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java index 02750f08882..8e2abb60897 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java @@ -148,7 +148,6 @@ public void shouldUseCorrectBlockNumberWhenEarliest() { public void shouldUseCorrectBlockNumberWhenSpecified() { final JsonRpcRequestContext request = ethCallRequest(privacyGroupId, callParameter(), Quantity.create(13L)); - when(blockchainQueries.headBlockNumber()).thenReturn(13L); method.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionCountTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionCountTest.java index e4a928153ce..5d9a507f7b8 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionCountTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionCountTest.java @@ -14,7 +14,6 @@ */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.privacy.methods.priv; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -36,7 +35,6 @@ import io.vertx.core.json.JsonObject; import io.vertx.ext.auth.User; import io.vertx.ext.auth.jwt.impl.JWTUser; -import org.apache.tuweni.bytes.Bytes; import org.junit.Before; import org.junit.Test; @@ -46,7 +44,7 @@ public class PrivGetTransactionCountTest { private final PrivacyParameters privacyParameters = mock(PrivacyParameters.class); private final PrivacyController privacyController = mock(PrivacyController.class); - private final String privacyGroupId = Bytes.wrap("0x123".getBytes(UTF_8)).toBase64String(); + private static final String PRIVACY_GROUP_ID = "DyAOiF/ynpc+JXa2YAGB0bCitSlOMNm+ShmB/7M6C4w="; private final Address senderAddress = Address.fromHexString("0x627306090abab3a6e1400e9345bc60c78a8bef57"); @@ -58,7 +56,7 @@ public class PrivGetTransactionCountTest { @Before public void before() { when(privacyParameters.isEnabled()).thenReturn(true); - when(privacyController.determineBesuNonce(senderAddress, privacyGroupId, ENCLAVE_PUBLIC_KEY)) + when(privacyController.determineBesuNonce(senderAddress, PRIVACY_GROUP_ID, ENCLAVE_PUBLIC_KEY)) .thenReturn(NONCE); } @@ -67,7 +65,7 @@ public void verifyTransactionCount() { final PrivGetTransactionCount privGetTransactionCount = new PrivGetTransactionCount(privacyController, enclavePublicKeyProvider); - final Object[] params = new Object[] {senderAddress, privacyGroupId}; + final Object[] params = new Object[] {senderAddress, PRIVACY_GROUP_ID}; final JsonRpcRequestContext request = new JsonRpcRequestContext( new JsonRpcRequest("1", "priv_getTransactionCount", params), user); @@ -76,7 +74,8 @@ public void verifyTransactionCount() { (JsonRpcSuccessResponse) privGetTransactionCount.response(request); assertThat(response.getResult()).isEqualTo(String.format("0x%X", NONCE)); - verify(privacyController).determineBesuNonce(senderAddress, privacyGroupId, ENCLAVE_PUBLIC_KEY); + verify(privacyController) + .determineBesuNonce(senderAddress, PRIVACY_GROUP_ID, ENCLAVE_PUBLIC_KEY); } @Test @@ -84,10 +83,10 @@ public void failsWithNonceErrorIfExceptionIsThrown() { final PrivGetTransactionCount privGetTransactionCount = new PrivGetTransactionCount(privacyController, enclavePublicKeyProvider); - when(privacyController.determineBesuNonce(senderAddress, privacyGroupId, ENCLAVE_PUBLIC_KEY)) + when(privacyController.determineBesuNonce(senderAddress, PRIVACY_GROUP_ID, ENCLAVE_PUBLIC_KEY)) .thenThrow(EnclaveClientException.class); - final Object[] params = new Object[] {senderAddress, privacyGroupId}; + final Object[] params = new Object[] {senderAddress, PRIVACY_GROUP_ID}; final JsonRpcRequestContext request = new JsonRpcRequestContext( new JsonRpcRequest("1", "priv_getTransactionCount", params), user); @@ -104,10 +103,10 @@ public void failsWithUnauthorizedErrorIfMultiTenancyValidationFails() { final PrivGetTransactionCount privGetTransactionCount = new PrivGetTransactionCount(privacyController, enclavePublicKeyProvider); - when(privacyController.determineBesuNonce(senderAddress, privacyGroupId, ENCLAVE_PUBLIC_KEY)) + when(privacyController.determineBesuNonce(senderAddress, PRIVACY_GROUP_ID, ENCLAVE_PUBLIC_KEY)) .thenThrow(new MultiTenancyValidationException("validation failed")); - final Object[] params = new Object[] {senderAddress, privacyGroupId}; + final Object[] params = new Object[] {senderAddress, PRIVACY_GROUP_ID}; final JsonRpcRequestContext request = new JsonRpcRequestContext( new JsonRpcRequest("1", "priv_getTransactionCount", params), user); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceiptTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceiptTest.java index c6a4650d432..4e1035bbd0c 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceiptTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivGetTransactionReceiptTest.java @@ -46,6 +46,7 @@ import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.privacy.PrivacyController; import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionReceipt; import org.hyperledger.besu.ethereum.privacy.Restriction; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.rlp.RLP; @@ -174,9 +175,12 @@ public void setUp() { when(privacyParameters.isEnabled()).thenReturn(true); when(privacyParameters.getPrivateStateStorage()).thenReturn(privateStateStorage); - when(privateStateStorage.getTransactionLogs(any(Bytes32.class))).thenReturn(Optional.empty()); - when(privateStateStorage.getTransactionOutput(any(Bytes32.class))).thenReturn(Optional.empty()); - when(privateStateStorage.getStatus(any(Bytes32.class))).thenReturn(Optional.of(Bytes.of(1))); + @SuppressWarnings("unchecked") + final PrivateTransactionReceipt receipt = + new PrivateTransactionReceipt( + 1, Collections.EMPTY_LIST, Bytes.EMPTY, Optional.ofNullable(null)); + when(privateStateStorage.getTransactionReceipt(any(Bytes32.class), any(Bytes32.class))) + .thenReturn(Optional.of(receipt)); } @Test @@ -255,8 +259,15 @@ public void enclaveConnectionIssueThrowsRuntimeException() { @Test public void transactionReceiptContainsRevertReasonWhenInvalidTransactionOccurs() { - when(privateStateStorage.getRevertReason(any(Bytes32.class))) - .thenReturn(Optional.of(Bytes.fromHexString("0x01"))); + @SuppressWarnings("unchecked") + final PrivateTransactionReceipt privateTransactionReceipt = + new PrivateTransactionReceipt( + 1, + Collections.EMPTY_LIST, + Bytes.EMPTY, + Optional.of(Bytes.wrap(new byte[] {(byte) 0x01}))); + when(privateStateStorage.getTransactionReceipt(any(Bytes32.class), any(Bytes32.class))) + .thenReturn(Optional.of(privateTransactionReceipt)); final PrivGetTransactionReceipt privGetTransactionReceipt = new PrivGetTransactionReceipt( diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractIntegrationTest.java index 5028a2e8e40..e22b7e493dc 100644 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractIntegrationTest.java +++ b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractIntegrationTest.java @@ -27,12 +27,15 @@ import org.hyperledger.besu.enclave.types.SendResponse; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.core.WorldUpdater; import org.hyperledger.besu.ethereum.mainnet.SpuriousDragonGasCalculator; import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.MessageFrame; @@ -79,6 +82,7 @@ public class PrivacyPrecompiledContractIntegrationTest { private static Enclave enclave; private static MessageFrame messageFrame; + private static Blockchain blockchain; private static OrionTestHarness testHarness; private static WorldStateArchive worldStateArchive; @@ -120,6 +124,17 @@ public static void setUpOnce() throws Exception { final EnclaveFactory factory = new EnclaveFactory(vertx); enclave = factory.createVertxEnclave(testHarness.clientUrl()); messageFrame = mock(MessageFrame.class); + blockchain = mock(Blockchain.class); + final BlockDataGenerator blockGenerator = new BlockDataGenerator(); + final Block genesis = blockGenerator.genesisBlock(); + final Block block = + blockGenerator.block( + new BlockDataGenerator.BlockOptions().setParentHash(genesis.getHeader().getHash())); + when(blockchain.getGenesisBlock()).thenReturn(genesis); + when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); + when(blockchain.getBlockByHash(genesis.getHash())).thenReturn(Optional.of(genesis)); + when(messageFrame.getBlockchain()).thenReturn(blockchain); + when(messageFrame.getBlockHeader()).thenReturn(block.getHeader()); worldStateArchive = mock(WorldStateArchive.class); final MutableWorldState mutableWorldState = mock(MutableWorldState.class); @@ -129,11 +144,13 @@ public static void setUpOnce() throws Exception { privateStateStorage = mock(PrivateStateStorage.class); final PrivateStateStorage.Updater storageUpdater = mock(PrivateStateStorage.Updater.class); - when(storageUpdater.putLatestStateRoot(nullable(Bytes32.class), any())) + when(privateStateStorage.getPrivacyGroupHeadBlockMap(any())) + .thenReturn(Optional.of(PrivacyGroupHeadBlockMap.EMPTY)); + when(storageUpdater.putPrivateBlockMetadata( + nullable(Bytes32.class), nullable(Bytes32.class), any())) .thenReturn(storageUpdater); - when(storageUpdater.putTransactionLogs(nullable(Bytes32.class), any())) - .thenReturn(storageUpdater); - when(storageUpdater.putTransactionResult(nullable(Bytes32.class), any())) + when(storageUpdater.putTransactionReceipt( + nullable(Bytes32.class), nullable(Bytes32.class), any())) .thenReturn(storageUpdater); when(privateStateStorage.updater()).thenReturn(storageUpdater); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 4dd937f2940..ba64659fc59 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -89,6 +89,15 @@ public boolean isSuccessful() { private final MiningBeneficiaryCalculator miningBeneficiaryCalculator; + public AbstractBlockProcessor(final AbstractBlockProcessor blockProcessor) { + this( + blockProcessor.transactionProcessor, + blockProcessor.transactionReceiptFactory, + blockProcessor.blockReward, + blockProcessor.miningBeneficiaryCalculator, + blockProcessor.skipZeroBlockRewards); + } + public AbstractBlockProcessor( final TransactionProcessor transactionProcessor, final MainnetBlockProcessor.TransactionReceiptFactory transactionReceiptFactory, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 9fa880392a8..7c195da373a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -248,6 +248,7 @@ public Result processTransaction( .blockHashLookup(blockHashLookup) .isPersistingState(isPersistingState) .maxStackSize(maxStackSize) + .transactionHash(transaction.getHash()) .build(); } else { @@ -279,6 +280,7 @@ public Result processTransaction( .blockHashLookup(blockHashLookup) .maxStackSize(maxStackSize) .isPersistingState(isPersistingState) + .transactionHash(transaction.getHash()) .build(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java new file mode 100644 index 00000000000..3f77dcadf62 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java @@ -0,0 +1,54 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; + +import java.util.List; + +public class PrivacyBlockProcessor implements BlockProcessor { + private final BlockProcessor blockProcessor; + private final PrivateStateStorage privateStateStorage; + + public PrivacyBlockProcessor( + final BlockProcessor blockProcessor, final PrivateStateStorage privateStateStorage) { + this.blockProcessor = blockProcessor; + this.privateStateStorage = privateStateStorage; + } + + @Override + public Result processBlock( + final Blockchain blockchain, + final MutableWorldState worldState, + final BlockHeader blockHeader, + final List transactions, + final List ommers) { + final PrivacyGroupHeadBlockMap privacyGroupHeadBlockMap = + new PrivacyGroupHeadBlockMap( + privateStateStorage + .getPrivacyGroupHeadBlockMap(blockHeader.getParentHash()) + .orElse(PrivacyGroupHeadBlockMap.EMPTY)); + privateStateStorage + .updater() + .putPrivacyGroupHeadBlockMap(blockHeader.getHash(), privacyGroupHeadBlockMap) + .commit(); + return blockProcessor.processBlock(blockchain, worldState, blockHeader, transactions, ommers); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolSpecBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolSpecBuilder.java index 67251aab581..524491c625b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolSpecBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolSpecBuilder.java @@ -281,6 +281,20 @@ public ProtocolSpec build(final ProtocolSchedule protocolSchedule) { transactionProcessorBuilder.apply( gasCalculator, transactionValidator, contractCreationProcessor, messageCallProcessor); + final BlockHeaderValidator blockHeaderValidator = + blockHeaderValidatorBuilder.apply(difficultyCalculator); + final BlockHeaderValidator ommerHeaderValidator = + ommerHeaderValidatorBuilder.apply(difficultyCalculator); + final BlockBodyValidator blockBodyValidator = + blockBodyValidatorBuilder.apply(protocolSchedule); + + BlockProcessor blockProcessor = + blockProcessorBuilder.apply( + transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards); // Set private Tx Processor PrivateTransactionProcessor privateTransactionProcessor = null; if (privacyParameters.isEnabled()) { @@ -298,21 +312,10 @@ public ProtocolSpec build(final ProtocolSchedule protocolSchedule) { (PrivacyPrecompiledContract) precompileContractRegistry.get(address, Account.DEFAULT_VERSION); privacyPrecompiledContract.setPrivateTransactionProcessor(privateTransactionProcessor); + blockProcessor = + new PrivacyBlockProcessor(blockProcessor, privacyParameters.getPrivateStateStorage()); } - final BlockHeaderValidator blockHeaderValidator = - blockHeaderValidatorBuilder.apply(difficultyCalculator); - final BlockHeaderValidator ommerHeaderValidator = - ommerHeaderValidatorBuilder.apply(difficultyCalculator); - final BlockBodyValidator blockBodyValidator = - blockBodyValidatorBuilder.apply(protocolSchedule); - final BlockProcessor blockProcessor = - blockProcessorBuilder.apply( - transactionProcessor, - transactionReceiptFactory, - blockReward, - miningBeneficiaryCalculator, - skipZeroBlockRewards); final BlockValidator blockValidator = blockValidatorBuilder.apply(blockHeaderValidator, blockBodyValidator, blockProcessor); final BlockImporter blockImporter = blockImporterBuilder.apply(blockValidator); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java index 0fee4c5746d..2a3c21b53d3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java @@ -21,28 +21,32 @@ import org.hyperledger.besu.enclave.EnclaveIOException; import org.hyperledger.besu.enclave.EnclaveServerException; import org.hyperledger.besu.enclave.types.ReceiveResponse; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Gas; import org.hyperledger.besu.ethereum.core.Hash; -import org.hyperledger.besu.ethereum.core.Log; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.core.WorldUpdater; import org.hyperledger.besu.ethereum.debug.TraceOptions; import org.hyperledger.besu.ethereum.mainnet.AbstractPrecompiledContract; -import org.hyperledger.besu.ethereum.mainnet.TransactionProcessor; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionReceipt; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateBlockMetadata; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateTransactionMetadata; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.ethereum.vm.GasCalculator; import org.hyperledger.besu.ethereum.vm.MessageFrame; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Base64; -import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -50,12 +54,11 @@ import org.apache.tuweni.bytes.Bytes32; public class PrivacyPrecompiledContract extends AbstractPrecompiledContract { - private final Enclave enclave; private final WorldStateArchive privateWorldStateArchive; private final PrivateStateStorage privateStateStorage; + private final PrivateStateRootResolver privateStateRootResolver; private PrivateTransactionProcessor privateTransactionProcessor; - private static final Hash EMPTY_ROOT_HASH = Hash.wrap(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); private static final Logger LOG = LogManager.getLogger(); @@ -77,6 +80,7 @@ public PrivacyPrecompiledContract( this.enclave = enclave; this.privateWorldStateArchive = worldStateArchive; this.privateStateStorage = privateStateStorage; + this.privateStateRootResolver = new PrivateStateRootResolver(privateStateStorage); } public void setPrivateTransactionProcessor( @@ -86,11 +90,23 @@ public void setPrivateTransactionProcessor( @Override public Gas gasRequirement(final Bytes input) { - return Gas.of(40_000L); // Not sure + return Gas.of(0L); } @Override public Bytes compute(final Bytes input, final MessageFrame messageFrame) { + final ProcessableBlockHeader currentBlockHeader = messageFrame.getBlockHeader(); + if (!BlockHeader.class.isAssignableFrom(currentBlockHeader.getClass())) { + if (!messageFrame.isPersistingState()) { + // We get in here from block mining. + return Bytes.EMPTY; + } else { + throw new IllegalArgumentException( + "The MessageFrame contains an illegal block header type. Cannot persist private block metadata without current block hash."); + } + } + final Hash currentBlockHash = ((BlockHeader) currentBlockHeader).getHash(); + final String key = input.toBase64String(); final ReceiveResponse receiveResponse; @@ -112,16 +128,21 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { Bytes.wrap(Base64.getDecoder().decode(receiveResponse.getPayload())), false); final PrivateTransaction privateTransaction = PrivateTransaction.readFrom(bytesValueRLPInput); final WorldUpdater publicWorldState = messageFrame.getWorldState(); - final Bytes privacyGroupId = Bytes.fromBase64String(receiveResponse.getPrivacyGroupId()); + final Bytes32 privacyGroupId = + Bytes32.wrap(Bytes.fromBase64String(receiveResponse.getPrivacyGroupId())); LOG.trace( "Processing private transaction {} in privacy group {}", - privateTransaction.hash(), + privateTransaction.getHash(), privacyGroupId); - // get the last world state root hash or create a new one + final PrivacyGroupHeadBlockMap privacyGroupHeadBlockMap = + privateStateStorage.getPrivacyGroupHeadBlockMap(currentBlockHash).orElseThrow(); + + final Blockchain currentBlockchain = messageFrame.getBlockchain(); + final Hash lastRootHash = - privateStateStorage.getLatestStateRoot(privacyGroupId).orElse(EMPTY_ROOT_HASH); + privateStateRootResolver.resolveLastStateRoot(privacyGroupId, currentBlockHash); final MutableWorldState disposablePrivateState = privateWorldStateArchive.getMutable(lastRootHash).get(); @@ -129,10 +150,10 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { final WorldUpdater privateWorldStateUpdater = disposablePrivateState.updater(); final PrivateTransactionProcessor.Result result = privateTransactionProcessor.processTransaction( - messageFrame.getBlockchain(), + currentBlockchain, publicWorldState, privateWorldStateUpdater, - messageFrame.getBlockHeader(), + currentBlockHeader, privateTransaction, messageFrame.getMiningBeneficiary(), new DebugOperationTracer(TraceOptions.DEFAULT), @@ -142,7 +163,7 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { if (result.isInvalid() || !result.isSuccessful()) { LOG.error( "Failed to process private transaction {}: {}", - privateTransaction.hash(), + privateTransaction.getHash(), result.getValidationResult().getErrorMessage()); return Bytes.EMPTY; } @@ -156,24 +177,52 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { disposablePrivateState.persist(); final PrivateStateStorage.Updater privateStateUpdater = privateStateStorage.updater(); - privateStateUpdater.putLatestStateRoot(privacyGroupId, disposablePrivateState.rootHash()); + + updatePrivateBlockMetadata( + messageFrame.getTransactionHash(), + currentBlockHash, + privacyGroupId, + disposablePrivateState.rootHash(), + privateStateUpdater); final Bytes32 txHash = keccak256(RLP.encode(privateTransaction::writeTo)); - final List logs = result.getLogs(); - if (!logs.isEmpty()) { - privateStateUpdater.putTransactionLogs(txHash, result.getLogs()); - } - if (result.getRevertReason().isPresent()) { - privateStateUpdater.putTransactionRevertReason(txHash, result.getRevertReason().get()); - } - privateStateUpdater.putTransactionStatus( - txHash, - Bytes.of(result.getStatus() == TransactionProcessor.Result.Status.SUCCESSFUL ? 1 : 0)); - privateStateUpdater.putTransactionResult(txHash, result.getOutput()); + final int txStatus = + result.getStatus() == PrivateTransactionProcessor.Result.Status.SUCCESSFUL ? 1 : 0; + + final PrivateTransactionReceipt privateTransactionReceipt = + new PrivateTransactionReceipt( + txStatus, result.getLogs(), result.getOutput(), result.getRevertReason()); + + privateStateUpdater.putTransactionReceipt( + currentBlockHash, txHash, privateTransactionReceipt); + + // TODO: this map could be passed through from @PrivacyBlockProcessor and saved once at the + // end of block processing + if (!privacyGroupHeadBlockMap.contains(Bytes32.wrap(privacyGroupId), currentBlockHash)) { + privacyGroupHeadBlockMap.put(Bytes32.wrap(privacyGroupId), currentBlockHash); + privateStateUpdater.putPrivacyGroupHeadBlockMap( + currentBlockHash, new PrivacyGroupHeadBlockMap(privacyGroupHeadBlockMap)); + } privateStateUpdater.commit(); } return result.getOutput(); } + + private void updatePrivateBlockMetadata( + final Hash markerTransactionHash, + final Hash currentBlockHash, + final Bytes32 privacyGroupId, + final Hash rootHash, + final PrivateStateStorage.Updater privateStateUpdater) { + final PrivateBlockMetadata privateBlockMetadata = + privateStateStorage + .getPrivateBlockMetadata(currentBlockHash, Bytes32.wrap(privacyGroupId)) + .orElseGet(PrivateBlockMetadata::empty); + privateBlockMetadata.addPrivateTransactionMetadata( + new PrivateTransactionMetadata(markerTransactionHash, rootHash)); + privateStateUpdater.putPrivateBlockMetadata( + Bytes32.wrap(currentBlockHash), Bytes32.wrap(privacyGroupId), privateBlockMetadata); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProvider.java new file mode 100644 index 00000000000..8d10f565a8e --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProvider.java @@ -0,0 +1,54 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; + +import org.apache.tuweni.bytes.Bytes32; + +public class ChainHeadPrivateNonceProvider implements PrivateNonceProvider { + private final Blockchain blockchain; + private final PrivateStateRootResolver privateStateRootResolver; + private final WorldStateArchive privateWorldStateArchive; + + public ChainHeadPrivateNonceProvider( + final Blockchain blockchain, + final PrivateStateRootResolver privateStateRootResolver, + final WorldStateArchive privateWorldStateArchive) { + this.blockchain = blockchain; + this.privateStateRootResolver = privateStateRootResolver; + this.privateWorldStateArchive = privateWorldStateArchive; + } + + @Override + public long getNonce(final Address sender, final Bytes32 privacyGroupId) { + final BlockHeader chainHeadHeader = blockchain.getChainHeadHeader(); + final Hash stateRoot = + privateStateRootResolver.resolveLastStateRoot(privacyGroupId, chainHeadHeader.getHash()); + return privateWorldStateArchive + .get(stateRoot) + .map( + privateWorldState -> { + final Account account = privateWorldState.get(sender); + return account == null ? 0L : account.getNonce(); + }) + .orElse(Account.DEFAULT_NONCE); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyController.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyController.java index 09a402aacc7..5c25200833b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyController.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyController.java @@ -19,17 +19,14 @@ import org.hyperledger.besu.enclave.types.PrivacyGroup.Type; import org.hyperledger.besu.enclave.types.ReceiveResponse; import org.hyperledger.besu.enclave.types.SendResponse; -import org.hyperledger.besu.ethereum.core.Account; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.mainnet.TransactionValidator; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.privacy.markertransaction.PrivateMarkerTransactionFactory; -import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.transaction.CallParameter; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.math.BigInteger; import java.util.List; @@ -41,45 +38,43 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; public class DefaultPrivacyController implements PrivacyController { private static final Logger LOG = LogManager.getLogger(); private final Enclave enclave; - private final PrivateStateStorage privateStateStorage; - private final WorldStateArchive privateWorldStateArchive; private final PrivateTransactionValidator privateTransactionValidator; private final PrivateMarkerTransactionFactory privateMarkerTransactionFactory; private final PrivateTransactionSimulator privateTransactionSimulator; + private final PrivateNonceProvider privateNonceProvider; public DefaultPrivacyController( final PrivacyParameters privacyParameters, final Optional chainId, final PrivateMarkerTransactionFactory privateMarkerTransactionFactory, - final PrivateTransactionSimulator privateTransactionSimulator) { + final PrivateTransactionSimulator privateTransactionSimulator, + final PrivateNonceProvider privateNonceProvider) { this( privacyParameters.getEnclave(), - privacyParameters.getPrivateStateStorage(), - privacyParameters.getPrivateWorldStateArchive(), new PrivateTransactionValidator(chainId), privateMarkerTransactionFactory, - privateTransactionSimulator); + privateTransactionSimulator, + privateNonceProvider); } public DefaultPrivacyController( final Enclave enclave, - final PrivateStateStorage privateStateStorage, - final WorldStateArchive privateWorldStateArchive, final PrivateTransactionValidator privateTransactionValidator, final PrivateMarkerTransactionFactory privateMarkerTransactionFactory, - final PrivateTransactionSimulator privateTransactionSimulator) { + final PrivateTransactionSimulator privateTransactionSimulator, + final PrivateNonceProvider privateNonceProvider) { this.enclave = enclave; - this.privateStateStorage = privateStateStorage; - this.privateWorldStateArchive = privateWorldStateArchive; this.privateTransactionValidator = privateTransactionValidator; this.privateMarkerTransactionFactory = privateMarkerTransactionFactory; this.privateTransactionSimulator = privateTransactionSimulator; + this.privateNonceProvider = privateNonceProvider; } @Override @@ -169,27 +164,8 @@ public long determineEeaNonce( @Override public long determineBesuNonce( final Address sender, final String privacyGroupId, final String enclavePublicKey) { - return privateStateStorage - .getLatestStateRoot(Bytes.fromBase64String(privacyGroupId)) - .map( - lastRootHash -> - privateWorldStateArchive - .getMutable(lastRootHash) - .map( - worldState -> { - final Account maybePrivateSender = worldState.get(sender); - - if (maybePrivateSender != null) { - return maybePrivateSender.getNonce(); - } - // account has not interacted in this private state - return Account.DEFAULT_NONCE; - }) - // private state does not exist - .orElse(Account.DEFAULT_NONCE)) - .orElse( - // private state does not exist - Account.DEFAULT_NONCE); + return privateNonceProvider.getNonce( + sender, Bytes32.wrap(Bytes.fromBase64String(privacyGroupId))); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateNonceProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateNonceProvider.java new file mode 100644 index 00000000000..5544bce92cd --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateNonceProvider.java @@ -0,0 +1,23 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import org.hyperledger.besu.ethereum.core.Address; + +import org.apache.tuweni.bytes.Bytes32; + +public interface PrivateNonceProvider { + long getNonce(Address sender, Bytes32 privacyGroupId); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolver.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolver.java new file mode 100644 index 00000000000..1c48ca7de64 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolver.java @@ -0,0 +1,70 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateBlockMetadata; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; + +public class PrivateStateRootResolver { + public static final Hash EMPTY_ROOT_HASH = Hash.wrap(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); + + private final PrivateStateStorage privateStateStorage; + + public PrivateStateRootResolver(final PrivateStateStorage privateStateStorage) { + this.privateStateStorage = privateStateStorage; + } + + public Hash resolveLastStateRoot(final Bytes32 privacyGroupId, final Hash blockHash) { + final Optional privateBlockMetadataOptional = + privateStateStorage.getPrivateBlockMetadata(blockHash, privacyGroupId); + if (privateBlockMetadataOptional.isPresent()) { + // Check if block already has meta data for the privacy group + return privateBlockMetadataOptional.get().getLatestStateRoot().orElse(EMPTY_ROOT_HASH); + } + + final Optional maybePrivacyGroupHeadBlockMap = + privateStateStorage.getPrivacyGroupHeadBlockMap(blockHash); + if (maybePrivacyGroupHeadBlockMap.isPresent()) { + return resolveLastStateRoot(privacyGroupId, maybePrivacyGroupHeadBlockMap.get()); + } else { + return EMPTY_ROOT_HASH; + } + } + + private Hash resolveLastStateRoot( + final Bytes32 privacyGroupId, final PrivacyGroupHeadBlockMap privacyGroupHeadBlockMap) { + final Hash lastRootHash; + if (privacyGroupHeadBlockMap.containsKey(privacyGroupId)) { + // Check this PG head block is being tracked + final Hash blockHashForLastBlockWithTx = privacyGroupHeadBlockMap.get(privacyGroupId); + lastRootHash = + privateStateStorage + .getPrivateBlockMetadata(blockHashForLastBlockWithTx, privacyGroupId) + .flatMap(PrivateBlockMetadata::getLatestStateRoot) + .orElse(EMPTY_ROOT_HASH); + } else { + // First transaction for this PG + lastRootHash = EMPTY_ROOT_HASH; + } + return lastRootHash; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStorageMigrationTransactionProcessorResult.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStorageMigrationTransactionProcessorResult.java new file mode 100644 index 00000000000..80c33f17394 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateStorageMigrationTransactionProcessorResult.java @@ -0,0 +1,76 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.mainnet.TransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidator; +import org.hyperledger.besu.ethereum.mainnet.ValidationResult; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor.Result; + +import java.util.Objects; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +public class PrivateStorageMigrationTransactionProcessorResult { + + private final PrivateTransactionProcessor.Result result; + private final Optional resultingRootHash; + + public PrivateStorageMigrationTransactionProcessorResult( + final Result result, final Optional resultingRootHash) { + this.result = result; + this.resultingRootHash = resultingRootHash; + } + + public boolean isSuccessful() { + return result.isSuccessful(); + } + + public Bytes getOutput() { + return result.getOutput(); + } + + public ValidationResult getValidationResult() { + return result.getValidationResult(); + } + + public TransactionProcessor.Result getResult() { + return result; + } + + public Optional getResultingRootHash() { + return resultingRootHash; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final PrivateStorageMigrationTransactionProcessorResult that = + (PrivateStorageMigrationTransactionProcessorResult) o; + return result.equals(that.result) && resultingRootHash.equals(that.resultingRootHash); + } + + @Override + public int hashCode() { + return Objects.hash(result, resultingRootHash); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransaction.java index e488bdcea82..21bb1021723 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransaction.java @@ -416,7 +416,7 @@ public BigInteger getV() { * * @return the transaction hash */ - public Hash hash() { + public Hash getHash() { if (hash == null) { final Bytes rlp = RLP.encode(this::writeTo); hash = Hash.hash(rlp); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java index 948975c342a..04054a772b0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java @@ -248,6 +248,7 @@ public Result processTransaction( .miningBeneficiary(miningBeneficiary) .blockHashLookup(blockHashLookup) .maxStackSize(maxStackSize) + .transactionHash(transaction.getHash()) .build(); } else { @@ -278,6 +279,7 @@ public Result processTransaction( .miningBeneficiary(miningBeneficiary) .blockHashLookup(blockHashLookup) .maxStackSize(maxStackSize) + .transactionHash(transaction.getHash()) .build(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java new file mode 100644 index 00000000000..a3595565809 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java @@ -0,0 +1,184 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import org.hyperledger.besu.ethereum.core.Log; +import org.hyperledger.besu.ethereum.mainnet.TransactionProcessor; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import com.google.common.base.MoreObjects; +import org.apache.tuweni.bytes.Bytes; + +/** + * A transaction receipt for a private transaction, containing information pertaining a transaction + * execution. + */ +public class PrivateTransactionReceipt { + + @SuppressWarnings("unchecked") + public static final PrivateTransactionReceipt EMPTY = + new PrivateTransactionReceipt( + 0, Collections.EMPTY_LIST, Bytes.EMPTY, Optional.ofNullable(null)); + + private final int status; + private final List logs; + private final Bytes output; + private final Optional revertReason; + + /** + * Creates an instance of a state root-encoded transaction receipt. + * + * @param status the state root for the world state after the transaction has been processed + * @param logs the total amount of gas consumed in the block after this transaction + * @param output output from the transaction + * @param revertReason the revert reason for a failed transaction (if applicable) + */ + public PrivateTransactionReceipt( + final int status, + final List logs, + final Bytes output, + final Optional revertReason) { + this.status = status; + this.logs = logs; + this.output = output; + this.revertReason = revertReason; + } + + public PrivateTransactionReceipt(final TransactionProcessor.Result result) { + this( + result.getStatus() == PrivateTransactionProcessor.Result.Status.SUCCESSFUL ? 1 : 0, + result.getLogs(), + result.getOutput(), + result.getRevertReason()); + } + + /** + * Write an RLP representation. + * + * @param out The RLP output to write to + */ + public void writeTo(final RLPOutput out) { + out.startList(); + + out.writeLongScalar(status); + out.writeList(logs, Log::writeTo); + out.writeBytes(output); + if (revertReason.isPresent()) { + out.writeBytes(revertReason.get()); + } + out.endList(); + } + + /** + * Creates a transaction receipt for the given RLP + * + * @param input the RLP-encoded transaction receipt + * @return the transaction receipt + */ + public static PrivateTransactionReceipt readFrom(final RLPInput input) { + input.enterList(); + + try { + // Get the first element to check later to determine the + // correct transaction receipt encoding to use. + final int status = input.readIntScalar(); + final List logs = input.readList(Log::readFrom); + final Bytes output = input.readBytes(); + final Optional revertReason; + if (input.isEndOfCurrentList()) { + revertReason = Optional.empty(); + } else { + revertReason = Optional.of(input.readBytes()); + } + return new PrivateTransactionReceipt(status, logs, output, revertReason); + } finally { + input.leaveList(); + } + } + + /** + * Returns the status code for the status-encoded transaction receipt + * + * @return the status code if the transaction receipt is status-encoded; otherwise {@code -1} + */ + public int getStatus() { + return status; + } + + /** + * Returns the logs generated by the transaction. + * + * @return the logs generated by the transaction + */ + public List getLogs() { + return logs; + } + + /** + * Returns the output generated by the transaction. + * + * @return the output generated by the transaction + */ + public Bytes getOutput() { + return output; + } + + /** + * Returns the revert reason generated by the transaction. + * + * @return the revert reason generated by the transaction + */ + public Optional getRevertReason() { + return revertReason; + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof PrivateTransactionReceipt)) { + return false; + } + final PrivateTransactionReceipt other = (PrivateTransactionReceipt) obj; + return logs.equals(other.getLogs()) + && status == other.status + && output.equals(other.output) + && revertReason.isPresent() + ? revertReason.get().equals(other.revertReason.get()) + : true; + } + + @Override + public int hashCode() { + return Objects.hash(status, logs, output, revertReason); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("status", status) + .add("logs", logs) + .add("output", output) + .add("revertReason", revertReason) + .toString(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionSimulator.java index acb87410971..2211f37e507 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionSimulator.java @@ -27,7 +27,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.transaction.CallParameter; -import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -35,6 +34,7 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; /* * Used to process transactions for priv_call. @@ -44,8 +44,6 @@ */ public class PrivateTransactionSimulator { - private static final Hash EMPTY_ROOT_HASH = Hash.wrap(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); - // Dummy signature for transactions to not fail being processed. private static final SECP256K1.Signature FAKE_SIGNATURE = SECP256K1.Signature.create(SECP256K1.HALF_CURVE_ORDER, SECP256K1.HALF_CURVE_ORDER, (byte) 0); @@ -57,6 +55,7 @@ public class PrivateTransactionSimulator { private final WorldStateArchive worldStateArchive; private final ProtocolSchedule protocolSchedule; private final PrivacyParameters privacyParameters; + private final PrivateStateRootResolver privateStateRootResolver; public PrivateTransactionSimulator( final Blockchain blockchain, @@ -67,6 +66,8 @@ public PrivateTransactionSimulator( this.worldStateArchive = worldStateArchive; this.protocolSchedule = protocolSchedule; this.privacyParameters = privacyParameters; + this.privateStateRootResolver = + new PrivateStateRootResolver(privacyParameters.getPrivateStateStorage()); } public Optional process( @@ -94,12 +95,9 @@ private Optional process( } // get the last world state root hash or create a new one - final Bytes privacyGroupId = Bytes.fromBase64String(privacyGroupIdString); + final Bytes32 privacyGroupId = Bytes32.wrap(Bytes.fromBase64String(privacyGroupIdString)); final Hash lastRootHash = - privacyParameters - .getPrivateStateStorage() - .getLatestStateRoot(privacyGroupId) - .orElse(EMPTY_ROOT_HASH); + privateStateRootResolver.resolveLastStateRoot(privacyGroupId, header.getHash()); final MutableWorldState disposablePrivateState = privacyParameters.getPrivateWorldStateArchive().getMutable(lastRootHash).get(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionValidator.java index c1a42e911a8..130dc0ea14e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionValidator.java @@ -36,25 +36,25 @@ public PrivateTransactionValidator(final Optional chainId) { public ValidationResult validate( final PrivateTransaction transaction, final Long accountNonce) { - LOG.debug("Validating private transaction fields of {}", transaction.hash()); + LOG.debug("Validating private transaction fields of {}", transaction.getHash()); final ValidationResult privateFieldsValidationResult = validatePrivateTransactionFields(transaction); if (!privateFieldsValidationResult.isValid()) { LOG.debug( "Private Transaction fields are invalid {}, {}", - transaction.hash(), + transaction.getHash(), privateFieldsValidationResult.getErrorMessage()); return privateFieldsValidationResult; } - LOG.debug("Validating the signature of Private Transaction {} ", transaction.hash()); + LOG.debug("Validating the signature of Private Transaction {} ", transaction.getHash()); final ValidationResult signatureValidationResult = validateTransactionSignature(transaction); if (!signatureValidationResult.isValid()) { LOG.debug( "Private Transaction {}, failed validation {}, {}", - transaction.hash(), + transaction.getHash(), signatureValidationResult.getInvalidReason(), signatureValidationResult.getErrorMessage()); return signatureValidationResult; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateKeyValueStorage.java new file mode 100644 index 00000000000..c3175c33b0f --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateKeyValueStorage.java @@ -0,0 +1,178 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.Log; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +@Deprecated +public class LegacyPrivateStateKeyValueStorage implements LegacyPrivateStateStorage { + + public static final Bytes EVENTS_KEY_SUFFIX = Bytes.of("EVENTS".getBytes(UTF_8)); + public static final Bytes LOGS_KEY_SUFFIX = Bytes.of("LOGS".getBytes(UTF_8)); + public static final Bytes OUTPUT_KEY_SUFFIX = Bytes.of("OUTPUT".getBytes(UTF_8)); + public static final Bytes METADATA_KEY_SUFFIX = Bytes.of("METADATA".getBytes(UTF_8)); + public static final Bytes STATUS_KEY_SUFFIX = Bytes.of("STATUS".getBytes(UTF_8)); + public static final Bytes REVERT_KEY_SUFFIX = Bytes.of("REVERT".getBytes(UTF_8)); + + private final KeyValueStorage keyValueStorage; + + public LegacyPrivateStateKeyValueStorage(final KeyValueStorage keyValueStorage) { + this.keyValueStorage = keyValueStorage; + } + + @Override + public Optional getLatestStateRoot(final Bytes privacyId) { + final byte[] id = privacyId.toArrayUnsafe(); + + if (keyValueStorage.get(id).isPresent()) { + return Optional.of(Hash.wrap(Bytes32.wrap(keyValueStorage.get(id).get()))); + } else { + return Optional.empty(); + } + } + + @Override + public Optional> getTransactionLogs(final Bytes32 transactionHash) { + final Optional> logs = get(transactionHash, LOGS_KEY_SUFFIX).map(this::rlpDecodeLog); + if (logs.isEmpty()) { + return get(transactionHash, EVENTS_KEY_SUFFIX).map(this::rlpDecodeLog); + } + return logs; + } + + @Override + public Optional getTransactionOutput(final Bytes32 transactionHash) { + return get(transactionHash, OUTPUT_KEY_SUFFIX); + } + + @Override + public Optional getStatus(final Bytes32 transactionHash) { + return get(transactionHash, STATUS_KEY_SUFFIX); + } + + @Override + public Optional getRevertReason(final Bytes32 transactionHash) { + return get(transactionHash, REVERT_KEY_SUFFIX); + } + + @Override + public Optional getTransactionMetadata( + final Bytes32 blockHash, final Bytes32 transactionHash) { + return get(Bytes.concatenate(blockHash, transactionHash), METADATA_KEY_SUFFIX) + .map(bytes -> PrivateTransactionMetadata.readFrom(new BytesValueRLPInput(bytes, false))); + } + + @Override + public boolean isPrivateStateAvailable(final Bytes32 transactionHash) { + return false; + } + + @Override + public boolean isWorldStateAvailable(final Bytes32 rootHash) { + return false; + } + + private Optional get(final Bytes key, final Bytes keySuffix) { + return keyValueStorage.get(Bytes.concatenate(key, keySuffix).toArrayUnsafe()).map(Bytes::wrap); + } + + private List rlpDecodeLog(final Bytes bytes) { + return RLP.input(bytes).readList(Log::readFrom); + } + + @Override + public LegacyPrivateStateStorage.Updater updater() { + return new LegacyPrivateStateKeyValueStorage.Updater(keyValueStorage.startTransaction()); + } + + public static class Updater implements LegacyPrivateStateStorage.Updater { + + private final KeyValueStorageTransaction transaction; + + private Updater(final KeyValueStorageTransaction transaction) { + this.transaction = transaction; + } + + @Override + public Updater putLatestStateRoot(final Bytes privacyId, final Hash privateStateHash) { + transaction.put(privacyId.toArrayUnsafe(), privateStateHash.toArray()); + return this; + } + + @Override + public Updater putTransactionLogs(final Bytes32 transactionHash, final List logs) { + set(transactionHash, LOGS_KEY_SUFFIX, RLP.encode(out -> out.writeList(logs, Log::writeTo))); + return this; + } + + @Override + public Updater putTransactionResult(final Bytes32 transactionHash, final Bytes events) { + set(transactionHash, OUTPUT_KEY_SUFFIX, events); + return this; + } + + @Override + public Updater putTransactionStatus(final Bytes32 transactionHash, final Bytes status) { + set(transactionHash, STATUS_KEY_SUFFIX, status); + return this; + } + + @Override + public Updater putTransactionRevertReason( + final Bytes32 transactionHash, final Bytes revertReason) { + set(transactionHash, REVERT_KEY_SUFFIX, revertReason); + return this; + } + + @Override + public Updater putTransactionMetadata( + final Bytes32 blockHash, + final Bytes32 transactionHash, + final PrivateTransactionMetadata metadata) { + set( + Bytes.concatenate(blockHash, transactionHash), + METADATA_KEY_SUFFIX, + RLP.encode(metadata::writeTo)); + return this; + } + + @Override + public void commit() { + transaction.commit(); + } + + @Override + public void rollback() { + transaction.rollback(); + } + + private void set(final Bytes key, final Bytes keySuffix, final Bytes value) { + transaction.put(Bytes.concatenate(key, keySuffix).toArrayUnsafe(), value.toArrayUnsafe()); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateStorage.java new file mode 100644 index 00000000000..4ea10e7dcc6 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/LegacyPrivateStateStorage.java @@ -0,0 +1,68 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.Log; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** This interface contains the methods used to access the private state until version 1.3 */ +@Deprecated +public interface LegacyPrivateStateStorage { + + Optional getLatestStateRoot(Bytes privacyId); + + Optional> getTransactionLogs(Bytes32 transactionHash); + + Optional getTransactionOutput(Bytes32 transactionHash); + + Optional getStatus(Bytes32 transactionHash); + + Optional getRevertReason(Bytes32 transactionHash); + + Optional getTransactionMetadata( + Bytes32 blockHash, Bytes32 transactionHash); + + boolean isPrivateStateAvailable(Bytes32 transactionHash); + + boolean isWorldStateAvailable(Bytes32 rootHash); + + Updater updater(); + + interface Updater { + + Updater putLatestStateRoot(Bytes privacyId, Hash privateStateHash); + + Updater putTransactionLogs(Bytes32 transactionHash, List logs); + + Updater putTransactionResult(Bytes32 transactionHash, Bytes events); + + Updater putTransactionStatus(Bytes32 transactionHash, Bytes status); + + Updater putTransactionRevertReason(Bytes32 txHash, Bytes bytesValue); + + Updater putTransactionMetadata( + Bytes32 blockHash, Bytes32 transactionHash, PrivateTransactionMetadata metadata); + + void commit(); + + void rollback(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyGroupHeadBlockMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyGroupHeadBlockMap.java new file mode 100644 index 00000000000..f0f8db26221 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyGroupHeadBlockMap.java @@ -0,0 +1,134 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import org.apache.tuweni.bytes.Bytes32; + +public class PrivacyGroupHeadBlockMap implements Map { + private final HashMap map; + + public static final PrivacyGroupHeadBlockMap EMPTY = + new PrivacyGroupHeadBlockMap(Collections.emptyMap()); + + public PrivacyGroupHeadBlockMap(final Map map) { + this.map = new HashMap<>(map); + } + + public void writeTo(final RLPOutput out) { + out.startList(); + + map.forEach((key, value) -> new RLPMapEntry(key, value).writeTo(out)); + + out.endList(); + } + + public static PrivacyGroupHeadBlockMap readFrom(final RLPInput input) { + final List entries = input.readList(RLPMapEntry::readFrom); + + final HashMap map = new HashMap<>(); + entries.forEach(e -> map.put(Bytes32.wrap(e.getKey()), Hash.wrap(Bytes32.wrap(e.getValue())))); + + return new PrivacyGroupHeadBlockMap(map); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final PrivacyGroupHeadBlockMap that = (PrivacyGroupHeadBlockMap) o; + return map.equals(that.map); + } + + public boolean contains(final Bytes32 key, final Hash value) { + return map.containsKey(key) && map.get(key).equals(value); + } + + @Override + public int hashCode() { + return Objects.hash(map); + } + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(final Object key) { + return map.containsKey(key); + } + + @Override + public boolean containsValue(final Object value) { + return map.containsValue(value); + } + + @Override + public Hash get(final Object key) { + return map.get(key); + } + + @Override + public Hash put(final Bytes32 key, final Hash value) { + return map.put(key, value); + } + + @Override + public Hash remove(final Object key) { + return map.remove(key); + } + + @Override + public void putAll(final Map m) { + map.putAll(m); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Set keySet() { + return map.keySet(); + } + + @Override + public Collection values() { + return map.values(); + } + + @Override + public Set> entrySet() { + return map.entrySet(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java index fd275d0c692..2dc167a884b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java @@ -27,5 +27,8 @@ public interface PrivacyStorageProvider extends Closeable { PrivateStateStorage createPrivateStateStorage(); + @Deprecated + LegacyPrivateStateStorage createLegacyPrivateStateStorage(); + int getFactoryVersion(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateBlockMetadata.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateBlockMetadata.java new file mode 100644 index 00000000000..ed69e9e89fa --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateBlockMetadata.java @@ -0,0 +1,68 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class PrivateBlockMetadata { + + public static PrivateBlockMetadata empty() { + return new PrivateBlockMetadata(new ArrayList<>()); + } + + private final List privateTransactionMetadataList; + + public PrivateBlockMetadata( + final List privateTransactionMetadataList) { + this.privateTransactionMetadataList = + privateTransactionMetadataList == null ? new ArrayList<>() : privateTransactionMetadataList; + } + + public List getPrivateTransactionMetadataList() { + return privateTransactionMetadataList; + } + + public void addPrivateTransactionMetadata( + final PrivateTransactionMetadata privateTransactionMetadata) { + privateTransactionMetadataList.add(privateTransactionMetadata); + } + + public void writeTo(final RLPOutput out) { + out.writeList(privateTransactionMetadataList, PrivateTransactionMetadata::writeTo); + } + + public static PrivateBlockMetadata readFrom(final RLPInput in) { + final List privateTransactionMetadataList = + in.readList(PrivateTransactionMetadata::readFrom); + return new PrivateBlockMetadata(privateTransactionMetadataList); + } + + public Optional getLatestStateRoot() { + if (privateTransactionMetadataList.size() > 0) { + return Optional.ofNullable( + privateTransactionMetadataList + .get(privateTransactionMetadataList.size() - 1) + .getStateRoot()); + } else { + return Optional.empty(); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorage.java index 0a8642b8796..c3ef13e7235 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorage.java @@ -16,28 +16,30 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import org.hyperledger.besu.ethereum.core.Hash; -import org.hyperledger.besu.ethereum.core.Log; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionReceipt; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; -import java.util.List; +import java.util.Arrays; import java.util.Optional; +import java.util.function.Predicate; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; public class PrivateStateKeyValueStorage implements PrivateStateStorage { - @Deprecated private static final Bytes EVENTS_KEY_SUFFIX = Bytes.of("EVENTS".getBytes(UTF_8)); + public static final int SCHEMA_VERSION_1_0_0 = 1; + public static final int SCHEMA_VERSION_1_4_0 = 2; - private static final Bytes LOGS_KEY_SUFFIX = Bytes.of("LOGS".getBytes(UTF_8)); - private static final Bytes OUTPUT_KEY_SUFFIX = Bytes.of("OUTPUT".getBytes(UTF_8)); + private static final Bytes DB_VERSION_KEY = Bytes.of("DBVERSION".getBytes(UTF_8)); + private static final Bytes TX_RECEIPT_SUFFIX = Bytes.of("RECEIPT".getBytes(UTF_8)); private static final Bytes METADATA_KEY_SUFFIX = Bytes.of("METADATA".getBytes(UTF_8)); - private static final Bytes STATUS_KEY_SUFFIX = Bytes.of("STATUS".getBytes(UTF_8)); - private static final Bytes REVERT_KEY_SUFFIX = Bytes.of("REVERT".getBytes(UTF_8)); + private static final Bytes PRIVACY_GROUP_HEAD_BLOCK_MAP_SUFFIX = + Bytes.of("PGHEADMAP".getBytes(UTF_8)); + private static final Bytes LEGACY_STATUS_KEY_SUFFIX = Bytes.of("STATUS".getBytes(UTF_8)); private final KeyValueStorage keyValueStorage; @@ -46,63 +48,52 @@ public PrivateStateKeyValueStorage(final KeyValueStorage keyValueStorage) { } @Override - public Optional getLatestStateRoot(final Bytes privacyId) { - final byte[] id = privacyId.toArrayUnsafe(); - - if (keyValueStorage.get(id).isPresent()) { - return Optional.of(Hash.wrap(Bytes32.wrap(keyValueStorage.get(id).get()))); - } else { - return Optional.empty(); - } - } - - @Override - public Optional> getTransactionLogs(final Bytes32 transactionHash) { - final Optional> logs = get(transactionHash, LOGS_KEY_SUFFIX).map(this::rlpDecodeLog); - if (logs.isEmpty()) { - return get(transactionHash, EVENTS_KEY_SUFFIX).map(this::rlpDecodeLog); - } - return logs; - } - - @Override - public Optional getTransactionOutput(final Bytes32 transactionHash) { - return get(transactionHash, OUTPUT_KEY_SUFFIX); + public Optional getTransactionReceipt( + final Bytes32 blockHash, final Bytes32 txHash) { + final Bytes blockHashTxHash = Bytes.concatenate(blockHash, txHash); + return get(blockHashTxHash, TX_RECEIPT_SUFFIX) + .map(b -> PrivateTransactionReceipt.readFrom(new BytesValueRLPInput(b, false))); } @Override - public Optional getStatus(final Bytes32 transactionHash) { - return get(transactionHash, STATUS_KEY_SUFFIX); + public Optional getPrivateBlockMetadata( + final Bytes32 blockHash, final Bytes32 privacyGroupId) { + return get(Bytes.concatenate(blockHash, privacyGroupId), METADATA_KEY_SUFFIX) + .map(this::rlpDecodePrivateBlockMetadata); } @Override - public Optional getRevertReason(final Bytes32 transactionHash) { - return get(transactionHash, REVERT_KEY_SUFFIX); + public Optional getPrivacyGroupHeadBlockMap(final Bytes32 blockHash) { + return get(blockHash, PRIVACY_GROUP_HEAD_BLOCK_MAP_SUFFIX) + .map(b -> PrivacyGroupHeadBlockMap.readFrom(new BytesValueRLPInput(b, false))); } @Override - public Optional getTransactionMetadata( - final Bytes32 blockHash, final Bytes32 transactionHash) { - return get(Bytes.concatenate(blockHash, transactionHash), METADATA_KEY_SUFFIX) - .map(bytes -> PrivateTransactionMetadata.readFrom(new BytesValueRLPInput(bytes, false))); + public int getSchemaVersion() { + return get(Bytes.EMPTY, DB_VERSION_KEY).map(Bytes::toInt).orElse(SCHEMA_VERSION_1_0_0); } @Override - public boolean isPrivateStateAvailable(final Bytes32 transactionHash) { - return false; + public boolean isEmpty() { + return keyValueStorage.getAllKeysThat(containsSuffix(LEGACY_STATUS_KEY_SUFFIX)).isEmpty() + && keyValueStorage.getAllKeysThat(containsSuffix(TX_RECEIPT_SUFFIX)).isEmpty() + && keyValueStorage.getAllKeysThat(containsSuffix(METADATA_KEY_SUFFIX)).isEmpty(); } - @Override - public boolean isWorldStateAvailable(final Bytes32 rootHash) { - return false; + private Predicate containsSuffix(final Bytes suffix) { + return key -> + key.length > suffix.toArrayUnsafe().length + && Arrays.equals( + Arrays.copyOfRange(key, key.length - suffix.toArrayUnsafe().length, key.length), + suffix.toArrayUnsafe()); } private Optional get(final Bytes key, final Bytes keySuffix) { return keyValueStorage.get(Bytes.concatenate(key, keySuffix).toArrayUnsafe()).map(Bytes::wrap); } - private List rlpDecodeLog(final Bytes bytes) { - return RLP.input(bytes).readList(Log::readFrom); + private PrivateBlockMetadata rlpDecodePrivateBlockMetadata(final Bytes bytes) { + return PrivateBlockMetadata.readFrom(RLP.input(bytes)); } @Override @@ -119,46 +110,37 @@ private Updater(final KeyValueStorageTransaction transaction) { } @Override - public Updater putLatestStateRoot(final Bytes privacyId, final Hash privateStateHash) { - transaction.put(privacyId.toArrayUnsafe(), privateStateHash.toArray()); - return this; - } - - @Override - public Updater putTransactionLogs(final Bytes32 transactionHash, final List logs) { - set(transactionHash, LOGS_KEY_SUFFIX, RLP.encode(out -> out.writeList(logs, Log::writeTo))); - return this; - } - - @Override - public Updater putTransactionResult(final Bytes32 transactionHash, final Bytes events) { - set(transactionHash, OUTPUT_KEY_SUFFIX, events); + public PrivateStateStorage.Updater putTransactionReceipt( + final Bytes32 blockHash, + final Bytes32 transactionHash, + final PrivateTransactionReceipt receipt) { + final Bytes blockHashTxHash = Bytes.concatenate(blockHash, transactionHash); + set(blockHashTxHash, TX_RECEIPT_SUFFIX, RLP.encode(receipt::writeTo)); return this; } @Override - public PrivateStateStorage.Updater putTransactionStatus( - final Bytes32 transactionHash, final Bytes status) { - set(transactionHash, STATUS_KEY_SUFFIX, status); + public PrivateStateStorage.Updater putPrivateBlockMetadata( + final Bytes32 blockHash, + final Bytes32 privacyGroupId, + final PrivateBlockMetadata metadata) { + set( + Bytes.concatenate(blockHash, privacyGroupId), + METADATA_KEY_SUFFIX, + RLP.encode(metadata::writeTo)); return this; } @Override - public PrivateStateStorage.Updater putTransactionRevertReason( - final Bytes32 transactionHash, final Bytes revertReason) { - set(transactionHash, REVERT_KEY_SUFFIX, revertReason); + public PrivateStateStorage.Updater putPrivacyGroupHeadBlockMap( + final Bytes32 blockHash, final PrivacyGroupHeadBlockMap map) { + set(blockHash, PRIVACY_GROUP_HEAD_BLOCK_MAP_SUFFIX, RLP.encode(map::writeTo)); return this; } @Override - public Updater putTransactionMetadata( - final Bytes32 blockHash, - final Bytes32 transactionHash, - final PrivateTransactionMetadata metadata) { - set( - Bytes.concatenate(blockHash, transactionHash), - METADATA_KEY_SUFFIX, - RLP.encode(metadata::writeTo)); + public PrivateStateStorage.Updater putDatabaseVersion(final int version) { + set(Bytes.EMPTY, DB_VERSION_KEY, Bytes.ofUnsignedInt(version)); return this; } @@ -175,5 +157,10 @@ public void rollback() { private void set(final Bytes key, final Bytes keySuffix, final Bytes value) { transaction.put(Bytes.concatenate(key, keySuffix).toArrayUnsafe(), value.toArrayUnsafe()); } + + @Override + public void remove(final Bytes key, final Bytes keySuffix) { + transaction.remove(Bytes.concatenate(key, keySuffix).toArrayUnsafe()); + } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateStorage.java index 1fc0f37df1f..29d8b57db53 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateStorage.java @@ -14,10 +14,8 @@ */ package org.hyperledger.besu.ethereum.privacy.storage; -import org.hyperledger.besu.ethereum.core.Hash; -import org.hyperledger.besu.ethereum.core.Log; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionReceipt; -import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; @@ -25,44 +23,34 @@ public interface PrivateStateStorage { - @Deprecated - Optional getLatestStateRoot(Bytes privacyId); + Optional getTransactionReceipt(Bytes32 blockHash, Bytes32 txHash); - Optional> getTransactionLogs(Bytes32 transactionHash); + Optional getPrivateBlockMetadata(Bytes32 blockHash, Bytes32 privacyGroupId); - Optional getTransactionOutput(Bytes32 transactionHash); + Optional getPrivacyGroupHeadBlockMap(Bytes32 blockHash); - Optional getStatus(Bytes32 transactionHash); + int getSchemaVersion(); - Optional getRevertReason(Bytes32 transactionHash); - - Optional getTransactionMetadata( - Bytes32 blockHash, Bytes32 transactionHash); - - boolean isPrivateStateAvailable(Bytes32 transactionHash); - - boolean isWorldStateAvailable(Bytes32 rootHash); + boolean isEmpty(); Updater updater(); interface Updater { - @Deprecated - Updater putLatestStateRoot(Bytes privacyId, Hash privateStateHash); + Updater putTransactionReceipt( + Bytes32 blockHash, Bytes32 transactionHash, PrivateTransactionReceipt receipt); - Updater putTransactionLogs(Bytes32 transactionHash, List logs); + Updater putPrivateBlockMetadata( + Bytes32 blockHash, Bytes32 privacyGroupId, PrivateBlockMetadata metadata); - Updater putTransactionResult(Bytes32 transactionHash, Bytes events); + Updater putPrivacyGroupHeadBlockMap(Bytes32 blockHash, PrivacyGroupHeadBlockMap map); - Updater putTransactionStatus(Bytes32 transactionHash, Bytes status); - - Updater putTransactionRevertReason(Bytes32 txHash, Bytes bytesValue); - - Updater putTransactionMetadata( - Bytes32 blockHash, Bytes32 transactionHash, PrivateTransactionMetadata metadata); + Updater putDatabaseVersion(int version); void commit(); void rollback(); + + void remove(final Bytes key, final Bytes keySuffix); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateTransactionMetadata.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateTransactionMetadata.java index b4e9233833c..01f95c2eaa3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateTransactionMetadata.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateTransactionMetadata.java @@ -20,9 +20,11 @@ /** Mined private transaction metadata. */ public class PrivateTransactionMetadata { + private final Hash privacyMarkerTransactionHash; private final Hash stateRoot; - public PrivateTransactionMetadata(final Hash stateRoot) { + public PrivateTransactionMetadata(final Hash privacyMarkerTransactionHash, final Hash stateRoot) { + this.privacyMarkerTransactionHash = privacyMarkerTransactionHash; this.stateRoot = stateRoot; } @@ -30,9 +32,14 @@ public Hash getStateRoot() { return stateRoot; } + public Hash getPrivacyMarkerTransactionHash() { + return privacyMarkerTransactionHash; + } + public void writeTo(final RLPOutput out) { out.startList(); + out.writeBytes(privacyMarkerTransactionHash); out.writeBytes(stateRoot); out.endList(); @@ -42,7 +49,8 @@ public static PrivateTransactionMetadata readFrom(final RLPInput input) { input.enterList(); final PrivateTransactionMetadata privateTransactionMetadata = - new PrivateTransactionMetadata(Hash.wrap(input.readBytes32())); + new PrivateTransactionMetadata( + Hash.wrap(input.readBytes32()), Hash.wrap(input.readBytes32())); input.leaveList(); return privateTransactionMetadata; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/RLPMapEntry.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/RLPMapEntry.java new file mode 100644 index 00000000000..d703e49121e --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/RLPMapEntry.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; + +public class RLPMapEntry { + private final Bytes key; + private final Bytes value; + + public RLPMapEntry(final Bytes key, final Bytes value) { + this.key = key; + this.value = value; + } + + public Bytes getKey() { + return key; + } + + public Bytes getValue() { + return value; + } + + public void writeTo(final RLPOutput out) { + out.startList(); + + out.writeBytes(key); + out.writeBytes(value); + + out.endList(); + } + + public static RLPMapEntry readFrom(final RLPInput input) { + input.enterList(); + + final RLPMapEntry rlpMapEntry = new RLPMapEntry(input.readBytes(), input.readBytes()); + + input.leaveList(); + return rlpMapEntry; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final RLPMapEntry rlpMapEntry = (RLPMapEntry) o; + return key.equals(rlpMapEntry.key) && value.equals(rlpMapEntry.value); + } + + @Override + public int hashCode() { + return Objects.hash(key, value); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java index 1b9b67831a8..d54a4ba164f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.privacy.storage.keyvalue; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateKeyValueStorage; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; @@ -27,36 +29,41 @@ public class PrivacyKeyValueStorageProvider implements PrivacyStorageProvider { - private final KeyValueStorage privateWorldStateStorage; - private final KeyValueStorage privateWorldStatePreimageStorage; - private final KeyValueStorage privateStateStorage; + private final KeyValueStorage privateWorldStateKeyValueStorage; + private final KeyValueStorage privateWorldStatePreimageKeyValueStorage; + private final KeyValueStorage privateStateKeyValueStorage; private final int factoryVersion; public PrivacyKeyValueStorageProvider( - final KeyValueStorage privateWorldStateStorage, - final KeyValueStorage privateWorldStatePreimageStorage, - final KeyValueStorage privateStateStorage, + final KeyValueStorage privateWorldStateKeyValueStorage, + final KeyValueStorage privateWorldStatePreimageKeyValueStorage, + final KeyValueStorage privateStateKeyValueStorage, final int factoryVersion) { - this.privateWorldStateStorage = privateWorldStateStorage; - this.privateWorldStatePreimageStorage = privateWorldStatePreimageStorage; - this.privateStateStorage = privateStateStorage; + this.privateWorldStateKeyValueStorage = privateWorldStateKeyValueStorage; + this.privateWorldStatePreimageKeyValueStorage = privateWorldStatePreimageKeyValueStorage; + this.privateStateKeyValueStorage = privateStateKeyValueStorage; this.factoryVersion = factoryVersion; } @Override public WorldStateStorage createWorldStateStorage() { - return new WorldStateKeyValueStorage(privateWorldStateStorage); + return new WorldStateKeyValueStorage(privateWorldStateKeyValueStorage); + } + + @Override + public WorldStatePreimageStorage createWorldStatePreimageStorage() { + return new WorldStatePreimageKeyValueStorage(privateWorldStatePreimageKeyValueStorage); } @Override public PrivateStateStorage createPrivateStateStorage() { - return new PrivateStateKeyValueStorage(privateStateStorage); + return new PrivateStateKeyValueStorage(privateStateKeyValueStorage); } @Override - public WorldStatePreimageStorage createWorldStatePreimageStorage() { - return new WorldStatePreimageKeyValueStorage(privateWorldStatePreimageStorage); + public LegacyPrivateStateStorage createLegacyPrivateStateStorage() { + return new LegacyPrivateStateKeyValueStorage(privateStateKeyValueStorage); } @Override @@ -66,8 +73,8 @@ public int getFactoryVersion() { @Override public void close() throws IOException { - privateWorldStateStorage.close(); - privateWorldStatePreimageStorage.close(); - privateStateStorage.close(); + privateWorldStateKeyValueStorage.close(); + privateWorldStatePreimageKeyValueStorage.close(); + privateStateKeyValueStorage.close(); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java new file mode 100644 index 00000000000..3e6bce0e0c2 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java @@ -0,0 +1,160 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableAccount; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.WorldUpdater; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.MiningBeneficiaryCalculator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.TransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; +import org.hyperledger.besu.ethereum.vm.BlockHashLookup; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class PrivateMigrationBlockProcessor { + + private static final Logger LOG = LogManager.getLogger(); + + static final int MAX_GENERATION = 6; + + private final TransactionProcessor transactionProcessor; + private final MainnetBlockProcessor.TransactionReceiptFactory transactionReceiptFactory; + final Wei blockReward; + private final boolean skipZeroBlockRewards; + private final MiningBeneficiaryCalculator miningBeneficiaryCalculator; + + public PrivateMigrationBlockProcessor( + final TransactionProcessor transactionProcessor, + final MainnetBlockProcessor.TransactionReceiptFactory transactionReceiptFactory, + final Wei blockReward, + final MiningBeneficiaryCalculator miningBeneficiaryCalculator, + final boolean skipZeroBlockRewards) { + this.transactionProcessor = transactionProcessor; + this.transactionReceiptFactory = transactionReceiptFactory; + this.blockReward = blockReward; + this.miningBeneficiaryCalculator = miningBeneficiaryCalculator; + this.skipZeroBlockRewards = skipZeroBlockRewards; + } + + public PrivateMigrationBlockProcessor(final ProtocolSpec protocolSpec) { + this( + protocolSpec.getTransactionProcessor(), + protocolSpec.getTransactionReceiptFactory(), + protocolSpec.getBlockReward(), + protocolSpec.getMiningBeneficiaryCalculator(), + protocolSpec.isSkipZeroBlockRewards()); + } + + public AbstractBlockProcessor.Result processBlock( + final Blockchain blockchain, + final MutableWorldState worldState, + final BlockHeader blockHeader, + final List transactions, + final List ommers) { + long gasUsed = 0; + final List receipts = new ArrayList<>(); + + for (final Transaction transaction : transactions) { + final long remainingGasBudget = blockHeader.getGasLimit() - gasUsed; + if (Long.compareUnsigned(transaction.getGasLimit(), remainingGasBudget) > 0) { + LOG.warn( + "Transaction processing error: transaction gas limit {} exceeds available block budget remaining {}", + transaction.getGasLimit(), + remainingGasBudget); + return AbstractBlockProcessor.Result.failed(); + } + + final WorldUpdater worldStateUpdater = worldState.updater(); + final BlockHashLookup blockHashLookup = new BlockHashLookup(blockHeader, blockchain); + final Address miningBeneficiary = + miningBeneficiaryCalculator.calculateBeneficiary(blockHeader); + + final TransactionProcessor.Result result = + transactionProcessor.processTransaction( + blockchain, + worldStateUpdater, + blockHeader, + transaction, + miningBeneficiary, + blockHashLookup, + true, + TransactionValidationParams.processingBlock()); + if (result.isInvalid()) { + return AbstractBlockProcessor.Result.failed(); + } + + worldStateUpdater.commit(); + gasUsed = transaction.getGasLimit() - result.getGasRemaining() + gasUsed; + final TransactionReceipt transactionReceipt = + transactionReceiptFactory.create(result, worldState, gasUsed); + receipts.add(transactionReceipt); + } + + if (!rewardCoinbase(worldState, blockHeader, ommers, skipZeroBlockRewards)) { + return AbstractBlockProcessor.Result.failed(); + } + + return AbstractBlockProcessor.Result.successful(receipts); + } + + private boolean rewardCoinbase( + final MutableWorldState worldState, + final ProcessableBlockHeader header, + final List ommers, + final boolean skipZeroBlockRewards) { + if (skipZeroBlockRewards && blockReward.isZero()) { + return true; + } + + final Wei coinbaseReward = blockReward.add(blockReward.multiply(ommers.size()).divide(32)); + final WorldUpdater updater = worldState.updater(); + final MutableAccount coinbase = updater.getOrCreate(header.getCoinbase()).getMutable(); + + coinbase.incrementBalance(coinbaseReward); + for (final BlockHeader ommerHeader : ommers) { + if (ommerHeader.getNumber() - header.getNumber() > MAX_GENERATION) { + LOG.warn( + "Block processing error: ommer block number {} more than {} generations current block number {}", + ommerHeader.getNumber(), + MAX_GENERATION, + header.getNumber()); + return false; + } + + final MutableAccount ommerCoinbase = + updater.getOrCreate(ommerHeader.getCoinbase()).getMutable(); + final long distance = header.getNumber() - ommerHeader.getNumber(); + final Wei ommerReward = blockReward.subtract(blockReward.multiply(distance).divide(8)); + ommerCoinbase.incrementBalance(ommerReward); + } + + return true; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigration.java new file mode 100644 index 00000000000..5f25246f8b8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigration.java @@ -0,0 +1,176 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes32; + +public class PrivateStorageMigration { + + private static final Logger LOG = LogManager.getLogger(); + + private final Blockchain blockchain; + private final Address privacyPrecompileAddress; + private final ProtocolSchedule protocolSchedule; + private final WorldStateArchive publicWorldStateArchive; + private final PrivateStateStorage privateStateStorage; + private final PrivateStateRootResolver privateStateRootResolver; + private final LegacyPrivateStateStorage legacyPrivateStateStorage; + private final Function, PrivateMigrationBlockProcessor> + privateMigrationBlockProcessorBuilder; + + public PrivateStorageMigration( + final Blockchain blockchain, + final Address privacyPrecompileAddress, + final ProtocolSchedule protocolSchedule, + final WorldStateArchive publicWorldStateArchive, + final PrivateStateStorage privateStateStorage, + final PrivateStateRootResolver privateStateRootResolver, + final LegacyPrivateStateStorage legacyPrivateStateStorage, + final Function, PrivateMigrationBlockProcessor> + privateMigrationBlockProcessorBuilder) { + this.privateStateStorage = privateStateStorage; + this.blockchain = blockchain; + this.privacyPrecompileAddress = privacyPrecompileAddress; + this.protocolSchedule = protocolSchedule; + this.publicWorldStateArchive = publicWorldStateArchive; + this.privateStateRootResolver = privateStateRootResolver; + this.legacyPrivateStateStorage = legacyPrivateStateStorage; + this.privateMigrationBlockProcessorBuilder = privateMigrationBlockProcessorBuilder; + } + + public void migratePrivateStorage() { + final long migrationStartTimestamp = System.currentTimeMillis(); + final long chainHeadBlockNumber = blockchain.getChainHeadBlockNumber(); + + LOG.info("Migrating private storage database..."); + + for (int blockNumber = 0; blockNumber <= chainHeadBlockNumber; blockNumber++) { + final Block block = + blockchain + .getBlockByNumber(blockNumber) + .orElseThrow(PrivateStorageMigrationException::new); + final Hash blockHash = block.getHash(); + final BlockHeader blockHeader = block.getHeader(); + LOG.info("Processing block {} ({}/{})", blockHash, blockNumber, chainHeadBlockNumber); + + createPrivacyGroupHeadBlockMap(blockHeader); + + final int lastPmtIndex = findLastPMTIndexInBlock(block); + if (lastPmtIndex >= 0) { + final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(blockNumber); + final PrivateMigrationBlockProcessor privateMigrationBlockProcessor = + privateMigrationBlockProcessorBuilder.apply(protocolSpec); + + final MutableWorldState publicWorldState = + blockchain + .getBlockHeader(blockHeader.getParentHash()) + .map(BlockHeader::getStateRoot) + .flatMap(publicWorldStateArchive::getMutable) + .orElseThrow(PrivateStorageMigrationException::new); + + final List transactionsToProcess = + block.getBody().getTransactions().subList(0, lastPmtIndex + 1); + final List ommers = block.getBody().getOmmers(); + + privateMigrationBlockProcessor.processBlock( + blockchain, publicWorldState, blockHeader, transactionsToProcess, ommers); + } + } + + if (isResultingPrivateStateRootAtHeadValid()) { + privateStateStorage.updater().putDatabaseVersion(SCHEMA_VERSION_1_4_0).commit(); + } else { + throw new PrivateStorageMigrationException("Inconsistent state root. Please re-sync."); + } + + final long migrationDuration = System.currentTimeMillis() - migrationStartTimestamp; + LOG.info("Migration took {} seconds", migrationDuration / 1000.0); + } + + /* + Returns the index of the last PMT in the block, or -1 if there are no PMTs in the block. + */ + private int findLastPMTIndexInBlock(final Block block) { + final List txs = block.getBody().getTransactions(); + int lastPmtIndex = -1; + for (int i = 0; i < txs.size(); i++) { + if (isPrivacyMarkerTransaction(txs.get(i))) { + lastPmtIndex = i; + } + } + return lastPmtIndex; + } + + private boolean isPrivacyMarkerTransaction(final Transaction tx) { + return tx.getTo().isPresent() && tx.getTo().get().equals(privacyPrecompileAddress); + } + + private boolean isResultingPrivateStateRootAtHeadValid() { + final Optional privacyGroupHeadBlockMap = + privateStateStorage.getPrivacyGroupHeadBlockMap(blockchain.getChainHeadHash()); + final Set privacyGroupIds = + privacyGroupHeadBlockMap.orElseThrow(PrivateStorageMigrationException::new).keySet(); + + privacyGroupIds.forEach( + pgId -> { + final Optional legacyStateRoot = legacyPrivateStateStorage.getLatestStateRoot(pgId); + final Hash newStateRoot = + privateStateRootResolver.resolveLastStateRoot(pgId, blockchain.getChainHeadHash()); + if (!newStateRoot.equals(legacyStateRoot.orElse(Hash.EMPTY))) { + throw new PrivateStorageMigrationException( + "Inconsistent state root. Please delete your database and re-sync your node to avoid inconsistencies in your database."); + } + }); + + return true; + } + + private void createPrivacyGroupHeadBlockMap(final BlockHeader blockHeader) { + final PrivacyGroupHeadBlockMap privacyGroupHeadBlockHash = + new PrivacyGroupHeadBlockMap( + privateStateStorage + .getPrivacyGroupHeadBlockMap(blockHeader.getParentHash()) + .orElse(PrivacyGroupHeadBlockMap.EMPTY)); + + privateStateStorage + .updater() + .putPrivacyGroupHeadBlockMap(blockHeader.getHash(), privacyGroupHeadBlockHash) + .commit(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationException.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationException.java new file mode 100644 index 00000000000..0175bb6a753 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationException.java @@ -0,0 +1,33 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +public class PrivateStorageMigrationException extends RuntimeException { + + private static final String MIGRATION_ERROR_MSG = + "Unexpected error during private database migration. Please re-sync your node to avoid data corruption."; + + public PrivateStorageMigrationException(final String message) { + super(message); + } + + public PrivateStorageMigrationException() { + super(MIGRATION_ERROR_MSG); + } + + public PrivateStorageMigrationException(final Throwable th) { + super(MIGRATION_ERROR_MSG, th); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationService.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationService.java new file mode 100644 index 00000000000..9b2c7c43653 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationService.java @@ -0,0 +1,78 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_0_0; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; + +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; + +import java.util.function.Supplier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class PrivateStorageMigrationService { + + private static final Logger LOG = LogManager.getLogger(); + + private final PrivateStateStorage privateStateStorage; + private final boolean migrationFlag; + private final Supplier migrationBuilder; + + public PrivateStorageMigrationService( + final PrivateStateStorage privateStateStorage, + final boolean migrationFlag, + final Supplier migrationBuilder) { + this.privateStateStorage = privateStateStorage; + this.migrationFlag = migrationFlag; + this.migrationBuilder = migrationBuilder; + } + + /** + * Migration only happens if the system detects that the private schema version is lower than + * version 2 (1.4.x), and the user has set the` privacy-enable-database-migration` option. + */ + public void runMigrationIfRequired() { + final int schemaVersion = privateStateStorage.getSchemaVersion(); + + if (schemaVersion >= SCHEMA_VERSION_1_4_0) { + LOG.debug("Private database metadata does not require migration."); + return; + } + + /* + If this is a new database, we need to set the version and no migration is required + */ + if (privateStateStorage.isEmpty()) { + privateStateStorage.updater().putDatabaseVersion(SCHEMA_VERSION_1_4_0).commit(); + LOG.debug("Private database metadata does not require migration."); + return; + } + + if (schemaVersion == SCHEMA_VERSION_1_0_0 && !migrationFlag) { + final String message = + "Private database metadata requires migration. For more information check the 1.4 changelog."; + LOG.warn(message); + throw new PrivateStorageMigrationException(message); + } + + if (schemaVersion == SCHEMA_VERSION_1_0_0 && migrationFlag) { + LOG.info( + "Private database metadata requires migration and `privacy-enable-database-migration` was set. Starting migration!"); + migrationBuilder.get().migratePrivateStorage(); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/MessageFrame.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/MessageFrame.java index 411f7294b79..f5e80cb7c73 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/MessageFrame.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/MessageFrame.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.Gas; +import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.Log; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; @@ -229,6 +230,9 @@ public enum Type { private final Boolean isPersistingState; private Optional revertReason; + // Privacy Execution Environment fields. + private final Hash transactionHash; + // Miscellaneous fields. private final EnumSet exceptionalHaltReasons = EnumSet.noneOf(ExceptionalHaltReason.class); @@ -264,6 +268,7 @@ private MessageFrame( final Address miningBeneficiary, final BlockHashLookup blockHashLookup, final Boolean isPersistingState, + final Hash transactionHash, final Optional revertReason, final int maxStackSize) { this.type = type; @@ -299,6 +304,7 @@ private MessageFrame( this.completer = completer; this.miningBeneficiary = miningBeneficiary; this.isPersistingState = isPersistingState; + this.transactionHash = transactionHash; this.revertReason = revertReason; } @@ -960,6 +966,15 @@ public Boolean isPersistingState() { return isPersistingState; } + /** + * Returns the transaction hash of the transaction being processed + * + * @return the transaction hash of the transaction being processed + */ + public Hash getTransactionHash() { + return transactionHash; + } + public void setCurrentOperation(final Operation currentOperation) { this.currentOperation = currentOperation; } @@ -1006,6 +1021,7 @@ public static class Builder { private Address miningBeneficiary; private BlockHashLookup blockHashLookup; private Boolean isPersistingState = false; + private Hash transactionHash; private Optional reason = Optional.empty(); public Builder type(final Type type) { @@ -1124,6 +1140,11 @@ public Builder isPersistingState(final Boolean isPersistingState) { return this; } + public Builder transactionHash(final Hash transactionHash) { + this.transactionHash = transactionHash; + return this; + } + public Builder reason(final Bytes reason) { this.reason = Optional.ofNullable(reason); return this; @@ -1179,6 +1200,7 @@ public MessageFrame build() { miningBeneficiary, blockHashLookup, isPersistingState, + transactionHash, reason, maxStackSize); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java index 13f416c366b..5f8d7a1be69 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.crypto.SECP256K1; import org.hyperledger.besu.crypto.SecureRandomProvider; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -226,9 +227,9 @@ public Block genesisBlock(final BlockOptions options) { public Block block(final BlockOptions options) { final long blockNumber = options.getBlockNumber(positiveLong()); - final BlockHeader header = header(blockNumber, options); final BlockBody body = blockNumber == BlockHeader.GENESIS_BLOCK_NUMBER ? BlockBody.empty() : body(options); + final BlockHeader header = header(blockNumber, body, options); return new Block(header, body); } @@ -248,30 +249,34 @@ public Block nextBlock(final Block afterBlock) { return block(options); } + public BlockHeader header(final long blockNumber, final BlockBody blockBody) { + return header(blockNumber, blockBody, new BlockOptions()); + } + public BlockHeader header(final long blockNumber) { - return header(blockNumber, blockOptionsSupplier.get()); + return header(blockNumber, body(), blockOptionsSupplier.get()); } public BlockHeader header() { - return header(positiveLong(), blockOptionsSupplier.get()); + return header(positiveLong(), body(), blockOptionsSupplier.get()); } - public BlockHeader header(final long number, final BlockOptions options) { + public BlockHeader header(final long number, final BlockBody body, final BlockOptions options) { final int gasLimit = random.nextInt() & Integer.MAX_VALUE; final int gasUsed = Math.max(0, gasLimit - 1); final long blockNonce = random.nextLong(); return BlockHeaderBuilder.create() .parentHash(options.getParentHash(hash())) - .ommersHash(hash()) + .ommersHash(BodyValidation.ommersHash(body.getOmmers())) .coinbase(address()) .stateRoot(options.getStateRoot(hash())) - .transactionsRoot(hash()) - .receiptsRoot(hash()) - .logsBloom(logsBloom()) + .transactionsRoot(BodyValidation.transactionsRoot(body.getTransactions())) + .receiptsRoot(options.getReceiptsRoot(hash())) + .logsBloom(options.getLogsBloom(logsBloom())) .difficulty(options.getDifficulty(Difficulty.of(uint256(4)))) .number(number) .gasLimit(gasLimit) - .gasUsed(gasUsed) + .gasUsed(options.getGasUsed(gasUsed)) .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) .extraData(options.getExtraData(bytes32())) .mixHash(hash()) @@ -286,37 +291,41 @@ public BlockBody body() { public BlockBody body(final BlockOptions options) { final List ommers = new ArrayList<>(); - final int ommerCount = random.nextInt(3); - for (int i = 0; i < ommerCount; i++) { - ommers.add(header()); + if (options.hasOmmers()) { + final int ommerCount = random.nextInt(3); + for (int i = 0; i < ommerCount; i++) { + ommers.add(ommer()); + } } final List defaultTxs = new ArrayList<>(); - defaultTxs.add(transaction()); - defaultTxs.add(transaction()); + if (options.hasTransactions()) { + defaultTxs.add(transaction()); + defaultTxs.add(transaction()); + } return new BlockBody(options.getTransactions(defaultTxs), ommers); } - public Transaction transaction(final Bytes payload) { - return Transaction.builder() - .nonce(positiveLong()) - .gasPrice(Wei.wrap(bytes32())) - .gasLimit(positiveLong()) - .to(address()) - .value(Wei.wrap(bytes32())) - .payload(payload) - .chainId(BigInteger.ONE) - .signAndBuild(generateKeyPair()); + private BlockHeader ommer() { + return header(positiveLong(), body(BlockOptions.create().hasOmmers(false))); } public Transaction transaction() { + return transaction(bytes32()); + } + + public Transaction transaction(final Bytes payload) { + return transaction(payload, address()); + } + + public Transaction transaction(final Bytes payload, final Address to) { return Transaction.builder() .nonce(positiveLong()) .gasPrice(Wei.wrap(bytes32())) .gasLimit(positiveLong()) - .to(address()) + .to(to) .value(Wei.wrap(bytes32())) - .payload(bytes32()) + .payload(payload) .chainId(BigInteger.ONE) .signAndBuild(generateKeyPair()); } @@ -496,8 +505,14 @@ public static class BlockOptions { private Optional stateRoot = Optional.empty(); private Optional difficulty = Optional.empty(); private final List transactions = new ArrayList<>(); + private final List ommers = new ArrayList<>(); private Optional extraData = Optional.empty(); private Optional blockHeaderFunctions = Optional.empty(); + private Optional receiptsRoot = Optional.empty(); + private Optional gasUsed = Optional.empty(); + private Optional logsBloom = Optional.empty(); + private boolean hasOmmers = true; + private boolean hasTransactions = true; public static BlockOptions create() { return new BlockOptions(); @@ -507,6 +522,10 @@ public List getTransactions(final List defaultValue) { return transactions.isEmpty() ? defaultValue : transactions; } + public List getOmmers(final List defaultValue) { + return ommers.isEmpty() ? defaultValue : ommers; + } + public long getBlockNumber(final long defaultValue) { return blockNumber.orElse(defaultValue); } @@ -531,11 +550,36 @@ public BlockHeaderFunctions getBlockHeaderFunctions(final BlockHeaderFunctions d return blockHeaderFunctions.orElse(defaultValue); } + public Hash getReceiptsRoot(final Hash defaultValue) { + return receiptsRoot.orElse(defaultValue); + } + + public long getGasUsed(final long defaultValue) { + return gasUsed.orElse(defaultValue); + } + + public LogsBloomFilter getLogsBloom(final LogsBloomFilter defaultValue) { + return logsBloom.orElse(defaultValue); + } + + public boolean hasTransactions() { + return hasTransactions; + } + + public boolean hasOmmers() { + return hasOmmers; + } + public BlockOptions addTransaction(final Transaction... tx) { transactions.addAll(Arrays.asList(tx)); return this; } + public BlockOptions addOmmers(final BlockHeader... headers) { + ommers.addAll(Arrays.asList(headers)); + return this; + } + public BlockOptions addTransaction(final Collection txs) { return addTransaction(txs.toArray(new Transaction[] {})); } @@ -569,5 +613,30 @@ public BlockOptions setBlockHeaderFunctions(final BlockHeaderFunctions blockHead this.blockHeaderFunctions = Optional.of(blockHeaderFunctions); return this; } + + public BlockOptions setReceiptsRoot(final Hash receiptsRoot) { + this.receiptsRoot = Optional.of(receiptsRoot); + return this; + } + + public BlockOptions setGasUsed(final long gasUsed) { + this.gasUsed = Optional.of(gasUsed); + return this; + } + + public BlockOptions setLogsBloom(final LogsBloomFilter logsBloom) { + this.logsBloom = Optional.of(logsBloom); + return this; + } + + public BlockOptions hasTransactions(final boolean hasTransactions) { + this.hasTransactions = hasTransactions; + return this; + } + + public BlockOptions hasOmmers(final boolean hasOmmers) { + this.hasOmmers = hasOmmers; + return this; + } } } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java index cf7bec9b6a3..cf7c2209ba5 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.core; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateKeyValueStorage; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; @@ -54,6 +56,11 @@ public PrivateStateStorage createPrivateStateStorage() { return new PrivateStateKeyValueStorage(new InMemoryKeyValueStorage()); } + @Override + public LegacyPrivateStateStorage createLegacyPrivateStateStorage() { + return new LegacyPrivateStateKeyValueStorage(new InMemoryKeyValueStorage()); + } + @Override public int getFactoryVersion() { return 1; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryStorageProvider.java index 525ea1e4e27..5c8b271f677 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryStorageProvider.java @@ -20,6 +20,8 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; @@ -59,6 +61,10 @@ public static MutableWorldState createInMemoryWorldState() { provider.createWorldStateStorage(), provider.createWorldStatePreimageStorage()); } + public static PrivateStateStorage createInMemoryPrivateStateStorage() { + return new PrivateStateKeyValueStorage(new InMemoryKeyValueStorage()); + } + @Override public BlockchainStorage createBlockchainStorage(final ProtocolSchedule protocolSchedule) { return new KeyValueStoragePrefixedKeyBlockchainStorage( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java new file mode 100644 index 00000000000..8d09c9d6d6f --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java @@ -0,0 +1,77 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import java.util.Collections; + +import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; +import org.junit.Test; + +public class PrivacyBlockProcessorTest { + + private PrivacyBlockProcessor privacyBlockProcessor; + private PrivateStateStorage privateStateStorage; + private AbstractBlockProcessor blockProcessor; + + @Before + public void setUp() { + blockProcessor = mock(AbstractBlockProcessor.class); + privateStateStorage = new PrivateStateKeyValueStorage(new InMemoryKeyValueStorage()); + this.privacyBlockProcessor = new PrivacyBlockProcessor(blockProcessor, privateStateStorage); + } + + @Test + public void mustCopyPreviousPrivacyGroupBlockHeadMap() { + final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); + final Blockchain blockchain = mock(Blockchain.class); + final MutableWorldState mutableWorldState = mock(MutableWorldState.class); + final PrivacyGroupHeadBlockMap expected = + new PrivacyGroupHeadBlockMap(Collections.singletonMap(Bytes32.ZERO, Hash.EMPTY)); + final Block firstBlock = blockDataGenerator.block(); + final Block secondBlock = + blockDataGenerator.block( + BlockDataGenerator.BlockOptions.create().setParentHash(firstBlock.getHash())); + privacyBlockProcessor.processBlock(blockchain, mutableWorldState, firstBlock); + privateStateStorage + .updater() + .putPrivacyGroupHeadBlockMap(firstBlock.getHash(), expected) + .commit(); + privacyBlockProcessor.processBlock(blockchain, mutableWorldState, secondBlock); + assertThat(privateStateStorage.getPrivacyGroupHeadBlockMap(secondBlock.getHash())) + .contains(expected); + verify(blockProcessor) + .processBlock( + blockchain, + mutableWorldState, + firstBlock.getHeader(), + firstBlock.getBody().getTransactions(), + firstBlock.getBody().getOmmers()); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractTest.java index f841cb634ff..1acd06d3793 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContractTest.java @@ -26,6 +26,8 @@ import org.hyperledger.besu.enclave.types.ReceiveResponse; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.Log; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; @@ -33,6 +35,7 @@ import org.hyperledger.besu.ethereum.mainnet.SpuriousDragonGasCalculator; import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.MessageFrame; @@ -57,6 +60,7 @@ public class PrivacyPrecompiledContractTest { private final String actual = "Test String"; private final Bytes key = Bytes.wrap(actual.getBytes(UTF_8)); private MessageFrame messageFrame; + private Blockchain blockchain; private final String DEFAULT_OUTPUT = "0x01"; private final WorldStateArchive worldStateArchive = mock(WorldStateArchive.class); final PrivateStateStorage privateStateStorage = mock(PrivateStateStorage.class); @@ -107,27 +111,46 @@ public void setUp() { when(worldStateArchive.getMutable(any())).thenReturn(Optional.of(mutableWorldState)); final PrivateStateStorage.Updater storageUpdater = mock(PrivateStateStorage.Updater.class); - when(storageUpdater.putLatestStateRoot(nullable(Bytes32.class), any())) + when(privateStateStorage.getPrivacyGroupHeadBlockMap(any())) + .thenReturn(Optional.of(PrivacyGroupHeadBlockMap.EMPTY)); + when(privateStateStorage.getPrivateBlockMetadata(any(), any())).thenReturn(Optional.empty()); + when(storageUpdater.putPrivateBlockMetadata( + nullable(Bytes32.class), nullable(Bytes32.class), any())) .thenReturn(storageUpdater); - when(storageUpdater.putTransactionLogs(nullable(Bytes32.class), any())) + when(storageUpdater.putPrivacyGroupHeadBlockMap(nullable(Bytes32.class), any())) .thenReturn(storageUpdater); - when(storageUpdater.putTransactionResult(nullable(Bytes32.class), any())) + when(storageUpdater.putTransactionReceipt( + nullable(Bytes32.class), nullable(Bytes32.class), any())) .thenReturn(storageUpdater); when(privateStateStorage.updater()).thenReturn(storageUpdater); messageFrame = mock(MessageFrame.class); + blockchain = mock(Blockchain.class); + final BlockDataGenerator blockGenerator = new BlockDataGenerator(); + final Block genesis = blockGenerator.genesisBlock(); + final Block block = + blockGenerator.block( + new BlockDataGenerator.BlockOptions().setParentHash(genesis.getHeader().getHash())); + when(blockchain.getGenesisBlock()).thenReturn(genesis); + when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); + when(blockchain.getBlockByHash(genesis.getHash())).thenReturn(Optional.of(genesis)); + when(messageFrame.getBlockchain()).thenReturn(blockchain); + when(messageFrame.getBlockHeader()).thenReturn(block.getHeader()); } @Test - public void testPayloadFoundInEnaclave() { - Enclave enclave = mock(Enclave.class); - PrivacyPrecompiledContract contract = + public void testPayloadFoundInEnclave() { + final Enclave enclave = mock(Enclave.class); + final PrivacyPrecompiledContract contract = new PrivacyPrecompiledContract( new SpuriousDragonGasCalculator(), enclave, worldStateArchive, privateStateStorage); contract.setPrivateTransactionProcessor(mockPrivateTxProcessor()); final ReceiveResponse response = - new ReceiveResponse(VALID_PRIVATE_TRANSACTION_RLP_BASE64, "", null); + new ReceiveResponse( + VALID_PRIVATE_TRANSACTION_RLP_BASE64, + "8lDVI66RZHIrBsolz6Kn88Rd+WsJ4hUjb4hsh29xW/o=", + null); when(enclave.receive(any(String.class))).thenReturn(response); final Bytes actual = contract.compute(key, messageFrame); @@ -137,9 +160,9 @@ public void testPayloadFoundInEnaclave() { @Test public void testPayloadNotFoundInEnclave() { - Enclave enclave = mock(Enclave.class); + final Enclave enclave = mock(Enclave.class); - PrivacyPrecompiledContract contract = + final PrivacyPrecompiledContract contract = new PrivacyPrecompiledContract( new SpuriousDragonGasCalculator(), enclave, worldStateArchive, privateStateStorage); @@ -151,9 +174,9 @@ public void testPayloadNotFoundInEnclave() { @Test(expected = RuntimeException.class) public void testEnclaveDown() { - Enclave enclave = mock(Enclave.class); + final Enclave enclave = mock(Enclave.class); - PrivacyPrecompiledContract contract = + final PrivacyPrecompiledContract contract = new PrivacyPrecompiledContract( new SpuriousDragonGasCalculator(), enclave, worldStateArchive, privateStateStorage); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProviderTest.java new file mode 100644 index 00000000000..fb578cb5eab --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/ChainHeadPrivateNonceProviderTest.java @@ -0,0 +1,102 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.WorldState; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; +import org.junit.Test; + +public class ChainHeadPrivateNonceProviderTest { + private static final Bytes32 PRIVACY_GROUP_ID = + Bytes32.wrap(Bytes.fromBase64String("DyAOiF/ynpc+JXa2YAGB0bCitSlOMNm+ShmB/7M6C4w=")); + private static final Address ADDRESS = Address.fromHexString("55");; + + private Account account; + private WorldState worldState; + private ChainHeadPrivateNonceProvider privateNonceProvider; + private WorldStateArchive privateWorldStateArchive; + private PrivateStateRootResolver privateStateRootResolver; + + @Before + public void setUp() { + final BlockDataGenerator gen = new BlockDataGenerator(); + + final Blockchain blockchain = mock(Blockchain.class); + when(blockchain.getChainHeadHeader()).thenReturn(gen.header()); + + account = mock(Account.class); + worldState = mock(WorldState.class); + + privateStateRootResolver = mock(PrivateStateRootResolver.class); + privateWorldStateArchive = mock(WorldStateArchive.class); + privateNonceProvider = + new ChainHeadPrivateNonceProvider( + blockchain, privateStateRootResolver, privateWorldStateArchive); + } + + @Test + public void determineNonceForPrivacyGroupRequestWhenPrivateStateDoesNotExist() { + when(privateStateRootResolver.resolveLastStateRoot(any(Bytes32.class), any(Hash.class))) + .thenReturn(Hash.ZERO); + when(privateWorldStateArchive.get(any(Hash.class))).thenReturn(Optional.empty()); + + final long nonce = privateNonceProvider.getNonce(ADDRESS, PRIVACY_GROUP_ID); + + assertThat(nonce).isEqualTo(Account.DEFAULT_NONCE); + } + + @Test + public void determineNonceForPrivacyGroupRequestWhenAccountExists() { + when(account.getNonce()).thenReturn(4L); + when(worldState.get(any(Address.class))).thenReturn(account); + when(privateStateRootResolver.resolveLastStateRoot(any(Bytes32.class), any(Hash.class))) + .thenReturn(Hash.ZERO); + when(privateWorldStateArchive.get(any(Hash.class))).thenReturn(Optional.of(worldState)); + + final long nonce = privateNonceProvider.getNonce(ADDRESS, PRIVACY_GROUP_ID); + + assertThat(nonce).isEqualTo(4L); + } + + @Test + public void determineNonceForPrivacyGroupRequestWhenAccountDoesNotExist() { + when(privateStateRootResolver.resolveLastStateRoot(any(Bytes32.class), any(Hash.class))) + .thenReturn(Hash.ZERO); + when(privateWorldStateArchive.get(any(Hash.class))).thenReturn(Optional.of(worldState)); + when(account.getNonce()).thenReturn(4L); + + final long nonce = privateNonceProvider.getNonce(ADDRESS, PRIVACY_GROUP_ID); + + assertThat(nonce).isEqualTo(Account.DEFAULT_NONCE); + verifyNoInteractions(account); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyControllerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyControllerTest.java index 6c3a7e54aae..3522edc7dd8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyControllerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/DefaultPrivacyControllerTest.java @@ -27,7 +27,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.crypto.SECP256K1; @@ -38,19 +37,14 @@ import org.hyperledger.besu.enclave.types.PrivacyGroup.Type; import org.hyperledger.besu.enclave.types.ReceiveResponse; import org.hyperledger.besu.enclave.types.SendResponse; -import org.hyperledger.besu.ethereum.core.Account; import org.hyperledger.besu.ethereum.core.Address; -import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.Log; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.mainnet.TransactionValidator.TransactionInvalidReason; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.privacy.markertransaction.FixedKeySigningPrivateMarkerTransactionFactory; -import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.transaction.CallParameter; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.orion.testutil.OrionKeyUtils; import java.math.BigInteger; @@ -59,6 +53,7 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.io.Base64; import org.junit.Before; import org.junit.Test; @@ -87,12 +82,8 @@ public class DefaultPrivacyControllerTest { private PrivacyController brokenPrivacyController; private PrivateTransactionValidator privateTransactionValidator; private Enclave enclave; - private Account account; private String enclavePublicKey; - private PrivateStateStorage privateStateStorage; - private WorldStateArchive worldStateArchive; - private MutableWorldState mutableWorldState; - private final Hash hash = mock(Hash.class); + private PrivateNonceProvider privateNonceProvider; private static final Transaction PUBLIC_TRANSACTION = Transaction.builder() @@ -109,7 +100,8 @@ public class DefaultPrivacyControllerTest { private Enclave mockEnclave() { final Enclave mockEnclave = mock(Enclave.class); final SendResponse response = new SendResponse(TRANSACTION_KEY); - final ReceiveResponse receiveResponse = new ReceiveResponse(new byte[0], "mock", null); + final ReceiveResponse receiveResponse = + new ReceiveResponse(new byte[0], PRIVACY_GROUP_ID, null); when(mockEnclave.send(anyString(), anyString(), anyList())).thenReturn(response); when(mockEnclave.send(anyString(), anyString(), anyString())).thenReturn(response); when(mockEnclave.receive(any(), any())).thenReturn(receiveResponse); @@ -141,14 +133,8 @@ private PrivateTransactionSimulator mockPrivateTransactionSimulator() { @Before public void setUp() throws Exception { - privateStateStorage = mock(PrivateStateStorage.class); - when(privateStateStorage.getLatestStateRoot(any(Bytes.class))).thenReturn(Optional.of(hash)); - worldStateArchive = mock(WorldStateArchive.class); - account = mock(Account.class); - when(account.getNonce()).thenReturn(1L); - mutableWorldState = mock(MutableWorldState.class); - when(worldStateArchive.getMutable(any(Hash.class))).thenReturn(Optional.of(mutableWorldState)); - when(mutableWorldState.get(any(Address.class))).thenReturn(account); + privateNonceProvider = mock(ChainHeadPrivateNonceProvider.class); + when(privateNonceProvider.getNonce(any(), any())).thenReturn(1L); enclavePublicKey = OrionKeyUtils.loadKey("orion_key_0.pub"); privateTransactionValidator = mockPrivateTransactionValidator(); @@ -159,26 +145,23 @@ public void setUp() throws Exception { privacyController = new DefaultPrivacyController( enclave, - privateStateStorage, - worldStateArchive, privateTransactionValidator, new FixedKeySigningPrivateMarkerTransactionFactory( Address.DEFAULT_PRIVACY, (address) -> 0, KEY_PAIR), - privateTransactionSimulator); + privateTransactionSimulator, + privateNonceProvider); brokenPrivacyController = new DefaultPrivacyController( brokenMockEnclave(), - privateStateStorage, - worldStateArchive, privateTransactionValidator, new FixedKeySigningPrivateMarkerTransactionFactory( Address.DEFAULT_PRIVACY, (address) -> 0, KEY_PAIR), - privateTransactionSimulator); + privateTransactionSimulator, + privateNonceProvider); } @Test public void sendsValidLegacyTransaction() { - final PrivateTransaction transaction = buildLegacyPrivateTransaction(1); final String enclaveKey = privacyController.sendTransaction(transaction, ENCLAVE_PUBLIC_KEY); @@ -201,7 +184,6 @@ public void sendsValidLegacyTransaction() { @Test public void sendValidBesuTransaction() { - final PrivateTransaction transaction = buildBesuPrivateTransaction(1); final String enclaveKey = privacyController.sendTransaction(transaction, ENCLAVE_PUBLIC_KEY); @@ -329,20 +311,21 @@ public void determinesNonceForEeaRequest() { final long reportedNonce = 8L; final PrivacyGroup[] returnedGroups = new PrivacyGroup[] { - new PrivacyGroup("Group1", Type.LEGACY, "Group1_Name", "Group1_Desc", emptyList()), + new PrivacyGroup( + PRIVACY_GROUP_ID, Type.LEGACY, "Group1_Name", "Group1_Desc", emptyList()), }; when(enclave.findPrivacyGroup(any())).thenReturn(returnedGroups); - when(account.getNonce()).thenReturn(8L); + when(privateNonceProvider.getNonce(any(Address.class), any(Bytes32.class))).thenReturn(8L); final long nonce = privacyController.determineEeaNonce( - "privateFrom", new String[] {"first", "second"}, address, ENCLAVE_PUBLIC_KEY); + ENCLAVE_PUBLIC_KEY, new String[] {ENCLAVE_KEY2}, address, ENCLAVE_PUBLIC_KEY); assertThat(nonce).isEqualTo(reportedNonce); verify(enclave) .findPrivacyGroup( - argThat((m) -> m.containsAll(newArrayList("first", "second", "privateFrom")))); + argThat((m) -> m.containsAll(newArrayList(ENCLAVE_PUBLIC_KEY, ENCLAVE_KEY2)))); } @Test @@ -381,62 +364,6 @@ public void determineNonceForEeaRequestWithMoreThanOneMatchingGroupThrowsExcepti "privateFrom", new String[] {"first", "second"}, address, ENCLAVE_PUBLIC_KEY)); } - @Test - public void determineNonceForPrivacyGroupRequestWhenAccountExists() { - final Address address = Address.fromHexString("55"); - - when(account.getNonce()).thenReturn(4L); - - final long nonce = privacyController.determineBesuNonce(address, "Group1", ENCLAVE_PUBLIC_KEY); - - assertThat(nonce).isEqualTo(4L); - verify(privateStateStorage).getLatestStateRoot(Base64.decode("Group1")); - verify(worldStateArchive).getMutable(hash); - verify(mutableWorldState).get(address); - } - - @Test - public void determineNonceForPrivacyGroupRequestWhenPrivateStateDoesNotExist() { - final Address address = Address.fromHexString("55"); - - when(privateStateStorage.getLatestStateRoot(Base64.decode("Group1"))) - .thenReturn(Optional.empty()); - - final long nonce = privacyController.determineBesuNonce(address, "Group1", ENCLAVE_PUBLIC_KEY); - - assertThat(nonce).isEqualTo(Account.DEFAULT_NONCE); - verifyNoInteractions(worldStateArchive, mutableWorldState, account); - } - - @Test - public void determineNonceForPrivacyGroupRequestWhenWorldStateDoesNotExist() { - final Address address = Address.fromHexString("55"); - - when(privateStateStorage.getLatestStateRoot(Base64.decode("Group1"))) - .thenReturn(Optional.of(hash)); - when(worldStateArchive.getMutable(hash)).thenReturn(Optional.empty()); - - final long nonce = privacyController.determineBesuNonce(address, "Group1", ENCLAVE_PUBLIC_KEY); - - assertThat(nonce).isEqualTo(Account.DEFAULT_NONCE); - verifyNoInteractions(mutableWorldState, account); - } - - @Test - public void determineNonceForPrivacyGroupRequestWhenAccountDoesNotExist() { - final Address address = Address.fromHexString("55"); - - when(privateStateStorage.getLatestStateRoot(Base64.decode("Group1"))) - .thenReturn(Optional.of(hash)); - when(worldStateArchive.getMutable(hash)).thenReturn(Optional.of(mutableWorldState)); - when(mutableWorldState.get(address)).thenReturn(null); - - final long nonce = privacyController.determineBesuNonce(address, "Group1", ENCLAVE_PUBLIC_KEY); - - assertThat(nonce).isEqualTo(Account.DEFAULT_NONCE); - verifyNoInteractions(account); - } - @Test public void simulatingPrivateTransactionWorks() { final CallParameter callParameter = mock(CallParameter.class); @@ -460,8 +387,8 @@ private static PrivateTransaction buildLegacyPrivateTransaction(final long nonce private static PrivateTransaction buildBesuPrivateTransaction(final long nonce) { return buildPrivateTransaction(nonce) - .privateFrom(Base64.decode(ENCLAVE_PUBLIC_KEY)) - .privacyGroupId(Base64.decode(PRIVACY_GROUP_ID)) + .privateFrom(Bytes.fromBase64String(ENCLAVE_PUBLIC_KEY)) + .privacyGroupId(Bytes.fromBase64String(PRIVACY_GROUP_ID)) .signAndBuild(KEY_PAIR); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolverTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolverTest.java new file mode 100644 index 00000000000..ac0e583e33b --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/PrivateStateRootResolverTest.java @@ -0,0 +1,180 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.InMemoryStorageProvider; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateBlockMetadata; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateTransactionMetadata; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PrivateStateRootResolverTest { + + private static final BlockDataGenerator BLOCK_GENERATOR = new BlockDataGenerator(); + private static MutableBlockchain BLOCKCHAIN; + + private static final Hash pmt1StateHash = + Hash.fromHexString("0x37659019840d6e04e740614d1ad93d62f0d9d7cc423b2178189f391db602a6a6"); + private static final Hash pmt2StateHash = + Hash.fromHexString("0x12d390c87b405e91523b5829002bf90095005366eb9aa168ff8a18540902e410"); + private static final Bytes32 privacyGroupId = + Bytes32.wrap(Bytes.fromBase64String("A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo=")); + private static final Bytes32 failingPrivacyGroupId = + Bytes32.wrap(Bytes.fromBase64String("Ko2bVqD+nNlNYL5EE7y3IdOnviftjiizpjRt+HTuFBs=")); + + private PrivateStateStorage privateStateStorage; + + @BeforeClass + public static void setupClass() { + BLOCKCHAIN = InMemoryStorageProvider.createInMemoryBlockchain(BLOCK_GENERATOR.genesisBlock()); + for (int i = 1; i <= 69; i++) { + final BlockDataGenerator.BlockOptions options = + new BlockDataGenerator.BlockOptions() + .setBlockNumber(i) + .setParentHash(BLOCKCHAIN.getBlockHashByNumber(i - 1).get()); + final Block block = BLOCK_GENERATOR.block(options); + final List receipts = BLOCK_GENERATOR.receipts(block); + BLOCKCHAIN.appendBlock(block, receipts); + } + } + + @Before + public void setUp() { + privateStateStorage = InMemoryStorageProvider.createInMemoryPrivateStateStorage(); + } + + @Test + public void mustResolveEmptyStateRootWhenChainHeadIsNotCommitted() { + final BlockDataGenerator.BlockOptions options = + new BlockDataGenerator.BlockOptions() + .setBlockNumber(BLOCKCHAIN.getChainHeadBlockNumber()) + .setParentHash(BLOCKCHAIN.getChainHeadHash()); + final Block block = BLOCK_GENERATOR.block(options); + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + assertThat( + privateStateRootResolver.resolveLastStateRoot( + privacyGroupId, block.getHeader().getHash())) + .isEqualTo(PrivateStateRootResolver.EMPTY_ROOT_HASH); + } + + @Test + public void resolveEmptyRootHashWhenNoCommitmentForPrivacyGroupExists() { + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + assertThat( + privateStateRootResolver.resolveLastStateRoot( + privacyGroupId, BLOCKCHAIN.getChainHeadHeader().getHash())) + .isEqualTo(PrivateStateRootResolver.EMPTY_ROOT_HASH); + } + + @Test + public void resolveExpectedRootHashWhenCommitmentForPrivacyGroupExists() { + final PrivateStateStorage.Updater updater = privateStateStorage.updater(); + updater.putPrivateBlockMetadata( + BLOCKCHAIN.getBlockByNumber(16).get().getHash(), + Bytes32.wrap(privacyGroupId), + new PrivateBlockMetadata( + Collections.singletonList( + new PrivateTransactionMetadata( + BLOCK_GENERATOR.transaction().getHash(), pmt1StateHash)))); + updater.putPrivacyGroupHeadBlockMap( + BLOCKCHAIN.getChainHeadHash(), + new PrivacyGroupHeadBlockMap( + Collections.singletonMap( + Bytes32.wrap(privacyGroupId), BLOCKCHAIN.getBlockByNumber(16).get().getHash()))); + updater.commit(); + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + assertThat( + privateStateRootResolver.resolveLastStateRoot( + privacyGroupId, BLOCKCHAIN.getChainHeadHash())) + .isEqualTo(pmt1StateHash); + } + + @Test + public void resolveCorrectRootHashWhenMultipleCommitmentsExistForPrivacyGroup() { + final PrivateStateStorage.Updater updater = privateStateStorage.updater(); + updater.putPrivateBlockMetadata( + BLOCKCHAIN.getBlockByNumber(16).get().getHash(), + Bytes32.wrap(privacyGroupId), + new PrivateBlockMetadata( + Collections.singletonList( + new PrivateTransactionMetadata( + BLOCK_GENERATOR.transaction().getHash(), pmt1StateHash)))); + updater.putPrivateBlockMetadata( + BLOCKCHAIN.getBlockByNumber(16).get().getHash(), + Bytes32.wrap(failingPrivacyGroupId), + new PrivateBlockMetadata( + Collections.singletonList( + new PrivateTransactionMetadata( + BLOCK_GENERATOR.transaction().getHash(), pmt2StateHash)))); + updater.putPrivacyGroupHeadBlockMap( + BLOCKCHAIN.getChainHeadHash(), + new PrivacyGroupHeadBlockMap( + Collections.singletonMap( + Bytes32.wrap(privacyGroupId), BLOCKCHAIN.getBlockByNumber(16).get().getHash()))); + updater.commit(); + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + assertThat( + privateStateRootResolver.resolveLastStateRoot( + privacyGroupId, BLOCKCHAIN.getChainHeadHash())) + .isEqualTo(pmt1StateHash); + } + + @Test + public void resolveLatestRootHashWhenMultipleCommitmentsForTheSamePrivacyGroupExist() { + final PrivateStateStorage.Updater updater = privateStateStorage.updater(); + updater.putPrivateBlockMetadata( + BLOCKCHAIN.getBlockByNumber(16).get().getHash(), + Bytes32.wrap(privacyGroupId), + new PrivateBlockMetadata( + Arrays.asList( + new PrivateTransactionMetadata( + BLOCK_GENERATOR.transaction().getHash(), pmt1StateHash), + new PrivateTransactionMetadata( + BLOCK_GENERATOR.transaction().getHash(), pmt2StateHash)))); + updater.putPrivacyGroupHeadBlockMap( + BLOCKCHAIN.getChainHeadHash(), + new PrivacyGroupHeadBlockMap( + Collections.singletonMap( + Bytes32.wrap(privacyGroupId), BLOCKCHAIN.getBlockByNumber(16).get().getHash()))); + updater.commit(); + final PrivateStateRootResolver privateStateRootResolver = + new PrivateStateRootResolver(privateStateStorage); + assertThat( + privateStateRootResolver.resolveLastStateRoot( + privacyGroupId, BLOCKCHAIN.getChainHeadHash())) + .isEqualTo(pmt2StateHash); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java new file mode 100644 index 00000000000..ce7fdd12311 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java @@ -0,0 +1,55 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_0_0; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; + +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import org.junit.Before; +import org.junit.Test; + +public class PrivateStateKeyValueStorageTest { + + private PrivateStateKeyValueStorage storage; + + @Before + public void before() { + storage = new PrivateStateKeyValueStorage(new InMemoryKeyValueStorage()); + } + + @Test + public void databaseWithoutVersionShouldReturn1_0_x() { + assertThat(storage.getSchemaVersion()).isEqualTo(SCHEMA_VERSION_1_0_0); + } + + @Test + public void databaseSetVersionShouldSetVersion() { + storage.updater().putDatabaseVersion(123).commit(); + assertThat(storage.getSchemaVersion()).isEqualTo(123); + } + + @Test + public void schemaVersion1_0_xHasCorrectValue() { + assertThat(SCHEMA_VERSION_1_0_0).isEqualTo(1); + } + + @Test + public void schemaVersion1_4_xHasCorrectValue() { + assertThat(SCHEMA_VERSION_1_4_0).isEqualTo(2); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java new file mode 100644 index 00000000000..51e57a64425 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java @@ -0,0 +1,315 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver.EMPTY_ROOT_HASH; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_0_0; +import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; +import static org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateTransactionDataFixture.privacyMarkerTransaction; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.SECP256K1.KeyPair; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.TransactionReceiptFactory; +import org.hyperledger.besu.ethereum.mainnet.MiningBeneficiaryCalculator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.TransactionProcessor; +import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; +import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; +import org.hyperledger.besu.ethereum.privacy.storage.PrivacyGroupHeadBlockMap; +import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@SuppressWarnings({"unchecked", "rawtypes"}) +public class PrivateStorageMigrationTest { + + private static final String PRIVACY_GROUP_ID = "tJw12cPM6EZRF5zfHv2zLePL0cqlaDjLn0x1T/V0yzE="; + public static final Bytes32 PRIVACY_GROUP_BYTES = + Bytes32.wrap(Bytes.fromBase64String(PRIVACY_GROUP_ID)); + private static final Address PRIVACY_ADDRESS = Address.DEFAULT_PRIVACY; + private static final String TRANSACTION_KEY = "93Ky7lXwFkMc7+ckoFgUMku5bpr9tz4zhmWmk9RlNng="; + + @Mock private Blockchain blockchain; + @Mock private ProtocolSchedule protocolSchedule; + @Mock private ProtocolSpec protocolSpec; + @Mock private WorldStateArchive publicWorldStateArchive; + @Mock private MutableWorldState publicMutableWorldState; + @Mock private LegacyPrivateStateStorage legacyPrivateStateStorage; + @Mock private TransactionProcessor transactionProcessor; + @Mock private TransactionReceiptFactory transactionReceiptFactory; + @Mock private MiningBeneficiaryCalculator miningBeneficiaryCalculator; + @Mock private PrivateMigrationBlockProcessor privateMigrationBlockProcessor; + + private PrivateStateKeyValueStorage privateStateStorage; + private PrivateStateRootResolver privateStateRootResolver; + private PrivateStorageMigration migration; + + @Before + public void setUp() { + final KeyValueStorage kvStorage = new InMemoryKeyValueStorage(); + + privateStateStorage = new PrivateStateKeyValueStorage(kvStorage); + privateStateRootResolver = new PrivateStateRootResolver(privateStateStorage); + + lenient().when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(protocolSpec); + lenient().when(protocolSpec.getTransactionProcessor()).thenReturn(transactionProcessor); + lenient() + .when(protocolSpec.getTransactionReceiptFactory()) + .thenReturn(transactionReceiptFactory); + lenient().when(protocolSpec.getBlockReward()).thenReturn(Wei.ZERO); + lenient() + .when(protocolSpec.getMiningBeneficiaryCalculator()) + .thenReturn(miningBeneficiaryCalculator); + lenient().when(protocolSpec.isSkipZeroBlockRewards()).thenReturn(false); + + migration = + new PrivateStorageMigration( + blockchain, + PRIVACY_ADDRESS, + protocolSchedule, + publicWorldStateArchive, + privateStateStorage, + privateStateRootResolver, + legacyPrivateStateStorage, + (protocolSpec) -> privateMigrationBlockProcessor); + } + + @Test + public void privateGroupHeadBlocKMapIsCopiedFromPreviousBlocks() { + mockBlockchainWithZeroTransactions(); + + // create existing map at block hash 'zero' (pre-genesis) + final PrivacyGroupHeadBlockMap existingPgHeadMap = + createPrivacyGroupHeadBlockInitialMap(PRIVACY_GROUP_BYTES); + + migration.migratePrivateStorage(); + + // check that for every block we have the existing mapping + for (long i = 0; i <= blockchain.getChainHeadBlockNumber(); i++) { + final Optional pgHeadMapAfterMigration = + privateStateStorage.getPrivacyGroupHeadBlockMap( + blockchain.getBlockByNumber(i).get().getHash()); + + assertThat(pgHeadMapAfterMigration).isPresent().hasValue(existingPgHeadMap); + } + } + + @Test + public void successfulMigrationBumpsSchemaVersion() { + final Transaction privacyMarkerTransaction = createPrivacyMarkerTransaction(); + mockBlockchainWithPrivacyMarkerTransaction(privacyMarkerTransaction); + assertThat(privateStateStorage.getSchemaVersion()).isEqualTo(SCHEMA_VERSION_1_0_0); + + migration.migratePrivateStorage(); + + assertThat(privateStateStorage.getSchemaVersion()).isEqualTo(SCHEMA_VERSION_1_4_0); + } + + @Test + public void failedMigrationThrowsErrorAndDoesNotBumpSchemaVersion() { + final Transaction privacyMarkerTransaction = createPrivacyMarkerTransaction(); + mockBlockchainWithPrivacyMarkerTransaction(privacyMarkerTransaction); + createPrivacyGroupHeadBlockInitialMap(PRIVACY_GROUP_BYTES); + + // final state root won't match the legacy state root + when(legacyPrivateStateStorage.getLatestStateRoot(any())).thenReturn(Optional.of(Hash.ZERO)); + + assertThat(privateStateStorage.getSchemaVersion()).isEqualTo(SCHEMA_VERSION_1_0_0); + + assertThatThrownBy(() -> migration.migratePrivateStorage()) + .isInstanceOf(PrivateStorageMigrationException.class) + .hasMessageContaining("Inconsistent state root"); + + assertThat(privateStateStorage.getSchemaVersion()).isEqualTo(SCHEMA_VERSION_1_0_0); + } + + @Test + public void migrationInBlockchainWithZeroPMTsDoesNotReprocessAnyBlocks() { + mockBlockchainWithZeroTransactions(); + + migration.migratePrivateStorage(); + + verifyNoInteractions(privateMigrationBlockProcessor); + } + + @Test + public void migrationReprocessBlocksWithPMT() { + final Transaction privacyMarkerTransaction = createPrivacyMarkerTransaction(); + mockBlockchainWithPrivacyMarkerTransaction(privacyMarkerTransaction); + final Block blockWithPMT = blockchain.getBlockByNumber(1L).orElseThrow(); + + migration.migratePrivateStorage(); + + verify(privateMigrationBlockProcessor) + .processBlock( + any(), + any(), + eq(blockWithPMT.getHeader()), + eq(blockWithPMT.getBody().getTransactions()), + eq(blockWithPMT.getBody().getOmmers())); + } + + /* + When processing a block, we only need to process up to the last PTM in the block. + */ + @Test + public void migrationOnlyProcessRequiredTransactions() { + final List transactions = new ArrayList<>(); + transactions.add(publicTransaction()); + transactions.add(createPrivacyMarkerTransaction()); + transactions.add(publicTransaction()); + + mockBlockchainWithTransactionsInABlock(transactions); + + migration.migratePrivateStorage(); + + final ArgumentCaptor txsCaptor = ArgumentCaptor.forClass(List.class); + + verify(privateMigrationBlockProcessor) + .processBlock(any(), any(), any(), txsCaptor.capture(), any()); + + // won't process transaction after PMT, that's why we only process 2 txs + final List processedTxs = txsCaptor.getValue(); + assertThat(processedTxs).hasSize(2); + } + + private PrivacyGroupHeadBlockMap createPrivacyGroupHeadBlockInitialMap( + final Bytes32 privacyGroupBytes) { + final PrivacyGroupHeadBlockMap existingPgHeadMap = + new PrivacyGroupHeadBlockMap(Map.of(privacyGroupBytes, Hash.ZERO)); + privateStateStorage + .updater() + .putPrivacyGroupHeadBlockMap(Hash.ZERO, existingPgHeadMap) + .commit(); + return existingPgHeadMap; + } + + private Transaction createPrivacyMarkerTransaction() { + final Transaction privacyMarkerTransaction = privacyMarkerTransaction(TRANSACTION_KEY); + mockBlockchainWithPrivacyMarkerTransaction(privacyMarkerTransaction); + return privacyMarkerTransaction; + } + + private void mockBlockchainWithZeroTransactions() { + final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); + + final Block genesis = blockDataGenerator.genesisBlock(); + mockBlockInBlockchain(genesis); + + final BlockDataGenerator.BlockOptions options = + BlockDataGenerator.BlockOptions.create() + .setParentHash(genesis.getHash()) + .setBlockNumber(1) + .hasTransactions(false); + final Block block = blockDataGenerator.block(options); + mockBlockInBlockchain(block); + mockChainHeadInBlockchain(block); + + when(legacyPrivateStateStorage.getLatestStateRoot(any())) + .thenReturn(Optional.of(EMPTY_ROOT_HASH)); + } + + private void mockBlockchainWithPrivacyMarkerTransaction(final Transaction transaction) { + final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); + + final Block genesis = blockDataGenerator.genesisBlock(); + mockBlockInBlockchain(genesis); + + final BlockDataGenerator.BlockOptions options = + BlockDataGenerator.BlockOptions.create() + .setParentHash(genesis.getHash()) + .setBlockNumber(1) + .addTransaction(transaction); + final Block block = blockDataGenerator.block(options); + mockBlockInBlockchain(block); + mockChainHeadInBlockchain(block); + } + + private void mockBlockchainWithTransactionsInABlock(final List transactions) { + final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); + + final Block genesis = blockDataGenerator.genesisBlock(); + mockBlockInBlockchain(genesis); + + final Block block = + blockDataGenerator.block( + BlockDataGenerator.BlockOptions.create() + .setParentHash(genesis.getHash()) + .setBlockNumber(1) + .addTransaction(transactions)); + mockBlockInBlockchain(block); + mockChainHeadInBlockchain(block); + } + + private void mockBlockInBlockchain(final Block block) { + when(blockchain.getBlockByNumber(block.getHeader().getNumber())).thenReturn(Optional.of(block)); + when(blockchain.getBlockHeader(block.getHash())).thenReturn(Optional.of(block.getHeader())); + when(blockchain.getBlockBody(block.getHash())).thenReturn(Optional.of(block.getBody())); + + when(publicWorldStateArchive.getMutable(block.getHeader().getStateRoot())) + .thenReturn(Optional.of(publicMutableWorldState)); + } + + private void mockChainHeadInBlockchain(final Block block) { + when(blockchain.getChainHeadBlockNumber()).thenReturn(block.getHeader().getNumber()); + when(blockchain.getChainHeadHash()).thenReturn(block.getHash()); + } + + private Transaction publicTransaction() { + return Transaction.builder() + .nonce(0) + .gasPrice(Wei.of(1000)) + .gasLimit(3000000) + .value(Wei.ZERO) + .payload(Bytes.EMPTY) + .sender(Address.fromHexString("0xfe3b557e8fb62b89f4916b721be55ceb828dbd73")) + .chainId(BigInteger.valueOf(2018)) + .signAndBuild(KeyPair.generate()); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateTransactionDataFixture.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateTransactionDataFixture.java new file mode 100644 index 00000000000..cb15b0ff75b --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateTransactionDataFixture.java @@ -0,0 +1,85 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.privacy.storage.migration; + +import org.hyperledger.besu.crypto.SECP256K1; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.mainnet.ValidationResult; +import org.hyperledger.besu.ethereum.privacy.PrivateTransaction; +import org.hyperledger.besu.ethereum.privacy.PrivateTransactionProcessor.Result; +import org.hyperledger.besu.ethereum.privacy.Restriction; + +import java.math.BigInteger; + +import org.apache.tuweni.bytes.Bytes; + +public class PrivateTransactionDataFixture { + + private static final SECP256K1.KeyPair KEY_PAIR = + SECP256K1.KeyPair.create( + SECP256K1.PrivateKey.create( + new BigInteger( + "8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63", 16))); + + private static final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); + + static Transaction privacyMarkerTransaction(final String transactionKey) { + return Transaction.builder() + .nonce(0) + .gasPrice(Wei.of(1000)) + .gasLimit(3000000) + .to(Address.DEFAULT_PRIVACY) + .value(Wei.ZERO) + .payload(Bytes.fromBase64String(transactionKey)) + .sender(Address.fromHexString("0xfe3b557e8fb62b89f4916b721be55ceb828dbd73")) + .chainId(BigInteger.valueOf(2018)) + .signAndBuild(KEY_PAIR); + } + + static PrivateTransaction privateTransaction(final String privacyGroupId) { + return PrivateTransaction.builder() + .nonce(0) + .gasPrice(Wei.of(1000)) + .gasLimit(3000000) + .to(null) + .value(Wei.ZERO) + .payload( + Bytes.fromHexString( + "0x608060405234801561001057600080fd5b5060d08061001f6000396000" + + "f3fe60806040526004361060485763ffffffff7c010000000000" + + "0000000000000000000000000000000000000000000000600035" + + "04166360fe47b18114604d5780636d4ce63c146075575b600080" + + "fd5b348015605857600080fd5b50607360048036036020811015" + + "606d57600080fd5b50356099565b005b348015608057600080fd" + + "5b506087609e565b60408051918252519081900360200190f35b" + + "600055565b6000549056fea165627a7a72305820cb1d0935d14b" + + "589300b12fcd0ab849a7e9019c81da24d6daa4f6b2f003d1b018" + + "0029")) + .sender(Address.wrap(Bytes.fromHexString("0x1c9a6e1ee3b7ac6028e786d9519ae3d24ee31e79"))) + .chainId(BigInteger.valueOf(4)) + .privateFrom(Bytes.fromBase64String("A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo=")) + .privacyGroupId(Bytes.fromBase64String(privacyGroupId)) + .restriction(Restriction.RESTRICTED) + .signAndBuild(KEY_PAIR); + } + + public static Result successfulPrivateTxProcessingResult() { + return Result.successful( + blockDataGenerator.logs(3, 1), 0, Bytes.EMPTY, ValidationResult.valid()); + } +}