diff --git a/core/src/main/java/net/consensys/shomei/storage/ZkWorldStateArchive.java b/core/src/main/java/net/consensys/shomei/storage/ZkWorldStateArchive.java index f9510e6f..ac01303b 100644 --- a/core/src/main/java/net/consensys/shomei/storage/ZkWorldStateArchive.java +++ b/core/src/main/java/net/consensys/shomei/storage/ZkWorldStateArchive.java @@ -16,7 +16,9 @@ import net.consensys.shomei.exception.MissingTrieLogException; import net.consensys.shomei.metrics.MetricsService; import net.consensys.shomei.observer.TrieLogObserver.TrieLogIdentifier; +import net.consensys.shomei.storage.worldstate.LayeredWorldStateStorage; import net.consensys.shomei.storage.worldstate.WorldStateStorage; +import net.consensys.shomei.trie.trace.Trace; import net.consensys.shomei.trielog.TrieLogLayer; import net.consensys.shomei.trielog.TrieLogLayerConverter; import net.consensys.shomei.worldview.ZkEvmWorldState; @@ -25,11 +27,13 @@ import java.io.IOException; import java.util.Collections; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentSkipListMap; import com.google.common.annotations.VisibleForTesting; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.ethereum.rlp.RLP; import org.slf4j.Logger; @@ -172,6 +176,72 @@ void applyTrieLog( headWorldState.commit(newBlockNumber, trieLogLayer.getBlockHash(), generateTrace); } + /** + * Result of generating a virtual trace. + */ + public record VirtualTraceResult(List> traces, Bytes32 zkEndStateRootHash) {} + + /** + * Generate a virtual trace from a trielog without persisting state changes. + * This is used for simulating transactions on a virtual block. + * + * @param parentBlockNumber the parent block number on which to base the virtual state + * @param trieLogLayer the trielog to apply + * @return the generated trace and resulting state root hash + * @throws IllegalStateException if the worldstate for the parent block is not cached + */ + public VirtualTraceResult generateVirtualTrace( + final long parentBlockNumber, final TrieLogLayer trieLogLayer) { + // Get the cached worldstate for the parent block + final WorldStateStorage parentStorage = cachedWorldStates.entrySet().stream() + .filter(entry -> entry.getKey().blockNumber().equals(parentBlockNumber)) + .map(Map.Entry::getValue) + .findFirst() + .orElseThrow(() -> new IllegalStateException( + "Worldstate for parent block " + parentBlockNumber + " is not cached")); + + // Create a layered storage that overlays in-memory writes on top of the parent snapshot + // This ensures we don't modify the cached parent state during simulation + try (final WorldStateStorage virtualStorage = new LayeredWorldStateStorage(parentStorage)) { + // Use an in-memory trace manager that won't persist to disk + final TraceManager ephemeralTraceManager = new InMemoryStorageProvider().getTraceManager(); + + final ZkEvmWorldState virtualWorldState = new ZkEvmWorldState(virtualStorage, ephemeralTraceManager); + + // Apply the trielog and generate trace for the virtual block + // Use the virtual block number from the trielog (parentBlockNumber + 1) + final long virtualBlockNumber = trieLogLayer.getBlockNumber(); + + virtualWorldState.getAccumulator().rollForward(trieLogLayer); + virtualWorldState.commit(virtualBlockNumber, trieLogLayer.getBlockHash(), true); + + // Retrieve the trace for the virtual block + final Optional traceBytes = ephemeralTraceManager.getTrace(virtualBlockNumber); + + if (traceBytes.isEmpty()) { + throw new IllegalStateException( + "Failed to generate trace for virtual block " + virtualBlockNumber + + " on parent " + parentBlockNumber); + } + + // Get the resulting state root hash after applying the virtual block + final Bytes32 zkEndStateRootHash = ephemeralTraceManager + .getZkStateRootHash(virtualBlockNumber) + .orElseThrow(() -> new IllegalStateException( + "Failed to get state root hash for virtual block " + virtualBlockNumber)); + + return new VirtualTraceResult( + List.of(Trace.deserialize(RLP.input(traceBytes.get()))), + zkEndStateRootHash); + } catch (Exception e) { + if (e instanceof IllegalStateException) { + throw (IllegalStateException) e; + } + throw new IllegalStateException( + "Failed to generate virtual trace for parent block " + parentBlockNumber, e); + } + } + public ZkEvmWorldState getHeadWorldState() { return headWorldState; } diff --git a/core/src/main/java/net/consensys/shomei/storage/worldstate/LayeredWorldStateStorage.java b/core/src/main/java/net/consensys/shomei/storage/worldstate/LayeredWorldStateStorage.java new file mode 100644 index 00000000..ee53e075 --- /dev/null +++ b/core/src/main/java/net/consensys/shomei/storage/worldstate/LayeredWorldStateStorage.java @@ -0,0 +1,256 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.storage.worldstate; + +import net.consensys.shomei.trie.model.FlattenedLeaf; + +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** + * LayeredWorldStateStorage composes an in-memory overlay and a base (parent) storage. + * All reads check the overlay first, then fall back to the parent. + * All writes go only to the overlay, leaving the parent unmodified. + * Deletes are tracked explicitly to prevent fallback to parent for deleted keys. + * + * This is useful for virtual/simulated blocks where we want to apply changes temporarily + * without modifying the cached parent state or persist the state permanently. + */ +public class LayeredWorldStateStorage extends InMemoryWorldStateStorage { + + private final WorldStateStorage parent; + + public LayeredWorldStateStorage(final WorldStateStorage parent) { + super(); + this.parent = parent; + } + + @Override + public Optional getFlatLeaf(final Bytes key) { + // null = key not in overlay at all → ask parent + // Optional.empty() = key explicitly deleted in overlay → return empty + // Optional.of(val) = key exists in overlay → return value + final Optional overlayValue = getFlatLeafStorage().get(key); + if (overlayValue == null) { + return parent.getFlatLeaf(key); + } + return overlayValue; + } + + @Override + public Optional getTrieNode(final Bytes location, final Bytes nodeHash) { + final Optional overlayValue = getTrieNodeStorage().get(location); + if (overlayValue == null) { + return parent.getTrieNode(location, nodeHash); + } + return overlayValue; + } + + @Override + public Range getNearestKeys(final Bytes hkey) { + + final Range parentRange = parent.getNearestKeys(hkey); + + // --- Center: check overlay directly, then parent --- + Optional> centerNode; + final Optional overlayCenter = getFlatLeafStorage().get(hkey); + if (overlayCenter != null && overlayCenter.isPresent()) { + centerNode = Optional.of(Map.entry(hkey, overlayCenter.get())); + } else if (overlayCenter != null && overlayCenter.isEmpty()) { + centerNode = Optional.empty(); + } else { + centerNode = parentRange.getCenterNode(); + } + + // --- Left: closest key < hkey --- + final Map.Entry leftNode = + resolveLeftNode(hkey, parentRange); + + // --- Right: closest key > hkey --- + final Map.Entry rightNode = + resolveRightNode(hkey, parentRange); + + return new Range(leftNode, centerNode, rightNode); + } + + private Map.Entry resolveLeftNode( + final Bytes hkey, final Range parentRange) { + + final Map.Entry parentLeft = + getValidParentNode( + parentRange.getLeftNodeKey(), parentRange.getLeftNodeValue(), true); + + Map.Entry> overlayLeft = + getFlatLeafStorage().lowerEntry(hkey); + // skip deleted entries + while (overlayLeft != null && overlayLeft.getValue().isEmpty()) { + overlayLeft = getFlatLeafStorage().lowerEntry(overlayLeft.getKey()); + } + + if (overlayLeft != null && overlayLeft.getValue().isPresent() + && overlayLeft.getKey().compareTo(parentLeft.getKey()) > 0) { + return Map.entry(overlayLeft.getKey(), overlayLeft.getValue().get()); + } + + return parentLeft; + } + + private Map.Entry resolveRightNode( + final Bytes hkey, final Range parentRange) { + + final Map.Entry parentRight = + getValidParentNode( + parentRange.getRightNodeKey(), parentRange.getRightNodeValue(), false); + + Map.Entry> overlayRight = + getFlatLeafStorage().higherEntry(hkey); + // skip deleted entries + while (overlayRight != null && overlayRight.getValue().isEmpty()) { + overlayRight = getFlatLeafStorage().higherEntry(overlayRight.getKey()); + } + + if (overlayRight != null && overlayRight.getKey().compareTo(parentRight.getKey()) < 0) { + return Map.entry(overlayRight.getKey(), overlayRight.getValue().get()); + } + + return parentRight; + } + + /** + * Check if a key has been explicitly deleted in the overlay. + */ + private boolean isDeletedInOverlay(final Bytes key) { + final Optional value = getFlatLeafStorage().get(key); + return value != null && value.isEmpty(); + } + + /** + * Walk through parent storage to find a node that hasn't been deleted in the overlay. + */ + private Map.Entry getValidParentNode( + final Bytes initialKey, + final FlattenedLeaf initialValue, + final boolean searchLeft) { + + Bytes currentKey = initialKey; + FlattenedLeaf currentValue = initialValue; + + while (isDeletedInOverlay(currentKey) && !currentKey.equals(Bytes.EMPTY)) { + final Range nextRange = parent.getNearestKeys(currentKey); + + if (searchLeft) { + final Bytes nextKey = nextRange.getLeftNodeKey(); + if (nextKey.equals(currentKey) || nextKey.equals(Bytes.EMPTY)) { + break; + } + currentKey = nextKey; + currentValue = nextRange.getLeftNodeValue(); + } else { + final Bytes nextKey = nextRange.getRightNodeKey(); + if (nextKey.equals(currentKey) || nextKey.equals(Bytes.EMPTY)) { + break; + } + currentKey = nextKey; + currentValue = nextRange.getRightNodeValue(); + } + } + + return Map.entry(currentKey, currentValue); + } + + @Override + public TrieUpdater updater() { + return new LayeredTrieUpdater(this); + } + + + /** + * Wrapper around the overlay's TrieUpdater that tracks deleted keys and handles re-insertions. + */ + private static class LayeredTrieUpdater implements WorldStateUpdater { + private final LayeredWorldStateStorage delegate; + + LayeredTrieUpdater(final LayeredWorldStateStorage delegate) { + this.delegate = delegate; + } + + @Override + public void putFlatLeaf(Bytes key, FlattenedLeaf value) { + delegate.putFlatLeaf(key, value); + } + + @Override + public void removeFlatLeafValue(Bytes key) { + delegate.removeFlatLeafValue(key); + } + + @Override + public void putTrieNode(Bytes location, Bytes nodeHash, Bytes value) { + delegate.putTrieNode(location, nodeHash, value); + } + + @Override + public void commit() { + delegate.commit(); + } + + @Override + public void setBlockHash(final Bytes32 blockHash) { + delegate.setBlockHash(blockHash); + } + + @Override + public void setBlockNumber(final long blockNumber) { + delegate.setBlockNumber(blockNumber); + } + } + + @Override + public Optional getWorldStateBlockNumber() { + final Optional overlayResult = super.getWorldStateBlockNumber(); + return overlayResult.isPresent() ? overlayResult : parent.getWorldStateBlockNumber(); + } + + @Override + public Optional getWorldStateBlockHash() { + final Optional overlayResult = super.getWorldStateBlockHash(); + return overlayResult.isPresent() ? overlayResult : parent.getWorldStateBlockHash(); + } + + @Override + public Optional getWorldStateRootHash() { + final Optional overlayResult = super.getWorldStateRootHash(); + return overlayResult.isPresent() ? overlayResult : parent.getWorldStateRootHash(); + } + + @Override + public Optional getZkStateRootHash(final long blockNumber) { + final Optional overlayResult = super.getZkStateRootHash(blockNumber); + return overlayResult.isPresent() ? overlayResult : parent.getZkStateRootHash(blockNumber); + } + + @Override + public WorldStateStorage snapshot() { + // Snapshots not supported on layered storage + throw new UnsupportedOperationException("Cannot snapshot a layered storage"); + } + + @Override + public void close() { + // Don't close parent — it's managed elsewhere + } +} diff --git a/core/src/main/java/net/consensys/shomei/storage/worldstate/WorldStateStorage.java b/core/src/main/java/net/consensys/shomei/storage/worldstate/WorldStateStorage.java index ab49bbc6..098560ea 100644 --- a/core/src/main/java/net/consensys/shomei/storage/worldstate/WorldStateStorage.java +++ b/core/src/main/java/net/consensys/shomei/storage/worldstate/WorldStateStorage.java @@ -25,7 +25,7 @@ * provides methods for accessing and modifying the state of accounts and storage in the world * state. */ -public interface WorldStateStorage extends TrieStorage { +public interface WorldStateStorage extends TrieStorage, AutoCloseable { /** key identifier of the block hash of the current world state. */ byte[] WORLD_BLOCK_HASH_KEY = "blockHash".getBytes(StandardCharsets.UTF_8); @@ -64,6 +64,7 @@ public interface WorldStateStorage extends TrieStorage { */ WorldStateStorage snapshot(); + @Override default void close() throws Exception { // no-op } diff --git a/core/src/main/java/net/consensys/shomei/trielog/TrieLogLayerConverter.java b/core/src/main/java/net/consensys/shomei/trielog/TrieLogLayerConverter.java index 847117d3..8351da62 100644 --- a/core/src/main/java/net/consensys/shomei/trielog/TrieLogLayerConverter.java +++ b/core/src/main/java/net/consensys/shomei/trielog/TrieLogLayerConverter.java @@ -55,7 +55,9 @@ public TrieLogLayer decodeTrieLog(final RLPInput input) { trieLogLayer.setBlockHash(input.readBytes32()); trieLogLayer.setBlockNumber(input.readLongScalar()); - while (!input.isEndOfCurrentList()) { + // Only process list items (account entries). Stop when encountering non-list items like + // the optional zkTraceComparisonFeature integer that may be appended by Besu's trielog format + while (!input.isEndOfCurrentList() && input.nextIsList()) { input.enterList(); final Address address = Address.readFrom(input); @@ -115,12 +117,13 @@ public TrieLogLayer decodeTrieLog(final RLPInput input) { oldValueFound = maybeAccountIndex .flatMap( - index -> - new StorageTrieRepositoryWrapper(index, worldStateStorage, null) + index -> { + return new StorageTrieRepositoryWrapper(index, worldStateStorage, null) .getFlatLeaf(storageSlotKey.slotHash()) .map(FlattenedLeaf::leafValue) - .map(UInt256::fromBytes)) - .orElse(null); + .map(UInt256::fromBytes); + }) + .orElse(null); // Return null for new accounts to match trielog's null representation } LOG.atTrace() .setMessage( @@ -158,6 +161,13 @@ public TrieLogLayer decodeTrieLog(final RLPInput input) { // lenient leave list for forward compatible additions. input.leaveListLenient(); } + + // zkTraceComparisonFeature is optional (read as last element in container, before leaving) + // This is written by Besu's ZkTrieLogFactory when includeMetadata=true + if (!input.isEndOfCurrentList()) { + input.readInt(); // consume but don't use + } + input.leaveListLenient(); trieLogLayer.freeze(); @@ -173,6 +183,7 @@ public PriorAccount preparePriorTrieLogAccount(final AccountKey accountKey, fina final Optional flatLeaf = worldStateStorage.getFlatLeaf(WRAP_ACCOUNT.apply(accountKey.accountHash())); + if (in.nextIsNull() && flatLeaf.isEmpty()) { in.skipNext(); @@ -189,7 +200,6 @@ public PriorAccount preparePriorTrieLogAccount(final AccountKey accountKey, fina .orElseThrow(); in.enterList(); - final UInt256 nonce = UInt256.valueOf(in.readLongScalar()); final Wei balance = Wei.of(in.readUInt256Scalar()); final Bytes32 evmStorageRoot; diff --git a/core/src/test/java/net/consensys/shomei/LayeredWorldStateTraceTest.java b/core/src/test/java/net/consensys/shomei/LayeredWorldStateTraceTest.java new file mode 100644 index 00000000..f8bf2b83 --- /dev/null +++ b/core/src/test/java/net/consensys/shomei/LayeredWorldStateTraceTest.java @@ -0,0 +1,801 @@ +/* + * Copyright ConsenSys Software Inc., 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei; + +import static net.consensys.shomei.ZkAccount.EMPTY_CODE_HASH; +import static net.consensys.shomei.ZkAccount.EMPTY_KECCAK_CODE_HASH; +import static net.consensys.shomei.trie.ZKTrie.DEFAULT_TRIE_ROOT; +import static net.consensys.shomei.util.TestFixtureGenerator.createDumAddress; +import static net.consensys.shomei.util.TestFixtureGenerator.createDumDigest; +import static net.consensys.shomei.util.TestFixtureGenerator.createDumFullBytes; +import static net.consensys.shomei.util.TestFixtureGenerator.getAccountOne; +import static net.consensys.shomei.util.bytes.MimcSafeBytes.unsafeFromBytes; +import static org.assertj.core.api.Assertions.assertThat; + +import net.consensys.shomei.storage.worldstate.InMemoryWorldStateStorage; +import net.consensys.shomei.storage.worldstate.LayeredWorldStateStorage; +import net.consensys.shomei.trie.ZKTrie; +import net.consensys.shomei.trie.json.JsonTraceParser; +import net.consensys.shomei.trie.storage.AccountTrieRepositoryWrapper; +import net.consensys.shomei.trie.storage.StorageTrieRepositoryWrapper; +import net.consensys.shomei.trie.storage.TrieStorage; +import net.consensys.shomei.trie.trace.Trace; +import net.consensys.shomei.trielog.AccountKey; +import net.consensys.shomei.util.bytes.MimcSafeBytes; +import net.consensys.zkevm.HashProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.hyperledger.besu.datatypes.Wei; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class LayeredWorldStateTraceTest { + + private static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); + + @BeforeEach + public void setup() { + JSON_OBJECT_MAPPER.registerModules(JsonTraceParser.modules); + } + + private ZKTrie loadAccountTrie(final Bytes32 stateRoot, final TrieStorage storage) { + if (storage.getTrieNode(Bytes.EMPTY, null).isEmpty()) { + return ZKTrie.createTrie(storage); + } else { + return ZKTrie.loadTrie(stateRoot, storage); + } + } + + private ZKTrie loadStorageTrie(final Bytes32 storageRoot, final TrieStorage storage) { + if (storage.getTrieNode(Bytes.EMPTY, null).isEmpty()) { + return ZKTrie.createTrie(storage); + } else { + return ZKTrie.loadTrie(storageRoot, storage); + } + } + + /** + * Helper: create an empty committed parent and return its root + storage. + */ + private static final class ParentState { + private final Bytes32 root; + private final InMemoryWorldStateStorage storage; + + /** + * + */ + private ParentState(Bytes32 root, InMemoryWorldStateStorage storage) { + this.root = root; + this.storage = storage; + } + + public Bytes32 root() { + return root; + } + + public InMemoryWorldStateStorage storage() { + return storage; + } + + } + + private ParentState createEmptyParent() { + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.commit(); + return new ParentState(parentTrie.getTopRootHash(), parentStorage); + } + + // =========================================================================== + // Common test accounts + // =========================================================================== + + private ZkAccount makeAccount2Simple() { + return new ZkAccount( + new AccountKey(createDumAddress(41)), + 42, + Wei.of(354), + DEFAULT_TRIE_ROOT, + EMPTY_CODE_HASH, + EMPTY_KECCAK_CODE_HASH, + 0L); + } + + private ZkAccount makeAccount2Contract() { + return new ZkAccount( + new AccountKey(createDumAddress(47)), + 41, + Wei.of(15353), + DEFAULT_TRIE_ROOT, + createDumDigest(75), + createDumFullBytes(15), + 7L); + } + + private MutableZkAccount makeMutableAccount2Contract() { + return new MutableZkAccount( + new AccountKey(createDumAddress(47)), + createDumFullBytes(15), + createDumDigest(75), + 7L, + 41, + Wei.of(15353), + DEFAULT_TRIE_ROOT); + } + + private ZkAccount makeAccount3() { + return new ZkAccount( + new AccountKey(createDumAddress(120)), + 48, + Wei.of(9835), + DEFAULT_TRIE_ROOT, + createDumDigest(54), + createDumFullBytes(85), + 19L); + } + + // =========================================================================== + // READ ZERO + // =========================================================================== + + /** Original: empty parent, single overlay, read zero. */ + @Test + public void testTraceReadZero() throws IOException { + final Bytes32 key = createDumDigest(36); + final Bytes32 hkey = HashProvider.trieHash(key); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.readWithTrace(hkey, MimcSafeBytes.safeByte32(key)); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(trace)) + .isEqualToIgnoringWhitespace(getResources("testTraceReadZero.json")); + } + + // =========================================================================== + // READ SIMPLE VALUE + // =========================================================================== + + /** Original: write in parent, read from overlay. */ + @Test + public void testTraceReadSimpleValueFromParent() throws IOException { + final MimcSafeBytes key = unsafeFromBytes(createDumDigest(36)); + final MimcSafeBytes value = unsafeFromBytes(createDumDigest(32)); + final Bytes32 hkey = HashProvider.trieHash(key); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace(hkey, key, value); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.readWithTrace(hkey, key); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(trace)) + .isEqualToIgnoringWhitespace(getResources("testTraceReadSimpleValue.json")); + } + + // =========================================================================== + // READ ACCOUNT + // =========================================================================== + + /** Original: write account in parent, read from overlay. */ + @Test + public void testTraceReadAccountFromParent() throws IOException { + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace(account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.readWithTrace(account.getHkey(), account.getAddress()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(trace)) + .isEqualToIgnoringWhitespace(getResources("testTraceReadAccount.json")); + } + + // =========================================================================== + // TWO ACCOUNTS + // =========================================================================== + + /** Original: account1 in parent, account2 in overlay. */ + @Test + public void testWorldStateWithTwoAccountsSplitBetweenParentAndOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Simple(); + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + Trace trace = + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace(getResources("testWorldStateWithTwoAccount.json")); + } + + /** Both accounts written in overlay, empty parent. */ + @Test + public void testWorldStateWithTwoAccounts_bothInOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Simple(); + MutableZkAccount account = getAccountOne(); + + ParentState parent = createEmptyParent(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parent.storage()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parent.root(), layeredRepo); + + Trace trace = + layeredTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace(getResources("testWorldStateWithTwoAccount.json")); + } + + // =========================================================================== + // ACCOUNT + CONTRACT + // =========================================================================== + + /** Original: account1 in parent, contract in overlay. */ + @Test + public void testWorldStateWithAccountInParentAndContractInOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + final Trace trace = + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + final Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithAccountAndContract.json")); + } + + /** Both account and contract written in overlay, empty parent. */ + @Test + public void testWorldStateWithAccountAndContract_bothInOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + ParentState parent = createEmptyParent(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parent.storage()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parent.root(), layeredRepo); + + final Trace trace = + layeredTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + final Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithAccountAndContract.json")); + } + + // =========================================================================== + // STORAGE UPDATE + // =========================================================================== + + /** Original: both accounts in parent, storage update in overlay. */ + @Test + public void testWorldStateWithStorageUpdateInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + final Trace trace = + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + final Trace trace2 = + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + + StorageTrieRepositoryWrapper storageRepo = + new StorageTrieRepositoryWrapper(zkAccount2.hashCode(), layeredStorage); + ZKTrie account2Storage = loadStorageTrie(zkAccount2.getStorageRoot(), storageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + final Trace trace3 = account2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + + zkAccount2.setStorageRoot(account2Storage.getTopRootHash()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + final Trace trace4 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2, trace3, trace4))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithUpdateContractStorage.json")); + } + + /** All writes (accounts + storage) in overlay, empty parent. */ + @Test + public void testWorldStateWithStorageUpdate_allInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + ParentState parent = createEmptyParent(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parent.storage()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parent.root(), layeredRepo); + + final Trace trace = + layeredTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + final Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + StorageTrieRepositoryWrapper storageRepo = + new StorageTrieRepositoryWrapper(zkAccount2.hashCode(), layeredStorage); + ZKTrie account2Storage = loadStorageTrie(zkAccount2.getStorageRoot(), storageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + final Trace trace3 = account2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + + zkAccount2.setStorageRoot(account2Storage.getTopRootHash()); + final Trace trace4 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2, trace3, trace4))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithUpdateContractStorage.json")); + } + + /** Account1 in parent, account2 + storage in overlay. */ + @Test + public void testWorldStateWithStorageUpdate_account1InParentRestInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + final Trace trace = + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + final Trace trace2 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + StorageTrieRepositoryWrapper storageRepo = + new StorageTrieRepositoryWrapper(zkAccount2.hashCode(), layeredStorage); + ZKTrie account2Storage = loadStorageTrie(zkAccount2.getStorageRoot(), storageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + final Trace trace3 = account2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + + zkAccount2.setStorageRoot(account2Storage.getTopRootHash()); + final Trace trace4 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2, trace3, trace4))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithUpdateContractStorage.json")); + } + + // =========================================================================== + // DELETE ACCOUNT + UPDATE STORAGE + // =========================================================================== + + /** Original: both accounts + storage in parent, delete + storage update in overlay. */ + @Test + public void testDeleteAccountInOverlayAndUpdateStorageInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + // --- PARENT: write both accounts + storage --- + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace(account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final long accountLeafIndex = + parentTrie.getLeafIndex(zkAccount2.getHkey()).orElse(parentTrie.getNextFreeNode()); + + StorageTrieRepositoryWrapper parentStorageRepo = + new StorageTrieRepositoryWrapper(accountLeafIndex, parentStorage); + ZKTrie parentAccount2Storage = + loadStorageTrie(zkAccount2.getStorageRoot(), parentStorageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + parentAccount2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + parentAccount2Storage.commit(); + final Bytes32 parentStorageRoot = parentAccount2Storage.getTopRootHash(); + + zkAccount2.setStorageRoot(parentStorageRoot); + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + // --- OVERLAY --- + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + + final long accountLayeredLeafIndex = + layeredTrie.getLeafIndex(zkAccount2.getHkey()).orElse(layeredTrie.getNextFreeNode()); + + StorageTrieRepositoryWrapper layeredStorageRepo = + new StorageTrieRepositoryWrapper(accountLayeredLeafIndex, layeredStorage); + ZKTrie layeredAccount2Storage = loadStorageTrie(parentStorageRoot, layeredStorageRepo); + + Trace trace2 = layeredAccount2Storage.removeWithTrace(slotKeyHash, slotKey); + trace2.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace3 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final MimcSafeBytes newSlotKey = createDumFullBytes(11); + final Bytes32 newSlotKeyHash = HashProvider.trieHash(newSlotKey); + final MimcSafeBytes newSlotValue = createDumFullBytes(78); + Trace trace4 = + layeredAccount2Storage.putWithTrace(newSlotKeyHash, newSlotKey, newSlotValue); + trace4.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace5 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString( + List.of(trace, trace2, trace3, trace4, trace5))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithDeleteAccountAndStorage.json")); + } + + /** Accounts in parent (no storage), storage + delete + new storage all in overlay. */ + @Test + public void testDeleteAccountAndStorage_accountsInParentAllOpsInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + // --- PARENT: write both accounts only (no storage) --- + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace(account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final long accountLeafIndex = + parentTrie.getLeafIndex(zkAccount2.getHkey()).orElse(parentTrie.getNextFreeNode()); + + // Write storage in parent too so we have the same initial state + StorageTrieRepositoryWrapper parentStorageRepo = + new StorageTrieRepositoryWrapper(accountLeafIndex, parentStorage); + ZKTrie parentAccount2Storage = + loadStorageTrie(zkAccount2.getStorageRoot(), parentStorageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + parentAccount2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + parentAccount2Storage.commit(); + final Bytes32 parentStorageRoot = parentAccount2Storage.getTopRootHash(); + + zkAccount2.setStorageRoot(parentStorageRoot); + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + // --- OVERLAY: delete account1, remove storage, add new storage --- + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + + final long accountLayeredLeafIndex = + layeredTrie.getLeafIndex(zkAccount2.getHkey()).orElse(layeredTrie.getNextFreeNode()); + + StorageTrieRepositoryWrapper layeredStorageRepo = + new StorageTrieRepositoryWrapper(accountLayeredLeafIndex, layeredStorage); + ZKTrie layeredAccount2Storage = loadStorageTrie(parentStorageRoot, layeredStorageRepo); + + Trace trace2 = layeredAccount2Storage.removeWithTrace(slotKeyHash, slotKey); + trace2.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace3 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final MimcSafeBytes newSlotKey = createDumFullBytes(11); + final Bytes32 newSlotKeyHash = HashProvider.trieHash(newSlotKey); + final MimcSafeBytes newSlotValue = createDumFullBytes(78); + Trace trace4 = + layeredAccount2Storage.putWithTrace(newSlotKeyHash, newSlotKey, newSlotValue); + trace4.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace5 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString( + List.of(trace, trace2, trace3, trace4, trace5))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithDeleteAccountAndStorage.json")); + } + + /** Everything in overlay, empty parent. */ + @Test + public void testDeleteAccountAndStorage_allInOverlay() throws IOException { + final MutableZkAccount zkAccount2 = makeMutableAccount2Contract(); + MutableZkAccount account = getAccountOne(); + + // --- Empty parent --- + ParentState parent = createEmptyParent(); + + // --- OVERLAY: build entire state then do delete + storage ops --- + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parent.storage()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parent.root(), layeredRepo); + + // Build initial state in overlay + layeredTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final long accountLeafIndex = + layeredTrie.getLeafIndex(zkAccount2.getHkey()).orElse(layeredTrie.getNextFreeNode()); + + StorageTrieRepositoryWrapper layeredStorageRepo = + new StorageTrieRepositoryWrapper(accountLeafIndex, layeredStorage); + ZKTrie layeredAccount2Storage = + loadStorageTrie(zkAccount2.getStorageRoot(), layeredStorageRepo); + + final MimcSafeBytes slotKey = createDumFullBytes(14); + final Bytes32 slotKeyHash = HashProvider.trieHash(slotKey); + final MimcSafeBytes slotValue = createDumFullBytes(18); + layeredAccount2Storage.putWithTrace(slotKeyHash, slotKey, slotValue); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + // Now do the traced operations + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + + Trace trace2 = layeredAccount2Storage.removeWithTrace(slotKeyHash, slotKey); + trace2.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace3 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + final MimcSafeBytes newSlotKey = createDumFullBytes(11); + final Bytes32 newSlotKeyHash = HashProvider.trieHash(newSlotKey); + final MimcSafeBytes newSlotValue = createDumFullBytes(78); + Trace trace4 = + layeredAccount2Storage.putWithTrace(newSlotKeyHash, newSlotKey, newSlotValue); + trace4.setLocation(zkAccount2.getAddress().getOriginalUnsafeValue()); + + zkAccount2.setStorageRoot(layeredAccount2Storage.getTopRootHash()); + Trace trace5 = + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + assertThat( + JSON_OBJECT_MAPPER.writeValueAsString( + List.of(trace, trace2, trace3, trace4, trace5))) + .isEqualToIgnoringWhitespace( + getResources("testWorldStateWithDeleteAccountAndStorage.json")); + } + + // =========================================================================== + // DELETE + ADD + // =========================================================================== + + /** Original: both accounts in parent, delete + add in overlay. */ + @Test + public void testDeleteInOverlayAndAddInOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Contract(); + final ZkAccount zkAccount3 = makeAccount3(); + MutableZkAccount account = getAccountOne(); + + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + Trace trace2 = + layeredTrie.putWithTrace( + zkAccount3.getHkey(), zkAccount3.getAddress(), zkAccount3.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace(getResources("testAddAndDeleteAccounts.json")); + } + + /** All in overlay: write account1+2, then delete account1, add account3. */ + @Test + public void testDeleteAndAdd_allInOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Contract(); + final ZkAccount zkAccount3 = makeAccount3(); + MutableZkAccount account = getAccountOne(); + + ParentState parent = createEmptyParent(); + + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parent.storage()); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parent.root(), layeredRepo); + + // Build initial state in overlay + layeredTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + // Now do the traced operations + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + Trace trace2 = + layeredTrie.putWithTrace( + zkAccount3.getHkey(), zkAccount3.getAddress(), zkAccount3.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace(getResources("testAddAndDeleteAccounts.json")); + } + + /** Account1 in parent, account2 in overlay, then delete account1 + add account3 in overlay. */ + @Test + public void testDeleteAndAdd_account1InParentAccount2InOverlay() throws IOException { + final ZkAccount zkAccount2 = makeAccount2Contract(); + final ZkAccount zkAccount3 = makeAccount3(); + MutableZkAccount account = getAccountOne(); + + // Parent: only account1 + InMemoryWorldStateStorage parentStorage = new InMemoryWorldStateStorage(); + AccountTrieRepositoryWrapper parentRepo = new AccountTrieRepositoryWrapper(parentStorage); + ZKTrie parentTrie = loadAccountTrie(DEFAULT_TRIE_ROOT, parentRepo); + parentTrie.putWithTrace( + account.getHkey(), account.getAddress(), account.getEncodedBytes()); + parentTrie.commit(); + final Bytes32 parentRoot = parentTrie.getTopRootHash(); + + // Overlay: add account2, then delete account1, add account3 + LayeredWorldStateStorage layeredStorage = new LayeredWorldStateStorage(parentStorage); + AccountTrieRepositoryWrapper layeredRepo = new AccountTrieRepositoryWrapper(layeredStorage); + ZKTrie layeredTrie = loadAccountTrie(parentRoot, layeredRepo); + + // Build state: add account2 + layeredTrie.putWithTrace( + zkAccount2.getHkey(), zkAccount2.getAddress(), zkAccount2.getEncodedBytes()); + + // Traced operations + Trace trace = layeredTrie.removeWithTrace(account.getHkey(), account.getAddress()); + Trace trace2 = + layeredTrie.putWithTrace( + zkAccount3.getHkey(), zkAccount3.getAddress(), zkAccount3.getEncodedBytes()); + + assertThat(JSON_OBJECT_MAPPER.writeValueAsString(List.of(trace, trace2))) + .isEqualToIgnoringWhitespace(getResources("testAddAndDeleteAccounts.json")); + } + + @SuppressWarnings({"SameParameterValue", "ConstantConditions", "resource"}) + private String getResources(final String fileName) throws IOException { + var classLoader = LayeredWorldStateTraceTest.class.getClassLoader(); + return new String( + classLoader.getResourceAsStream(fileName).readAllBytes(), StandardCharsets.UTF_8); + } +} diff --git a/core/src/test/java/net/consensys/shomei/storage/LayeredWorldStateStorageTest.java b/core/src/test/java/net/consensys/shomei/storage/LayeredWorldStateStorageTest.java new file mode 100644 index 00000000..bee3ab78 --- /dev/null +++ b/core/src/test/java/net/consensys/shomei/storage/LayeredWorldStateStorageTest.java @@ -0,0 +1,234 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.storage; + +import static org.assertj.core.api.Assertions.assertThat; + +import net.consensys.shomei.storage.worldstate.InMemoryWorldStateStorage; +import net.consensys.shomei.storage.worldstate.LayeredWorldStateStorage; +import net.consensys.shomei.storage.worldstate.WorldStateStorage; +import net.consensys.shomei.trie.ZKTrie; +import net.consensys.shomei.trie.model.FlattenedLeaf; +import net.consensys.shomei.trie.storage.TrieStorage; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class LayeredWorldStateStorageTest { + + private WorldStateStorage parentStorage; + private LayeredWorldStateStorage layeredStorage; + + @BeforeEach + public void setup() { + parentStorage = new InMemoryWorldStateStorage(); + layeredStorage = new LayeredWorldStateStorage(parentStorage); + // Initialize storage with HEAD and TAIL like a real trie would + ZKTrie.createTrie(parentStorage); + ZKTrie.createTrie(layeredStorage); + + + } + + @Test + public void testGetFlatLeaf_ReturnsFromOverlayWhenPresent() { + // Setup: put value in overlay + final Bytes key = Bytes.of(1); + final FlattenedLeaf overlayValue = new FlattenedLeaf(1L, Bytes.fromHexString("0x0100")); + layeredStorage.updater().putFlatLeaf(key, overlayValue); + + // Assert: reads from overlay + assertThat(layeredStorage.getFlatLeaf(key)).contains(overlayValue); + } + + @Test + public void testGetFlatLeaf_FallsBackToParentWhenNotInOverlay() { + // Setup: put value in parent only + final Bytes key = Bytes.of(1); + final FlattenedLeaf parentValue = new FlattenedLeaf(2L, Bytes.fromHexString("0x0200")); + parentStorage.updater().putFlatLeaf(key, parentValue); + + // Assert: reads from parent + assertThat(layeredStorage.getFlatLeaf(key)).contains(parentValue); + } + + @Test + public void testGetFlatLeaf_DeletedKeyReturnsEmpty() { + // Setup: put value in parent + final Bytes key = Bytes.of(1); + final FlattenedLeaf parentValue = new FlattenedLeaf(2L, Bytes.fromHexString("0x0200")); + parentStorage.updater().putFlatLeaf(key, parentValue); + + // Delete in overlay + layeredStorage.updater().removeFlatLeafValue(key); + + // Assert: deleted key returns empty, doesn't fall back to parent + assertThat(layeredStorage.getFlatLeaf(key)).isEmpty(); + } + + @Test + public void testGetFlatLeaf_ReinsertionAfterDeletion() { + // Setup: put value in parent + final Bytes key = Bytes.of(1); + final FlattenedLeaf parentValue = new FlattenedLeaf(2L, Bytes.fromHexString("0x0200")); + parentStorage.updater().putFlatLeaf(key, parentValue); + + // Delete in overlay + layeredStorage.updater().removeFlatLeafValue(key); + assertThat(layeredStorage.getFlatLeaf(key)).isEmpty(); + + // Re-insert with new value + final FlattenedLeaf newValue = new FlattenedLeaf(3L, Bytes.fromHexString("0x0300")); + layeredStorage.updater().putFlatLeaf(key, newValue); + + // Assert: re-inserted key is readable + assertThat(layeredStorage.getFlatLeaf(key)).contains(newValue); + } + + @Test + public void testGetNearestKeys_ExcludesDeletedCenterNode() { + // Setup: put values in parent + final Bytes key1 = Bytes.of(1); + final Bytes key2 = Bytes.of(2); + final Bytes key3 = Bytes.of(3); + parentStorage.updater().putFlatLeaf(key1, new FlattenedLeaf(1L, Bytes.fromHexString("0x0100"))); + parentStorage.updater().putFlatLeaf(key2, new FlattenedLeaf(2L, Bytes.fromHexString("0x0200"))); + parentStorage.updater().putFlatLeaf(key3, new FlattenedLeaf(3L, Bytes.fromHexString("0x0300"))); + + // Delete key2 in overlay + layeredStorage.updater().removeFlatLeafValue(key2); + + // Query for nearest keys at key2 + TrieStorage.Range range = layeredStorage.getNearestKeys(key2); + + // Assert: center node should be empty (deleted) + assertThat(range.getCenterNode()).isEmpty(); + + // Assert: left and right nodes should still be present + assertThat(range.getLeftNodeKey()).isEqualTo(key1); + assertThat(range.getRightNodeKey()).isEqualTo(key3); + } + + @Test + public void testGetNearestKeys_ExcludesDeletedLeftNode() { + // Setup: put values in parent - keys spaced out for testing + final Bytes key1 = Bytes.of(10); + final Bytes key2 = Bytes.of(20); + final Bytes key3 = Bytes.of(30); + parentStorage.updater().putFlatLeaf(key1, new FlattenedLeaf(1L, Bytes.fromHexString("0x0100"))); + parentStorage.updater().putFlatLeaf(key2, new FlattenedLeaf(2L, Bytes.fromHexString("0x0200"))); + parentStorage.updater().putFlatLeaf(key3, new FlattenedLeaf(3L, Bytes.fromHexString("0x0300"))); + + // Delete key2 in overlay + layeredStorage.updater().removeFlatLeafValue(key2); + + // Query for nearest keys at key3 + TrieStorage.Range range = layeredStorage.getNearestKeys(key3); + + // Assert: left node should skip deleted key2 and return key1 + assertThat(range.getLeftNodeKey()).isEqualTo(key1); + assertThat(range.getCenterNode()).isPresent(); + assertThat(range.getCenterNode().get().getKey()).isEqualTo(key3); + } + + @Test + public void testGetNearestKeys_ExcludesDeletedRightNode() { + // Setup: put values in parent + final Bytes key1 = Bytes.of(10); + final Bytes key2 = Bytes.of(20); + final Bytes key3 = Bytes.of(30); + parentStorage.updater().putFlatLeaf(key1, new FlattenedLeaf(1L, Bytes.fromHexString("0x0100"))); + parentStorage.updater().putFlatLeaf(key2, new FlattenedLeaf(2L, Bytes.fromHexString("0x0200"))); + parentStorage.updater().putFlatLeaf(key3, new FlattenedLeaf(3L, Bytes.fromHexString("0x0300"))); + + // Delete key2 in overlay + layeredStorage.updater().removeFlatLeafValue(key2); + + // Query for nearest keys at key1 + TrieStorage.Range range = layeredStorage.getNearestKeys(key1); + + // Assert: right node should skip deleted key2 and return key3 + assertThat(range.getRightNodeKey()).isEqualTo(key3); + assertThat(range.getCenterNode()).isPresent(); + assertThat(range.getCenterNode().get().getKey()).isEqualTo(key1); + } + + @Test + public void testGetNearestKeys_OverlayOverridesParent() { + // Setup: put value in parent + final Bytes key = Bytes.of(1); + final FlattenedLeaf parentValue = new FlattenedLeaf(1L, Bytes.fromHexString("0x0100")); + parentStorage.updater().putFlatLeaf(key, parentValue); + + // Override with value in overlay + final FlattenedLeaf overlayValue = new FlattenedLeaf(2L, Bytes.fromHexString("0x0200")); + layeredStorage.updater().putFlatLeaf(key, overlayValue); + + // Query for nearest keys + TrieStorage.Range range = layeredStorage.getNearestKeys(key); + + // Assert: should return overlay value, not parent value + assertThat(range.getCenterNode()).isPresent(); + assertThat(range.getCenterNode().get().getValue()).isEqualTo(overlayValue); + } + + @Test + public void testGetNearestKeys_MultipleDeletesSkipsToValidKey() { + // Setup: put multiple values in parent + final Bytes key1 = Bytes.of(10); + final Bytes key2 = Bytes.of(20); + final Bytes key3 = Bytes.of(30); + final Bytes key4 = Bytes.of(40); + final Bytes key5 = Bytes.of(50); + parentStorage.updater().putFlatLeaf(key1, new FlattenedLeaf(1L, Bytes.fromHexString("0x0100"))); + parentStorage.updater().putFlatLeaf(key2, new FlattenedLeaf(2L, Bytes.fromHexString("0x0200"))); + parentStorage.updater().putFlatLeaf(key3, new FlattenedLeaf(3L, Bytes.fromHexString("0x0300"))); + parentStorage.updater().putFlatLeaf(key4, new FlattenedLeaf(4L, Bytes.fromHexString("0x0400"))); + parentStorage.updater().putFlatLeaf(key5, new FlattenedLeaf(5L, Bytes.fromHexString("0x0500"))); + + // Delete multiple consecutive keys + layeredStorage.updater().removeFlatLeafValue(key2); + layeredStorage.updater().removeFlatLeafValue(key3); + + // Query for nearest keys at key4 + TrieStorage.Range range = layeredStorage.getNearestKeys(key4); + + // Assert: left node should skip both deleted keys and return key1 + assertThat(range.getLeftNodeKey()).isEqualTo(key1); + assertThat(range.getCenterNode().get().getKey()).isEqualTo(key4); + assertThat(range.getRightNodeKey()).isEqualTo(key5); + } + + @Test + public void testParentStorageUnmodified() { + // Setup: put value in parent + final Bytes key = Bytes.of(1); + final FlattenedLeaf parentValue = new FlattenedLeaf(1L, Bytes.fromHexString("0x0100")); + parentStorage.updater().putFlatLeaf(key, parentValue); + + // Delete in overlay + layeredStorage.updater().removeFlatLeafValue(key); + + // Assert: parent storage should still have the value + assertThat(parentStorage.getFlatLeaf(key)).contains(parentValue); + + // Add new value in overlay + final FlattenedLeaf overlayValue = new FlattenedLeaf(2L, Bytes.fromHexString("0x0200")); + layeredStorage.updater().putFlatLeaf(key, overlayValue); + + // Assert: parent storage should still have original value + assertThat(parentStorage.getFlatLeaf(key)).contains(parentValue); + } +} diff --git a/services/rpc/client/build.gradle b/services/rpc/client/build.gradle index 3cf79434..fc2539c4 100644 --- a/services/rpc/client/build.gradle +++ b/services/rpc/client/build.gradle @@ -20,6 +20,10 @@ dependencies { // Hyperledger Besu dependencies implementation 'org.hyperledger.besu.internal:besu-ethereum-api' + implementation 'org.hyperledger.besu.internal:besu-ethereum-core' + implementation 'org.hyperledger.besu.internal:besu-ethereum-rlp' + implementation 'org.hyperledger.besu:besu-datatypes' + implementation 'org.hyperledger.besu:besu-plugin-api' // Tuweni implementation 'io.consensys.tuweni:tuweni-bytes' diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuRpcMethod.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuRpcMethod.java index 83165126..6045609c 100644 --- a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuRpcMethod.java +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuRpcMethod.java @@ -14,7 +14,8 @@ package net.consensys.shomei.rpc.client; public enum BesuRpcMethod { - BESU_GET_TRIE_LOGS_BY_RANGE("shomei_getTrieLogsByRange"); + BESU_GET_TRIE_LOGS_BY_RANGE("shomei_getTrieLogsByRange"), + BESU_ETH_SIMULATE_V1("eth_simulateV1"); private final String methodName; public String getMethodName() { diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuSimulateClient.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuSimulateClient.java new file mode 100644 index 00000000..e2b2422a --- /dev/null +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/BesuSimulateClient.java @@ -0,0 +1,276 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.client; + +import net.consensys.shomei.rpc.client.model.SimulateV1Request; +import net.consensys.shomei.rpc.client.model.SimulateV1Response; + +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.client.WebClientOptions; +import org.apache.tuweni.bytes.Bytes; +import org.hyperledger.besu.datatypes.BytesHolder; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BesuSimulateClient { + + private static final String APPLICATION_JSON = "application/json"; + private static final Logger LOG = LoggerFactory.getLogger(BesuSimulateClient.class); + private static final SecureRandom RANDOM; + + static { + try { + RANDOM = SecureRandom.getInstanceStrong(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("UnusedVariable") // Stored for reference; used to create WebClient + private final Vertx vertx; + private final WebClient webClient; + private final String besuHttpHost; + private final int besuHttpPort; + + public BesuSimulateClient( + final Vertx vertx, final String besuHttpHost, final int besuHttpPort) { + this.vertx = vertx; + final WebClientOptions options = new WebClientOptions(); + this.webClient = WebClient.create(vertx, options); + this.besuHttpHost = besuHttpHost; + this.besuHttpPort = besuHttpPort; + } + + /** + * Close the client and release resources. + * Note: This does not close the shared Vertx instance. + */ + public void close() { + // WebClient resources are automatically cleaned up when the Vertx instance closes + // We don't close the vertx instance here since it's shared with Runner + } + + public CompletableFuture simulateTransaction( + final long parentBlockNumber, final String transactionRlp) { + + final CompletableFuture completableFuture = new CompletableFuture<>(); + final int requestId = RANDOM.nextInt(); + + try { + // Decode the RLP transaction to extract fields + final Bytes transactionBytes = Bytes.fromHexString(transactionRlp); + final Transaction transaction = + Transaction.readFrom(new BytesValueRLPInput(transactionBytes, false)); + + // Create block overrides with parent block number + final Map blockOverrides = + Collections.singletonMap("number", "0x" + Long.toHexString(parentBlockNumber + 1)); + + // Extract transaction fields for the call + final String chainId = + transaction.getChainId().map(cid -> "0x" + cid.toString(16)).orElse(null); + final String from = transaction.getSender().toHexString(); + final String to = transaction.getTo().map(address -> address.toHexString()).orElse(null); + final String gas = "0x" + Long.toHexString(transaction.getGasLimit()); + final String value = transaction.getValue().toHexString(); + final String nonce = "0x" + Long.toHexString(transaction.getNonce()); + final String input = transaction.getPayload().toHexString(); + + // Handle gas pricing fields based on transaction type + String gasPrice = null; + String maxPriorityFeePerGas = null; + String maxFeePerGas = null; + String maxFeePerBlobGas = null; + + if (transaction.getGasPrice().isPresent()) { + gasPrice = transaction.getGasPrice().get().toHexString(); + } + if (transaction.getMaxPriorityFeePerGas().isPresent()) { + maxPriorityFeePerGas = transaction.getMaxPriorityFeePerGas().get().toHexString(); + } + if (transaction.getMaxFeePerGas().isPresent()) { + maxFeePerGas = transaction.getMaxFeePerGas().get().toHexString(); + } + if (transaction.getMaxFeePerBlobGas().isPresent()) { + maxFeePerBlobGas = transaction.getMaxFeePerBlobGas().get().toHexString(); + } + + // Extract access list if present + final List> accessList = + transaction.getAccessList().map( + al -> + al.stream() + .map( + entry -> { + final Map accessEntry = new HashMap<>(); + accessEntry.put("address", entry.address().toHexString()); + accessEntry.put( + "storageKeys", + entry.storageKeys().stream() + .map(Bytes::toHexString) + .collect(Collectors.toList())); + return accessEntry; + }) + .collect(Collectors.toList())) + .orElse(null); + + // Extract blob versioned hashes if present + final List blobVersionedHashes = + transaction.getBlobsWithCommitments().isPresent() + ? transaction.getBlobsWithCommitments().get().getVersionedHashes().stream() + .map(BytesHolder::toHexString) + .collect(Collectors.toList()) + : null; + + // Create transaction call with decoded fields + final SimulateV1Request.TransactionCall transactionCall = + new SimulateV1Request.TransactionCall( + chainId, + from, + to, + gas, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + value, + nonce, + input, + accessList, + blobVersionedHashes); + + // Create block state call + final SimulateV1Request.BlockStateCall blockStateCall = + new SimulateV1Request.BlockStateCall(blockOverrides, List.of(transactionCall)); + + // Create params with returnTrieLog=true + final SimulateV1Request.SimulateV1Params params = + new SimulateV1Request.SimulateV1Params(List.of(blockStateCall), true, true); + + // Specify the parent block number as the block parameter for simulation context + final String blockParameter = "0x" + Long.toHexString(parentBlockNumber); + + final SimulateV1Request jsonRpcRequest = new SimulateV1Request(requestId, params, blockParameter); + + // Send the request to the JSON-RPC service + webClient + .request(HttpMethod.POST, besuHttpPort, besuHttpHost, "/") + .putHeader("Content-Type", APPLICATION_JSON) + .timeout(TimeUnit.SECONDS.toMillis(30)) + .sendJson( + jsonRpcRequest, + response -> { + if (response.succeeded()) { + try { + final SimulateV1Response responseBody = + response.result().bodyAsJson(SimulateV1Response.class); + LOG.atDebug() + .setMessage("response received for eth_simulateV1 {}") + .addArgument(responseBody) + .log(); + + // Check if the response contains an error + if (responseBody.getError() != null) { + final String errorMessage = + String.format( + "eth_simulateV1 error [code=%d]: %s", + responseBody.getError().getCode(), + responseBody.getError().getMessage()); + LOG.atDebug() + .setMessage("eth_simulateV1 returned error: {}") + .addArgument(errorMessage) + .log(); + completableFuture.completeExceptionally(new RuntimeException(errorMessage)); + return; + } + + if (responseBody.getResult() != null + && !responseBody.getResult().isEmpty()) { + final SimulateV1Response.BlockResult blockResult = + responseBody.getResult().get(0); + + // Check if any calls in the block result have errors + if (blockResult.getCalls() != null && !blockResult.getCalls().isEmpty()) { + final SimulateV1Response.CallResult firstCall = + blockResult.getCalls().get(0); + if (firstCall.getError() != null) { + final String errorMessage = + String.format( + "Transaction simulation failed [code=%d]: %s", + firstCall.getError().getCode(), + firstCall.getError().getMessage()); + LOG.atDebug() + .setMessage("eth_simulateV1 call error: {}") + .addArgument(errorMessage) + .log(); + completableFuture.completeExceptionally(new RuntimeException(errorMessage)); + return; + } + } + + final String trieLog = blockResult.getTrieLog(); + + if (trieLog != null) { + LOG.atInfo() + .setMessage("eth_simulateV1 returned trielog: length={}, prefix={}") + .addArgument(trieLog.length()) + .addArgument(trieLog.length() > 100 ? trieLog.substring(0, 100) : trieLog) + .log(); + completableFuture.complete(trieLog); + } else { + completableFuture.completeExceptionally( + new RuntimeException("No trielog in eth_simulateV1 response")); + } + } else { + completableFuture.completeExceptionally( + new RuntimeException("Empty result in eth_simulateV1 response")); + } + + } catch (RuntimeException e) { + LOG.error("failed to handle eth_simulateV1 response {}", e.getMessage()); + LOG.debug("exception handling eth_simulateV1 response", e); + completableFuture.completeExceptionally(e); + } + } else { + LOG.trace( + "failed to call eth_simulateV1 on besu {}", response.cause().getMessage()); + completableFuture.completeExceptionally( + new RuntimeException( + "cannot call eth_simulateV1 on besu: " + response.cause().getMessage())); + } + }); + } catch (Exception e) { + LOG.error("Failed to decode transaction RLP: {}", e.getMessage()); + LOG.debug("Exception decoding transaction RLP", e); + completableFuture.completeExceptionally( + new RuntimeException("Failed to decode transaction RLP: " + e.getMessage(), e)); + } + + return completableFuture; + } +} diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/GetRawTrieLogClient.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/GetRawTrieLogClient.java index 3e69f647..18b12ec4 100644 --- a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/GetRawTrieLogClient.java +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/GetRawTrieLogClient.java @@ -50,6 +50,7 @@ public class GetRawTrieLogClient { } } + private final Vertx vertx; private final WebClient webClient; private final String besuHttpHost; private final int besuHttpPort; @@ -57,8 +58,11 @@ public class GetRawTrieLogClient { private final TrieLogManager trieLogManager; public GetRawTrieLogClient( - final TrieLogManager trieLogManager, final String besuHttpHost, final int besuHttpPort) { - final Vertx vertx = Vertx.vertx(); + final Vertx vertx, + final TrieLogManager trieLogManager, + final String besuHttpHost, + final int besuHttpPort) { + this.vertx = vertx; final WebClientOptions options = new WebClientOptions(); this.webClient = WebClient.create(vertx, options); this.trieLogManager = trieLogManager; @@ -66,6 +70,15 @@ public GetRawTrieLogClient( this.besuHttpPort = besuHttpPort; } + /** + * Close the client and release resources. + * Note: This does not close the shared Vertx instance. + */ + public void close() { + // WebClient resources are automatically cleaned up when the Vertx instance closes + // We don't close the vertx instance here since it's shared with Runner + } + public CompletableFuture> getTrieLog( final long startBlockNumber, final long endBlockNumber) { @@ -90,6 +103,28 @@ public CompletableFuture> getTrieLog( .addArgument(responseBody) .log(); + // Check if the response contains an error + if (responseBody.getError() != null) { + final String errorMessage = + String.format( + "getTrieLogsByRange error [code=%d]: %s", + responseBody.getError().getCode(), + responseBody.getError().getMessage()); + LOG.atDebug() + .setMessage("getTrieLogsByRange returned error: {}") + .addArgument(errorMessage) + .log(); + completableFuture.completeExceptionally(new RuntimeException(errorMessage)); + return; + } + + // Check if result is present + if (responseBody.getResult() == null) { + completableFuture.completeExceptionally( + new RuntimeException("getTrieLogsByRange returned null result")); + return; + } + try { TrieLogManager.TrieLogManagerUpdater trieLogManagerTransaction = trieLogManager.updater(); diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/GetRawTrieLogRpcResponse.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/GetRawTrieLogRpcResponse.java index 78029c9c..57871683 100644 --- a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/GetRawTrieLogRpcResponse.java +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/GetRawTrieLogRpcResponse.java @@ -29,6 +29,9 @@ public class GetRawTrieLogRpcResponse { @JsonProperty("result") private List result; + @JsonProperty("error") + private JsonRpcError error; + @JsonProperty("jsonrpc") public String getJsonrpc() { return jsonrpc; @@ -59,6 +62,16 @@ public void setResult(List result) { this.result = result; } + @JsonProperty("error") + public JsonRpcError getError() { + return error; + } + + @JsonProperty("error") + public void setError(JsonRpcError error) { + this.error = error; + } + @Override public String toString() { return "GetRawTrieLogRpcResponse{" @@ -69,6 +82,8 @@ public String toString() { + id + ", result=" + result + + ", error=" + + error + '}'; } } diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/JsonRpcError.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/JsonRpcError.java new file mode 100644 index 00000000..82227bce --- /dev/null +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/JsonRpcError.java @@ -0,0 +1,65 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.client.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** JSON-RPC error response model. */ +public class JsonRpcError { + @JsonProperty("code") + private Integer code; + + @JsonProperty("message") + private String message; + + @JsonProperty("data") + private Object data; + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Object getData() { + return data; + } + + public void setData(Object data) { + this.data = data; + } + + @Override + public String toString() { + return "JsonRpcError{" + + "code=" + + code + + ", message='" + + message + + '\'' + + ", data=" + + data + + '}'; + } +} diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Request.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Request.java new file mode 100644 index 00000000..e46a222a --- /dev/null +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Request.java @@ -0,0 +1,211 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.client.model; + +import net.consensys.shomei.rpc.client.BesuRpcMethod; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestId; + +public class SimulateV1Request extends JsonRpcRequest { + + public SimulateV1Request( + final long requestId, final SimulateV1Params params, final String blockParameter) { + super( + "2.0", + BesuRpcMethod.BESU_ETH_SIMULATE_V1.getMethodName(), + new Object[] {params, blockParameter}); + setId(new JsonRpcRequestId(requestId)); + } + + public static class SimulateV1Params { + @JsonProperty("blockStateCalls") + private final List blockStateCalls; + + @JsonProperty("validation") + private final boolean validation; + + @JsonProperty("returnTrieLog") + private final boolean returnTrieLog; + + public SimulateV1Params( + final List blockStateCalls, + final boolean validation, + final boolean returnTrieLog) { + this.blockStateCalls = blockStateCalls; + this.validation = validation; + this.returnTrieLog = returnTrieLog; + } + + public List getBlockStateCalls() { + return blockStateCalls; + } + + public boolean isValidation() { + return validation; + } + + public boolean isReturnTrieLog() { + return returnTrieLog; + } + } + + public static class BlockStateCall { + @JsonProperty("blockOverrides") + private final Map blockOverrides; + + @JsonProperty("calls") + private final List calls; + + public BlockStateCall( + final Map blockOverrides, final List calls) { + this.blockOverrides = blockOverrides; + this.calls = calls; + } + + public Map getBlockOverrides() { + return blockOverrides; + } + + public List getCalls() { + return calls; + } + } + + public static class TransactionCall { + @JsonProperty("chainId") + private final String chainId; + + @JsonProperty("from") + private final String from; + + @JsonProperty("to") + private final String to; + + @JsonProperty("gas") + private final String gas; + + @JsonProperty("gasPrice") + private final String gasPrice; + + @JsonProperty("maxPriorityFeePerGas") + private final String maxPriorityFeePerGas; + + @JsonProperty("maxFeePerGas") + private final String maxFeePerGas; + + @JsonProperty("maxFeePerBlobGas") + private final String maxFeePerBlobGas; + + @JsonProperty("value") + private final String value; + + @JsonProperty("nonce") + private final String nonce; + + @JsonProperty("input") + private final String input; + + @JsonProperty("accessList") + private final List> accessList; + + @JsonProperty("blobVersionedHashes") + private final List blobVersionedHashes; + + public TransactionCall( + final String chainId, + final String from, + final String to, + final String gas, + final String gasPrice, + final String maxPriorityFeePerGas, + final String maxFeePerGas, + final String maxFeePerBlobGas, + final String value, + final String nonce, + final String input, + final List> accessList, + final List blobVersionedHashes) { + this.chainId = chainId; + this.from = from; + this.to = to; + this.gas = gas; + this.gasPrice = gasPrice; + this.maxPriorityFeePerGas = maxPriorityFeePerGas; + this.maxFeePerGas = maxFeePerGas; + this.maxFeePerBlobGas = maxFeePerBlobGas; + this.value = value; + this.nonce = nonce; + this.input = input; + this.accessList = accessList; + this.blobVersionedHashes = blobVersionedHashes; + } + + public String getChainId() { + return chainId; + } + + public String getFrom() { + return from; + } + + public String getTo() { + return to; + } + + public String getGas() { + return gas; + } + + public String getGasPrice() { + return gasPrice; + } + + public String getMaxPriorityFeePerGas() { + return maxPriorityFeePerGas; + } + + public String getMaxFeePerGas() { + return maxFeePerGas; + } + + public String getMaxFeePerBlobGas() { + return maxFeePerBlobGas; + } + + public String getValue() { + return value; + } + + public String getNonce() { + return nonce; + } + + public String getInput() { + return input; + } + + public List> getAccessList() { + return accessList; + } + + public List getBlobVersionedHashes() { + return blobVersionedHashes; + } + } +} diff --git a/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Response.java b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Response.java new file mode 100644 index 00000000..09c732d2 --- /dev/null +++ b/services/rpc/client/src/main/java/net/consensys/shomei/rpc/client/model/SimulateV1Response.java @@ -0,0 +1,229 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.client.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SimulateV1Response { + @JsonProperty("jsonrpc") + private String jsonrpc; + + @JsonProperty("id") + private Integer id; + + @JsonProperty("result") + private List result; + + @JsonProperty("error") + private JsonRpcError error; + + public String getJsonrpc() { + return jsonrpc; + } + + public void setJsonrpc(String jsonrpc) { + this.jsonrpc = jsonrpc; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public List getResult() { + return result; + } + + public void setResult(List result) { + this.result = result; + } + + public JsonRpcError getError() { + return error; + } + + public void setError(JsonRpcError error) { + this.error = error; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class BlockResult { + @JsonProperty("number") + private String number; + + @JsonProperty("hash") + private String hash; + + @JsonProperty("stateRoot") + private String stateRoot; + + @JsonProperty("gasUsed") + private String gasUsed; + + @JsonProperty("blobGasUsed") + private String blobGasUsed; + + @JsonProperty("baseFeePerGas") + private String baseFeePerGas; + + @JsonProperty("trieLog") + private String trieLog; + + @JsonProperty("calls") + private List calls; + + public String getNumber() { + return number; + } + + public void setNumber(String number) { + this.number = number; + } + + public String getHash() { + return hash; + } + + public void setHash(String hash) { + this.hash = hash; + } + + public String getStateRoot() { + return stateRoot; + } + + public void setStateRoot(String stateRoot) { + this.stateRoot = stateRoot; + } + + public String getGasUsed() { + return gasUsed; + } + + public void setGasUsed(String gasUsed) { + this.gasUsed = gasUsed; + } + + public String getBlobGasUsed() { + return blobGasUsed; + } + + public void setBlobGasUsed(String blobGasUsed) { + this.blobGasUsed = blobGasUsed; + } + + public String getBaseFeePerGas() { + return baseFeePerGas; + } + + public void setBaseFeePerGas(String baseFeePerGas) { + this.baseFeePerGas = baseFeePerGas; + } + + public String getTrieLog() { + return trieLog; + } + + public void setTrieLog(String trieLog) { + this.trieLog = trieLog; + } + + public List getCalls() { + return calls; + } + + public void setCalls(List calls) { + this.calls = calls; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CallResult { + @JsonProperty("status") + private String status; + + @JsonProperty("gasUsed") + private String gasUsed; + + @JsonProperty("logs") + private List logs; + + @JsonProperty("returnData") + private String returnData; + + @JsonProperty("error") + private JsonRpcError error; + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getGasUsed() { + return gasUsed; + } + + public void setGasUsed(String gasUsed) { + this.gasUsed = gasUsed; + } + + public List getLogs() { + return logs; + } + + public void setLogs(List logs) { + this.logs = logs; + } + + public String getReturnData() { + return returnData; + } + + public void setReturnData(String returnData) { + this.returnData = returnData; + } + + public JsonRpcError getError() { + return error; + } + + public void setError(JsonRpcError error) { + this.error = error; + } + } + + @Override + public String toString() { + return "SimulateV1Response{" + + "jsonrpc='" + + jsonrpc + + '\'' + + ", id=" + + id + + ", result=" + + result + + ", error=" + + error + + '}'; + } +} diff --git a/services/rpc/common/src/main/java/net/consensys/shomei/rpc/model/TrieLogElement.java b/services/rpc/common/src/main/java/net/consensys/shomei/rpc/model/TrieLogElement.java index e0ec352e..3f3e21a5 100644 --- a/services/rpc/common/src/main/java/net/consensys/shomei/rpc/model/TrieLogElement.java +++ b/services/rpc/common/src/main/java/net/consensys/shomei/rpc/model/TrieLogElement.java @@ -19,25 +19,46 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.tuweni.bytes.Bytes32; -public record TrieLogElement( - Long blockNumber, Bytes32 blockHash, boolean isInitialSync, String trieLog) { +public class TrieLogElement { - public TrieLogIdentifier getTrieLogIdentifier() { - return new TrieLogIdentifier(blockNumber, blockHash, isInitialSync); - } + private Long blockNumber; + private Bytes32 blockHash; + private boolean isInitialSync; + private String trieLog; + + public TrieLogElement() {} @JsonCreator public TrieLogElement( @JsonProperty("blockNumber") final Long blockNumber, @JsonProperty("blockHash") final Bytes32 blockHash, - @JsonProperty("syncing") final boolean isInitialSync, @JsonProperty("trieLog") final String trieLog) { this.blockNumber = blockNumber; this.blockHash = blockHash; - this.isInitialSync = isInitialSync; + this.isInitialSync = false; this.trieLog = trieLog; } + public Long blockNumber() { + return blockNumber; + } + + public Bytes32 blockHash() { + return blockHash; + } + + public boolean isInitialSync() { + return isInitialSync; + } + + public String trieLog() { + return trieLog; + } + + public TrieLogIdentifier getTrieLogIdentifier() { + return new TrieLogIdentifier(blockNumber, blockHash, isInitialSync); + } + @Override public String toString() { return "SendRawTrieLogParameter{" diff --git a/services/rpc/server/build.gradle b/services/rpc/server/build.gradle index 060d1df0..71534945 100644 --- a/services/rpc/server/build.gradle +++ b/services/rpc/server/build.gradle @@ -18,6 +18,7 @@ dependencies { implementation project(':util') implementation project(':core') implementation project(':crypto') + implementation project(':services:rpc:client') implementation project(':services:rpc:common') implementation project(':services:metrics') implementation project(':sync') @@ -62,6 +63,8 @@ dependencies { // Test dependencies testImplementation 'org.junit.jupiter:junit-jupiter-api' + testImplementation project(':services:rpc:client') + testImplementation 'org.hyperledger.besu.internal:besu-crypto-algorithms' testImplementation 'org.assertj:assertj-core' testImplementation 'org.mockito:mockito-core' diff --git a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/JsonRpcService.java b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/JsonRpcService.java index 420c4071..f224f25f 100644 --- a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/JsonRpcService.java +++ b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/JsonRpcService.java @@ -17,10 +17,12 @@ import net.consensys.shomei.fullsync.FullSyncDownloader; import net.consensys.shomei.metrics.MetricsService; +import net.consensys.shomei.rpc.client.BesuSimulateClient; import net.consensys.shomei.rpc.server.method.LineaGetProof; import net.consensys.shomei.rpc.server.method.LineaGetTrielogProof; import net.consensys.shomei.rpc.server.method.RollupDeleteZkEVMStateMerkleProofByRange; import net.consensys.shomei.rpc.server.method.RollupForkChoiceUpdated; +import net.consensys.shomei.rpc.server.method.RollupGetVirtualZkEVMStateMerkleProofV0; import net.consensys.shomei.rpc.server.method.RollupGetZkEVMBlockNumber; import net.consensys.shomei.rpc.server.method.RollupGetZkEVMStateMerkleProofV0; import net.consensys.shomei.rpc.server.method.SendRawTrieLog; @@ -96,7 +98,8 @@ public JsonRpcService( final Integer rpcHttpPort, final Optional> hostAllowList, final FullSyncDownloader fullSyncDownloader, - final ZkWorldStateArchive worldStateArchive) { + final ZkWorldStateArchive worldStateArchive, + final BesuSimulateClient besuSimulateClient) { this.config = JsonRpcConfiguration.createDefault(); config.setHost(rpcHttpHost); config.setPort(rpcHttpPort); @@ -111,7 +114,8 @@ public JsonRpcService( new RollupGetZkEVMBlockNumber(worldStateArchive), new RollupDeleteZkEVMStateMerkleProofByRange(worldStateArchive.getTraceManager()), new RollupForkChoiceUpdated(worldStateArchive, fullSyncDownloader), - new RollupGetZkEVMStateMerkleProofV0(worldStateArchive.getTraceManager()))); + new RollupGetZkEVMStateMerkleProofV0(worldStateArchive.getTraceManager()), + new RollupGetVirtualZkEVMStateMerkleProofV0(worldStateArchive, besuSimulateClient))); this.maxActiveConnections = config.getMaxActiveConnections(); this.livenessService = new HealthService(new LivenessCheck()); } diff --git a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/ShomeiRpcMethod.java b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/ShomeiRpcMethod.java index 85b637c5..6fef963c 100644 --- a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/ShomeiRpcMethod.java +++ b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/ShomeiRpcMethod.java @@ -23,6 +23,7 @@ public enum ShomeiRpcMethod { LINEA_GET_TRIELOG_PROOF("linea_getTrielogProof"), ROLLUP_GET_ZKEVM_STATE_MERKLE_PROOF_V0("rollup_getZkEVMStateMerkleProofV0"), + ROLLUP_GET_VIRTUAL_ZKEVM_STATE_MERKLE_PROOF_V0("rollup_getVirtualZkEVMStateMerkleProofV0"), ROLLUP_DELETE_ZKEVM_STATE_MERKLE_PROOF_BY_RANGE("rollup_deleteZkEVMStateMerkleProofByRange"), ROLLUP_FORK_CHOICE_UPDATED("rollup_forkChoiceUpdated"); diff --git a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0.java b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0.java new file mode 100644 index 00000000..04821b2f --- /dev/null +++ b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0.java @@ -0,0 +1,128 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.server.method; + +import static net.consensys.shomei.rpc.server.ShomeiVersion.IMPL_VERSION; + +import net.consensys.shomei.rpc.client.BesuSimulateClient; +import net.consensys.shomei.rpc.server.ShomeiRpcMethod; +import net.consensys.shomei.rpc.server.error.ShomeiJsonRpcErrorResponse; +import net.consensys.shomei.rpc.server.model.RollupGetVirtualZkEVMStateMerkleProofV0Response; +import net.consensys.shomei.rpc.server.model.RollupGetVirtualZkEvmStateMerkleProofV0Parameter; +import net.consensys.shomei.storage.ZkWorldStateArchive; +import net.consensys.shomei.trie.ZKTrie; +import net.consensys.shomei.trielog.TrieLogLayer; + +import org.apache.tuweni.bytes.Bytes; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.rlp.RLP; + +public class RollupGetVirtualZkEVMStateMerkleProofV0 implements JsonRpcMethod { + + final ZkWorldStateArchive worldStateArchive; + final BesuSimulateClient besuSimulateClient; + + public RollupGetVirtualZkEVMStateMerkleProofV0( + final ZkWorldStateArchive worldStateArchive, final BesuSimulateClient besuSimulateClient) { + this.worldStateArchive = worldStateArchive; + this.besuSimulateClient = besuSimulateClient; + } + + @Override + public String getName() { + return ShomeiRpcMethod.ROLLUP_GET_VIRTUAL_ZKEVM_STATE_MERKLE_PROOF_V0.getMethodName(); + } + + @Override + public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { + final RollupGetVirtualZkEvmStateMerkleProofV0Parameter param; + try { + param = + requestContext.getRequiredParameter( + 0, RollupGetVirtualZkEvmStateMerkleProofV0Parameter.class); + } catch (JsonRpcParameter.JsonRpcParameterException e) { + throw new RuntimeException(e); + } + + final long blockNumber = param.getBlockNumber(); + final long parentBlockNumber = blockNumber - 1; + final String transactionRlp = param.getTransaction(); + + + try { + + // blockNumber represents the virtual block we want to build, fail early if we do not have it + if (worldStateArchive.getTrieLogManager().getTrieLog(parentBlockNumber).isEmpty()) { + return new ShomeiJsonRpcErrorResponse( + requestContext.getRequest().getId(), + RpcErrorType.INVALID_PARAMS, + "BLOCK_MISSING_IN_CHAIN - block %d is missing".formatted(parentBlockNumber)); + } + + // do not have it in cache + if (worldStateArchive.getCachedWorldState(parentBlockNumber).isEmpty()) { + return new ShomeiJsonRpcErrorResponse( + requestContext.getRequest().getId(), + RpcErrorType.INVALID_PARAMS, + "Worldstate for parent block %d is not cached".formatted(parentBlockNumber)); + } + + // Call eth_simulateV1 to get the trielog for the virtual block + // Simulate on top of parentBlockNumber state to build virtual block at blockNumber + final String trieLogHex = + besuSimulateClient.simulateTransaction(parentBlockNumber, transactionRlp).join(); + final Bytes trieLogBytes = Bytes.fromHexString(trieLogHex); + + // Decode the trielog + final TrieLogLayer trieLogLayer = + worldStateArchive.getTrieLogLayerConverter().decodeTrieLog(RLP.input(trieLogBytes)); + + // Apply the virtual trielog and generate the trace + // This generates a trace without persisting the state + // Use parentBlockNumber as the base state for applying the virtual trielog + final var virtualTraceResult = + worldStateArchive.generateVirtualTrace(parentBlockNumber, trieLogLayer); + + // Get the parent state root hash (state at parentBlockNumber that we're building on) + final String zkParentStateRootHash = + worldStateArchive + .getTraceManager() + .getZkStateRootHash(parentBlockNumber) + .orElse(ZKTrie.DEFAULT_TRIE_ROOT) + .toHexString(); + + // Get the resulting state root hash (state after applying the virtual block) + final String zkEndStateRootHash = virtualTraceResult.zkEndStateRootHash().toHexString(); + + return new JsonRpcSuccessResponse( + requestContext.getRequest().getId(), + new RollupGetVirtualZkEVMStateMerkleProofV0Response( + zkParentStateRootHash, + zkEndStateRootHash, + virtualTraceResult.traces(), + IMPL_VERSION)); + + } catch (Exception e) { + return new ShomeiJsonRpcErrorResponse( + requestContext.getRequest().getId(), + RpcErrorType.INTERNAL_ERROR, + "Error processing virtual block: " + e.getMessage()); + } + } +} diff --git a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEVMStateMerkleProofV0Response.java b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEVMStateMerkleProofV0Response.java new file mode 100644 index 00000000..02e6f078 --- /dev/null +++ b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEVMStateMerkleProofV0Response.java @@ -0,0 +1,47 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.server.model; + +import net.consensys.shomei.trie.trace.Trace; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +@SuppressWarnings("unused") +public class RollupGetVirtualZkEVMStateMerkleProofV0Response { + + @JsonProperty("zkParentStateRootHash") + private final String zkParentStateRootHash; + + @JsonProperty("zkEndStateRootHash") + private final String zkEndStateRootHash; + + @JsonProperty("zkStateMerkleProof") + private final List> zkStateMerkleProof; + + @JsonProperty("zkStateManagerVersion") + private final String zkStateManagerVersion; + + public RollupGetVirtualZkEVMStateMerkleProofV0Response( + final String zkParentStateRootHash, + final String zkEndStateRootHash, + final List> zkStateMerkleProof, + final String zkStateManagerVersion) { + this.zkParentStateRootHash = zkParentStateRootHash; + this.zkEndStateRootHash = zkEndStateRootHash; + this.zkStateMerkleProof = zkStateMerkleProof; + this.zkStateManagerVersion = zkStateManagerVersion; + } +} diff --git a/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEvmStateMerkleProofV0Parameter.java b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEvmStateMerkleProofV0Parameter.java new file mode 100644 index 00000000..829ad5e7 --- /dev/null +++ b/services/rpc/server/src/main/java/net/consensys/shomei/rpc/server/model/RollupGetVirtualZkEvmStateMerkleProofV0Parameter.java @@ -0,0 +1,51 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.server.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RollupGetVirtualZkEvmStateMerkleProofV0Parameter { + + private final String blockNumber; + + private final String transaction; + + @JsonCreator + public RollupGetVirtualZkEvmStateMerkleProofV0Parameter( + @JsonProperty("blockNumber") final String blockNumber, + @JsonProperty("transaction") final String transaction) { + this.blockNumber = blockNumber; + this.transaction = transaction; + } + + public long getBlockNumber() { + return Long.decode(blockNumber); + } + + public String getTransaction() { + return transaction; + } + + @Override + public String toString() { + return "RollupGetVirtualZkEvmStateMerkleProofV0Parameter{" + + "blockNumber=" + + blockNumber + + ", transaction='" + + transaction + + '\'' + + '}'; + } +} diff --git a/services/rpc/server/src/test/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0Test.java b/services/rpc/server/src/test/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0Test.java new file mode 100644 index 00000000..410808ff --- /dev/null +++ b/services/rpc/server/src/test/java/net/consensys/shomei/rpc/server/method/RollupGetVirtualZkEVMStateMerkleProofV0Test.java @@ -0,0 +1,232 @@ +/* + * Copyright ConsenSys Software Inc., 2026 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.rpc.server.method; + +import static net.consensys.zkevm.HashProvider.KECCAK_HASH_ZERO; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import net.consensys.shomei.rpc.client.BesuSimulateClient; +import net.consensys.shomei.rpc.server.error.ShomeiJsonRpcErrorResponse; +import net.consensys.shomei.rpc.server.model.RollupGetVirtualZkEvmStateMerkleProofV0Parameter; +import net.consensys.shomei.storage.TraceManager; +import net.consensys.shomei.storage.TrieLogManager; +import net.consensys.shomei.storage.ZkWorldStateArchive; +import net.consensys.shomei.trie.trace.Trace; +import net.consensys.shomei.trielog.TrieLogLayer; +import net.consensys.shomei.trielog.TrieLogLayerConverter; +import net.consensys.shomei.worldview.ZkEvmWorldState; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import org.apache.tuweni.bytes.Bytes; +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class RollupGetVirtualZkEVMStateMerkleProofV0Test { + + @Mock public ZkWorldStateArchive worldStateArchive; + @Mock public BesuSimulateClient besuSimulateClient; + @Mock public TrieLogManager trieLogManager; + @Mock public TraceManager traceManager; + @Mock public TrieLogLayerConverter trieLogLayerConverter; + @Mock public TrieLogLayer trieLogLayer; + @Mock public ZkEvmWorldState mockZkWorldState; + + public RollupGetVirtualZkEVMStateMerkleProofV0 method; + + // private static final String TEST_ACCOUNT_ADDRESS = "f17f52151EbEF6C7334FAD080c5704D77216b732"; + private static final String TEST_ACCOUNT_PRIVATE_KEY = + "0xae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f"; + + @BeforeEach + public void setup() { + method = new RollupGetVirtualZkEVMStateMerkleProofV0(worldStateArchive, besuSimulateClient); + } + + /** + * Creates a signed transaction RLP using the test account that has balance in the genesis file. + * Balance: 0x09184e72a000 (10000000000000 wei = 0.00001 ETH) Gas cost: 21000 * 100000000 = + * 2100000000000 wei Value: 1000000000000 wei Total: 3100000000000 wei (fits in budget) + */ + private String createTestTransactionRlp() { + final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); + final KeyPair keyPair = + signatureAlgorithm.createKeyPair( + signatureAlgorithm.createPrivateKey( + Bytes.fromHexString(TEST_ACCOUNT_PRIVATE_KEY).toUnsignedBigInteger())); + + final Transaction transaction = + Transaction.builder() + .type(TransactionType.FRONTIER) + .nonce(0) + .gasPrice(Wei.of(100000000)) // 0.1 Gwei + .gasLimit(21000) + .to(Address.fromHexString("0x3535353535353535353535353535353535353535")) + .value(Wei.of(1000000000000L)) // 0.000001 ETH + .payload(Bytes.EMPTY) + .chainId(BigInteger.ONE) + .signAndBuild(keyPair); + + return transaction.encoded().toHexString(); + } + + @Test + public void shouldReturnCorrectMethodName() { + assertThat(method.getName()).isEqualTo("rollup_getVirtualZkEVMStateMerkleProofV0"); + } + + @Test + public void shouldReturnBlockMissingWhenParentBlockUnavailable() { + when(worldStateArchive.getTrieLogManager()).thenReturn(trieLogManager); + when(trieLogManager.getTrieLog(7L)).thenReturn(Optional.empty()); + + final JsonRpcRequestContext request = request(8L, createTestTransactionRlp()); + final JsonRpcResponse response = method.response(request); + + assertThat(response).isInstanceOf(ShomeiJsonRpcErrorResponse.class); + final ShomeiJsonRpcErrorResponse errorResponse = (ShomeiJsonRpcErrorResponse) response; + assertThat(errorResponse.getError().getCode()).isEqualTo(RpcErrorType.INVALID_PARAMS.getCode()); + assertThat(errorResponse.getJsonError().message()).contains("BLOCK_MISSING_IN_CHAIN"); + assertThat(errorResponse.getJsonError().message()).contains("block 7 is missing"); + } + + @Test + public void shouldReturnValidResponseWhenSimulationSucceeds() throws Exception { + // Mock trielog bytes (RLP encoded trielog layer) + final String mockTrieLogHex = createMockTrieLogHex(); + final String testTxRlp = createTestTransactionRlp(); + + // Mock the BesuSimulateClient to return a trielog + when(besuSimulateClient.simulateTransaction(eq(7L), eq(testTxRlp))) + .thenReturn(CompletableFuture.completedFuture(mockTrieLogHex)); + + // Mock world state archive behavior + when(worldStateArchive.getTrieLogManager()).thenReturn(trieLogManager); + when(trieLogManager.getTrieLog(7L)) + .thenReturn(Optional.of(Bytes.fromHexString("0x01"))); // Parent block exists + when(worldStateArchive.getCachedWorldState(7L)).thenReturn(Optional.of(mockZkWorldState)); + + when(worldStateArchive.getTraceManager()).thenReturn(traceManager); + when(traceManager.getZkStateRootHash(7L)).thenReturn(Optional.of(KECCAK_HASH_ZERO)); + + // Mock the trielog layer converter + when(worldStateArchive.getTrieLogLayerConverter()).thenReturn(trieLogLayerConverter); + when(trieLogLayerConverter.decodeTrieLog(any())).thenReturn(trieLogLayer); + + // Mock virtual trace generation + final List> mockTraces = List.of(List.of()); + final ZkWorldStateArchive.VirtualTraceResult mockResult = + new ZkWorldStateArchive.VirtualTraceResult(mockTraces, KECCAK_HASH_ZERO); + when(worldStateArchive.generateVirtualTrace(eq(7L), any(TrieLogLayer.class))) + .thenReturn(mockResult); + + final JsonRpcRequestContext request = request(8L, testTxRlp); + final JsonRpcResponse response = method.response(request); + + assertThat(response).isInstanceOf(JsonRpcSuccessResponse.class); + final JsonRpcSuccessResponse successResponse = (JsonRpcSuccessResponse) response; + assertThat(successResponse.getResult()).isNotNull(); + } + + @Test + public void shouldReturnErrorWhenSimulationFails() { + when(worldStateArchive.getTrieLogManager()).thenReturn(trieLogManager); + when(trieLogManager.getTrieLog(7L)) + .thenReturn(Optional.of(Bytes.fromHexString("0x01"))); + when(worldStateArchive.getCachedWorldState(7L)).thenReturn(Optional.of(mockZkWorldState)); + + // Mock simulation failure + when(besuSimulateClient.simulateTransaction(anyLong(), anyString())) + .thenReturn( + CompletableFuture.failedFuture(new RuntimeException("Simulation failed"))); + + final JsonRpcRequestContext request = request(8L, createTestTransactionRlp()); + final JsonRpcResponse response = method.response(request); + + assertThat(response).isInstanceOf(ShomeiJsonRpcErrorResponse.class); + final ShomeiJsonRpcErrorResponse errorResponse = (ShomeiJsonRpcErrorResponse) response; + assertThat(errorResponse.getError().getCode()).isEqualTo(RpcErrorType.INTERNAL_ERROR.getCode()); + assertThat(errorResponse.getJsonError().message()).contains("Error processing virtual block"); + } + + @Test + public void shouldReturnErrorWithDetailsWhenSimulationReturnsError() { + when(worldStateArchive.getTrieLogManager()).thenReturn(trieLogManager); + when(trieLogManager.getTrieLog(7L)) + .thenReturn(Optional.of(Bytes.fromHexString("0x01"))); + when(worldStateArchive.getCachedWorldState(7L)).thenReturn(Optional.of(mockZkWorldState)); + + // Mock simulation returning a detailed error from eth_simulateV1 + final String detailedErrorMessage = "eth_simulateV1 error [code=-32000]: insufficient funds"; + when(besuSimulateClient.simulateTransaction(anyLong(), anyString())) + .thenReturn(CompletableFuture.failedFuture(new RuntimeException(detailedErrorMessage))); + + final JsonRpcRequestContext request = request(8L, createTestTransactionRlp()); + final JsonRpcResponse response = method.response(request); + + assertThat(response).isInstanceOf(ShomeiJsonRpcErrorResponse.class); + final ShomeiJsonRpcErrorResponse errorResponse = (ShomeiJsonRpcErrorResponse) response; + assertThat(errorResponse.getError().getCode()).isEqualTo(RpcErrorType.INTERNAL_ERROR.getCode()); + assertThat(errorResponse.getJsonError().message()).contains("Error processing virtual block"); + } + + private JsonRpcRequestContext request(final long blockNumber, final String transactionRlp) { + return new JsonRpcRequestContext( + new JsonRpcRequest( + "2.0", + "rollup_getVirtualZkEVMStateMerkleProofV0", + new Object[] { + new RollupGetVirtualZkEvmStateMerkleProofV0Parameter( + String.valueOf(blockNumber), transactionRlp) + })); + } + + private String createMockTrieLogHex() { + // Create a simple RLP-encoded trielog + final BytesValueRLPOutput output = new BytesValueRLPOutput(); + output.startList(); + output.writeBytes(KECCAK_HASH_ZERO); // blockHash + output.startList(); // account changes + output.endList(); + output.startList(); // storage changes + output.endList(); + output.endList(); + return output.encoded().toHexString(); + } +} diff --git a/shomei/src/main/java/net/consensys/shomei/Runner.java b/shomei/src/main/java/net/consensys/shomei/Runner.java index bf3376a2..ded4ef3a 100644 --- a/shomei/src/main/java/net/consensys/shomei/Runner.java +++ b/shomei/src/main/java/net/consensys/shomei/Runner.java @@ -22,6 +22,7 @@ import net.consensys.shomei.fullsync.rules.FullSyncRules; import net.consensys.shomei.metrics.MetricsService; import net.consensys.shomei.metrics.PrometheusMetricsService; +import net.consensys.shomei.rpc.client.BesuSimulateClient; import net.consensys.shomei.rpc.client.GetRawTrieLogClient; import net.consensys.shomei.rpc.server.JsonRpcService; import net.consensys.shomei.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder; @@ -53,6 +54,8 @@ public class Runner { private final MetricsService.VertxMetricsService metricsService; private final ZkWorldStateArchive worldStateArchive; + private final GetRawTrieLogClient getRawTrieLogClient; + private final BesuSimulateClient besuSimulateClient; public Runner( final DataStorageOption dataStorageOption, @@ -73,12 +76,17 @@ public Runner( worldStateArchive = new ZkWorldStateArchive(storageProvider, syncOption.isEnableFinalizedBlockLimit()); - final GetRawTrieLogClient getRawTrieLog = + this.getRawTrieLogClient = new GetRawTrieLogClient( + vertx, worldStateArchive.getTrieLogManager(), jsonRpcOption.getBesuRpcHttpHost(), jsonRpcOption.getBesuRHttpPort()); + this.besuSimulateClient = + new BesuSimulateClient( + vertx, jsonRpcOption.getBesuRpcHttpHost(), jsonRpcOption.getBesuRHttpPort()); + final FullSyncRules fullSyncRules = new FullSyncRules( syncOption.isTraceGenerationEnabled(), @@ -88,7 +96,7 @@ public Runner( Optional.ofNullable(syncOption.getFinalizedBlockNumberLimit()), Optional.ofNullable(syncOption.getFinalizedBlockHashLimit()).map(Bytes32::fromHexString)); - fullSyncDownloader = new FullSyncDownloader(worldStateArchive, getRawTrieLog, fullSyncRules); + fullSyncDownloader = new FullSyncDownloader(worldStateArchive, getRawTrieLogClient, fullSyncRules); this.jsonRpcService = new JsonRpcService( @@ -96,7 +104,8 @@ public Runner( jsonRpcOption.getRpcHttpPort(), Optional.of(jsonRpcOption.getRpcHttpHostAllowList()), fullSyncDownloader, - worldStateArchive); + worldStateArchive, + besuSimulateClient); } private void setupHashFunction(HashFunctionOption hashFunctionOption) { @@ -154,6 +163,14 @@ public void start() { } public void stop() throws IOException { + // Close RPC clients to release their Vertx instances and non-daemon threads + if (getRawTrieLogClient != null) { + getRawTrieLogClient.close(); + } + if (besuSimulateClient != null) { + besuSimulateClient.close(); + } + worldStateArchive.close(); vertx.close(); } diff --git a/sync/src/main/java/net/consensys/shomei/fullsync/TrieLogBlockingQueue.java b/sync/src/main/java/net/consensys/shomei/fullsync/TrieLogBlockingQueue.java index 845a5c9c..e434c406 100644 --- a/sync/src/main/java/net/consensys/shomei/fullsync/TrieLogBlockingQueue.java +++ b/sync/src/main/java/net/consensys/shomei/fullsync/TrieLogBlockingQueue.java @@ -19,8 +19,10 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Supplier; @@ -72,45 +74,76 @@ private Optional distance( public TrieLogObserver.TrieLogIdentifier waitForNewElement() { long distance; CompletableFuture foundBlockFuture; - try { - do { - // remove deprecated trielog (already imported block) - iterator() - .forEachRemaining( - trieLogIdentifier -> { - if (trieLogIdentifier.blockNumber() <= currentShomeiHeadSupplier.get()) { - remove(trieLogIdentifier); - } - }); - - if (importValidators.stream() - .anyMatch(blockImportValidator -> !blockImportValidator.canImportBlock())) { - /* - * We wait until all the rules allow us to import the block (minimum block confirmations, max limit, or others) - */ - clear(); - // just wait a second and check again: - foundBlockFuture = - CompletableFuture.supplyAsync( - () -> false, CompletableFuture.delayedExecutor(1, TimeUnit.SECONDS)); - } else { - final TrieLogObserver.TrieLogIdentifier trieLogIdentifier = peek(); - distance = - distance(trieLogIdentifier, currentShomeiHeadSupplier.get()) - .orElse(INITIAL_SYNC_BLOCK_NUMBER_RANGE); - if (distance == 1) { - remove(trieLogIdentifier); - return trieLogIdentifier; - } else { // missing trielog we need to import them - foundBlockFuture = onTrieLogMissing.apply(distance); - } + Boolean blocksFound; + + while (!completableFuture.isDone()) { + // remove deprecated trielog (already imported block) + iterator() + .forEachRemaining( + trieLogIdentifier -> { + if (trieLogIdentifier.blockNumber() <= currentShomeiHeadSupplier.get()) { + remove(trieLogIdentifier); + } + }); + + if (importValidators.stream() + .anyMatch(blockImportValidator -> !blockImportValidator.canImportBlock())) { + /* + * We wait until all the rules allow us to import the block (minimum block confirmations, max limit, or others) + */ + clear(); + // just wait a second and check again: + foundBlockFuture = + CompletableFuture.supplyAsync( + () -> false, CompletableFuture.delayedExecutor(1, TimeUnit.SECONDS)); + } else { + final TrieLogObserver.TrieLogIdentifier trieLogIdentifier = peek(); + distance = + distance(trieLogIdentifier, currentShomeiHeadSupplier.get()) + .orElse(INITIAL_SYNC_BLOCK_NUMBER_RANGE); + if (distance == 1) { + remove(trieLogIdentifier); + return trieLogIdentifier; + } else { // missing trielog we need to import them + foundBlockFuture = onTrieLogMissing.apply(distance); + } + } + + // Wait for the future to complete with a short timeout to allow frequent shutdown checks + try { + blocksFound = foundBlockFuture.get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // Timeout is normal, continue loop to check completableFuture.isDone() + blocksFound = false; + } catch (InterruptedException e) { + // Restore interrupted status and exit cleanly + Thread.currentThread().interrupt(); + return null; + } catch (ExecutionException e) { + // Log and continue on execution failure + blocksFound = false; + } + + // If blocks were found, exit and let caller process them + if (blocksFound) { + return null; + } + + // If no blocks were found, add a delay before retrying to avoid hammering Besu + // when we've caught up to the chain tip. Without this delay, Shomei would + // immediately retry in a tight loop since Besu responds quickly with empty results. + if (!completableFuture.isDone()) { + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException e) { + // Restore interrupted status and exit cleanly + Thread.currentThread().interrupt(); + return null; } - } while (!completableFuture.isDone() - && !foundBlockFuture.completeOnTimeout(false, 5, TimeUnit.SECONDS).get()); - return null; - } catch (Exception ex) { - return null; + } } + + return null; } public void stop() { diff --git a/sync/src/test/java/net/consensys/shomei/fullsync/TrieLogBlockingQueueTest.java b/sync/src/test/java/net/consensys/shomei/fullsync/TrieLogBlockingQueueTest.java new file mode 100644 index 00000000..9fe93aac --- /dev/null +++ b/sync/src/test/java/net/consensys/shomei/fullsync/TrieLogBlockingQueueTest.java @@ -0,0 +1,644 @@ +/* + * Copyright ConsenSys Software Inc., 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package net.consensys.shomei.fullsync; + +import static org.assertj.core.api.Assertions.assertThat; + +import net.consensys.shomei.fullsync.rules.BlockImportValidator; +import net.consensys.shomei.observer.TrieLogObserver.TrieLogIdentifier; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(value = 30, unit = TimeUnit.SECONDS) +public class TrieLogBlockingQueueTest { + + private ExecutorService executor; + + @BeforeEach + public void setUp() { + executor = Executors.newSingleThreadExecutor(); + } + + @AfterEach + public void tearDown() { + executor.shutdownNow(); + } + + // =========================================================================== + // Helpers + // =========================================================================== + + private static TrieLogIdentifier makeTrieLog(final long blockNumber) { + return new TrieLogIdentifier(blockNumber, Bytes32.wrap(Bytes.random(32)), false); + } + + private static BlockImportValidator alwaysAllow() { + return () -> true; + } + + private static BlockImportValidator alwaysBlock() { + return () -> false; + } + + // =========================================================================== + // OFFER / CAPACITY + // =========================================================================== + + @Test + public void testOffer_withinCapacity() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + assertThat(queue.offer(makeTrieLog(1))).isTrue(); + assertThat(queue.offer(makeTrieLog(2))).isTrue(); + assertThat(queue.offer(makeTrieLog(3))).isTrue(); + assertThat(queue).hasSize(3); + } + + @Test + public void testOffer_atCapacity_higherPriorityReplacesLowest() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 3, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(1)); + queue.offer(makeTrieLog(2)); + queue.offer(makeTrieLog(3)); + assertThat(queue).hasSize(3); + + // Block 4 has higher priority (greater block number), should evict block 1 + assertThat(queue.offer(makeTrieLog(4))).isTrue(); + assertThat(queue).hasSize(3); + + // Block 1 should have been evicted; smallest remaining is block 2 + assertThat(queue.peek().blockNumber()).isEqualTo(2); + } + + @Test + public void testOffer_atCapacity_lowerPriorityRejected() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 3, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(5)); + queue.offer(makeTrieLog(6)); + queue.offer(makeTrieLog(7)); + + // Block 3 is lower priority than everything in queue — should be rejected + assertThat(queue.offer(makeTrieLog(3))).isFalse(); + assertThat(queue).hasSize(3); + assertThat(queue.peek().blockNumber()).isEqualTo(5); + } + + @Test + public void testOffer_duplicateBlockNumbers() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(5)); + queue.offer(makeTrieLog(5)); + assertThat(queue).hasSize(2); + } + + // =========================================================================== + // STOP + // =========================================================================== + + @Test + public void testStop_causesWaitForNewElementToReturnNull() throws Exception { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(500); + + queue.stop(); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNull(); + } + + @Test + public void testStop_idempotent() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.stop(); + queue.stop(); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — happy path (distance == 1) + // =========================================================================== + + @Test + public void testWaitForNewElement_returnsWhenNextBlockAvailable() throws Exception { + final AtomicLong head = new AtomicLong(5); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(6)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(6); + assertThat(queue).isEmpty(); + } + + @Test + public void testWaitForNewElement_skipsAlreadyImportedBlocks() throws Exception { + final AtomicLong head = new AtomicLong(10); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(8)); + queue.offer(makeTrieLog(9)); + queue.offer(makeTrieLog(10)); + queue.offer(makeTrieLog(11)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(11); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — missing trie logs (distance > 1) + // =========================================================================== + + @Test + public void testWaitForNewElement_callsOnTrieLogMissingWhenGap() throws Exception { + final AtomicLong head = new AtomicLong(5); + final CountDownLatch missingCalled = new CountDownLatch(1); + final AtomicReference capturedDistance = new AtomicReference<>(); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + capturedDistance.set(dist); + missingCalled.countDown(); + return CompletableFuture.completedFuture(true); + }); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + assertThat(missingCalled.await(5, TimeUnit.SECONDS)).isTrue(); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNull(); + assertThat(capturedDistance.get()).isEqualTo(5L); + } + + @Test + public void testWaitForNewElement_retriesWhenOnTrieLogMissingReturnsFalse() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicLong missingCallCount = new AtomicLong(0); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + long count = missingCallCount.incrementAndGet(); + if (count >= 2) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(15, TimeUnit.SECONDS); + assertThat(result).isNull(); + assertThat(missingCallCount.get()).isGreaterThanOrEqualTo(2); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — blocked by import validator + // =========================================================================== + + @Test + public void testWaitForNewElement_blockedByValidator_thenUnblocked() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicReference canImport = new AtomicReference<>(false); + + final BlockImportValidator validator = canImport::get; + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(validator), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(6)); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(2000); + assertThat(future.isDone()).isFalse(); + + canImport.set(true); + queue.offer(makeTrieLog(6)); + + TrieLogIdentifier result = future.get(10, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(6); + } + + @Test + public void testWaitForNewElement_validatorClearsQueue() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicReference canImport = new AtomicReference<>(false); + + final BlockImportValidator validator = canImport::get; + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(validator), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(6)); + queue.offer(makeTrieLog(7)); + queue.offer(makeTrieLog(8)); + assertThat(queue).hasSize(3); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(2000); + + queue.stop(); + future.get(5, TimeUnit.SECONDS); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — empty queue + // =========================================================================== + + @Test + public void testWaitForNewElement_emptyQueue_usesInitialSyncRange() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicReference capturedDistance = new AtomicReference<>(); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + capturedDistance.set(dist); + return CompletableFuture.completedFuture(true); + }); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNull(); + assertThat(capturedDistance.get()) + .isEqualTo(TrieLogBlockingQueue.INITIAL_SYNC_BLOCK_NUMBER_RANGE); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — onTrieLogMissing throws / times out + // =========================================================================== + + @Test + public void testWaitForNewElement_onTrieLogMissingThrows_continuesLoop() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicLong callCount = new AtomicLong(0); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + long count = callCount.incrementAndGet(); + if (count == 1) { + CompletableFuture f = new CompletableFuture<>(); + f.completeExceptionally(new RuntimeException("simulated failure")); + return f; + } + return CompletableFuture.completedFuture(true); + }); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(15, TimeUnit.SECONDS); + assertThat(result).isNull(); + assertThat(callCount.get()).isGreaterThanOrEqualTo(2); + } + + @Test + public void testWaitForNewElement_onTrieLogMissingTimesOut_continuesLoop() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicLong callCount = new AtomicLong(0); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + long count = callCount.incrementAndGet(); + if (count == 1) { + return new CompletableFuture<>(); + } + return CompletableFuture.completedFuture(true); + }); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(15, TimeUnit.SECONDS); + assertThat(result).isNull(); + assertThat(callCount.get()).isGreaterThanOrEqualTo(2); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — interrupted + // =========================================================================== + + @Test + public void testWaitForNewElement_interruptedReturnsNull() throws Exception { + final AtomicLong head = new AtomicLong(5); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> new CompletableFuture<>()); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(500); + + executor.shutdownNow(); + + TrieLogIdentifier result = future.get(10, TimeUnit.SECONDS); + assertThat(result).isNull(); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — head advances during wait + // =========================================================================== + + @Test + public void testWaitForNewElement_headAdvancesDuringWait() throws Exception { + final AtomicLong head = new AtomicLong(5); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + head.set(9); + return CompletableFuture.completedFuture(false); + }); + + queue.offer(makeTrieLog(8)); + queue.offer(makeTrieLog(9)); + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + TrieLogIdentifier result = future.get(15, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(10); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — multiple validators + // =========================================================================== + + @Test + public void testWaitForNewElement_multipleValidators_anyBlockingPreventsImport() + throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicReference secondValidatorAllows = new AtomicReference<>(false); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow(), secondValidatorAllows::get), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(6)); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(2000); + assertThat(future.isDone()).isFalse(); + + secondValidatorAllows.set(true); + queue.offer(makeTrieLog(6)); + + TrieLogIdentifier result = future.get(10, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(6); + } + + // =========================================================================== + // WAIT FOR NEW ELEMENT — block added after wait starts + // =========================================================================== + + @Test + public void testWaitForNewElement_blockAddedLaterDuringWait() throws Exception { + final AtomicLong head = new AtomicLong(5); + final AtomicLong missingCallCount = new AtomicLong(0); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + missingCallCount.incrementAndGet(); + return CompletableFuture.completedFuture(false); + }); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(4000); + queue.offer(makeTrieLog(6)); + + TrieLogIdentifier result = future.get(15, TimeUnit.SECONDS); + assertThat(result).isNotNull(); + assertThat(result.blockNumber()).isEqualTo(6); + } + + // =========================================================================== + // DISTANCE / PEEK + // =========================================================================== + + @Test + public void testDistance_emptyQueue_returnsEmpty() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + Collections.emptyList(), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + assertThat(queue.peek()).isNull(); + assertThat(queue).isEmpty(); + } + + @Test + public void testPeek_returnsSmallestBlockNumber() { + final AtomicLong head = new AtomicLong(0); + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + Collections.emptyList(), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(10)); + queue.offer(makeTrieLog(5)); + queue.offer(makeTrieLog(8)); + + assertThat(queue.peek().blockNumber()).isEqualTo(5); + } + + // =========================================================================== + // STOP during various states + // =========================================================================== + + @Test + public void testStop_duringOnTrieLogMissing() throws Exception { + final AtomicLong head = new AtomicLong(5); + final CountDownLatch insideMissing = new CountDownLatch(1); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysAllow()), + head::get, + dist -> { + insideMissing.countDown(); + return new CompletableFuture<>(); + }); + + queue.offer(makeTrieLog(10)); + + Future future = executor.submit(queue::waitForNewElement); + + assertThat(insideMissing.await(5, TimeUnit.SECONDS)).isTrue(); + + queue.stop(); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNull(); + } + + @Test + public void testStop_duringValidatorBlock() throws Exception { + final AtomicLong head = new AtomicLong(5); + + final TrieLogBlockingQueue queue = + new TrieLogBlockingQueue( + 10, + List.of(alwaysBlock()), + head::get, + dist -> CompletableFuture.completedFuture(false)); + + queue.offer(makeTrieLog(6)); + + Future future = executor.submit(queue::waitForNewElement); + + Thread.sleep(2000); + assertThat(future.isDone()).isFalse(); + + queue.stop(); + + TrieLogIdentifier result = future.get(5, TimeUnit.SECONDS); + assertThat(result).isNull(); + } +} diff --git a/trie/src/main/java/net/consensys/shomei/trie/ZKTrie.java b/trie/src/main/java/net/consensys/shomei/trie/ZKTrie.java index 5cdff04d..4d07e1bc 100644 --- a/trie/src/main/java/net/consensys/shomei/trie/ZKTrie.java +++ b/trie/src/main/java/net/consensys/shomei/trie/ZKTrie.java @@ -91,7 +91,7 @@ public static ZKTrie generateEmptyTrie() { new InMemoryStorage() { @Override public Optional getTrieNode(final Bytes location, final Bytes nodeHash) { - return Optional.ofNullable(super.getTrieNodeStorage().get(nodeHash)); + return super.getTrieNodeStorage().get(nodeHash); // In a sparse Merkle trie, the hash value of a parent node is computed based on the // hashes of its children nodes. // so the hash value of a parent node at each level will be the same as long as its @@ -103,7 +103,7 @@ public Optional getTrieNode(final Bytes location, final Bytes nodeHash) { @Override public void putTrieNode(final Bytes location, final Bytes nodeHash, final Bytes value) { - super.getTrieNodeStorage().put(nodeHash, value); + super.getTrieNodeStorage().put(nodeHash, Optional.of(value)); } }; return new ZKTrie(initWorldState(inMemoryStorage::putTrieNode).getHash(), inMemoryStorage); @@ -116,7 +116,7 @@ public static ZKTrie createTrie(final TrieStorage worldStateStorage) { } public static ZKTrie loadTrie(final Bytes32 rootHash, final TrieStorage worldStateStorage) { - return new ZKTrie(rootHash, worldStateStorage); + return new ZKTrie(rootHash, worldStateStorage); } /** @@ -154,16 +154,16 @@ private static Node initWorldState(final NodeUpdater nodeUpdater) { } public void setHeadAndTail() { - // head - final Long headIndex = pathResolver.getNextFreeLeafNodeIndex(); - updater.putFlatLeaf(LeafOpening.HEAD.getHkey(), FlattenedLeaf.HEAD); - state.put(pathResolver.getLeafPath(headIndex), LeafOpening.HEAD.getEncodesBytes()); - pathResolver.incrementNextFreeLeafNodeIndex(); - // tail - final Long tailIndex = pathResolver.getNextFreeLeafNodeIndex(); - updater.putFlatLeaf(LeafOpening.TAIL.getHkey(), FlattenedLeaf.TAIL); - state.put(pathResolver.getLeafPath(tailIndex), LeafOpening.TAIL.getEncodesBytes()); - pathResolver.incrementNextFreeLeafNodeIndex(); + // head + final Long headIndex = pathResolver.getNextFreeLeafNodeIndex(); + updater.putFlatLeaf(LeafOpening.HEAD.getHkey(), FlattenedLeaf.HEAD); + state.put(pathResolver.getLeafPath(headIndex), LeafOpening.HEAD.getEncodesBytes()); + pathResolver.incrementNextFreeLeafNodeIndex(); + // tail + final Long tailIndex = pathResolver.getNextFreeLeafNodeIndex(); + updater.putFlatLeaf(LeafOpening.TAIL.getHkey(), FlattenedLeaf.TAIL); + state.put(pathResolver.getLeafPath(tailIndex), LeafOpening.TAIL.getEncodesBytes()); + pathResolver.incrementNextFreeLeafNodeIndex(); } public Node getSubRootNode() { diff --git a/trie/src/main/java/net/consensys/shomei/trie/storage/AccountTrieRepositoryWrapper.java b/trie/src/main/java/net/consensys/shomei/trie/storage/AccountTrieRepositoryWrapper.java index 7314ff94..58c40dfa 100644 --- a/trie/src/main/java/net/consensys/shomei/trie/storage/AccountTrieRepositoryWrapper.java +++ b/trie/src/main/java/net/consensys/shomei/trie/storage/AccountTrieRepositoryWrapper.java @@ -70,6 +70,10 @@ public Range getNearestKeys(final Bytes hkey) { return new Range(left, center, right); } + public TrieStorage getTrieStorage() { + return trieStorage; + } + @Override public Optional getTrieNode(final Bytes location, final Bytes nodeHash) { return trieStorage.getTrieNode(location, nodeHash); diff --git a/trie/src/main/java/net/consensys/shomei/trie/storage/InMemoryStorage.java b/trie/src/main/java/net/consensys/shomei/trie/storage/InMemoryStorage.java index 0937a76f..ee0a7d67 100644 --- a/trie/src/main/java/net/consensys/shomei/trie/storage/InMemoryStorage.java +++ b/trie/src/main/java/net/consensys/shomei/trie/storage/InMemoryStorage.java @@ -18,6 +18,7 @@ import java.util.Comparator; import java.util.Iterator; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -27,27 +28,30 @@ public class InMemoryStorage implements TrieStorage, TrieStorage.TrieUpdater { - private final TreeMap flatLeafStorage = - new TreeMap<>(Comparator.naturalOrder()); - private final Map trieNodeStorage = new ConcurrentHashMap<>(); + private final TreeMap> flatLeafStorage = + new TreeMap<>(Comparator.naturalOrder()); + private final Map> trieNodeStorage = new ConcurrentHashMap<>(); - public TreeMap getFlatLeafStorage() { +public TreeMap> getFlatLeafStorage() { return flatLeafStorage; } - public Map getTrieNodeStorage() { + public Map> getTrieNodeStorage() { return trieNodeStorage; } @Override public Optional getFlatLeaf(final Bytes hkey) { - return Optional.ofNullable(flatLeafStorage.get(hkey)); + final Optional value = flatLeafStorage.get(hkey); + return Objects.requireNonNullElseGet(value, Optional::empty); } @Override public Range getNearestKeys(final Bytes hkey) { - final Iterator> iterator = - flatLeafStorage.entrySet().iterator(); + final Iterator>> iterator = + flatLeafStorage.entrySet().stream() + .filter(e -> e.getValue().isPresent()) + .iterator(); Map.Entry next = Map.entry(Bytes32.ZERO, FlattenedLeaf.HEAD); Map.Entry left = next; Optional> maybeMiddle = Optional.empty(); @@ -58,7 +62,8 @@ public Range getNearestKeys(final Bytes hkey) { } else { left = next; } - next = iterator.next(); + var rawEntry = iterator.next(); + next = Map.entry(rawEntry.getKey(), rawEntry.getValue().get()); } return new Range( Map.entry(left.getKey(), left.getValue()), @@ -66,29 +71,36 @@ public Range getNearestKeys(final Bytes hkey) { Map.entry(next.getKey(), next.getValue())); } + @Override public Optional getTrieNode(final Bytes location, final Bytes nodeHash) { - return Optional.ofNullable(trieNodeStorage.get(location)); + final Optional value = trieNodeStorage.get(location); + if (value == null) { + // Key was never inserted + return Optional.empty(); + } + // Returns the Optional: present = exists, empty = deleted + return value; } @Override public void putTrieNode(final Bytes location, final Bytes nodeHash, final Bytes value) { - trieNodeStorage.put(location, value); + trieNodeStorage.put(location, Optional.of(value)); } @Override - public InMemoryStorage updater() { + public TrieUpdater updater() { return this; } @Override public void putFlatLeaf(final Bytes key, final FlattenedLeaf value) { - flatLeafStorage.put(key, value); + flatLeafStorage.put(key, Optional.of(value)); } @Override public void removeFlatLeafValue(final Bytes key) { - flatLeafStorage.remove(key); + flatLeafStorage.put(key, Optional.empty()); } @Override