diff --git a/src/Nethermind/Ethereum.Trie.Test/StorageTrieTests.cs b/src/Nethermind/Ethereum.Trie.Test/StorageTrieTests.cs index cbe281c7f576..f6ea02fa8585 100644 --- a/src/Nethermind/Ethereum.Trie.Test/StorageTrieTests.cs +++ b/src/Nethermind/Ethereum.Trie.Test/StorageTrieTests.cs @@ -17,7 +17,6 @@ */ using Nethermind.Core.Crypto; -using Nethermind.Core.Encoding; using Nethermind.Store; using NUnit.Framework; @@ -33,6 +32,7 @@ public void Storage_trie_set_reset_with_empty() Keccak rootBefore = tree.RootHash; tree.Set(1, new byte[] { 1 }); tree.Set(1, new byte[] { }); + tree.UpdateRootHash(); Keccak rootAfter = tree.RootHash; Assert.AreEqual(rootBefore, rootAfter); } @@ -44,6 +44,7 @@ public void Storage_trie_set_reset_with_long_zero() Keccak rootBefore = tree.RootHash; tree.Set(1, new byte[] { 1 }); tree.Set(1, new byte[] { 0, 0, 0, 0, 0 }); + tree.UpdateRootHash(); Keccak rootAfter = tree.RootHash; Assert.AreEqual(rootBefore, rootAfter); } @@ -55,6 +56,7 @@ public void Storage_trie_set_reset_with_short_zero() Keccak rootBefore = tree.RootHash; tree.Set(1, new byte[] { 1 }); tree.Set(1, new byte[] { 0 }); + tree.UpdateRootHash(); Keccak rootAfter = tree.RootHash; Assert.AreEqual(rootBefore, rootAfter); } diff --git a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs index 5ac2deb2c14a..2cf5a8b107f4 100644 --- a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs @@ -158,6 +158,7 @@ private void RunTest(TrieTest test, bool secure) patriciaTree.Set(key, value); } + patriciaTree.UpdateRootHash(); Assert.AreEqual(test.ExpectedRoot, patriciaTree.RootHash.ToString()); } @@ -177,6 +178,7 @@ public void Tutorial_test_1() new byte[] { 0x01, 0x01, 0x02 }, Rlp.Encode(new object[] { "hello" })); + patriciaTree.UpdateRootHash(); Assert.AreEqual("0x15da97c42b7ed2e1c0c8dab6a6d7e3d9dc0a75580bbc4f1f29c33996d1415dcc", patriciaTree.RootHash.ToString()); } @@ -189,6 +191,7 @@ public void Tutorial_test_1_keccak() new byte[] { 0x01, 0x01, 0x02 }, Rlp.Encode(new object[] { "hello" })); + patriciaTree.Commit(); PatriciaTree another = new PatriciaTree(_db, patriciaTree.RootHash); Assert.AreEqual(((Leaf)patriciaTree.Root).Key.ToString(), ((Leaf)another.Root).Key.ToString()); Assert.AreEqual(Keccak.Compute(((Leaf)patriciaTree.Root).Value), @@ -207,6 +210,7 @@ public void Tutorial_test_2() new byte[] { 0x01, 0x01, 0x02 }, Rlp.Encode(new object[] { "hellothere" })); + patriciaTree.Commit(); Assert.AreEqual("0x05e13d8be09601998499c89846ec5f3101a1ca09373a5f0b74021261af85d396", patriciaTree.RootHash.ToString()); } @@ -225,9 +229,10 @@ public void Tutorial_test_2b() Extension extension = patriciaTree.Root as Extension; Assert.NotNull(extension); - Branch branch = patriciaTree.GetNode(extension.NextNode) as Branch; + Branch branch = extension.NextNodeRef?.Node as Branch; Assert.NotNull(branch); + patriciaTree.UpdateRootHash(); Assert.AreEqual("0xb5e187f15f1a250e51a78561e29ccfc0a7f48e06d19ce02f98dd61159e81f71d", patriciaTree.RootHash.ToString()); } @@ -246,7 +251,7 @@ public void Tutorial_test_2c() Extension extension = patriciaTree.Root as Extension; Assert.NotNull(extension); - Branch branch = patriciaTree.GetNode(extension.NextNode) as Branch; + Branch branch = extension.NextNodeRef?.Node as Branch; Assert.NotNull(branch); } @@ -264,9 +269,10 @@ public void Tutorial_test_2d() Extension extension = patriciaTree.Root as Extension; Assert.NotNull(extension); - Branch branch = patriciaTree.GetNode(extension.NextNode) as Branch; + Branch branch = extension.NextNodeRef?.Node as Branch; Assert.NotNull(branch); + patriciaTree.UpdateRootHash(); Assert.AreEqual("0x17fe8af9c6e73de00ed5fd45d07e88b0c852da5dd4ee43870a26c39fc0ec6fb3", patriciaTree.RootHash.ToString()); } @@ -287,6 +293,7 @@ public void Tutorial_test_3() new byte[] { 0x01, 0x01, 0x02, 0x57 }, Rlp.Encode(new object[] { "jimbojones" })); + patriciaTree.Commit(); Assert.AreEqual("0xfcb2e3098029e816b04d99d7e1bba22d7b77336f9fe8604f2adfb04bcf04a727", patriciaTree.RootHash.ToString()); } @@ -303,6 +310,7 @@ public void Delete_on_empty() { PatriciaTree patriciaTree = new PatriciaTree(_db); patriciaTree.Set(Keccak.Compute("1").Bytes, new byte[0]); + patriciaTree.Commit(); Assert.AreEqual(PatriciaTree.EmptyTreeHash, patriciaTree.RootHash); } @@ -310,7 +318,7 @@ public void Delete_on_empty() public void Assigning_null_value_in_branch_throws_an_exception() { // ReSharper disable once ObjectCreationAsStatement - Assert.Throws(() => new Branch(new KeccakOrRlp[16], null)); + Assert.Throws(() => new Branch(new NodeRef[16], null)); Branch branch = new Branch(); Assert.Throws(() => branch.Value = null); @@ -333,8 +341,10 @@ public void Delete_missing_resolved_on_extension() PatriciaTree patriciaTree = new PatriciaTree(_db); patriciaTree.Set(new Nibble[] { 1, 2, 3, 4 }, new byte[] { 1 }); patriciaTree.Set(new Nibble[] { 1, 2, 3, 4, 5 }, new byte[] { 2 }); + patriciaTree.UpdateRootHash(); Keccak rootBefore = patriciaTree.RootHash; patriciaTree.Set(new Nibble[] { 1, 2, 3 }, new byte[] { }); + patriciaTree.UpdateRootHash(); Assert.AreEqual(rootBefore, patriciaTree.RootHash); } @@ -344,8 +354,10 @@ public void Delete_missing_resolved_on_leaf() PatriciaTree patriciaTree = new PatriciaTree(_db); patriciaTree.Set(Keccak.Compute("1234567").Bytes, new byte[] { 1 }); patriciaTree.Set(Keccak.Compute("1234501").Bytes, new byte[] { 2 }); + patriciaTree.UpdateRootHash(); Keccak rootBefore = patriciaTree.RootHash; patriciaTree.Set(Keccak.Compute("1234502").Bytes, new byte[0]); + patriciaTree.UpdateRootHash(); Assert.AreEqual(rootBefore, patriciaTree.RootHash); } @@ -353,9 +365,9 @@ public void Delete_missing_resolved_on_leaf() public void Lookup_in_empty_tree() { PatriciaTree tree = new PatriciaTree(new MemDb()); - Assert.AreEqual(tree.Root, null); + Assert.AreEqual(tree.RootRef, null); tree.Get(new byte[] { 1 }); - Assert.AreEqual(tree.Root, null); + Assert.AreEqual(tree.RootRef, null); } public class TrieTestJson diff --git a/src/Nethermind/Nethermind.Blockchain/BlockProcessor.cs b/src/Nethermind/Nethermind.Blockchain/BlockProcessor.cs index 14d0ee4e6413..b10a2bbecb2b 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockProcessor.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockProcessor.cs @@ -98,6 +98,8 @@ private void SetReceipts(Block block, TransactionReceipt[] receipts) receiptTree?.Set(Rlp.Encode(i).Bytes, receiptRlp); } + receiptTree?.UpdateRootHash(); + block.Receipts = receipts; block.Header.ReceiptsRoot = receiptTree?.RootHash ?? PatriciaTree.EmptyTreeHash; block.Header.Bloom = receipts.Length > 0 ? TransactionProcessor.BuildBloom(receipts.SelectMany(r => r.Logs).ToArray()) : Bloom.Empty; // TODO not tested anywhere at the time of writing @@ -117,6 +119,7 @@ private Keccak GetTransactionsRoot(Transaction[] transactions) txTree.Set(Rlp.Encode(i).Bytes, transactionRlp); } + txTree.UpdateRootHash(); return txTree.RootHash; } @@ -271,6 +274,8 @@ private Block ProcessOne(Block suggestedBlock, bool tryOnly) // TODO: refactor _logger.Debug($"Committing block - state root {_stateProvider.StateRoot}"); } + _stateProvider.CommitTree(); + _storageProvider.CommitTrees(); _dbProvider.Commit(_specProvider.GetSpec(suggestedBlock.Number)); return processedBlock; } diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index cda9a07257b3..2f7d08e3d1af 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -73,7 +73,7 @@ public Address(Hex hex) if (hex.ByteLength != AddressLengthInBytes) { - throw new ArgumentException($"{nameof(Address)} should be {AddressLengthInBytes} bytes long", nameof(hex)); + throw new ArgumentException($"{nameof(Address)} should be {AddressLengthInBytes} bytes long and is {hex.ByteLength} bytes long", nameof(hex)); } Hex = hex; diff --git a/src/Nethermind/Nethermind.Core/BlockHeader.cs b/src/Nethermind/Nethermind.Core/BlockHeader.cs index 41020513e98c..e67d9ed9f233 100644 --- a/src/Nethermind/Nethermind.Core/BlockHeader.cs +++ b/src/Nethermind/Nethermind.Core/BlockHeader.cs @@ -65,6 +65,11 @@ public BlockHeader(Keccak parentHash, Keccak ommersHash, Address beneficiary, Bi public BigInteger? TotalDifficulty { get; set; } public BigInteger? TotalTransactions { get; set; } + public static Keccak CalculateHash(Rlp headerRlp) + { + return Keccak.Compute(headerRlp); + } + public static Keccak CalculateHash(BlockHeader header) { return Keccak.Compute(Rlp.Encode(header)); diff --git a/src/Nethermind/Nethermind.Core/Bloom.cs b/src/Nethermind/Nethermind.Core/Bloom.cs index d0a0d8780af4..56db706bb91e 100644 --- a/src/Nethermind/Nethermind.Core/Bloom.cs +++ b/src/Nethermind/Nethermind.Core/Bloom.cs @@ -18,7 +18,6 @@ using System; using System.Collections; -using System.Diagnostics; using System.Text; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; diff --git a/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs b/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs index 4226806d6eb7..8a7f0942897c 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs @@ -35,7 +35,7 @@ public Keccak(Hex hex) { if (hex.ByteLength != Size) { - throw new ArgumentException($"{nameof(Keccak)} must be {Size} bytes", nameof(hex)); + throw new ArgumentException($"{nameof(Keccak)} must be {Size} bytes and was {hex.ByteLength} bytes", nameof(hex)); } Bytes = hex; @@ -45,7 +45,7 @@ public Keccak(byte[] bytes) { if (bytes.Length != Size) { - throw new ArgumentException($"{nameof(Keccak)} must be {Size} bytes", nameof(bytes)); + throw new ArgumentException($"{nameof(Keccak)} must be {Size} bytes and was {bytes.Length} bytes", nameof(bytes)); } Bytes = bytes; diff --git a/src/Nethermind/Nethermind.Core/Encoding/NewHeaderDecoder.cs b/src/Nethermind/Nethermind.Core/Encoding/NewHeaderDecoder.cs index 6d3d25ddfc10..0178525f581d 100644 --- a/src/Nethermind/Nethermind.Core/Encoding/NewHeaderDecoder.cs +++ b/src/Nethermind/Nethermind.Core/Encoding/NewHeaderDecoder.cs @@ -26,8 +26,12 @@ public class NewHeaderDecoder : INewRlpDecoder { public BlockHeader Decode(NewRlp.DecoderContext context, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { + byte[] headerRlp = context.ReadSequenceRlp(); + context.Position -= headerRlp.Length; + long headerSequenceLength = context.ReadSequenceLength(); long headerCheck = context.Position + headerSequenceLength; + Keccak parentHash = context.ReadKeccak(); Keccak ommersHash = context.ReadKeccak(); Address beneficiary = context.ReadAddress(); @@ -66,7 +70,7 @@ public BlockHeader Decode(NewRlp.DecoderContext context, RlpBehaviors rlpBehavio blockHeader.GasUsed = (long)gasUsed; blockHeader.MixHash = mixHash; blockHeader.Nonce = (ulong)nonce; - blockHeader.Hash = BlockHeader.CalculateHash(blockHeader); + blockHeader.Hash = BlockHeader.CalculateHash(new Rlp(headerRlp)); return blockHeader; } } diff --git a/src/Nethermind/Nethermind.Core/Encoding/Rlp.cs b/src/Nethermind/Nethermind.Core/Encoding/Rlp.cs index 38db5c92bb6c..608a8419e05f 100644 --- a/src/Nethermind/Nethermind.Core/Encoding/Rlp.cs +++ b/src/Nethermind/Nethermind.Core/Encoding/Rlp.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Numerics; using Nethermind.Core.Crypto; @@ -26,9 +27,9 @@ namespace Nethermind.Core.Encoding { /// - /// https://github.com/ethereum/wiki/wiki/RLP + /// https://github.com/ethereum/wiki/wiki/RLP /// - //[DebuggerStepThrough] + [DebuggerStepThrough] public class Rlp : IEquatable { public static readonly Rlp OfEmptyByteArray = new Rlp(128); @@ -94,13 +95,13 @@ public static T Decode(DecodedRlp rlp, RlpBehaviors rlpBehaviors = RlpBehavio return rlp.As(); } - + public static T[] DecodeArray(Rlp rlp, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { DecodedRlp decodedRlp = Decode(rlp); return DecodeArray(decodedRlp, rlpBehaviors); } - + public static T[] DecodeArray(DecodedRlp rlp, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { T[] array = new T[rlp.Items.Count]; @@ -126,7 +127,7 @@ public static Rlp[] ExtractRlpList(Rlp rlp) private static Rlp[] ExtractRlpList(DecoderContext context) { - var result = new List(); + List result = new List(); while (context.CurrentIndex < context.MaxIndex) { @@ -156,7 +157,7 @@ private static Rlp[] ExtractRlpList(DecoderContext context) if (prefix <= 183) { int length = prefix - 128; - var content = context.Pop(length); + byte[] content = context.Pop(length); if (content.Length == 1 && content[0] < 128) { throw new RlpException($"Unexpected byte value {content[0]}"); @@ -187,8 +188,8 @@ private static Rlp[] ExtractRlpList(DecoderContext context) } } - var data = context.Pop(concatenationLength); - var itemBytes = new[] {prefix}; + byte[] data = context.Pop(concatenationLength); + byte[] itemBytes = {prefix}; if (lenghtBytes != null) { itemBytes = itemBytes.Concat(lenghtBytes).ToArray(); @@ -415,12 +416,14 @@ public static Rlp Encode(bool value) { return value ? new Rlp(1) : new Rlp(128); } - + + public static Rlp Encode(byte value) { return EncodeNumber(value); } + public static Rlp Encode(long value) { return EncodeNumber(value); @@ -626,7 +629,7 @@ public static Rlp Encode(ChainLevelInfo levelInfo) elements[1] = Encode(levelInfo.BlockInfos); return Encode(elements); } - + public static Rlp Encode(BlockInfo blockInfo) { Rlp[] elements = new Rlp[4]; @@ -673,7 +676,6 @@ public static Rlp Encode(Bloom bloom) public static Rlp Encode(LogEntry logEntry) { - // TODO: can be slightly optimized in place return Encode( Encode(logEntry.LoggersAddress), Encode(logEntry.Topics), diff --git a/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs index 0dd930b8dd09..ce965d498cd5 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs @@ -121,7 +121,8 @@ public static byte[] SliceWithZeroPaddingEmptyOnError(this byte[] bytes, BigInte public static int GetXxHashCode(this byte[] bytes) { LazyInitializer.EnsureInitialized(ref _xxHash, XXHash32.Create); - return BitConverter.ToInt32(_xxHash.ComputeHash(bytes), 0); + byte[] hash = _xxHash.ComputeHash(bytes); + return (hash[0] >> 24) | (hash[1] >> 16) | (hash[2] >> 8) | hash[3]; } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 8b90b424cc3e..ebd71505e63a 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -26,6 +26,7 @@ namespace Nethermind.Core.Extensions { // TODO: move to ByteArrayExtensions and ByteExtensions + [DebuggerStepThrough] public static class Bytes { public static readonly IEqualityComparer EqualityComparer = new BytesEqualityComparer(); diff --git a/src/Nethermind/Nethermind.Db/RocksDbProvider.cs b/src/Nethermind/Nethermind.Db/RocksDbProvider.cs index 3a22c59a1a1a..b45104d7f4e7 100644 --- a/src/Nethermind/Nethermind.Db/RocksDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/RocksDbProvider.cs @@ -29,6 +29,7 @@ public class RocksDbProvider : IDbProvider { private readonly ISnapshotableDb _stateDb; private readonly ISnapshotableDb _codeDb; + // TODO: replace with one storage DB private readonly Dictionary _storageDbs = new Dictionary(); private IEnumerable AllDbs { @@ -68,8 +69,6 @@ public ISnapshotableDb GetOrCreateCodeDb() private readonly string _dbBasePath; private readonly ILogger _logger; - internal Stack> Snapshots { get; } = new Stack>(); - public RocksDbProvider(string dbBasePath, ILogger logger) { _logger = logger; @@ -81,50 +80,24 @@ public RocksDbProvider(string dbBasePath, ILogger logger) public void Restore(int snapshot) { if (_logger.IsDebugEnabled) _logger.Debug($"Restoring all DBs to {snapshot}"); - - while (Snapshots.Count != snapshot) - { - Snapshots.Pop(); - } - - Dictionary dbSnapshots = Snapshots.Pop(); foreach (ISnapshotableDb db in AllDbs) { - db.Restore(dbSnapshots.ContainsKey(db) ? dbSnapshots[db] : -1); + db.Restore(-1); } } public void Commit(IReleaseSpec spec) { if (_logger.IsDebugEnabled) _logger.Debug("Committing all DBs"); - foreach (ISnapshotableDb db in AllDbs) { db.Commit(spec); } - - Snapshots.Pop(); } public int TakeSnapshot() { - Dictionary dbSnapshots = new Dictionary(); - foreach (ISnapshotableDb db in AllDbs) - { - int dbSnapshot = db.TakeSnapshot(); - if (dbSnapshot == -1) - { - continue; - } - - dbSnapshots.Add(db, dbSnapshot); - } - - Snapshots.Push(dbSnapshots); - - int snapshot = Snapshots.Count; - if (_logger.IsDebugEnabled) _logger.Debug($"Taking DB snapshot at {snapshot}"); - return snapshot; + return -1; } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index c7756176c2ca..96820dce105d 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -19,6 +19,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Numerics; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -73,6 +74,12 @@ public VirtualMachine(ISpecProvider specProvider, IStateProvider stateProvider, // can refactor and integrate the other call public (byte[] output, TransactionSubstate) Run(EvmState state, IReleaseSpec releaseSpec, TransactionTrace trace) { + // TODO: review the concept commented below + //if (state.Env.CodeInfo.MachineCode.Length == 0 && state.ExecutionType == ExecutionType.Transaction) + //{ + // return (Bytes.Empty, new TransactionSubstate(0, new Collection
(), new Collection(), false)); + //} + _instructionCounter = 0; _traceEntry = null; _trace = trace; diff --git a/src/Nethermind/Nethermind.Store.Test/StateTreeTests.cs b/src/Nethermind/Nethermind.Store.Test/StateTreeTests.cs index 97bb2bcb0e19..ace450932971 100644 --- a/src/Nethermind/Nethermind.Store.Test/StateTreeTests.cs +++ b/src/Nethermind/Nethermind.Store.Test/StateTreeTests.cs @@ -1,4 +1,5 @@ using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Test.Builders; using NUnit.Framework; @@ -41,7 +42,251 @@ public void Minimal_writes_when_setting_on_empty() tree.Set(TestObject.AddressB, _account0); tree.Set(TestObject.AddressC, _account0); tree.Commit(); - Assert.AreEqual(4, db.WritesCount, "writes"); // extension, branch, two leaves + Assert.AreEqual(5, db.WritesCount, "writes"); // branch, branch, two leaves (one is stored as RLP) + } + + [Test] + public void Minimal_writes_when_setting_on_empty_scenario_2() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); + tree.Commit(); + Assert.AreEqual(7, db.WritesCount, "writes"); // extension, branch, leaf, extension, branch, leaf, leaf + Assert.AreEqual(7, Metrics.TreeNodeHashCalculations, "hashes"); + Assert.AreEqual(7, Metrics.TreeNodeRlpEncodings, "encodings"); + } + + [Test] + public void Minimal_writes_when_setting_on_empty_scenario_3() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); + tree.Commit(); + Assert.AreEqual(4, db.WritesCount, "writes"); // extension, branch, 2x leaf + Assert.AreEqual(4, Metrics.TreeNodeHashCalculations, "hashes"); + Assert.AreEqual(4, Metrics.TreeNodeRlpEncodings, "encodings"); + } + + [Test] + public void Minimal_writes_when_setting_on_empty_scenario_4() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); + tree.Commit(); + Assert.AreEqual(1, db.WritesCount, "writes"); // extension, branch, 2x leaf + Assert.AreEqual(1, Metrics.TreeNodeHashCalculations, "hashes"); + Assert.AreEqual(1, Metrics.TreeNodeRlpEncodings, "encodings"); + } + + [Test] + public void Minimal_writes_when_setting_on_empty_scenario_5() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); + tree.Commit(); + Assert.AreEqual(0, db.WritesCount, "writes"); // extension, branch, 2x leaf + Assert.AreEqual(0, Metrics.TreeNodeHashCalculations, "hashes"); + Assert.AreEqual(0, Metrics.TreeNodeRlpEncodings, "encodings"); + } + + [Test] + public void Scenario_traverse_extension_read_full_match() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"), _account1); + Account account = tree.Get(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111")); + Assert.AreEqual(0, db.ReadsCount); + Assert.AreEqual(_account1.Balance, account.Balance); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_extension_read_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"), _account1); + Account account = tree.Get(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd")); + Assert.Null(account); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_extension_new_branching() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"), _account1); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), _account2); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_extension_delete_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"), _account1); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null); + Assert.AreEqual(0, db.ReadsCount); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_extension_create_new_extension() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"), _account1); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab00000000"), _account2); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), _account3); + Assert.AreEqual(0, db.ReadsCount); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_update_new_value() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account1); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_update_no_change() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_read_matching_leaf() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), null); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_delete_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + tree.Set(new Keccak("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_update_with_extension() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111111111111111111111111111"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), _account1); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_delete_matching_leaf() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + Account account = tree.Get(new Keccak("1111111111111111111111111111111111111111111111111111111111111111")); + Assert.NotNull(account); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_leaf_read_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("1111111111111111111111111111111111111111111111111111111111111111"), _account0); + Account account = tree.Get(new Keccak("111111111111111111111111111111111111111111111111111111111ddddddd")); + Assert.Null(account); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_branch_update_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111"), _account1); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), _account2); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_branch_read_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111"), _account1); + Account account = tree.Get(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222")); + Assert.Null(account); + tree.UpdateRootHash(); + tree.Commit(); + } + + [Test] + public void Scenario_traverse_branch_delete_missing() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000"), _account0); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111"), _account1); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null); + tree.UpdateRootHash(); + tree.Commit(); } [Test] @@ -53,7 +298,7 @@ public void Minimal_hashes_when_setting_on_empty() tree.Set(TestObject.AddressB, _account0); tree.Set(TestObject.AddressC, _account0); tree.Commit(); - Assert.AreEqual(4, Metrics.TreeNodeHashCalculations, "hashes"); // extension, branch, two leaves + Assert.AreEqual(5, Metrics.TreeNodeHashCalculations, "hashes"); // branch, branch, three leaves } [Test] @@ -65,7 +310,7 @@ public void Minimal_encodings_when_setting_on_empty() tree.Set(TestObject.AddressB, _account0); tree.Set(TestObject.AddressC, _account0); tree.Commit(); - Assert.AreEqual(4, Metrics.TreeNodeRlpEncodings, "encodings"); // extension, branch, two leaves + Assert.AreEqual(5, Metrics.TreeNodeRlpEncodings, "encodings"); // branch, branch, three leaves } [Test] @@ -93,6 +338,7 @@ public void No_writes_on_continues_update() Assert.AreEqual(1, db.WritesCount, "writes"); // extension, branch, two leaves } + [Ignore("This is not critical")] [Test] public void No_writes_on_reverted_update() { @@ -123,7 +369,32 @@ public void Can_ask_about_root_hash_without_commiting() StateTree tree = new StateTree(db); tree.Set(TestObject.AddressA, _account0); tree.UpdateRootHash(); - Assert.AreNotEqual("0x545a417202afcb10925b2afddb70a698710bb1cf4ab32942c42e9f019d564fdc", tree.RootHash); + Assert.AreEqual("0x545a417202afcb10925b2afddb70a698710bb1cf4ab32942c42e9f019d564fdc", tree.RootHash.ToString(true)); + } + + [Test] + public void Can_ask_about_root_hash_without_when_emptied() + { + MemDb db = new MemDb(); + StateTree tree = new StateTree(db); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); + tree.UpdateRootHash(); + Assert.AreNotEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); + tree.UpdateRootHash(); + Assert.AreNotEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); + tree.UpdateRootHash(); + Assert.AreNotEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); + tree.UpdateRootHash(); + Assert.AreNotEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); + tree.UpdateRootHash(); + Assert.AreNotEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); + tree.Set(new Keccak("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); + tree.UpdateRootHash(); + Assert.AreEqual(PatriciaTree.EmptyTreeHash, tree.RootHash); } [Test] diff --git a/src/Nethermind/Nethermind.Store.Test/StorageProviderTests.cs b/src/Nethermind/Nethermind.Store.Test/StorageProviderTests.cs index 8b72b87a9241..d0f5601bf6ad 100644 --- a/src/Nethermind/Nethermind.Store.Test/StorageProviderTests.cs +++ b/src/Nethermind/Nethermind.Store.Test/StorageProviderTests.cs @@ -31,11 +31,12 @@ public class StorageProviderTests private static readonly ILogger Logger = NullLogger.Instance; private readonly Address _address1 = new Address(Keccak.Compute("1")); private readonly Address _address2 = new Address(Keccak.Compute("2")); - private readonly IStateProvider _stateProvider = new StateProvider(new StateTree(new MemDb()), Logger, Substitute.For()); + private IStateProvider _stateProvider; [SetUp] public void Setup() { + _stateProvider = new StateProvider(new StateTree(new MemDb()), Logger, Substitute.For()); _stateProvider.CreateAccount(_address1, 0); _stateProvider.CreateAccount(_address2, 0); _stateProvider.Commit(Frontier.Instance); diff --git a/src/Nethermind/Nethermind.Store/Branch.cs b/src/Nethermind/Nethermind.Store/Branch.cs index d3b2650a5628..2735e71be90a 100644 --- a/src/Nethermind/Nethermind.Store/Branch.cs +++ b/src/Nethermind/Nethermind.Store/Branch.cs @@ -25,11 +25,11 @@ namespace Nethermind.Store internal class Branch : Node { public Branch() - : this(new KeccakOrRlp[16], new byte[0]) + : this(new NodeRef[16], new byte[0]) { } - public Branch(KeccakOrRlp[] nodes, byte[] value) + public Branch(NodeRef[] nodes, byte[] value) { Value = value ?? throw new ArgumentNullException(nameof(value)); Nodes = nodes ?? throw new ArgumentNullException(nameof(nodes)); @@ -40,7 +40,7 @@ public Branch(KeccakOrRlp[] nodes, byte[] value) } } - public KeccakOrRlp[] Nodes { get; private set; } + public NodeRef[] Nodes { get; private set; } private byte[] _value; diff --git a/src/Nethermind/Nethermind.Store/Extension.cs b/src/Nethermind/Nethermind.Store/Extension.cs index 049c1398d90b..f98bdbe407ad 100644 --- a/src/Nethermind/Nethermind.Store/Extension.cs +++ b/src/Nethermind/Nethermind.Store/Extension.cs @@ -28,20 +28,20 @@ public Extension(HexPrefix key) } [DebuggerStepThrough] - public Extension(HexPrefix key, KeccakOrRlp nextNode) + public Extension(HexPrefix key, NodeRef nodeRef) { Key = key; - NextNode = nextNode; + NextNodeRef = nodeRef; } public byte[] Path => Key.Path; public HexPrefix Key { get; } - public KeccakOrRlp NextNode { get; set; } + public NodeRef NextNodeRef { get; set; } public override string ToString() { - return $"[{Key}, {NextNode}]"; + return $"[{Key}, {NextNodeRef}]"; } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/IStateProvider.cs b/src/Nethermind/Nethermind.Store/IStateProvider.cs index 2e030425f009..fea1d920d96e 100644 --- a/src/Nethermind/Nethermind.Store/IStateProvider.cs +++ b/src/Nethermind/Nethermind.Store/IStateProvider.cs @@ -58,5 +58,7 @@ public interface IStateProvider : ISnapshotable Keccak UpdateCode(byte[] code); void ClearCaches(); // TODO: temp while designing DB <-> store interaction + + void CommitTree(); } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/IStorageProvider.cs b/src/Nethermind/Nethermind.Store/IStorageProvider.cs index fe7a9c158cec..48e520c1df12 100644 --- a/src/Nethermind/Nethermind.Store/IStorageProvider.cs +++ b/src/Nethermind/Nethermind.Store/IStorageProvider.cs @@ -30,5 +30,7 @@ public interface IStorageProvider : ISnapshotable Keccak GetRoot(Address address); void ClearCaches(); // TODO: temp while designing DB <-> store interaction + + void CommitTrees(); } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/Node.cs b/src/Nethermind/Nethermind.Store/Node.cs index fcda46ef088e..076eb5fe4ede 100644 --- a/src/Nethermind/Nethermind.Store/Node.cs +++ b/src/Nethermind/Nethermind.Store/Node.cs @@ -24,5 +24,6 @@ namespace Nethermind.Store { internal abstract class Node { + public bool IsDirty { get; set; } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/NodeRef.cs b/src/Nethermind/Nethermind.Store/NodeRef.cs new file mode 100644 index 000000000000..e70444772b9b --- /dev/null +++ b/src/Nethermind/Nethermind.Store/NodeRef.cs @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018 Demerzel Solutions Limited + * This file is part of the Nethermind library. + * + * The Nethermind library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The Nethermind library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the Nethermind. If not, see . + */ + +using System.Diagnostics; +using Nethermind.Core.Encoding; + +namespace Nethermind.Store +{ + [DebuggerDisplay("KeccakOrRlp={KeccakOrRlp}, Node={Node}")] + internal class NodeRef + { + public NodeRef(KeccakOrRlp keccakOrRlp, bool isRoot = false) + { + KeccakOrRlp = keccakOrRlp; + IsRoot = isRoot; + } + + public NodeRef(Node node, bool isRoot = false) + { + Node = node; + IsRoot = isRoot; + } + + public void ResolveNode(PatriciaTree tree) + { + if (Node == null) + { + Node = tree.GetNode(KeccakOrRlp); + } + } + + public void ResolveKey() + { + if (KeccakOrRlp == null) + { + _fullRlp = PatriciaTree.RlpEncode(Node); + KeccakOrRlp = new KeccakOrRlp(_fullRlp); + } + } + + public KeccakOrRlp KeccakOrRlp { get; set; } + private Rlp _fullRlp; + public Rlp FullRlp => KeccakOrRlp.IsKeccak ? _fullRlp : KeccakOrRlp.GetOrEncodeRlp(); + + public Node Node { get; set; } + public bool IsRoot { get; set; } + + public bool IsDirty + { + get + { + // either unresolved (so just a reference to something in DB) or a new node (marked dirty) + if (Node == null) + { + return false; + } + + return Node.IsDirty; + } + } + } +} \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/PatriciaTree.cs b/src/Nethermind/Nethermind.Store/PatriciaTree.cs index 059fce2e6598..5c1817ef5098 100644 --- a/src/Nethermind/Nethermind.Store/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Store/PatriciaTree.cs @@ -31,12 +31,60 @@ public class PatriciaTree { public void Commit() { + if (RootRef == null) + { + return; + } + + if (RootRef.IsDirty) + { + Commit(RootRef, true); + + // reset objects + Keccak keccak = RootRef.KeccakOrRlp.GetOrComputeKeccak(); + SetRootHash(keccak, true); + } } - + + private void Commit(NodeRef nodeRef, bool isRoot) + { + if (nodeRef.IsDirty) + { + Node node = nodeRef.Node; + if (node is Branch branch) + { + for (int i = 0; i < 16; i++) + { + NodeRef subnode = branch.Nodes[i]; + if (subnode?.IsDirty ?? false) + { + Commit(branch.Nodes[i], false); + } + } + } + else if (node is Extension extension) + { + if (extension.NextNodeRef.IsDirty) + { + Commit(extension.NextNodeRef, false); + } + } + + nodeRef.Node.IsDirty = false; + nodeRef.ResolveKey(); + if (nodeRef.KeccakOrRlp.IsKeccak || isRoot) + { + _db.Set(nodeRef.KeccakOrRlp.GetOrComputeKeccak(), nodeRef.FullRlp.Bytes); + } + } + } + public void UpdateRootHash() { + RootRef?.ResolveKey(); + SetRootHash(RootRef?.KeccakOrRlp.GetOrComputeKeccak() ?? EmptyTreeHash, false); } - + /// /// 0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421 /// @@ -44,7 +92,16 @@ public void UpdateRootHash() private readonly IDb _db; - internal Node Root; + internal NodeRef RootRef; + + internal Node Root + { + get + { + RootRef?.ResolveNode(this); + return RootRef?.Node; + } + } public PatriciaTree() : this(new MemDb(), EmptyTreeHash) @@ -67,27 +124,38 @@ public PatriciaTree(IDb db, Keccak rootHash) public Keccak RootHash { get => _rootHash; - set + set => SetRootHash(value, true); + } + + private void SetRootHash(Keccak value, bool resetObjects) + { + _rootHash = value; + if (_rootHash == Keccak.EmptyTreeHash) { - _rootHash = value; - if (_rootHash == Keccak.EmptyTreeHash) - { - Root = null; - } - else + RootRef = null; + } + else + { + if (resetObjects) { - Rlp rootRlp = new Rlp(_db[_rootHash.Bytes]); - Root = RlpDecode(rootRlp); // TODO: needed? + RootRef = new NodeRef(new KeccakOrRlp(_rootHash), true); } } } - private static Rlp RlpEncode(KeccakOrRlp keccakOrRlp) + private static Rlp RlpEncode(NodeRef nodeRef) { - return keccakOrRlp == null ? Rlp.OfEmptyByteArray : keccakOrRlp.GetOrEncodeRlp(); + if (nodeRef == null) + { + return Rlp.OfEmptyByteArray; + } + + nodeRef.ResolveKey(); + + return nodeRef.KeccakOrRlp.GetOrEncodeRlp(); } - private static Rlp RlpEncode(Node node) + internal static Rlp RlpEncode(Node node) { Metrics.TreeNodeRlpEncodings++; if (node is Leaf leaf) @@ -123,12 +191,12 @@ private static Rlp RlpEncode(Node node) if (node is Extension extension) { - return Rlp.Encode(extension.Key.ToBytes(), RlpEncode(extension.NextNode)); + return Rlp.Encode(extension.Key.ToBytes(), RlpEncode(extension.NextNodeRef)); } - throw new InvalidOperationException("Unknown node type"); + throw new InvalidOperationException($"Unknown node type {node?.GetType().Name}"); } - + internal static Node RlpDecode(Rlp bytes) { Metrics.TreeNodeRlpDecodings++; @@ -140,8 +208,7 @@ internal static Node RlpDecode(Rlp bytes) Node result; if (numberOfItems == 17) { - - KeccakOrRlp[] nodes = new KeccakOrRlp[16]; + NodeRef[] nodes = new NodeRef[16]; for (int i = 0; i < 16; i++) { nodes[i] = DecodeChildNode(context); @@ -174,15 +241,16 @@ internal static Node RlpDecode(Rlp bytes) return result; } - private static KeccakOrRlp DecodeChildNode(NewRlp.DecoderContext decoderContext) + private static NodeRef DecodeChildNode(NewRlp.DecoderContext decoderContext) { if (decoderContext.IsSequenceNext()) { - return new KeccakOrRlp(new Rlp(decoderContext.ReadSequenceRlp())); + KeccakOrRlp keccakOrRlp = new KeccakOrRlp(new Rlp(decoderContext.ReadSequenceRlp())); + return new NodeRef(keccakOrRlp); } byte[] bytes = decoderContext.ReadByteArray(); - return bytes.Length == 0 ? null : new KeccakOrRlp(new Keccak(bytes)); + return bytes.Length == 0 ? null : new NodeRef(new KeccakOrRlp(new Keccak(bytes))); } [DebuggerStepThrough] @@ -225,6 +293,7 @@ internal Node GetNode(KeccakOrRlp keccakOrRlp) { Console.WriteLine(e); } + return RlpDecode(rlp); } @@ -247,7 +316,7 @@ internal void DeleteNode(KeccakOrRlp hash, bool ignoreChildren = false) // // if (node is Extension extension) // { -// DeleteNode(extension.NextNode, true); +// DeleteNode(extension.NextNodeRef, true); // _db.Remove(hash.GetOrComputeKeccak()); // } // @@ -260,37 +329,37 @@ internal void DeleteNode(KeccakOrRlp hash, bool ignoreChildren = false) // } } - internal KeccakOrRlp StoreNode(Node node, bool isRoot = false) - { - if (isRoot && node == null) - { -// DeleteNode(new KeccakOrRlp(RootHash)); - Root = null; -// _db.Remove(RootHash); - RootHash = EmptyTreeHash; - return new KeccakOrRlp(EmptyTreeHash); - } - - if (node == null) - { - return null; - } - - Rlp rlp = RlpEncode(node); - KeccakOrRlp key = new KeccakOrRlp(rlp); - if (key.IsKeccak || isRoot) - { - Keccak keyKeccak = key.GetOrComputeKeccak(); - _db[keyKeccak.Bytes] = rlp.Bytes; - - if (isRoot) - { - Root = node; - RootHash = keyKeccak; - } - } - - return key; - } +// internal KeccakOrRlp StoreNode(Node node, bool isRoot = false) +// { +// if (isRoot && node == null) +// { +//// DeleteNode(new KeccakOrRlp(RootHash)); +// RootRef = null; +//// _db.Remove(RootHash); +// RootHash = EmptyTreeHash; +// return new KeccakOrRlp(EmptyTreeHash); +// } +// +// if (node == null) +// { +// return null; +// } +// +// Rlp rlp = RlpEncode(node); +// KeccakOrRlp key = new KeccakOrRlp(rlp); +// if (key.IsKeccak || isRoot) +// { +// Keccak keyKeccak = key.GetOrComputeKeccak(); +// _db[keyKeccak.Bytes] = rlp.Bytes; +// +// if (isRoot) +// { +// RootRef = node; +// RootHash = keyKeccak; +// } +// } +// +// return key; +// } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/StateProvider.cs b/src/Nethermind/Nethermind.Store/StateProvider.cs index 4ca63d38ddce..85000854fc08 100644 --- a/src/Nethermind/Nethermind.Store/StateProvider.cs +++ b/src/Nethermind/Nethermind.Store/StateProvider.cs @@ -22,7 +22,6 @@ using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Core.Encoding; using Nethermind.Core.Specs; [assembly: InternalsVisibleTo("Nethermind.Store.Test")] @@ -433,6 +432,7 @@ public void Commit(IReleaseSpec releaseSpec) _currentPosition = -1; _committedThisRound.Clear(); _cache.Clear(); + _state.UpdateRootHash(); } private Account GetState(Address address) @@ -572,5 +572,10 @@ public void ClearCaches() _committedThisRound.Clear(); Array.Clear(_changes, 0, _changes.Length); } + + public void CommitTree() + { + _state.Commit(); + } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/StateTree.cs b/src/Nethermind/Nethermind.Store/StateTree.cs index aa3d785f1f30..5badc422f41d 100644 --- a/src/Nethermind/Nethermind.Store/StateTree.cs +++ b/src/Nethermind/Nethermind.Store/StateTree.cs @@ -16,6 +16,7 @@ * along with the Nethermind. If not, see . */ +using System.Diagnostics; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Encoding; @@ -27,14 +28,17 @@ public class StateTree : PatriciaTree { private readonly NewAccountDecoder _decoder = new NewAccountDecoder(); + [DebuggerStepThrough] public StateTree(IDb db) : base(db) { } + [DebuggerStepThrough] public StateTree(Keccak rootHash, IDb db) : base(db, rootHash) { } + [DebuggerStepThrough] public Account Get(Address address) { byte[] bytes = Get(Keccak.Compute((byte[])address.Hex).Bytes); @@ -45,15 +49,30 @@ public Account Get(Address address) return _decoder.Decode(bytes.AsRlpContext()); } + + [DebuggerStepThrough] + internal Account Get(Keccak keccak) // for testing + { + byte[] bytes = Get(keccak.Bytes); + if (bytes == null) + { + return null; + } + return _decoder.Decode(bytes.AsRlpContext()); + } + + [DebuggerStepThrough] public void Set(Address address, Account account) { - Set(Keccak.Compute((byte[])address.Hex), account == null ? null : Rlp.Encode(account)); + Keccak keccak = Keccak.Compute((byte[])address.Hex); + Set(keccak.Bytes, account == null ? null : Rlp.Encode(account)); } - - public void Set(Keccak addressHash, Rlp rlp) + + [DebuggerStepThrough] + internal void Set(Keccak keccak, Account account) // for testing { - base.Set(addressHash.Bytes, rlp); + Set(keccak.Bytes, account == null ? null : Rlp.Encode(account)); } } } \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Store/StorageProvider.cs b/src/Nethermind/Nethermind.Store/StorageProvider.cs index bf4ad7bf3646..93440c1f0aaf 100644 --- a/src/Nethermind/Nethermind.Store/StorageProvider.cs +++ b/src/Nethermind/Nethermind.Store/StorageProvider.cs @@ -65,7 +65,9 @@ public void Set(StorageAddress storageAddress, byte[] newValue) public Keccak GetRoot(Address address) { - return GetOrCreateStorage(address).RootHash; + StorageTree storageTree = GetOrCreateStorage(address); + storageTree.UpdateRootHash(); + return storageTree.RootHash; } public int TakeSnapshot() @@ -239,6 +241,14 @@ public void ClearCaches() _storages.Clear(); } + public void CommitTrees() + { + foreach (KeyValuePair storage in _storages) + { + storage.Value.Commit(); + } + } + private StorageTree GetOrCreateStorage(Address address) { if (!_storages.ContainsKey(address)) diff --git a/src/Nethermind/Nethermind.Store/TreeOperation.cs b/src/Nethermind/Nethermind.Store/TreeOperation.cs index a7be487d9de1..b8c095298e59 100644 --- a/src/Nethermind/Nethermind.Store/TreeOperation.cs +++ b/src/Nethermind/Nethermind.Store/TreeOperation.cs @@ -18,7 +18,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -51,7 +50,7 @@ public TreeOperation(PatriciaTree tree, Nibble[] updatePath, byte[] updateValue, public byte[] Run() { - if (_tree.Root == null) + if (_tree.RootRef == null) { if (!_isUpdate || _updateValue == null) { @@ -59,11 +58,13 @@ public byte[] Run() } Leaf leaf = new Leaf(new HexPrefix(true, _updatePath), _updateValue); - _tree.StoreNode(leaf, true); + leaf.IsDirty = true; + _tree.RootRef = new NodeRef(leaf, true); return _updateValue; } - return TraverseNode(_tree.Root); + _tree.RootRef.ResolveNode(_tree); + return TraverseNode(_tree.RootRef.Node); } private byte[] TraverseNode(Node node) @@ -102,12 +103,13 @@ public StackedNode(Node node, int pathIndex) public int PathIndex { get; } } - private void UpdateHashes(Node node) + // TODO: this can be removed now but is lower priority temporarily while the patricia rewrite testing is in progress + private void ConnectNodes(Node node) { // Keccak previousRootHash = _tree.RootHash; bool isRoot = _nodeStack.Count == 0; - KeccakOrRlp nextNodeHash = _tree.StoreNode(node, isRoot); + NodeRef nextNodeRef = node == null ? null : new NodeRef(node, isRoot); Node nextNode = node; // nodes should immutable here I guess @@ -120,16 +122,17 @@ private void UpdateHashes(Node node) if (node is Leaf leaf) { - throw new InvalidOperationException($"{nameof(Leaf)} {leaf} cannot be a parent of {nextNodeHash}"); + throw new InvalidOperationException($"{nameof(Leaf)} {leaf} cannot be a parent of {nextNodeRef}"); } if (node is Branch branch) { + node.IsDirty = true; // _tree.DeleteNode(branch.Nodes[parentOnStack.PathIndex], true); - branch.Nodes[parentOnStack.PathIndex] = nextNodeHash; + branch.Nodes[parentOnStack.PathIndex] = nextNodeRef; if (branch.IsValid) { - nextNodeHash = _tree.StoreNode(branch, isRoot); + nextNodeRef = new NodeRef(branch, isRoot); nextNode = branch; } else @@ -137,38 +140,46 @@ private void UpdateHashes(Node node) if (branch.Value.Length != 0) { Leaf leafFromBranch = new Leaf(new HexPrefix(true), branch.Value); - nextNodeHash = _tree.StoreNode(leafFromBranch, isRoot); + nextNodeRef = new NodeRef(leafFromBranch, isRoot); nextNode = leafFromBranch; } else { int childNodeIndex = Array.FindIndex(branch.Nodes, n => n != null); - KeccakOrRlp childNodeHash = branch.Nodes[childNodeIndex]; - if (childNodeHash == null) + NodeRef childNodeRef = branch.Nodes[childNodeIndex]; + if (childNodeRef == null) { throw new InvalidOperationException("Before updating branch should have had at least two non-empty children"); } - + // need to restore this node now? - Node childNode = _tree.GetNode(childNodeHash); + if (childNodeRef.Node == null) + { + } + + childNodeRef.ResolveNode(_tree); + Node childNode = childNodeRef.Node; if (childNode is Branch) { - Extension extensionFromBranch = new Extension(new HexPrefix(false, (byte)childNodeIndex), childNodeHash); - nextNodeHash = _tree.StoreNode(extensionFromBranch, isRoot); + Extension extensionFromBranch = new Extension(new HexPrefix(false, (byte)childNodeIndex), childNodeRef); + extensionFromBranch.IsDirty = true; + nextNodeRef = new NodeRef(extensionFromBranch, isRoot); nextNode = extensionFromBranch; } else if (childNode is Extension childExtension) { // _tree.DeleteNode(childNodeHash, true); - Extension extensionFromBranch = new Extension(new HexPrefix(false, Bytes.Concat((byte)childNodeIndex, childExtension.Path)), childExtension.NextNode); - nextNodeHash = _tree.StoreNode(extensionFromBranch, isRoot); + Extension extensionFromBranch = new Extension(new HexPrefix(false, Bytes.Concat((byte)childNodeIndex, childExtension.Path)), childExtension.NextNodeRef); + extensionFromBranch.IsDirty = true; + nextNodeRef = new NodeRef(extensionFromBranch, isRoot); nextNode = extensionFromBranch; } else if (childNode is Leaf childLeaf) { // _tree.DeleteNode(childNodeHash, true); Leaf leafFromBranch = new Leaf(new HexPrefix(true, Bytes.Concat((byte)childNodeIndex, childLeaf.Path)), childLeaf.Value); - nextNodeHash = _tree.StoreNode(leafFromBranch, isRoot); + leafFromBranch.IsDirty = true; + nextNodeRef = new NodeRef(leafFromBranch, isRoot); nextNode = leafFromBranch; } else @@ -180,23 +191,27 @@ private void UpdateHashes(Node node) } else if (node is Extension extension) { -// _tree.DeleteNode(extension.NextNode, true); +// _tree.DeleteNode(extension.NextNodeRef, true); if (nextNode is Leaf childLeaf) { Leaf leafFromExtension = new Leaf(new HexPrefix(true, Bytes.Concat(extension.Path, childLeaf.Path)), childLeaf.Value); - nextNodeHash = _tree.StoreNode(leafFromExtension, isRoot); + leafFromExtension.IsDirty = true; + nextNodeRef = new NodeRef(leafFromExtension, isRoot); nextNode = leafFromExtension; } else if (nextNode is Extension childExtension) { - Extension extensionFromExtension = new Extension(new HexPrefix(false, Bytes.Concat(extension.Path, childExtension.Path)), childExtension.NextNode); - nextNodeHash = _tree.StoreNode(extensionFromExtension, isRoot); + Extension extensionFromExtension = new Extension(new HexPrefix(false, Bytes.Concat(extension.Path, childExtension.Path)), childExtension.NextNodeRef); + extensionFromExtension.IsDirty = true; + nextNodeRef = new NodeRef(extensionFromExtension, isRoot); nextNode = extensionFromExtension; } else if (nextNode is Branch) { - extension.NextNode = nextNodeHash; - nextNodeHash = _tree.StoreNode(extension, isRoot); + // TODO: review modification of an existing node... + extension.NextNodeRef = nextNodeRef; + extension.IsDirty = true; + nextNodeRef = new NodeRef(extension, isRoot); nextNode = extension; } else @@ -210,6 +225,13 @@ private void UpdateHashes(Node node) } } + if (!nextNodeRef?.IsRoot ?? false) + { + throw new InvalidOperationException("Non-root being made root"); + } + + _tree.RootRef = nextNodeRef; + // _tree.DeleteNode(new KeccakOrRlp(previousRootHash), true); } @@ -224,22 +246,23 @@ private byte[] TraverseBranch(Branch node) if (_updateValue == null) { - UpdateHashes(null); + ConnectNodes(null); } else { Branch newBranch = new Branch(node.Nodes, _updateValue); - UpdateHashes(newBranch); + newBranch.IsDirty = true; + ConnectNodes(newBranch); } return _updateValue; } - KeccakOrRlp nextHash = node.Nodes[_updatePath[_currentIndex]]; + NodeRef nextNodeRef = node.Nodes[_updatePath[_currentIndex]]; _nodeStack.Push(new StackedNode(node, _updatePath[_currentIndex])); _currentIndex++; - if (nextHash == null) + if (nextNodeRef == null) { if (!_isUpdate) { @@ -258,12 +281,14 @@ private byte[] TraverseBranch(Branch node) byte[] leafPath = _updatePath.Slice(_currentIndex, _updatePath.Length - _currentIndex); Leaf leaf = new Leaf(new HexPrefix(true, leafPath), _updateValue); - UpdateHashes(leaf); + leaf.IsDirty = true; + ConnectNodes(leaf); return _updateValue; } - Node nextNode = _tree.GetNode(nextHash); + nextNodeRef.ResolveNode(_tree); + Node nextNode = nextNodeRef.Node; return TraverseNode(nextNode); } @@ -301,14 +326,15 @@ private byte[] TraverseLeaf(Leaf node) if (_updateValue == null) { - UpdateHashes(null); + ConnectNodes(null); return _updateValue; } if (!Bytes.UnsafeCompare(node.Value, _updateValue)) { Leaf newLeaf = new Leaf(new HexPrefix(true, RemainingUpdatePath), _updateValue); - UpdateHashes(newLeaf); + newLeaf.IsDirty = true; + ConnectNodes(newLeaf); return _updateValue; } @@ -334,10 +360,12 @@ private byte[] TraverseLeaf(Leaf node) { byte[] extensionPath = longerPath.Slice(0, extensionLength); Extension extension = new Extension(new HexPrefix(false, extensionPath)); + extension.IsDirty = true; _nodeStack.Push(new StackedNode(extension, 0)); } Branch branch = new Branch(); + branch.IsDirty = true; if (extensionLength == shorterPath.Length) { branch.Value = shorterPathValue; @@ -346,13 +374,15 @@ private byte[] TraverseLeaf(Leaf node) { byte[] shortLeafPath = shorterPath.Slice(extensionLength + 1, shorterPath.Length - extensionLength - 1); Leaf shortLeaf = new Leaf(new HexPrefix(true, shortLeafPath), shorterPathValue); - branch.Nodes[shorterPath[extensionLength]] = _tree.StoreNode(shortLeaf); + shortLeaf.IsDirty = true; + branch.Nodes[shorterPath[extensionLength]] = new NodeRef(shortLeaf); } - + byte[] leafPath = longerPath.Slice(extensionLength + 1, longerPath.Length - extensionLength - 1); Leaf leaf = new Leaf(new HexPrefix(true, leafPath), longerPathValue); + leaf.IsDirty = true; _nodeStack.Push(new StackedNode(branch, longerPath[extensionLength])); - UpdateHashes(leaf); + ConnectNodes(leaf); return _updateValue; } @@ -368,8 +398,8 @@ private byte[] TraverseExtension(Extension node) { _currentIndex += extensionLength; _nodeStack.Push(new StackedNode(node, 0)); - Node nextNode = _tree.GetNode(node.NextNode); - return TraverseNode(nextNode); + node.NextNodeRef.ResolveNode(_tree); + return TraverseNode(node.NextNodeRef.Node); } if (!_isUpdate) @@ -389,13 +419,14 @@ private byte[] TraverseExtension(Extension node) if (extensionLength != 0) { - byte[] extensionPath = node.Path.Slice(0, extensionLength); Extension extension = new Extension(new HexPrefix(false, extensionPath)); + extension.IsDirty = true; _nodeStack.Push(new StackedNode(extension, 0)); } Branch branch = new Branch(); + branch.IsDirty = true; if (extensionLength == RemainingUpdatePath.Length) { branch.Value = _updateValue; @@ -404,21 +435,23 @@ private byte[] TraverseExtension(Extension node) { byte[] path = RemainingUpdatePath.Slice(extensionLength + 1, RemainingUpdatePath.Length - extensionLength - 1); Leaf shortLeaf = new Leaf(new HexPrefix(true, path), _updateValue); - branch.Nodes[RemainingUpdatePath[extensionLength]] = _tree.StoreNode(shortLeaf); + shortLeaf.IsDirty = true; + branch.Nodes[RemainingUpdatePath[extensionLength]] = new NodeRef(shortLeaf); } if (node.Path.Length - extensionLength > 1) { byte[] extensionPath = node.Path.Slice(extensionLength + 1, node.Path.Length - extensionLength - 1); - Extension secondExtension = new Extension(new HexPrefix(false, extensionPath), node.NextNode); - branch.Nodes[node.Path[extensionLength]] = _tree.StoreNode(secondExtension); + Extension secondExtension = new Extension(new HexPrefix(false, extensionPath), node.NextNodeRef); + secondExtension.IsDirty = true; + branch.Nodes[node.Path[extensionLength]] = new NodeRef(secondExtension); } else { - branch.Nodes[node.Path[extensionLength]] = node.NextNode; + branch.Nodes[node.Path[extensionLength]] = node.NextNodeRef; } - - UpdateHashes(branch); + + ConnectNodes(branch); return _updateValue; } }