From bcba2ebf417318643ac6fce2e53d06404d3b918f Mon Sep 17 00:00:00 2001 From: Pathorn Date: Tue, 8 Oct 2019 17:34:39 +0700 Subject: [PATCH 01/10] modify trie.Database and state.Database --- core/blockchain.go | 2 +- core/state/database.go | 10 +++++++++- trie/database.go | 5 +++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f4a818f4c054..445001b046d8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -43,7 +43,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) var ( diff --git a/core/state/database.go b/core/state/database.go index c1b630991ce0..7875b48149e4 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -57,6 +57,9 @@ type Database interface { // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database + + // SetTrieDB set the low level trie database + SetTrieDB(trieDB *trie.Database) } // Trie is a Ethereum Merkle Trie. @@ -77,10 +80,11 @@ type Trie interface { // high level trie abstraction. func NewDatabase(db ethdb.Database) Database { csc, _ := lru.New(codeSizeCacheSize) - return &cachingDB{ + c := &cachingDB{ db: trie.NewDatabase(db), codeSizeCache: csc, } + return c } type cachingDB struct { @@ -90,6 +94,10 @@ type cachingDB struct { codeSizeCache *lru.Cache } +func (db *cachingDB) SetTrieDB(trieDB *trie.Database) { + db.db = trieDB +} + // OpenTrie opens the main account trie. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { db.mu.Lock() diff --git a/trie/database.go b/trie/database.go index 7a755c043aef..5166ae8da1da 100644 --- a/trie/database.go +++ b/trie/database.go @@ -278,6 +278,11 @@ func (db *Database) DiskDB() DatabaseReader { return db.diskdb } +// DiskDB retrieves the persistent storage backing the trie database. +func (db *Database) SetDiskDB(diskdb ethdb.Database) { + db.diskdb = diskdb +} + // InsertBlob writes a new reference tracked blob to the memory database if it's // yet unknown. This method should only be used for non-trie nodes that require // reference counting, since trie nodes are garbage collected directly through From 45c19daeab443e6ded4b7321c5c8ee487fb70369 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Wed, 9 Oct 2019 00:30:38 +0700 Subject: [PATCH 02/10] revert unnecessary change --- core/state/database.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 7875b48149e4..d4279c21092d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -80,11 +80,10 @@ type Trie interface { // high level trie abstraction. func NewDatabase(db ethdb.Database) Database { csc, _ := lru.New(codeSizeCacheSize) - c := &cachingDB{ + return &cachingDB{ db: trie.NewDatabase(db), codeSizeCache: csc, } - return c } type cachingDB struct { From b5251b8193f460424cfb3755bf6cf4935b3d0a67 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Wed, 9 Oct 2019 00:31:34 +0700 Subject: [PATCH 03/10] fix comment --- trie/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/database.go b/trie/database.go index 5166ae8da1da..3e38fbd41595 100644 --- a/trie/database.go +++ b/trie/database.go @@ -278,7 +278,7 @@ func (db *Database) DiskDB() DatabaseReader { return db.diskdb } -// DiskDB retrieves the persistent storage backing the trie database. +// SetDiskDB sets the persistent storage backing the trie database. func (db *Database) SetDiskDB(diskdb ethdb.Database) { db.diskdb = diskdb } From 30dc35bbd77aeb96b9bd401bc136b3279e0c6209 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Fri, 1 Nov 2019 13:15:27 +0700 Subject: [PATCH 04/10] Revert "Force stateObject to update storage trie deterministically" This reverts commit d58e7a3079f30f4c87509836ace69a8e775dde21. --- core/state/state_object.go | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/core/state/state_object.go b/core/state/state_object.go index b120ea70182a..f41ab0409221 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -77,9 +77,8 @@ type stateObject struct { trie Trie // storage trie, which becomes non-nil on first access code Code // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites - dirtyStorage Storage // Storage entries that need to be flushed to disk - dirtyStorageKeys []common.Hash // Dirty storage keys in order of insertion into dirtyStorage + originStorage Storage // Storage cache of original entries to dedup rewrites + dirtyStorage Storage // Storage entries that need to be flushed to disk // Cache flags. // When an object is marked suicided it will be delete from the trie @@ -112,13 +111,12 @@ func newObject(db *StateDB, address common.Address, data Account) *stateObject { data.CodeHash = emptyCodeHash } return &stateObject{ - db: db, - address: address, - addrHash: crypto.Keccak256Hash(address[:]), - data: data, - originStorage: make(Storage), - dirtyStorage: make(Storage), - dirtyStorageKeys: nil, + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + data: data, + originStorage: make(Storage), + dirtyStorage: make(Storage), } } @@ -214,21 +212,12 @@ func (self *stateObject) SetState(db Database, key, value common.Hash) { func (self *stateObject) setState(key, value common.Hash) { self.dirtyStorage[key] = value - for _, k := range self.dirtyStorageKeys { - if k == key { - return - } - } - self.dirtyStorageKeys = append(self.dirtyStorageKeys, key) } // updateTrie writes cached storage modifications into the object's storage trie. func (self *stateObject) updateTrie(db Database) Trie { tr := self.getTrie(db) - // Iterate through the storage keys in deterministic order to ensure the storage trie is - // identical across machines - for _, key := range self.dirtyStorageKeys { - value := self.dirtyStorage[key] + for key, value := range self.dirtyStorage { delete(self.dirtyStorage, key) // Skip noop changes, persist actual changes @@ -314,7 +303,6 @@ func (self *stateObject) deepCopy(db *StateDB) *stateObject { } stateObject.code = self.code stateObject.dirtyStorage = self.dirtyStorage.Copy() - stateObject.dirtyStorageKeys = append([]common.Hash{}, self.dirtyStorageKeys...) stateObject.originStorage = self.originStorage.Copy() stateObject.suicided = self.suicided stateObject.dirtyCode = self.dirtyCode From c85c31105f263dd05074548dada1bcf55ec06413 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Mon, 4 Nov 2019 19:10:53 +0700 Subject: [PATCH 05/10] Revert "Make trie.Database.Commit() write out preimages in deterministic order" This reverts commit 500eef96c8c93a6d6a2103419fb3d4aca7d023ac. --- trie/database.go | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/trie/database.go b/trie/database.go index 3e38fbd41595..582b018206b1 100644 --- a/trie/database.go +++ b/trie/database.go @@ -17,10 +17,8 @@ package trie import ( - "bytes" "fmt" "io" - "sort" "sync" "time" @@ -622,26 +620,9 @@ func (db *Database) Commit(node common.Hash, report bool) error { start := time.Now() batch := db.diskdb.NewBatch() - type kvPair struct { - key []byte - value []byte - } - - // Sort preimages to ensure they are written out in deterministic order - orderedPreimages := make([]kvPair, 0, len(db.preimages)) - for hash := range db.preimages { - orderedPreimages = append(orderedPreimages, kvPair{ - key: common.CopyBytes(hash[:]), - value: db.preimages[hash], - }) - } - sort.Slice(orderedPreimages, func(j, k int) bool { - return bytes.Compare(orderedPreimages[j].key, orderedPreimages[k].key) < 0 - }) - // Move all of the accumulated preimages into a write batch - for _, preimage := range orderedPreimages { - if err := batch.Put(db.secureKey(preimage.key), preimage.value); err != nil { + for hash, preimage := range db.preimages { + if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil { log.Error("Failed to commit preimage from trie database", "err", err) db.lock.RUnlock() return err From ab89bd9faa3ae3904011fd405ccb8dfe8ca5ecf9 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Mon, 4 Nov 2019 19:14:16 +0700 Subject: [PATCH 06/10] Revert "Revert "Make trie.Database.Commit() write out preimages in deterministic order"" This reverts commit c85c31105f263dd05074548dada1bcf55ec06413. --- trie/database.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/trie/database.go b/trie/database.go index 582b018206b1..3e38fbd41595 100644 --- a/trie/database.go +++ b/trie/database.go @@ -17,8 +17,10 @@ package trie import ( + "bytes" "fmt" "io" + "sort" "sync" "time" @@ -620,9 +622,26 @@ func (db *Database) Commit(node common.Hash, report bool) error { start := time.Now() batch := db.diskdb.NewBatch() + type kvPair struct { + key []byte + value []byte + } + + // Sort preimages to ensure they are written out in deterministic order + orderedPreimages := make([]kvPair, 0, len(db.preimages)) + for hash := range db.preimages { + orderedPreimages = append(orderedPreimages, kvPair{ + key: common.CopyBytes(hash[:]), + value: db.preimages[hash], + }) + } + sort.Slice(orderedPreimages, func(j, k int) bool { + return bytes.Compare(orderedPreimages[j].key, orderedPreimages[k].key) < 0 + }) + // Move all of the accumulated preimages into a write batch - for hash, preimage := range db.preimages { - if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil { + for _, preimage := range orderedPreimages { + if err := batch.Put(db.secureKey(preimage.key), preimage.value); err != nil { log.Error("Failed to commit preimage from trie database", "err", err) db.lock.RUnlock() return err From 9b39158c0886f0783bfb41b19c05b584a48625c3 Mon Sep 17 00:00:00 2001 From: Pathorn Date: Fri, 8 Nov 2019 15:09:04 +0700 Subject: [PATCH 07/10] add ResetLogs --- core/state/statedb.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/state/statedb.go b/core/state/statedb.go index 216667ce983f..94f4906d0977 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -146,6 +146,11 @@ func (self *StateDB) GetLogs(hash common.Hash) []*types.Log { return self.logs[hash] } +func (self *StateDB) ResetLogs() { + self.logs = make(map[common.Hash][]*types.Log) + self.logSize = 0 +} + func (self *StateDB) Logs() []*types.Log { var logs []*types.Log for _, lgs := range self.logs { From 0a555c6498c2b57a8aaf9b15e96b46610fa3a58f Mon Sep 17 00:00:00 2001 From: Pathorn Date: Thu, 28 Nov 2019 13:03:40 +0700 Subject: [PATCH 08/10] remove resetlog and change SetTrieDB to WithTrieDB --- core/state/database.go | 7 ++++--- core/state/statedb.go | 5 ----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index d4279c21092d..e7b7eae4f07d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -58,8 +58,8 @@ type Database interface { // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database - // SetTrieDB set the low level trie database - SetTrieDB(trieDB *trie.Database) + // WithTrieDB set the low level trie database + WithTrieDB(trieDB *trie.Database) Database } // Trie is a Ethereum Merkle Trie. @@ -93,8 +93,9 @@ type cachingDB struct { codeSizeCache *lru.Cache } -func (db *cachingDB) SetTrieDB(trieDB *trie.Database) { +func (db *cachingDB) WithTrieDB(trieDB *trie.Database) Database { db.db = trieDB + return db } // OpenTrie opens the main account trie. diff --git a/core/state/statedb.go b/core/state/statedb.go index 94f4906d0977..216667ce983f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -146,11 +146,6 @@ func (self *StateDB) GetLogs(hash common.Hash) []*types.Log { return self.logs[hash] } -func (self *StateDB) ResetLogs() { - self.logs = make(map[common.Hash][]*types.Log) - self.logSize = 0 -} - func (self *StateDB) Logs() []*types.Log { var logs []*types.Log for _, lgs := range self.logs { From e790de078d174ba7341681673cfe34ce2f4fd241 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Mon, 2 Dec 2019 13:15:02 +0700 Subject: [PATCH 09/10] Tweak the API to avoid creating useless instances of trie.Database --- core/state/database.go | 17 +++++++++-------- trie/database.go | 5 ----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index e7b7eae4f07d..fdfc6ec38b6f 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -57,9 +57,6 @@ type Database interface { // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database - - // WithTrieDB set the low level trie database - WithTrieDB(trieDB *trie.Database) Database } // Trie is a Ethereum Merkle Trie. @@ -86,6 +83,15 @@ func NewDatabase(db ethdb.Database) Database { } } +// NewDatabaseWithTrieDB creates a backing store for state from a previously created trie database. +func NewDatabaseWithTrieDB(db *trie.Database) Database { + csc, _ := lru.New(codeSizeCacheSize) + return &cachingDB{ + db: db, + codeSizeCache: csc, + } +} + type cachingDB struct { db *trie.Database mu sync.Mutex @@ -93,11 +99,6 @@ type cachingDB struct { codeSizeCache *lru.Cache } -func (db *cachingDB) WithTrieDB(trieDB *trie.Database) Database { - db.db = trieDB - return db -} - // OpenTrie opens the main account trie. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { db.mu.Lock() diff --git a/trie/database.go b/trie/database.go index e0a0c5e4e2f5..65bc34370f2d 100644 --- a/trie/database.go +++ b/trie/database.go @@ -280,11 +280,6 @@ func (db *Database) DiskDB() DatabaseReader { return db.diskdb } -// SetDiskDB sets the persistent storage backing the trie database. -func (db *Database) SetDiskDB(diskdb ethdb.Database) { - db.diskdb = diskdb -} - // InsertBlob writes a new reference tracked blob to the memory database if it's // yet unknown. This method should only be used for non-trie nodes that require // reference counting, since trie nodes are garbage collected directly through From 0c27835b75107cc3e49729c547b6d13f5fec5f68 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Wed, 4 Dec 2019 11:08:48 +0700 Subject: [PATCH 10/10] Revert irrelevant changes --- core/blockchain.go | 2 +- core/state/state_object.go | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 445001b046d8..f4a818f4c054 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -43,7 +43,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - lru "github.com/hashicorp/golang-lru" + "github.com/hashicorp/golang-lru" ) var ( diff --git a/core/state/state_object.go b/core/state/state_object.go index 602107366ab7..5291ae25fd16 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -78,9 +78,10 @@ type stateObject struct { trie Trie // storage trie, which becomes non-nil on first access code Code // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites - dirtyStorage Storage // Storage entries that need to be flushed to disk - dirtyStorageKeys []common.Hash + originStorage Storage // Storage cache of original entries to dedup rewrites + dirtyStorage Storage // Storage entries that need to be flushed to disk + dirtyStorageKeys []common.Hash // Dirty storage keys in order of insertion into dirtyStorage + // Cache flags. // When an object is marked suicided it will be delete from the trie // during the "update" phase of the state transition. @@ -112,12 +113,13 @@ func newObject(db *StateDB, address common.Address, data Account) *stateObject { data.CodeHash = emptyCodeHash } return &stateObject{ - db: db, - address: address, - addrHash: crypto.Keccak256Hash(address[:]), - data: data, - originStorage: make(Storage), - dirtyStorage: make(Storage), + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + data: data, + originStorage: make(Storage), + dirtyStorage: make(Storage), + dirtyStorageKeys: nil, } }