From 6eb15bc01c496502b4f6bf058b56c76b3b46e3b7 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 3 Aug 2020 14:34:01 +0800 Subject: [PATCH 1/8] cmd, core, eth, light, trie: disable recording preimage by default --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 ++++++++ core/blockchain.go | 15 ++++++++++----- core/genesis.go | 2 +- core/state/database.go | 10 +++++----- eth/api_tracer.go | 4 ++-- eth/backend.go | 1 + eth/config.go | 1 + eth/gen_config.go | 6 ++++++ light/postprocess.go | 4 ++-- trie/database.go | 31 ++++++++++++++++++++++--------- trie/secure_trie.go | 11 ++++++----- 13 files changed, 66 insertions(+), 29 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 38e48534dcb1..6473ced660ab 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -113,6 +113,7 @@ var ( utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, + utils.CacheRecordPreimageFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 237cb8d5165d..c252f6e1ecaa 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -114,6 +114,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, + utils.CacheRecordPreimageFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e5ccfd7435d4..5852dbea1802 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -383,6 +383,10 @@ var ( Name: "cache.noprefetch", Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", } + CacheRecordPreimageFlag = cli.BoolFlag{ + Name: "cache.recordpreimage", + Usage: "Enable recording of SHA3/keccak preimages of trie keys", + } // Miner settings MiningEnabledFlag = cli.BoolFlag{ Name: "mine", @@ -1526,6 +1530,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) } + if ctx.GlobalIsSet(CacheRecordPreimageFlag.Name) { + cfg.RecordPreimage = ctx.GlobalBool(CacheRecordPreimageFlag.Name) + } if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) } @@ -1835,6 +1842,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", TrieTimeLimit: eth.DefaultConfig.TrieTimeout, SnapshotLimit: eth.DefaultConfig.SnapshotCache, + RecordPreimage: ctx.GlobalBool(CacheRecordPreimageFlag.Name), } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled diff --git a/core/blockchain.go b/core/blockchain.go index 1c8a7fe60a9f..8989c6d4df69 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -129,6 +129,7 @@ type CacheConfig struct { TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + RecordPreimage bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -229,11 +230,15 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par badBlocks, _ := lru.New(badBlockLimit) bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triegc: prque.New(nil), - stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit, cacheConfig.TrieCleanJournal), + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triegc: prque.New(nil), + stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + RecordPreimage: cacheConfig.RecordPreimage, + }), quit: make(chan struct{}), shouldPreserve: shouldPreserve, bodyCache: bodyCache, diff --git a/core/genesis.go b/core/genesis.go index 0535d7ee3a18..908a969afd12 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -175,7 +175,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // We have the genesis block in database(perhaps in ancient database) // but the corresponding state is missing. header := rawdb.ReadHeader(db, stored, 0) - if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0, ""), nil); err != nil { + if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, nil), nil); err != nil { if genesis == nil { genesis = DefaultGenesisBlock() } diff --git a/core/state/database.go b/core/state/database.go index a9342f5179ca..83f7b2a839c5 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -104,18 +104,18 @@ type Trie interface { // NewDatabase creates a backing store for state. The returned database is safe for // concurrent use, but does not retain any recent trie nodes in memory. To keep some -// historical state in memory, use the NewDatabaseWithCache constructor. +// historical state in memory, use the NewDatabaseWithConfig constructor. func NewDatabase(db ethdb.Database) Database { - return NewDatabaseWithCache(db, 0, "") + return NewDatabaseWithConfig(db, nil) } -// NewDatabaseWithCache creates a backing store for state. The returned database +// NewDatabaseWithConfig creates a backing store for state. The returned database // is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a // large memory cache. -func NewDatabaseWithCache(db ethdb.Database, cache int, journal string) Database { +func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ - db: trie.NewDatabaseWithCache(db, cache, journal), + db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: fastcache.New(codeCacheSize), } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 90d4a95c147b..3b757a66289a 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -148,7 +148,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Ensure we have a valid starting state before doing any work origin := start.NumberU64() - database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16, "") // Chain tracing will probably start at genesis + database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16}) // Chain tracing will probably start at genesis if number := start.NumberU64(); number > 0 { start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) @@ -663,7 +663,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } // Otherwise try to reexec blocks until we find a state or reach our limit origin := block.NumberU64() - database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16, "") + database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16}) for i := uint64(0); i < reexec; i++ { block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/eth/backend.go b/eth/backend.go index 3fd027137c7f..2c3727d9e759 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -169,6 +169,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, + RecordPreimage: config.RecordPreimage, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) diff --git a/eth/config.go b/eth/config.go index 0d99c2a3f1e6..4140a42e21d9 100644 --- a/eth/config.go +++ b/eth/config.go @@ -149,6 +149,7 @@ type Config struct { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int + RecordPreimage bool // Mining options Miner miner.Config diff --git a/eth/gen_config.go b/eth/gen_config.go index 0093439d141b..6459d8063d0c 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -43,6 +43,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int + RecordPreimage bool Miner miner.Config Ethash ethash.Config TxPool core.TxPoolConfig @@ -83,6 +84,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache + enc.RecordPreimage = c.RecordPreimage enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -127,6 +129,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieDirtyCache *int TrieTimeout *time.Duration SnapshotCache *int + RecordPreimage *bool Miner *miner.Config Ethash *ethash.Config TxPool *core.TxPoolConfig @@ -222,6 +225,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SnapshotCache != nil { c.SnapshotCache = *dec.SnapshotCache } + if dec.RecordPreimage != nil { + c.RecordPreimage = *dec.RecordPreimage + } if dec.Miner != nil { c.Miner = *dec.Miner } diff --git a/light/postprocess.go b/light/postprocess.go index de207ad4a3bf..891c8a5869db 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -147,7 +147,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithCache(trieTable, 1, ""), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down trieset: mapset.NewSet(), sectionSize: size, disablePruning: disablePruning, @@ -340,7 +340,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithCache(trieTable, 1, ""), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down trieset: mapset.NewSet(), parentSize: parentSize, size: size, diff --git a/trie/database.go b/trie/database.go index c0c8870f8f1f..5e2cee53a082 100644 --- a/trie/database.go +++ b/trie/database.go @@ -74,7 +74,8 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + recordPreimage bool // Flag whether the preimage is recorded + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit @@ -272,32 +273,44 @@ func expandNode(hash hashNode, n node) node { } } +// Config defines all necessary options for database. +type Config struct { + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts + RecordPreimage bool // Flag whether the preimage of trie key is recorded +} + // NewDatabase creates a new trie database to store ephemeral trie content before // its written out to disk or garbage collected. No read cache is created, so all // data retrievals will hit the underlying disk database. func NewDatabase(diskdb ethdb.KeyValueStore) *Database { - return NewDatabaseWithCache(diskdb, 0, "") + return NewDatabaseWithConfig(diskdb, nil) } -// NewDatabaseWithCache creates a new trie database to store ephemeral trie content +// NewDatabaseWithConfig creates a new trie database to store ephemeral trie content // before its written out to disk or garbage collected. It also acts as a read cache // for nodes loaded from disk. -func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int, journal string) *Database { +func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database { var cleans *fastcache.Cache - if cache > 0 { - if journal == "" { - cleans = fastcache.New(cache * 1024 * 1024) + if config != nil && config.Cache > 0 { + if config.Journal == "" { + cleans = fastcache.New(config.Cache * 1024 * 1024) } else { - cleans = fastcache.LoadFromFileOrNew(journal, cache*1024*1024) + cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + var recordPreimage bool + if config != nil { + recordPreimage = config.RecordPreimage + } return &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - preimages: make(map[common.Hash][]byte), + recordPreimage: recordPreimage, + preimages: make(map[common.Hash][]byte), } } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 87b364fb1bba..9878aa09715e 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -147,12 +147,13 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - t.trie.db.lock.Lock() - for hk, key := range t.secKeyCache { - t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + if t.trie.db.recordPreimage { + t.trie.db.lock.Lock() + for hk, key := range t.secKeyCache { + t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + } + t.trie.db.lock.Unlock() } - t.trie.db.lock.Unlock() - t.secKeyCache = make(map[string][]byte) } // Commit the trie to its intermediate node database From a326cc65cf6d3cc428ddd92028c4aa3b70cbb192 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 3 Aug 2020 14:58:02 +0800 Subject: [PATCH 2/8] core, eth: fix unit tests --- core/state/state_test.go | 9 ++++++++- eth/api_test.go | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/state/state_test.go b/core/state/state_test.go index 0dc4c0ad63c3..90b729dbd441 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -18,6 +18,7 @@ package state import ( "bytes" + "github.com/ethereum/go-ethereum/trie" "math/big" "testing" @@ -40,8 +41,14 @@ func newStateTest() *stateTest { return &stateTest{db: db, state: sdb} } +func newStateRecordingPreimage() *stateTest { + db := rawdb.NewMemoryDatabase() + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{RecordPreimage: true}), nil) + return &stateTest{db: db, state: sdb} +} + func TestDump(t *testing.T) { - s := newStateTest() + s := newStateRecordingPreimage() // generate a few entries obj1 := s.state.GetOrNewStateObject(toAddr([]byte{0x01})) diff --git a/eth/api_test.go b/eth/api_test.go index 42f71e261e60..c1adb9cd9133 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -58,7 +59,7 @@ func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j func TestAccountRange(t *testing.T) { var ( - statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{RecordPreimage: true}) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} From 8cc63dae069704f7907e2a26e517d8a18cf561eb Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 3 Aug 2020 15:08:38 +0800 Subject: [PATCH 3/8] core: fix import --- core/state/state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/state_test.go b/core/state/state_test.go index 90b729dbd441..d4d595800f33 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -18,7 +18,6 @@ package state import ( "bytes" - "github.com/ethereum/go-ethereum/trie" "math/big" "testing" @@ -26,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) var toAddr = common.BytesToAddress From b30bd5e6c4a4c3ff96a0767af480ebd62ac36f67 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 29 Sep 2020 20:42:09 +0800 Subject: [PATCH 4/8] all: change to nopreimage --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 12 ++++++------ core/blockchain.go | 8 ++++---- core/state/state_test.go | 11 +++-------- eth/api_test.go | 3 +-- eth/backend.go | 2 +- eth/config.go | 2 +- eth/gen_config.go | 10 +++++----- trie/database.go | 18 +++++++----------- trie/secure_trie.go | 2 +- 11 files changed, 31 insertions(+), 41 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 6473ced660ab..c8961e8f8a3b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -113,7 +113,7 @@ var ( utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, - utils.CacheRecordPreimageFlag, + utils.CacheNoPreimageFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index c252f6e1ecaa..2fb60aca3892 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -114,7 +114,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, - utils.CacheRecordPreimageFlag, + utils.CacheNoPreimageFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5852dbea1802..7df73d89bbff 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -383,9 +383,9 @@ var ( Name: "cache.noprefetch", Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", } - CacheRecordPreimageFlag = cli.BoolFlag{ - Name: "cache.recordpreimage", - Usage: "Enable recording of SHA3/keccak preimages of trie keys", + CacheNoPreimageFlag = cli.BoolFlag{ + Name: "cache.nopreimage", + Usage: "Disable recording the SHA3/keccak preimages of trie keys", } // Miner settings MiningEnabledFlag = cli.BoolFlag{ @@ -1530,8 +1530,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) } - if ctx.GlobalIsSet(CacheRecordPreimageFlag.Name) { - cfg.RecordPreimage = ctx.GlobalBool(CacheRecordPreimageFlag.Name) + if ctx.GlobalIsSet(CacheNoPreimageFlag.Name) { + cfg.NoPreimage = ctx.GlobalBool(CacheNoPreimageFlag.Name) } if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) @@ -1842,7 +1842,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", TrieTimeLimit: eth.DefaultConfig.TrieTimeout, SnapshotLimit: eth.DefaultConfig.SnapshotCache, - RecordPreimage: ctx.GlobalBool(CacheRecordPreimageFlag.Name), + NoPreimage: ctx.GlobalBool(CacheNoPreimageFlag.Name), } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled diff --git a/core/blockchain.go b/core/blockchain.go index 8989c6d4df69..822d37d21cc1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -129,7 +129,7 @@ type CacheConfig struct { TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory - RecordPreimage bool // Whether to store preimage of trie key to the disk + NoPreimage bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -235,9 +235,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par db: db, triegc: prque.New(nil), stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - RecordPreimage: cacheConfig.RecordPreimage, + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + NoPreimage: cacheConfig.NoPreimage, }), quit: make(chan struct{}), shouldPreserve: shouldPreserve, diff --git a/core/state/state_test.go b/core/state/state_test.go index d4d595800f33..526d7f8177cd 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie" ) var toAddr = common.BytesToAddress @@ -41,14 +40,10 @@ func newStateTest() *stateTest { return &stateTest{db: db, state: sdb} } -func newStateRecordingPreimage() *stateTest { - db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{RecordPreimage: true}), nil) - return &stateTest{db: db, state: sdb} -} - func TestDump(t *testing.T) { - s := newStateRecordingPreimage() + db := rawdb.NewMemoryDatabase() + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, nil), nil) + s := &stateTest{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(toAddr([]byte{0x01})) diff --git a/eth/api_test.go b/eth/api_test.go index c1adb9cd9133..2c9a2e54e8ab 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -59,7 +58,7 @@ func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j func TestAccountRange(t *testing.T) { var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{RecordPreimage: true}) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} diff --git a/eth/backend.go b/eth/backend.go index 2c3727d9e759..9553d2d144ce 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -169,7 +169,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, - RecordPreimage: config.RecordPreimage, + NoPreimage: config.NoPreimage, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) diff --git a/eth/config.go b/eth/config.go index 4140a42e21d9..d578e9718ec4 100644 --- a/eth/config.go +++ b/eth/config.go @@ -149,7 +149,7 @@ type Config struct { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int - RecordPreimage bool + NoPreimage bool // Mining options Miner miner.Config diff --git a/eth/gen_config.go b/eth/gen_config.go index 6459d8063d0c..dd41015e4870 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -43,7 +43,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int - RecordPreimage bool + NoPreimage bool Miner miner.Config Ethash ethash.Config TxPool core.TxPoolConfig @@ -84,7 +84,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache - enc.RecordPreimage = c.RecordPreimage + enc.NoPreimage = c.NoPreimage enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -129,7 +129,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieDirtyCache *int TrieTimeout *time.Duration SnapshotCache *int - RecordPreimage *bool + NoPreimage *bool Miner *miner.Config Ethash *ethash.Config TxPool *core.TxPoolConfig @@ -225,8 +225,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SnapshotCache != nil { c.SnapshotCache = *dec.SnapshotCache } - if dec.RecordPreimage != nil { - c.RecordPreimage = *dec.RecordPreimage + if dec.NoPreimage != nil { + c.NoPreimage = *dec.NoPreimage } if dec.Miner != nil { c.Miner = *dec.Miner diff --git a/trie/database.go b/trie/database.go index 5e2cee53a082..589638c1b933 100644 --- a/trie/database.go +++ b/trie/database.go @@ -74,8 +74,8 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - recordPreimage bool // Flag whether the preimage is recorded - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + noPreimage bool // Flag whether the preimage should be recorded + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit @@ -275,9 +275,9 @@ func expandNode(hash hashNode, n node) node { // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - RecordPreimage bool // Flag whether the preimage of trie key is recorded + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts + NoPreimage bool // Flag whether the preimage of trie key is recorded } // NewDatabase creates a new trie database to store ephemeral trie content before @@ -299,18 +299,14 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } - var recordPreimage bool - if config != nil { - recordPreimage = config.RecordPreimage - } return &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - recordPreimage: recordPreimage, - preimages: make(map[common.Hash][]byte), + noPreimage: config != nil && config.NoPreimage, + preimages: make(map[common.Hash][]byte), } } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 9878aa09715e..28405daa3df8 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -147,7 +147,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.recordPreimage { + if !t.trie.db.noPreimage { t.trie.db.lock.Lock() for hk, key := range t.secKeyCache { t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) From 3f250c3ec07dc71075fd075ffcf4e3538c9e02a5 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 13 Nov 2020 14:30:59 +0800 Subject: [PATCH 5/8] cmd, core, eth, trie: use cache.preimages flag --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 14 +++++++------- core/blockchain.go | 16 ++++++++-------- eth/api_tracer.go | 4 ++-- eth/backend.go | 2 +- eth/config.go | 2 +- eth/gen_config.go | 10 +++++----- trie/database.go | 19 ++++++++++++------- trie/secure_trie.go | 2 +- 10 files changed, 39 insertions(+), 34 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c8961e8f8a3b..ca67859a91bd 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -113,7 +113,7 @@ var ( utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, - utils.CacheNoPreimageFlag, + utils.CachePreimagesFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2fb60aca3892..a9b6f53e72a2 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -114,7 +114,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.CacheGCFlag, utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, - utils.CacheNoPreimageFlag, + utils.CachePreimagesFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7df73d89bbff..111c6a392aff 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -383,9 +383,9 @@ var ( Name: "cache.noprefetch", Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", } - CacheNoPreimageFlag = cli.BoolFlag{ - Name: "cache.nopreimage", - Usage: "Disable recording the SHA3/keccak preimages of trie keys", + CachePreimagesFlag = cli.BoolTFlag{ + Name: "cache.preimages", + Usage: "Enable recording the SHA3/keccak preimages of trie keys", } // Miner settings MiningEnabledFlag = cli.BoolFlag{ @@ -1530,9 +1530,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) } - if ctx.GlobalIsSet(CacheNoPreimageFlag.Name) { - cfg.NoPreimage = ctx.GlobalBool(CacheNoPreimageFlag.Name) - } + // Read the value from the flag no matter it's set or not. + // The default value is true. + cfg.Preimages = ctx.GlobalBool(CachePreimagesFlag.Name) if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) } @@ -1842,7 +1842,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", TrieTimeLimit: eth.DefaultConfig.TrieTimeout, SnapshotLimit: eth.DefaultConfig.SnapshotCache, - NoPreimage: ctx.GlobalBool(CacheNoPreimageFlag.Name), + Preimages: ctx.GlobalBool(CachePreimagesFlag.Name), } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled diff --git a/core/blockchain.go b/core/blockchain.go index 822d37d21cc1..28f29d1abcef 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -125,11 +125,11 @@ type CacheConfig struct { TrieCleanJournal string // Disk journal for saving clean cache entries. TrieCleanRejournal time.Duration // Time interval to dump clean cache to disk periodically TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks - TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk - TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory - NoPreimage bool // Whether to store preimage of trie key to the disk + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -235,9 +235,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par db: db, triegc: prque.New(nil), stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - NoPreimage: cacheConfig.NoPreimage, + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + Preimages: cacheConfig.Preimages, }), quit: make(chan struct{}), shouldPreserve: shouldPreserve, diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 3b757a66289a..793d88113a54 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -148,7 +148,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Ensure we have a valid starting state before doing any work origin := start.NumberU64() - database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16}) // Chain tracing will probably start at genesis + database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16, Preimages: true}) if number := start.NumberU64(); number > 0 { start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) @@ -663,7 +663,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } // Otherwise try to reexec blocks until we find a state or reach our limit origin := block.NumberU64() - database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16}) + database := state.NewDatabaseWithConfig(api.eth.ChainDb(), &trie.Config{Cache: 16, Preimages: true}) for i := uint64(0); i < reexec; i++ { block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/eth/backend.go b/eth/backend.go index 9553d2d144ce..01e6cadd1f6c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -169,7 +169,7 @@ func New(stack *node.Node, config *Config) (*Ethereum, error) { TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, - NoPreimage: config.NoPreimage, + Preimages: config.Preimages, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) diff --git a/eth/config.go b/eth/config.go index d578e9718ec4..0d90376d9496 100644 --- a/eth/config.go +++ b/eth/config.go @@ -149,7 +149,7 @@ type Config struct { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int - NoPreimage bool + Preimages bool // Mining options Miner miner.Config diff --git a/eth/gen_config.go b/eth/gen_config.go index dd41015e4870..b0674c7d77e0 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -43,7 +43,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int - NoPreimage bool + Preimages bool Miner miner.Config Ethash ethash.Config TxPool core.TxPoolConfig @@ -84,7 +84,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache - enc.NoPreimage = c.NoPreimage + enc.Preimages = c.Preimages enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -129,7 +129,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieDirtyCache *int TrieTimeout *time.Duration SnapshotCache *int - NoPreimage *bool + Preimages *bool Miner *miner.Config Ethash *ethash.Config TxPool *core.TxPoolConfig @@ -225,8 +225,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SnapshotCache != nil { c.SnapshotCache = *dec.SnapshotCache } - if dec.NoPreimage != nil { - c.NoPreimage = *dec.NoPreimage + if dec.Preimages != nil { + c.Preimages = *dec.Preimages } if dec.Miner != nil { c.Miner = *dec.Miner diff --git a/trie/database.go b/trie/database.go index 589638c1b933..a0692d51a302 100644 --- a/trie/database.go +++ b/trie/database.go @@ -74,8 +74,8 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - noPreimage bool // Flag whether the preimage should be recorded - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + persistPreimages bool // Flag whether the preimage should be recorded + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit @@ -275,9 +275,9 @@ func expandNode(hash hashNode, n node) node { // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - NoPreimage bool // Flag whether the preimage of trie key is recorded + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts + Preimages bool // Flag whether the preimage of trie key is recorded } // NewDatabase creates a new trie database to store ephemeral trie content before @@ -299,14 +299,19 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + persistPreimages := true // Enable the preimages recording by default + if config != nil && !config.Preimages { + persistPreimages = false + log.Info("Disable recording the key preimages") + } return &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - noPreimage: config != nil && config.NoPreimage, - preimages: make(map[common.Hash][]byte), + persistPreimages: persistPreimages, + preimages: make(map[common.Hash][]byte), } } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 28405daa3df8..6437052b91cf 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -147,7 +147,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if !t.trie.db.noPreimage { + if t.trie.db.persistPreimages { t.trie.db.lock.Lock() for hk, key := range t.secKeyCache { t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) From 1772f39675127294e9c9fc0b9ceab1419d62f651 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 13 Nov 2020 14:57:34 +0800 Subject: [PATCH 6/8] cmd: enable preimages for archive node --- cmd/utils/flags.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 111c6a392aff..6e3604f8eccc 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -385,7 +385,7 @@ var ( } CachePreimagesFlag = cli.BoolTFlag{ Name: "cache.preimages", - Usage: "Enable recording the SHA3/keccak preimages of trie keys", + Usage: "Enable recording the SHA3/keccak preimages of trie keys (enable by default)", } // Miner settings MiningEnabledFlag = cli.BoolFlag{ @@ -1533,6 +1533,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Read the value from the flag no matter it's set or not. // The default value is true. cfg.Preimages = ctx.GlobalBool(CachePreimagesFlag.Name) + if cfg.NoPruning && !cfg.Preimages { + cfg.Preimages = true + log.Info("Enable recording the key preimages for archive node") + } if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) } @@ -1844,6 +1848,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B SnapshotLimit: eth.DefaultConfig.SnapshotCache, Preimages: ctx.GlobalBool(CachePreimagesFlag.Name), } + if cache.TrieDirtyDisabled && !cache.Preimages { + cache.Preimages = true + log.Info("Enable recording the key preimages for archive node") + } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled } From 14e6805872368f0472b1ffe136b46342c663c379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 17 Nov 2020 18:49:30 +0200 Subject: [PATCH 7/8] cmd/utils, trie: simplify preimage tracking a bit --- cmd/utils/flags.go | 9 +++--- trie/database.go | 69 +++++++++++++++++++++++++++------------------ trie/secure_trie.go | 2 +- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6e3604f8eccc..1b86771db894 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -385,7 +385,7 @@ var ( } CachePreimagesFlag = cli.BoolTFlag{ Name: "cache.preimages", - Usage: "Enable recording the SHA3/keccak preimages of trie keys (enable by default)", + Usage: "Enable recording the SHA3/keccak preimages of trie keys (default: true)", } // Miner settings MiningEnabledFlag = cli.BoolFlag{ @@ -1530,12 +1530,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) } - // Read the value from the flag no matter it's set or not. - // The default value is true. + // Read the value from the flag no matter if it's set or not. cfg.Preimages = ctx.GlobalBool(CachePreimagesFlag.Name) if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true - log.Info("Enable recording the key preimages for archive node") + log.Info("Enabling recording of key preimages since archive mode is used") } if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) @@ -1850,7 +1849,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B } if cache.TrieDirtyDisabled && !cache.Preimages { cache.Preimages = true - log.Info("Enable recording the key preimages for archive node") + log.Info("Enabling recording of key preimages since archive mode is used") } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled diff --git a/trie/database.go b/trie/database.go index a0692d51a302..d8fe45f44452 100644 --- a/trie/database.go +++ b/trie/database.go @@ -74,8 +74,7 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - persistPreimages bool // Flag whether the preimage should be recorded - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit @@ -299,20 +298,17 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } - persistPreimages := true // Enable the preimages recording by default - if config != nil && !config.Preimages { - persistPreimages = false - log.Info("Disable recording the key preimages") - } - return &Database{ + db := &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - persistPreimages: persistPreimages, - preimages: make(map[common.Hash][]byte), } + if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future + db.preimages = make(map[common.Hash][]byte) + } + return db } // DiskDB retrieves the persistent storage backing the trie database. @@ -359,6 +355,11 @@ func (db *Database) insert(hash common.Hash, size int, node node) { // // Note, this method assumes that the database's lock is held! func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { + // Short circuit if preimage collection is disabled + if db.preimages == nil { + return + } + // Track the preimage if a yet unknown one if _, ok := db.preimages[hash]; ok { return } @@ -445,6 +446,10 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { // preimage retrieves a cached trie node pre-image from memory. If it cannot be // found cached, the method queries the persistent database for the content. func (db *Database) preimage(hash common.Hash) []byte { + // Short circuit if preimage collection is disabled + if db.preimages == nil { + return nil + } // Retrieve the node from cache if available db.lock.RLock() preimage := db.preimages[hash] @@ -602,12 +607,16 @@ func (db *Database) Cap(limit common.StorageSize) error { // leave for later to deduplicate writes. flushPreimages := db.preimagesSize > 4*1024*1024 if flushPreimages { - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err + if db.preimages == nil { + log.Error("Attempted to write preimages whilst disabled") + } else { + rawdb.WritePreimages(batch, db.preimages) + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return err + } + batch.Reset() } - batch.Reset() } } // Keep committing nodes from the flush-list until we're below allowance @@ -644,7 +653,11 @@ func (db *Database) Cap(limit common.StorageSize) error { defer db.lock.Unlock() if flushPreimages { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 + if db.preimages == nil { + log.Error("Attempted to reset preimage cache whilst disabled") + } else { + db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 + } } for db.oldest != oldest { node := db.dirties[db.oldest] @@ -688,20 +701,21 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch := db.diskdb.NewBatch() // Move all of the accumulated preimages into a write batch - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { + if db.preimages != nil { + rawdb.WritePreimages(batch, db.preimages) + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + } + // Since we're going to replay trie node writes into the clean cache, flush out + // any batched pre-images before continuing. if err := batch.Write(); err != nil { return err } batch.Reset() } - // Since we're going to replay trie node writes into the clean cache, flush out - // any batched pre-images before continuing. - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize @@ -723,8 +737,9 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Reset() // Reset the storage counters and bumpd metrics - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - + if db.preimages != nil { + db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 + } memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 6437052b91cf..e38471c1b76f 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -147,7 +147,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.persistPreimages { + if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock t.trie.db.lock.Lock() for hk, key := range t.secKeyCache { t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) From 8f30c4d5b34ecc135925dc6373f71e9ea4856f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 18 Nov 2020 11:13:13 +0200 Subject: [PATCH 8/8] core: fix linter --- core/blockchain.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 28f29d1abcef..c52be6835429 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -125,11 +125,11 @@ type CacheConfig struct { TrieCleanJournal string // Disk journal for saving clean cache entries. TrieCleanRejournal time.Duration // Time interval to dump clean cache to disk periodically TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks - TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk - TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory - Preimages bool // Whether to store preimage of trie key to the disk + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it }