chore: updated by review tips

eth/backend.go

@@ -161,6 +161,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
+		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+			"adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
+		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024,
+			"adjusted", common.StorageSize(config.TrieCleanCache+config.TrieDirtyCache-pathdb.MaxDirtyBufferSize/1024/1024)*1024*1024)
 		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}

triedb/pathdb/asyncnodebuffer.go

@@ -226,8 +226,7 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr
 	}
 	if n.Hash != hash {
 		dirtyFalseMeter.Mark(1)
-		// difflayer cache miss maybe hit this, which is normal case, and the caller can retry it.
-		log.Debug("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+		log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
 		return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
 	}
 	return n, nil

triedb/pathdb/difflayer.go

@@ -26,12 +26,17 @@ import (
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
-type HashIndex struct {
+type RefTrieNode struct {
+	refCount uint32
+	node     *trienode.Node
+}
+
+type HashNodeCache struct {
 	lock  sync.RWMutex
-	cache map[common.Hash]*trienode.Node
+	cache map[common.Hash]*RefTrieNode
 }
 
-func (h *HashIndex) length() int {
+func (h *HashNodeCache) length() int {
 	if h == nil {
 		return 0
 	}
@@ -40,54 +45,68 @@ func (h *HashIndex) length() int {
 	return len(h.cache)
 }
 
-func (h *HashIndex) set(hash common.Hash, node *trienode.Node) {
+func (h *HashNodeCache) set(hash common.Hash, node *trienode.Node) {
 	if h == nil {
 		return
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-	h.cache[hash] = node
+	if n, ok := h.cache[hash]; ok {
+		n.refCount++
+	} else {
+		h.cache[hash] = &RefTrieNode{1, node}
+	}
 }
 
-func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
+func (h *HashNodeCache) Get(hash common.Hash) *trienode.Node {
 	if h == nil {
 		return nil
 	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
 	if n, ok := h.cache[hash]; ok {
-		return n
+		return n.node
 	}
 	return nil
 }
 
-func (h *HashIndex) del(hash common.Hash) {
+func (h *HashNodeCache) del(hash common.Hash) {
 	if h == nil {
 		return
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-	delete(h.cache, hash)
+	n, ok := h.cache[hash]
+	if !ok {
+		return
+	}
+	if n.refCount > 0 {
+		n.refCount--
+	}
+	if n.refCount == 0 {
+		delete(h.cache, hash)
+	}
 }
 
-func (h *HashIndex) Add(ly layer) {
+func (h *HashNodeCache) Add(ly layer) {
 	if h == nil {
 		return
 	}
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
 	}
+	beforeAdd := h.length()
 	for _, subset := range dl.nodes {
 		for _, node := range subset {
 			h.set(node.Hash, node)
 		}
 	}
 	diffHashCacheLengthGauge.Update(int64(h.length()))
-	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
+	log.Info("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd)
 }
 
-func (h *HashIndex) Remove(ly layer) {
+func (h *HashNodeCache) Remove(ly layer) {
 	if h == nil {
 		return
 	}
@@ -96,13 +115,14 @@ func (h *HashIndex) Remove(ly layer) {
 		return
 	}
 	go func() {
+		beforeDel := h.length()
 		for _, subset := range dl.nodes {
 			for _, node := range subset {
 				h.del(node.Hash)
 			}
 		}
 		diffHashCacheLengthGauge.Update(int64(h.length()))
-		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.length())
+		log.Info("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length())
 	}()
 }
 
@@ -119,7 +139,7 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
-	cache  *HashIndex                                // trienode cache by hash key.
+	cache  *HashNodeCache                            // trienode cache by hash key.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -139,13 +159,15 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		states: states,
 		parent: parent,
 	}
+
 	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
 		dl.cache = pdl.cache
 	} else {
-		dl.cache = &HashIndex{
-			cache: make(map[common.Hash]*trienode.Node),
+		dl.cache = &HashNodeCache{
+			cache: make(map[common.Hash]*RefTrieNode),
 		}
 	}
+
 	for _, subset := range nodes {
 		for path, n := range subset {
 			dl.memory += uint64(n.Size() + len(path))
@@ -233,13 +255,14 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	parent := dl.parent
 	for {
 		if disk, ok := parent.(*diskLayer); ok {
-			blob, err := disk.NodeByLogger(owner, path, hash, log.Debug)
+			blob, err := disk.Node(owner, path, hash)
 			if err != nil {
-				// This is a bad case with a very low probability. The same trienode exists
-				// in different difflayers and can be cleared from the map in advance. In
-				// this case, the 128-layer difflayer is queried again.
+				// This is a bad case with a very low probability.
+				// Reading the difflayer cache and reading the disklayer are not in the same lock,
+				// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
+				// In this case, fallback to the original 128-layer recursive difflayer query path.
 				diffHashCacheSlowPathMeter.Mark(1)
-				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
 				return dl.node(owner, path, hash, 0)
 			} else {
 				return blob, nil

triedb/pathdb/disklayer.go

@@ -150,9 +150,9 @@ func (dl *diskLayer) markStale() {
 	dl.stale = true
 }
 
-type loggerFunc func(string, ...interface{})
-
-func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Hash, logger loggerFunc) ([]byte, error) {
+// Node implements the layer interface, retrieving the trie node with the
+// provided node info. No error will be returned if the node is not found.
+func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
@@ -165,7 +165,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	// layer as stale.
 	n, err := dl.buffer.node(owner, path, hash)
 	if err != nil {
-		logger("Unexpected trie node in clean cache", "error", err)
+		log.Error("Unexpected trie node in clean cache", "error", err)
 		return nil, err
 	}
 	if n != nil {
@@ -189,7 +189,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 				return blob, nil
 			}
 			cleanFalseMeter.Mark(1)
-			logger("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
+			log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
 		}
 		cleanMissMeter.Mark(1)
 	}
@@ -205,7 +205,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	}
 	if nHash != hash {
 		diskFalseMeter.Mark(1)
-		logger("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
+		log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
 		return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
 	}
 	if dl.cleans != nil && len(nBlob) > 0 {
@@ -215,12 +215,6 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	return nBlob, nil
 }
 
-// Node implements the layer interface, retrieving the trie node with the
-// provided node info. No error will be returned if the node is not found.
-func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
-	return dl.NodeByLogger(owner, path, hash, log.Error)
-}
-
 // update implements the layer interface, returning a new diff layer on top
 // with the given state set.
 func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {

triedb/pathdb/layertree.go

@@ -51,9 +51,20 @@ func (tree *layerTree) reset(head layer) {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
+	for _, ly := range tree.layers {
+		if dl, ok := ly.(*diffLayer); ok {
+			// Clean up the hash cache of the child difflayer corresponding to difflayer.
+			dl.cache.Remove(dl)
+		}
+	}
+
 	var layers = make(map[common.Hash]layer)
 	for head != nil {
 		layers[head.rootHash()] = head
+		if dl, ok := head.(*diffLayer); ok {
+			// Add the hash cache of the child difflayer corresponding to difflayer.
+			dl.cache.Add(dl)
+		}
 		head = head.parentLayer()
 	}
 	tree.layers = layers
@@ -129,14 +140,32 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
+	defer func() {
+		log.Info("Succeed to cap", "root", root.String(), "layers", layers, "layer_tree_len", len(tree.layers))
+		for _, ly := range tree.layers {
+			if dl, ok := ly.(*diffLayer); ok {
+				log.Info("Print cache", "cache_length", dl.cache.length())
+				break
+			}
+		}
+	}()
+
 	// If full commit was requested, flatten the diffs and merge onto disk
 	if layers == 0 {
 		base, err := diff.persist(true)
 		if err != nil {
 			return err
 		}
+		for _, ly := range tree.layers {
+			if dl, ok := ly.(*diffLayer); ok {
+				// Clean up difflayer hash cache.
+				dl.cache.Remove(dl)
+				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String(), "diff_block_number", dl.block)
+			}
+		}
 		// Replace the entire layer tree with the flat base
 		tree.layers = map[common.Hash]layer{base.rootHash(): base}
+		log.Info("Cap all difflayers to disklayer", "disk_root", base.rootHash().String())
 		return nil
 	}
 	// Dive until we run out of layers or reach the persistent database
@@ -186,7 +215,9 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		if df, exist := tree.layers[root]; exist {
 			if dl, ok := df.(*diffLayer); ok {
 				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
+				// include re-org case.
 				dl.cache.Remove(dl)
+				log.Info("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
 		}
 		delete(tree.layers, root)
@@ -198,6 +229,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	for root, layer := range tree.layers {
 		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {
 			remove(root)
+			log.Info("Remove stale the disklayer", "disk_root", dl.root.String())
 		}
 	}
 	return nil