perf: speedup pbss trienode read

eth/backend.go

@@ -161,12 +161,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
-		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024, "adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
-		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024)
+		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
-	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
-
+	log.Info("Allocated memory caches",
+		"state_scheme", config.StateScheme,
+		"trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024,
+		"trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+		"snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)
 	// Try to recover offline state pruning only in hash-based.
 	if config.StateScheme == rawdb.HashScheme {
 		if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil {

triedb/pathdb/asyncnodebuffer.go

@@ -226,7 +226,8 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr
 	}
 	if n.Hash != hash {
 		dirtyFalseMeter.Mark(1)
-		log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+		// difflayer cache miss maybe hit this, which is normal case, and the caller can retry it.
+		log.Debug("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
 		return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
 	}
 	return n, nil

triedb/pathdb/difflayer.go

@@ -26,6 +26,86 @@ import (
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
+type HashIndex struct {
+	lock  sync.RWMutex
+	cache map[common.Hash]*trienode.Node
+}
+
+func (h *HashIndex) length() int {
+	if h == nil {
+		return 0
+	}
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+	return len(h.cache)
+}
+
+func (h *HashIndex) set(hash common.Hash, node *trienode.Node) {
+	if h == nil {
+		return
+	}
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	h.cache[hash] = node
+}
+
+func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
+	if h == nil {
+		return nil
+	}
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+	if n, ok := h.cache[hash]; ok {
+		return n
+	}
+	return nil
+}
+
+func (h *HashIndex) del(hash common.Hash) {
+	if h == nil {
+		return
+	}
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	delete(h.cache, hash)
+}
+
+func (h *HashIndex) Add(ly layer) {
+	if h == nil {
+		return
+	}
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	for _, subset := range dl.nodes {
+		for _, node := range subset {
+			h.set(node.Hash, node)
+		}
+	}
+	diffHashCacheLengthGauge.Update(int64(h.length()))
+	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
+}
+
+func (h *HashIndex) Remove(ly layer) {
+	if h == nil {
+		return
+	}
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	go func() {
+		for _, subset := range dl.nodes {
+			for _, node := range subset {
+				h.del(node.Hash)
+			}
+		}
+		diffHashCacheLengthGauge.Update(int64(h.length()))
+		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.length())
+	}()
+}
+
 // diffLayer represents a collection of modifications made to the in-memory tries
 // along with associated state changes after running a block on top.
 //
@@ -39,6 +119,7 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
+	cache  *HashIndex                                // trienode cache by hash key.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -58,6 +139,13 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		states: states,
 		parent: parent,
 	}
+	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
+		dl.cache = pdl.cache
+	} else {
+		dl.cache = &HashIndex{
+			cache: make(map[common.Hash]*trienode.Node),
+		}
+	}
 	for _, subset := range nodes {
 		for path, n := range subset {
 			dl.memory += uint64(n.Size() + len(path))
@@ -133,7 +221,32 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 // Node implements the layer interface, retrieving the trie node blob with the
 // provided node information. No error will be returned if the node is not found.
 func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
-	return dl.node(owner, path, hash, 0)
+	if n := dl.cache.Get(hash); n != nil {
+		// The query from the hash map is fastpath,
+		// avoiding recursive query of 128 difflayers.
+		diffHashCacheHitMeter.Mark(1)
+		diffHashCacheReadMeter.Mark(int64(len(n.Blob)))
+		return n.Blob, nil
+	}
+	diffHashCacheMissMeter.Mark(1)
+
+	parent := dl.parent
+	for {
+		if disk, ok := parent.(*diskLayer); ok {
+			blob, err := disk.NodeByLogger(owner, path, hash, log.Debug)
+			if err != nil {
+				// This is a bad case with a very low probability. The same trienode exists
+				// in different difflayers and can be cleared from the map in advance. In
+				// this case, the 128-layer difflayer is queried again.
+				diffHashCacheSlowPathMeter.Mark(1)
+				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+				return dl.node(owner, path, hash, 0)
+			} else {
+				return blob, nil
+			}
+		}
+		parent = parent.parentLayer()
+	}
 }
 
 // update implements the layer interface, creating a new layer on top of the

triedb/pathdb/disklayer.go

@@ -150,9 +150,9 @@ func (dl *diskLayer) markStale() {
 	dl.stale = true
 }
 
-// Node implements the layer interface, retrieving the trie node with the
-// provided node info. No error will be returned if the node is not found.
-func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
+type loggerFunc func(string, ...interface{})
+
+func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Hash, logger loggerFunc) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
@@ -165,6 +165,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	// layer as stale.
 	n, err := dl.buffer.node(owner, path, hash)
 	if err != nil {
+		logger("Unexpected trie node in clean cache", "error", err)
 		return nil, err
 	}
 	if n != nil {
@@ -188,7 +189,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 				return blob, nil
 			}
 			cleanFalseMeter.Mark(1)
-			log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
+			logger("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
 		}
 		cleanMissMeter.Mark(1)
 	}
@@ -204,7 +205,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	}
 	if nHash != hash {
 		diskFalseMeter.Mark(1)
-		log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
+		logger("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
 		return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
 	}
 	if dl.cleans != nil && len(nBlob) > 0 {
@@ -214,6 +215,12 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	return nBlob, nil
 }
 
+// Node implements the layer interface, retrieving the trie node with the
+// provided node info. No error will be returned if the node is not found.
+func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
+	return dl.NodeByLogger(owner, path, hash, log.Error)
+}
+
 // update implements the layer interface, returning a new diff layer on top
 // with the given state set.
 func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {
@@ -286,6 +293,9 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 		}
 		log.Debug("Pruned state history", "items", pruned, "tailid", oldest)
 	}
+
+	// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
+	bottom.cache.Remove(bottom)
 	return ndl, nil
 }
 

triedb/pathdb/layertree.go

@@ -104,6 +104,9 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	}
 	l := parent.update(root, parent.stateID()+1, block, nodes.Flatten(), states)
 
+	// Before adding layertree, update the hash cache.
+	l.cache.Add(l)
+
 	tree.lock.Lock()
 	tree.layers[l.rootHash()] = l
 	tree.lock.Unlock()
@@ -180,6 +183,12 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
+		if df, exist := tree.layers[root]; exist {
+			if dl, ok := df.(*diffLayer); ok {
+				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
+				dl.cache.Remove(dl)
+			}
+		}
 		delete(tree.layers, root)
 		for _, child := range children[root] {
 			remove(child)

triedb/pathdb/metrics.go

@@ -47,4 +47,10 @@ var (
 	historyBuildTimeMeter  = metrics.NewRegisteredTimer("pathdb/history/time", nil)
 	historyDataBytesMeter  = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
 	historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
+
+	diffHashCacheHitMeter      = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/hit", nil)
+	diffHashCacheReadMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil)
+	diffHashCacheMissMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil)
+	diffHashCacheSlowPathMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/slowpath", nil)
+	diffHashCacheLengthGauge   = metrics.NewRegisteredGauge("pathdb/difflayer/hashcache/size", nil)
 )