From 88d505edffae7b3082962b1783af6ef208aedd63 Mon Sep 17 00:00:00 2001
From: joeycli <joeycli0919@gmail.com>
Date: Tue, 4 Jun 2024 18:08:16 +0800
Subject: [PATCH 01/22] test: add hash index cache in difflayer

---
 triedb/pathdb/difflayer.go | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index ccd8d36764..9a0425cf09 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie/trienode"
@@ -40,6 +41,8 @@ type diffLayer struct {
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
 
+	cache *fastcache.Cache
+
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
 }
@@ -58,8 +61,14 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		states: states,
 		parent: parent,
 	}
+	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
+		dl.cache = pdl.cache
+	} else {
+		dl.cache = fastcache.New(1024 * 1024 * 1024)
+	}
 	for _, subset := range nodes {
 		for path, n := range subset {
+			dl.cache.Set(n.Hash[:], n.Blob)
 			dl.memory += uint64(n.Size() + len(path))
 			size += int64(len(n.Blob) + len(path))
 		}
@@ -133,7 +142,24 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 // Node implements the layer interface, retrieving the trie node blob with the
 // provided node information. No error will be returned if the node is not found.
 func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
-	return dl.node(owner, path, hash, 0)
+	if blob := dl.cache.Get(nil, hash[:]); blob != nil {
+		dirtyHitMeter.Mark(1)
+		dirtyReadMeter.Mark(int64(len(blob)))
+		return blob, nil
+	}
+
+	for {
+		parent := dl.parent
+		if disk, ok := parent.(*diskLayer); ok {
+			blob, err := disk.Node(owner, path, hash)
+			if err != nil {
+				return dl.node(owner, path, hash, 0)
+			} else {
+				return blob, nil
+			}
+		}
+	}
+	//return dl.node(owner, path, hash, 0)
 }
 
 // update implements the layer interface, creating a new layer on top of the

From a8f340dd9891346baa016d4e24c73b9259f41547 Mon Sep 17 00:00:00 2001
From: joeycli <joeycli0919@gmail.com>
Date: Wed, 5 Jun 2024 10:31:41 +0800
Subject: [PATCH 02/22] test: change lru cahce to map for hash index cahce

---
 triedb/pathdb/difflayer.go | 62 +++++++++++++++++++++++++++++++++-----
 triedb/pathdb/layertree.go |  6 ++++
 2 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 9a0425cf09..67f1580837 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -20,13 +20,55 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
+type HashIndex struct {
+	cache map[common.Hash]*trienode.Node
+	lock  sync.RWMutex
+}
+
+func (h *HashIndex) Set(hash common.Hash, node *trienode.Node) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	h.cache[hash] = node
+}
+
+func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
+	h.lock.RUnlock()
+	defer h.lock.RUnlock()
+
+	if n, ok := h.cache[hash]; ok {
+		return n
+	}
+	return nil
+}
+
+func (h *HashIndex) Del(hash common.Hash) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	delete(h.cache, hash)
+}
+
+func (h *HashIndex) Remove(ly layer) {
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	go func() {
+		for _, subset := range dl.nodes {
+			for _, node := range subset {
+				h.Del(node.Hash)
+			}
+		}
+	}()
+}
+
 // diffLayer represents a collection of modifications made to the in-memory tries
 // along with associated state changes after running a block on top.
 //
@@ -41,7 +83,7 @@ type diffLayer struct {
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
 
-	cache *fastcache.Cache
+	cache *HashIndex
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -64,11 +106,13 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
 		dl.cache = pdl.cache
 	} else {
-		dl.cache = fastcache.New(1024 * 1024 * 1024)
+		dl.cache = &HashIndex{
+			cache: make(map[common.Hash]*trienode.Node),
+		}
 	}
 	for _, subset := range nodes {
 		for path, n := range subset {
-			dl.cache.Set(n.Hash[:], n.Blob)
+			dl.cache.Set(n.Hash, n)
 			dl.memory += uint64(n.Size() + len(path))
 			size += int64(len(n.Blob) + len(path))
 		}
@@ -142,22 +186,24 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 // Node implements the layer interface, retrieving the trie node blob with the
 // provided node information. No error will be returned if the node is not found.
 func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
-	if blob := dl.cache.Get(nil, hash[:]); blob != nil {
+	if n := dl.cache.Get(hash); n != nil {
 		dirtyHitMeter.Mark(1)
-		dirtyReadMeter.Mark(int64(len(blob)))
-		return blob, nil
+		dirtyReadMeter.Mark(int64(len(n.Blob)))
+		return n.Blob, nil
 	}
 
+	parent := dl.parent
 	for {
-		parent := dl.parent
 		if disk, ok := parent.(*diskLayer); ok {
 			blob, err := disk.Node(owner, path, hash)
 			if err != nil {
+				log.Warn("hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash)
 				return dl.node(owner, path, hash, 0)
 			} else {
 				return blob, nil
 			}
 		}
+		parent = parent.parentLayer()
 	}
 	//return dl.node(owner, path, hash, 0)
 }
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index ed94d2e19e..fabc4ee3d8 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -180,6 +180,12 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
+		df, exit := tree.layers[root]
+		if exit {
+			if dl, ok := df.(*diffLayer); ok {
+				dl.cache.Remove(dl)
+			}
+		}
 		delete(tree.layers, root)
 		for _, child := range children[root] {
 			remove(child)

From 483225e9b09d7bcb87a6a140ac6efef867d44bc1 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 14:24:54 +0800
Subject: [PATCH 03/22] perf: polish pbss difflayer hash cache

---
 triedb/pathdb/difflayer.go | 34 +++++++++++++++++++++++++++++-----
 triedb/pathdb/disklayer.go |  5 +++++
 triedb/pathdb/layertree.go | 10 +++++++++-
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 67f1580837..b280aedc92 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -27,8 +27,15 @@ import (
 )
 
 type HashIndex struct {
-	cache map[common.Hash]*trienode.Node
 	lock  sync.RWMutex
+	cache map[common.Hash]*trienode.Node
+}
+
+func (h *HashIndex) Length() int {
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+
+	return len(h.cache)
 }
 
 func (h *HashIndex) Set(hash common.Hash, node *trienode.Node) {
@@ -39,7 +46,7 @@ func (h *HashIndex) Set(hash common.Hash, node *trienode.Node) {
 }
 
 func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
-	h.lock.RUnlock()
+	h.lock.RLock()
 	defer h.lock.RUnlock()
 
 	if n, ok := h.cache[hash]; ok {
@@ -55,6 +62,19 @@ func (h *HashIndex) Del(hash common.Hash) {
 	delete(h.cache, hash)
 }
 
+func (h *HashIndex) Add(ly layer) {
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	for _, subset := range dl.nodes {
+		for _, node := range subset {
+			h.Set(node.Hash, node)
+		}
+	}
+	log.Info("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.Length())
+}
+
 func (h *HashIndex) Remove(ly layer) {
 	dl, ok := ly.(*diffLayer)
 	if !ok {
@@ -66,6 +86,7 @@ func (h *HashIndex) Remove(ly layer) {
 				h.Del(node.Hash)
 			}
 		}
+		log.Info("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.Length())
 	}()
 }
 
@@ -112,7 +133,6 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 	}
 	for _, subset := range nodes {
 		for path, n := range subset {
-			dl.cache.Set(n.Hash, n)
 			dl.memory += uint64(n.Size() + len(path))
 			size += int64(len(n.Blob) + len(path))
 		}
@@ -187,6 +207,8 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 // provided node information. No error will be returned if the node is not found.
 func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
 	if n := dl.cache.Get(hash); n != nil {
+		// The query from the hash map is fastpath,
+		// avoiding recursive query of 128 difflayers.
 		dirtyHitMeter.Mark(1)
 		dirtyReadMeter.Mark(int64(len(n.Blob)))
 		return n.Blob, nil
@@ -197,7 +219,10 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 		if disk, ok := parent.(*diskLayer); ok {
 			blob, err := disk.Node(owner, path, hash)
 			if err != nil {
-				log.Warn("hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash)
+				// This is a bad case with a very low probability. The same trienode exists
+				// in different difflayers and can be cleared from the map in advance. In
+				// this case, the 128-layer difflayer is queried again.
+				log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
 				return dl.node(owner, path, hash, 0)
 			} else {
 				return blob, nil
@@ -205,7 +230,6 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 		}
 		parent = parent.parentLayer()
 	}
-	//return dl.node(owner, path, hash, 0)
 }
 
 // update implements the layer interface, creating a new layer on top of the
diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index f0f8dbe84b..6d632e1b84 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -286,7 +286,12 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 		}
 		log.Debug("Pruned state history", "items", pruned, "tailid", oldest)
 	}
+	if bottom.cache != nil {
+		// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
+		bottom.cache.Remove(bottom)
+	}
 	return ndl, nil
+
 }
 
 // revert applies the given state history and return a reverted disk layer.
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index fabc4ee3d8..d19c8c92eb 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -104,6 +104,11 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	}
 	l := parent.update(root, parent.stateID()+1, block, nodes.Flatten(), states)
 
+	if l.cache != nil {
+		// Before adding layertree, update the hash cache.
+		l.cache.Add(l)
+	}
+
 	tree.lock.Lock()
 	tree.layers[l.rootHash()] = l
 	tree.lock.Unlock()
@@ -183,7 +188,10 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		df, exit := tree.layers[root]
 		if exit {
 			if dl, ok := df.(*diffLayer); ok {
-				dl.cache.Remove(dl)
+				if dl.cache != nil {
+					// Clean up the hash cache of the child difflayer corresponding to the stale parent.
+					dl.cache.Remove(dl)
+				}
 			}
 		}
 		delete(tree.layers, root)

From 0fc07db9f0c3f4d236c1d3f485161d1be2206b51 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 14:41:19 +0800
Subject: [PATCH 04/22] chore: refine some code

---
 triedb/pathdb/difflayer.go | 34 ++++++++++++++++++++++++++--------
 triedb/pathdb/disklayer.go |  9 ++++-----
 triedb/pathdb/layertree.go | 15 +++++----------
 3 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index b280aedc92..1539208372 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -31,14 +31,20 @@ type HashIndex struct {
 	cache map[common.Hash]*trienode.Node
 }
 
-func (h *HashIndex) Length() int {
+func (h *HashIndex) length() int {
+	if h == nil {
+		return 0
+	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
 
 	return len(h.cache)
 }
 
-func (h *HashIndex) Set(hash common.Hash, node *trienode.Node) {
+func (h *HashIndex) set(hash common.Hash, node *trienode.Node) {
+	if h == nil {
+		return
+	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
 
@@ -46,6 +52,9 @@ func (h *HashIndex) Set(hash common.Hash, node *trienode.Node) {
 }
 
 func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
+	if h == nil {
+		return nil
+	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
 
@@ -55,7 +64,10 @@ func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
 	return nil
 }
 
-func (h *HashIndex) Del(hash common.Hash) {
+func (h *HashIndex) del(hash common.Hash) {
+	if h == nil {
+		return
+	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
 
@@ -63,19 +75,25 @@ func (h *HashIndex) Del(hash common.Hash) {
 }
 
 func (h *HashIndex) Add(ly layer) {
+	if h == nil {
+		return
+	}
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
 	}
 	for _, subset := range dl.nodes {
 		for _, node := range subset {
-			h.Set(node.Hash, node)
+			h.set(node.Hash, node)
 		}
 	}
-	log.Info("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.Length())
+	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
 }
 
 func (h *HashIndex) Remove(ly layer) {
+	if h == nil {
+		return
+	}
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
@@ -83,10 +101,10 @@ func (h *HashIndex) Remove(ly layer) {
 	go func() {
 		for _, subset := range dl.nodes {
 			for _, node := range subset {
-				h.Del(node.Hash)
+				h.del(node.Hash)
 			}
 		}
-		log.Info("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.Length())
+		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.length())
 	}()
 }
 
@@ -222,7 +240,7 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 				// This is a bad case with a very low probability. The same trienode exists
 				// in different difflayers and can be cleared from the map in advance. In
 				// this case, the 128-layer difflayer is queried again.
-				log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
 				return dl.node(owner, path, hash, 0)
 			} else {
 				return blob, nil
diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index 6d632e1b84..d6144f1010 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -286,12 +286,11 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 		}
 		log.Debug("Pruned state history", "items", pruned, "tailid", oldest)
 	}
-	if bottom.cache != nil {
-		// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
-		bottom.cache.Remove(bottom)
-	}
-	return ndl, nil
 
+	// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
+	bottom.cache.Remove(bottom)
+
+	return ndl, nil
 }
 
 // revert applies the given state history and return a reverted disk layer.
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index d19c8c92eb..4270f2594e 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -104,10 +104,8 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	}
 	l := parent.update(root, parent.stateID()+1, block, nodes.Flatten(), states)
 
-	if l.cache != nil {
-		// Before adding layertree, update the hash cache.
-		l.cache.Add(l)
-	}
+	// Before adding layertree, update the hash cache.
+	l.cache.Add(l)
 
 	tree.lock.Lock()
 	tree.layers[l.rootHash()] = l
@@ -185,13 +183,10 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
-		df, exit := tree.layers[root]
-		if exit {
+		if df, exist := tree.layers[root]; exist {
 			if dl, ok := df.(*diffLayer); ok {
-				if dl.cache != nil {
-					// Clean up the hash cache of the child difflayer corresponding to the stale parent.
-					dl.cache.Remove(dl)
-				}
+				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
+				dl.cache.Remove(dl)
 			}
 		}
 		delete(tree.layers, root)

From 923bc07c07bd64f50496b88f9a6b010765f14dda Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 14:43:07 +0800
Subject: [PATCH 05/22] perf: adjust pbss cleancache size

---
 eth/backend.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/eth/backend.go b/eth/backend.go
index 23136d719c..be62213b4e 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -161,12 +161,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
-		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024, "adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
-		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024)
+		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
-	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
-
+	log.Info("Allocated memory caches",
+		"state_schema", config.StateScheme,
+		"trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024,
+		"trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+		"snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)
 	// Try to recover offline state pruning only in hash-based.
 	if config.StateScheme == rawdb.HashScheme {
 		if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil {

From 9069b76396b30d833e15dd21c46ed0db5d2bd184 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 15:13:17 +0800
Subject: [PATCH 06/22] chore: add more metrics

---
 triedb/pathdb/difflayer.go | 11 +++++++----
 triedb/pathdb/disklayer.go |  1 -
 triedb/pathdb/metrics.go   |  5 +++++
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 1539208372..7c660ff43c 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -78,6 +78,7 @@ func (h *HashIndex) Add(ly layer) {
 	if h == nil {
 		return
 	}
+
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
@@ -94,6 +95,7 @@ func (h *HashIndex) Remove(ly layer) {
 	if h == nil {
 		return
 	}
+	
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
@@ -121,8 +123,7 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
-
-	cache *HashIndex
+	cache  *HashIndex                                // trienode cache by hash key.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -227,10 +228,11 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	if n := dl.cache.Get(hash); n != nil {
 		// The query from the hash map is fastpath,
 		// avoiding recursive query of 128 difflayers.
-		dirtyHitMeter.Mark(1)
-		dirtyReadMeter.Mark(int64(len(n.Blob)))
+		diffHashCacheHitMeter.Mark(1)
+		diffHashCacheReadMeter.Mark(int64(len(n.Blob)))
 		return n.Blob, nil
 	}
+	diffHashCacheMissMeter.Mark(1)
 
 	parent := dl.parent
 	for {
@@ -240,6 +242,7 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 				// This is a bad case with a very low probability. The same trienode exists
 				// in different difflayers and can be cleared from the map in advance. In
 				// this case, the 128-layer difflayer is queried again.
+				diffHashCacheSlowPathMeter.Mark(1)
 				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
 				return dl.node(owner, path, hash, 0)
 			} else {
diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index d6144f1010..d14b29e189 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -289,7 +289,6 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 
 	// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
 	bottom.cache.Remove(bottom)
-
 	return ndl, nil
 }
 
diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go
index 9e2b1dcbf5..0d716800e3 100644
--- a/triedb/pathdb/metrics.go
+++ b/triedb/pathdb/metrics.go
@@ -47,4 +47,9 @@ var (
 	historyBuildTimeMeter  = metrics.NewRegisteredTimer("pathdb/history/time", nil)
 	historyDataBytesMeter  = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
 	historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
+
+	diffHashCacheHitMeter      = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/hit", nil)
+	diffHashCacheReadMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil)
+	diffHashCacheMissMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil)
+	diffHashCacheSlowPathMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/slowpath", nil)
 )

From 4a9380326076264b0b468b0525fc662b09958437 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 15:45:57 +0800
Subject: [PATCH 07/22] chore: refine log print

---
 triedb/pathdb/asyncnodebuffer.go |  3 ++-
 triedb/pathdb/difflayer.go       |  8 +-------
 triedb/pathdb/disklayer.go       | 17 ++++++++++++-----
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go
index a94fab4461..a864582a49 100644
--- a/triedb/pathdb/asyncnodebuffer.go
+++ b/triedb/pathdb/asyncnodebuffer.go
@@ -226,7 +226,8 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr
 	}
 	if n.Hash != hash {
 		dirtyFalseMeter.Mark(1)
-		log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+		// difflayer cache miss maybe hit this, which is normal case, and the caller can retry it.
+		log.Debug("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
 		return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
 	}
 	return n, nil
diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 7c660ff43c..a5b533944c 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -37,7 +37,6 @@ func (h *HashIndex) length() int {
 	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
-
 	return len(h.cache)
 }
 
@@ -47,7 +46,6 @@ func (h *HashIndex) set(hash common.Hash, node *trienode.Node) {
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-
 	h.cache[hash] = node
 }
 
@@ -57,7 +55,6 @@ func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
 	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
-
 	if n, ok := h.cache[hash]; ok {
 		return n
 	}
@@ -70,7 +67,6 @@ func (h *HashIndex) del(hash common.Hash) {
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-
 	delete(h.cache, hash)
 }
 
@@ -78,7 +74,6 @@ func (h *HashIndex) Add(ly layer) {
 	if h == nil {
 		return
 	}
-
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
@@ -95,7 +90,6 @@ func (h *HashIndex) Remove(ly layer) {
 	if h == nil {
 		return
 	}
-	
 	dl, ok := ly.(*diffLayer)
 	if !ok {
 		return
@@ -237,7 +231,7 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	parent := dl.parent
 	for {
 		if disk, ok := parent.(*diskLayer); ok {
-			blob, err := disk.Node(owner, path, hash)
+			blob, err := disk.NodeByLogger(owner, path, hash, log.Debug)
 			if err != nil {
 				// This is a bad case with a very low probability. The same trienode exists
 				// in different difflayers and can be cleared from the map in advance. In
diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index d14b29e189..67e852267f 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -150,9 +150,9 @@ func (dl *diskLayer) markStale() {
 	dl.stale = true
 }
 
-// Node implements the layer interface, retrieving the trie node with the
-// provided node info. No error will be returned if the node is not found.
-func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
+type loggerFunc func(string, ...interface{})
+
+func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Hash, logger loggerFunc) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
@@ -165,6 +165,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	// layer as stale.
 	n, err := dl.buffer.node(owner, path, hash)
 	if err != nil {
+		logger("Unexpected trie node in clean cache", "error", err)
 		return nil, err
 	}
 	if n != nil {
@@ -188,7 +189,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 				return blob, nil
 			}
 			cleanFalseMeter.Mark(1)
-			log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
+			logger("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
 		}
 		cleanMissMeter.Mark(1)
 	}
@@ -204,7 +205,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	}
 	if nHash != hash {
 		diskFalseMeter.Mark(1)
-		log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
+		logger("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
 		return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
 	}
 	if dl.cleans != nil && len(nBlob) > 0 {
@@ -214,6 +215,12 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	return nBlob, nil
 }
 
+// Node implements the layer interface, retrieving the trie node with the
+// provided node info. No error will be returned if the node is not found.
+func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
+	return dl.NodeByLogger(owner, path, hash, log.Error)
+}
+
 // update implements the layer interface, returning a new diff layer on top
 // with the given state set.
 func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {

From 99e9b7323d34274596c3e53d79cb0f961675a1a9 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Thu, 6 Jun 2024 15:58:11 +0800
Subject: [PATCH 08/22] chore: add some metrics

---
 triedb/pathdb/difflayer.go | 2 ++
 triedb/pathdb/metrics.go   | 1 +
 2 files changed, 3 insertions(+)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index a5b533944c..f6b604b509 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -83,6 +83,7 @@ func (h *HashIndex) Add(ly layer) {
 			h.set(node.Hash, node)
 		}
 	}
+	diffHashCacheLengthGauge.Update(int64(h.length()))
 	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
 }
 
@@ -100,6 +101,7 @@ func (h *HashIndex) Remove(ly layer) {
 				h.del(node.Hash)
 			}
 		}
+		diffHashCacheLengthGauge.Update(int64(h.length()))
 		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.length())
 	}()
 }
diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go
index 0d716800e3..67267b666a 100644
--- a/triedb/pathdb/metrics.go
+++ b/triedb/pathdb/metrics.go
@@ -52,4 +52,5 @@ var (
 	diffHashCacheReadMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil)
 	diffHashCacheMissMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil)
 	diffHashCacheSlowPathMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/slowpath", nil)
+	diffHashCacheLengthGauge   = metrics.NewRegisteredGauge("pathdb/difflayer/hashcache/size", nil)
 )

From aec5cfcb0ea538318843df99ce72c02377ad5442 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 09:34:57 +0800
Subject: [PATCH 09/22] chore: fix typo

---
 eth/backend.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eth/backend.go b/eth/backend.go
index be62213b4e..b3e90654a2 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -165,7 +165,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
 	log.Info("Allocated memory caches",
-		"state_schema", config.StateScheme,
+		"state_scheme", config.StateScheme,
 		"trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024,
 		"trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024,
 		"snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)

From 6cb2322b4e412205341a0762d71becf9968c5fb7 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 13:57:26 +0800
Subject: [PATCH 10/22] chore: updated by review tips

---
 eth/backend.go                   |  4 ++
 triedb/pathdb/asyncnodebuffer.go |  3 +-
 triedb/pathdb/difflayer.go       | 91 +++++++++++++++++++++-----------
 triedb/pathdb/disklayer.go       | 18 +++----
 triedb/pathdb/layertree.go       | 12 +++++
 5 files changed, 82 insertions(+), 46 deletions(-)

diff --git a/eth/backend.go b/eth/backend.go
index b3e90654a2..27f8fbbad2 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -161,6 +161,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
+		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+			"adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
+		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024,
+			"adjusted", common.StorageSize(config.TrieCleanCache+config.TrieDirtyCache-pathdb.MaxDirtyBufferSize/1024/1024)*1024*1024)
 		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go
index a864582a49..a94fab4461 100644
--- a/triedb/pathdb/asyncnodebuffer.go
+++ b/triedb/pathdb/asyncnodebuffer.go
@@ -226,8 +226,7 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr
 	}
 	if n.Hash != hash {
 		dirtyFalseMeter.Mark(1)
-		// difflayer cache miss maybe hit this, which is normal case, and the caller can retry it.
-		log.Debug("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+		log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
 		return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
 	}
 	return n, nil
diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index f6b604b509..0589a0a33d 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -26,12 +26,17 @@ import (
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
-type HashIndex struct {
+type RefTrieNode struct {
+	refCount uint32
+	node     *trienode.Node
+}
+
+type HashNodeCache struct {
 	lock  sync.RWMutex
-	cache map[common.Hash]*trienode.Node
+	cache map[common.Hash]*RefTrieNode
 }
 
-func (h *HashIndex) length() int {
+func (h *HashNodeCache) length() int {
 	if h == nil {
 		return 0
 	}
@@ -40,37 +45,50 @@ func (h *HashIndex) length() int {
 	return len(h.cache)
 }
 
-func (h *HashIndex) set(hash common.Hash, node *trienode.Node) {
+func (h *HashNodeCache) set(hash common.Hash, node *trienode.Node) {
 	if h == nil {
 		return
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-	h.cache[hash] = node
+	if n, ok := h.cache[hash]; ok {
+		n.refCount++
+	} else {
+		h.cache[hash] = &RefTrieNode{1, node}
+	}
 }
 
-func (h *HashIndex) Get(hash common.Hash) *trienode.Node {
+func (h *HashNodeCache) Get(hash common.Hash) *trienode.Node {
 	if h == nil {
 		return nil
 	}
 	h.lock.RLock()
 	defer h.lock.RUnlock()
 	if n, ok := h.cache[hash]; ok {
-		return n
+		return n.node
 	}
 	return nil
 }
 
-func (h *HashIndex) del(hash common.Hash) {
+func (h *HashNodeCache) del(hash common.Hash) {
 	if h == nil {
 		return
 	}
 	h.lock.Lock()
 	defer h.lock.Unlock()
-	delete(h.cache, hash)
+	n, ok := h.cache[hash]
+	if !ok {
+		return
+	}
+	if n.refCount > 0 {
+		n.refCount--
+	}
+	if n.refCount == 0 {
+		delete(h.cache, hash)
+	}
 }
 
-func (h *HashIndex) Add(ly layer) {
+func (h *HashNodeCache) Add(ly layer) {
 	if h == nil {
 		return
 	}
@@ -87,7 +105,7 @@ func (h *HashIndex) Add(ly layer) {
 	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
 }
 
-func (h *HashIndex) Remove(ly layer) {
+func (h *HashNodeCache) Remove(ly layer) {
 	if h == nil {
 		return
 	}
@@ -119,7 +137,8 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
-	cache  *HashIndex                                // trienode cache by hash key.
+	origin *diskLayer                                // The disklayer corresponding to the current difflayer, initialized when created and never modified.
+	cache  *HashNodeCache                            // trienode cache by hash key.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -139,13 +158,22 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		states: states,
 		parent: parent,
 	}
-	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
-		dl.cache = pdl.cache
-	} else {
-		dl.cache = &HashIndex{
-			cache: make(map[common.Hash]*trienode.Node),
+
+	switch l := parent.(type) {
+	case *diskLayer:
+		dl.origin = l
+	case *diffLayer:
+		dl.origin = l.origin
+		dl.cache = l.cache
+	default:
+		panic("unknown parent type")
+	}
+	if dl.cache == nil {
+		dl.cache = &HashNodeCache{
+			cache: make(map[common.Hash]*RefTrieNode),
 		}
 	}
+
 	for _, subset := range nodes {
 		for path, n := range subset {
 			dl.memory += uint64(n.Size() + len(path))
@@ -230,23 +258,22 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	}
 	diffHashCacheMissMeter.Mark(1)
 
-	parent := dl.parent
-	for {
-		if disk, ok := parent.(*diskLayer); ok {
-			blob, err := disk.NodeByLogger(owner, path, hash, log.Debug)
-			if err != nil {
-				// This is a bad case with a very low probability. The same trienode exists
-				// in different difflayers and can be cleared from the map in advance. In
-				// this case, the 128-layer difflayer is queried again.
-				diffHashCacheSlowPathMeter.Mark(1)
-				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
-				return dl.node(owner, path, hash, 0)
-			} else {
-				return blob, nil
-			}
+	if dl.origin != nil {
+		blob, err := dl.origin.Node(owner, path, hash)
+		if err != nil {
+			// This is a bad case with a very low probability.
+			// Reading the difflayer cache and reading the disklayer are not in the same lock,
+			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
+			// In this case, fallback to the original 128-layer recursive difflayer query path.
+			diffHashCacheSlowPathMeter.Mark(1)
+			log.Warn("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+			return dl.node(owner, path, hash, 0)
+		} else {
+			// skip difflayer query, which is fastpath.
+			return blob, nil
 		}
-		parent = parent.parentLayer()
 	}
+	return dl.node(owner, path, hash, 0)
 }
 
 // update implements the layer interface, creating a new layer on top of the
diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index 67e852267f..46d67011e1 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -150,9 +150,9 @@ func (dl *diskLayer) markStale() {
 	dl.stale = true
 }
 
-type loggerFunc func(string, ...interface{})
-
-func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Hash, logger loggerFunc) ([]byte, error) {
+// Node implements the layer interface, retrieving the trie node with the
+// provided node info. No error will be returned if the node is not found.
+func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
@@ -165,7 +165,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	// layer as stale.
 	n, err := dl.buffer.node(owner, path, hash)
 	if err != nil {
-		logger("Unexpected trie node in clean cache", "error", err)
+		log.Error("Unexpected trie node in clean cache", "error", err)
 		return nil, err
 	}
 	if n != nil {
@@ -189,7 +189,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 				return blob, nil
 			}
 			cleanFalseMeter.Mark(1)
-			logger("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
+			log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
 		}
 		cleanMissMeter.Mark(1)
 	}
@@ -205,7 +205,7 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	}
 	if nHash != hash {
 		diskFalseMeter.Mark(1)
-		logger("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
+		log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
 		return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
 	}
 	if dl.cleans != nil && len(nBlob) > 0 {
@@ -215,12 +215,6 @@ func (dl *diskLayer) NodeByLogger(owner common.Hash, path []byte, hash common.Ha
 	return nBlob, nil
 }
 
-// Node implements the layer interface, retrieving the trie node with the
-// provided node info. No error will be returned if the node is not found.
-func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
-	return dl.NodeByLogger(owner, path, hash, log.Error)
-}
-
 // update implements the layer interface, returning a new diff layer on top
 // with the given state set.
 func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 4270f2594e..3a16bea584 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -51,9 +51,20 @@ func (tree *layerTree) reset(head layer) {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
+	for _, ly := range tree.layers {
+		if dl, ok := ly.(*diffLayer); ok {
+			// Clean up the hash cache of the child difflayer corresponding to difflayer.
+			dl.cache.Remove(dl)
+		}
+	}
+
 	var layers = make(map[common.Hash]layer)
 	for head != nil {
 		layers[head.rootHash()] = head
+		if dl, ok := head.(*diffLayer); ok {
+			// Add the hash cache of the child difflayer corresponding to difflayer.
+			dl.cache.Add(dl)
+		}
 		head = head.parentLayer()
 	}
 	tree.layers = layers
@@ -186,6 +197,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		if df, exist := tree.layers[root]; exist {
 			if dl, ok := df.(*diffLayer); ok {
 				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
+				// include re-org case.
 				dl.cache.Remove(dl)
 			}
 		}

From 5d5b309405afe1e0992976203310113e6b03b4ff Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 14:36:21 +0800
Subject: [PATCH 11/22] chore: refine log/metrics

---
 triedb/pathdb/difflayer.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 0589a0a33d..a0f7bba5cd 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -266,13 +266,14 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
 			// In this case, fallback to the original 128-layer recursive difflayer query path.
 			diffHashCacheSlowPathMeter.Mark(1)
-			log.Warn("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+			log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
 			return dl.node(owner, path, hash, 0)
 		} else {
 			// skip difflayer query, which is fastpath.
 			return blob, nil
 		}
 	}
+	diffHashCacheSlowPathMeter.Mark(1)
 	return dl.node(owner, path, hash, 0)
 }
 

From e2b366fe02217c760de381dd89b0b472c34438ee Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 15:41:39 +0800
Subject: [PATCH 12/22] fix: fix disklayer pointer

---
 triedb/pathdb/difflayer.go | 45 ++++++++++++++++----------------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index a0f7bba5cd..f23b5544d1 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -137,7 +137,6 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
-	origin *diskLayer                                // The disklayer corresponding to the current difflayer, initialized when created and never modified.
 	cache  *HashNodeCache                            // trienode cache by hash key.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
@@ -159,16 +158,9 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		parent: parent,
 	}
 
-	switch l := parent.(type) {
-	case *diskLayer:
-		dl.origin = l
-	case *diffLayer:
-		dl.origin = l.origin
-		dl.cache = l.cache
-	default:
-		panic("unknown parent type")
-	}
-	if dl.cache == nil {
+	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
+		dl.cache = pdl.cache
+	} else {
 		dl.cache = &HashNodeCache{
 			cache: make(map[common.Hash]*RefTrieNode),
 		}
@@ -258,23 +250,24 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	}
 	diffHashCacheMissMeter.Mark(1)
 
-	if dl.origin != nil {
-		blob, err := dl.origin.Node(owner, path, hash)
-		if err != nil {
-			// This is a bad case with a very low probability.
-			// Reading the difflayer cache and reading the disklayer are not in the same lock,
-			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
-			// In this case, fallback to the original 128-layer recursive difflayer query path.
-			diffHashCacheSlowPathMeter.Mark(1)
-			log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
-			return dl.node(owner, path, hash, 0)
-		} else {
-			// skip difflayer query, which is fastpath.
-			return blob, nil
+	parent := dl.parent
+	for {
+		if disk, ok := parent.(*diskLayer); ok {
+			blob, err := disk.Node(owner, path, hash)
+			if err != nil {
+				// This is a bad case with a very low probability.
+				// Reading the difflayer cache and reading the disklayer are not in the same lock,
+				// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
+				// In this case, fallback to the original 128-layer recursive difflayer query path.
+				diffHashCacheSlowPathMeter.Mark(1)
+				log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+				return dl.node(owner, path, hash, 0)
+			} else {
+				return blob, nil
+			}
 		}
+		parent = parent.parentLayer()
 	}
-	diffHashCacheSlowPathMeter.Mark(1)
-	return dl.node(owner, path, hash, 0)
 }
 
 // update implements the layer interface, creating a new layer on top of the

From 0d229af99ee37f2da3806c04e83ee5f865e8232d Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 15:44:09 +0800
Subject: [PATCH 13/22] chore: refine logs

---
 triedb/pathdb/difflayer.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index f23b5544d1..763e25822b 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -260,7 +260,7 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 				// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
 				// In this case, fallback to the original 128-layer recursive difflayer query path.
 				diffHashCacheSlowPathMeter.Mark(1)
-				log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String())
+				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
 				return dl.node(owner, path, hash, 0)
 			} else {
 				return blob, nil

From a38c161bc1851eb25ccbec4d28adb11cbe172112 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 20:30:03 +0800
Subject: [PATCH 14/22] fix: refine cache cleanup workflow

---
 triedb/pathdb/layertree.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 3a16bea584..021eb9ded1 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -146,8 +146,16 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		if err != nil {
 			return err
 		}
+		for _, ly := range tree.layers {
+			if dl, ok := ly.(*diffLayer); ok {
+				// Clean up difflayer hash cache.
+				dl.cache.Remove(dl)
+				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String())
+			}
+		}
 		// Replace the entire layer tree with the flat base
 		tree.layers = map[common.Hash]layer{base.rootHash(): base}
+		log.Info("Cap all difflayers to disklayer", "disk_root", base.rootHash().String())
 		return nil
 	}
 	// Dive until we run out of layers or reach the persistent database
@@ -199,6 +207,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
 				// include re-org case.
 				dl.cache.Remove(dl)
+				log.Info("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String())
 			}
 		}
 		delete(tree.layers, root)
@@ -206,6 +215,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			remove(child)
 		}
 		delete(children, root)
+		log.Info("Remove stale child layer due to reorg", "disk_root", root.String())
 	}
 	for root, layer := range tree.layers {
 		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {

From 958b3bfceeca80b56a2b18d79ac25776d7460c45 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Tue, 11 Jun 2024 21:09:26 +0800
Subject: [PATCH 15/22] chore: refine some log

---
 triedb/pathdb/difflayer.go |  6 ++++--
 triedb/pathdb/layertree.go | 16 +++++++++++++---
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 763e25822b..50b95448b6 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -96,13 +96,14 @@ func (h *HashNodeCache) Add(ly layer) {
 	if !ok {
 		return
 	}
+	beforeAdd := h.length()
 	for _, subset := range dl.nodes {
 		for _, node := range subset {
 			h.set(node.Hash, node)
 		}
 	}
 	diffHashCacheLengthGauge.Update(int64(h.length()))
-	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "map_len", h.length())
+	log.Info("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd)
 }
 
 func (h *HashNodeCache) Remove(ly layer) {
@@ -114,13 +115,14 @@ func (h *HashNodeCache) Remove(ly layer) {
 		return
 	}
 	go func() {
+		beforeDel := h.length()
 		for _, subset := range dl.nodes {
 			for _, node := range subset {
 				h.del(node.Hash)
 			}
 		}
 		diffHashCacheLengthGauge.Update(int64(h.length()))
-		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "map_len", h.length())
+		log.Info("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length())
 	}()
 }
 
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 021eb9ded1..7ff369bd1d 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -140,6 +140,16 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
+	defer func() {
+		log.Info("Succeed to cap", "root", root.String(), "layers", layers, "layer_tree_len", len(tree.layers))
+		for _, ly := range tree.layers {
+			if dl, ok := ly.(*diffLayer); ok {
+				log.Info("Print cache", "cache_length", dl.cache.length())
+				break
+			}
+		}
+	}()
+
 	// If full commit was requested, flatten the diffs and merge onto disk
 	if layers == 0 {
 		base, err := diff.persist(true)
@@ -150,7 +160,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			if dl, ok := ly.(*diffLayer); ok {
 				// Clean up difflayer hash cache.
 				dl.cache.Remove(dl)
-				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String())
+				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
 		}
 		// Replace the entire layer tree with the flat base
@@ -207,7 +217,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
 				// include re-org case.
 				dl.cache.Remove(dl)
-				log.Info("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String())
+				log.Info("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
 		}
 		delete(tree.layers, root)
@@ -215,11 +225,11 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			remove(child)
 		}
 		delete(children, root)
-		log.Info("Remove stale child layer due to reorg", "disk_root", root.String())
 	}
 	for root, layer := range tree.layers {
 		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {
 			remove(root)
+			log.Info("Remove stale the disklayer", "disk_root", dl.root.String())
 		}
 	}
 	return nil

From 25e0c9a50c56eddb4376d2e1d8f7eb09af06a6af Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 09:15:03 +0800
Subject: [PATCH 16/22] chore: check repeated difflayer

---
 triedb/pathdb/difflayer.go |  4 ++--
 triedb/pathdb/layertree.go | 22 ++++++++--------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 50b95448b6..75bbaec2b6 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -103,7 +103,7 @@ func (h *HashNodeCache) Add(ly layer) {
 		}
 	}
 	diffHashCacheLengthGauge.Update(int64(h.length()))
-	log.Info("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd)
+	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd)
 }
 
 func (h *HashNodeCache) Remove(ly layer) {
@@ -122,7 +122,7 @@ func (h *HashNodeCache) Remove(ly layer) {
 			}
 		}
 		diffHashCacheLengthGauge.Update(int64(h.length()))
-		log.Info("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length())
+		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length())
 	}()
 }
 
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 7ff369bd1d..636f5984dd 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -53,7 +53,7 @@ func (tree *layerTree) reset(head layer) {
 
 	for _, ly := range tree.layers {
 		if dl, ok := ly.(*diffLayer); ok {
-			// Clean up the hash cache of the child difflayer corresponding to difflayer.
+			// Clean up the hash cache of difflayers due to reset.
 			dl.cache.Remove(dl)
 		}
 	}
@@ -62,7 +62,7 @@ func (tree *layerTree) reset(head layer) {
 	for head != nil {
 		layers[head.rootHash()] = head
 		if dl, ok := head.(*diffLayer); ok {
-			// Add the hash cache of the child difflayer corresponding to difflayer.
+			// Add the hash cache of difflayers due to reset.
 			dl.cache.Add(dl)
 		}
 		head = head.parentLayer()
@@ -109,6 +109,10 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	if root == parentRoot {
 		return errors.New("layer cycle")
 	}
+	if tree.get(root) != nil {
+		log.Info("Skip add repeated difflayer", "root", root.String(), "block_id", block)
+		return nil
+	}
 	parent := tree.get(parentRoot)
 	if parent == nil {
 		return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot)
@@ -140,16 +144,6 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
-	defer func() {
-		log.Info("Succeed to cap", "root", root.String(), "layers", layers, "layer_tree_len", len(tree.layers))
-		for _, ly := range tree.layers {
-			if dl, ok := ly.(*diffLayer); ok {
-				log.Info("Print cache", "cache_length", dl.cache.length())
-				break
-			}
-		}
-	}()
-
 	// If full commit was requested, flatten the diffs and merge onto disk
 	if layers == 0 {
 		base, err := diff.persist(true)
@@ -158,7 +152,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		}
 		for _, ly := range tree.layers {
 			if dl, ok := ly.(*diffLayer); ok {
-				// Clean up difflayer hash cache.
+				// Clean up difflayer hash cache due to cap all.
 				dl.cache.Remove(dl)
 				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
@@ -229,7 +223,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	for root, layer := range tree.layers {
 		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {
 			remove(root)
-			log.Info("Remove stale the disklayer", "disk_root", dl.root.String())
+			log.Debug("Remove stale the disklayer", "disk_root", dl.root.String())
 		}
 	}
 	return nil

From 00e8f9d9c63334fbc61c307d999e5175bfaba2d3 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 10:54:15 +0800
Subject: [PATCH 17/22] chore: add origin disklayer to difflayer

---
 triedb/pathdb/difflayer.go | 56 ++++++++++++++++++++++++--------------
 triedb/pathdb/layertree.go | 21 ++++++++++----
 2 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 75bbaec2b6..f70de494d8 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -139,7 +139,10 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
-	cache  *HashNodeCache                            // trienode cache by hash key.
+
+	// mutables
+	cache  *HashNodeCache // trienode cache by hash key.
+	origin *diskLayer     // The current difflayer corresponds to the underlying disklayer and is updated during cap.
 
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
@@ -160,9 +163,17 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		parent: parent,
 	}
 
-	if pdl, ok := parent.(*diffLayer); ok && pdl.cache != nil {
-		dl.cache = pdl.cache
-	} else {
+	switch l := parent.(type) {
+	case *diskLayer:
+		dl.origin = l
+	case *diffLayer:
+		dl.origin = l.origin
+		dl.cache = l.cache
+	default:
+		panic("unknown parent type")
+	}
+
+	if dl.cache == nil {
 		dl.cache = &HashNodeCache{
 			cache: make(map[common.Hash]*RefTrieNode),
 		}
@@ -185,6 +196,12 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 	return dl
 }
 
+func (dl *diffLayer) originDiskLayer() *diskLayer {
+	dl.lock.RLock()
+	defer dl.lock.RUnlock()
+	return dl.origin
+}
+
 // rootHash implements the layer interface, returning the root hash of
 // corresponding state.
 func (dl *diffLayer) rootHash() common.Hash {
@@ -252,24 +269,23 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	}
 	diffHashCacheMissMeter.Mark(1)
 
-	parent := dl.parent
-	for {
-		if disk, ok := parent.(*diskLayer); ok {
-			blob, err := disk.Node(owner, path, hash)
-			if err != nil {
-				// This is a bad case with a very low probability.
-				// Reading the difflayer cache and reading the disklayer are not in the same lock,
-				// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
-				// In this case, fallback to the original 128-layer recursive difflayer query path.
-				diffHashCacheSlowPathMeter.Mark(1)
-				log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
-				return dl.node(owner, path, hash, 0)
-			} else {
-				return blob, nil
-			}
+	persistLayer := dl.originDiskLayer()
+	if persistLayer != nil {
+		blob, err := persistLayer.Node(owner, path, hash)
+		if err != nil {
+			// This is a bad case with a very low probability.
+			// Reading the difflayer cache and reading the disklayer are not in the same lock,
+			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
+			// In this case, fallback to the original 128-layer recursive difflayer query path.
+			diffHashCacheSlowPathMeter.Mark(1)
+			log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
+			return dl.node(owner, path, hash, 0)
+		} else {
+			return blob, nil
 		}
-		parent = parent.parentLayer()
 	}
+	diffHashCacheSlowPathMeter.Mark(1)
+	return dl.node(owner, path, hash, 0)
 }
 
 // update implements the layer interface, creating a new layer on top of the
diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 636f5984dd..32bfc3b386 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -152,14 +152,13 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		}
 		for _, ly := range tree.layers {
 			if dl, ok := ly.(*diffLayer); ok {
-				// Clean up difflayer hash cache due to cap all.
 				dl.cache.Remove(dl)
-				log.Info("Cleanup difflayer hash cache", "diff_root", dl.root.String(), "diff_block_number", dl.block)
+				log.Debug("Cleanup difflayer hash cache due to cap all", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
 		}
 		// Replace the entire layer tree with the flat base
 		tree.layers = map[common.Hash]layer{base.rootHash(): base}
-		log.Info("Cap all difflayers to disklayer", "disk_root", base.rootHash().String())
+		log.Debug("Cap all difflayers to disklayer", "disk_root", base.rootHash().String())
 		return nil
 	}
 	// Dive until we run out of layers or reach the persistent database
@@ -172,6 +171,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			return nil
 		}
 	}
+	var persisted *diskLayer
 	// We're out of layers, flatten anything below, stopping if it's the disk or if
 	// the memory limit is not yet exceeded.
 	switch parent := diff.parentLayer().(type) {
@@ -192,6 +192,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		diff.parent = base
 
 		diff.lock.Unlock()
+		persisted = base.(*diskLayer)
 
 	default:
 		panic(fmt.Sprintf("unknown data layer in triedb: %T", parent))
@@ -208,10 +209,9 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	remove = func(root common.Hash) {
 		if df, exist := tree.layers[root]; exist {
 			if dl, ok := df.(*diffLayer); ok {
-				// Clean up the hash cache of the child difflayer corresponding to the stale parent.
-				// include re-org case.
+				// Clean up the hash cache of the child difflayer corresponding to the stale parent, include the re-org case.
 				dl.cache.Remove(dl)
-				log.Info("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block)
+				log.Debug("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block)
 			}
 		}
 		delete(tree.layers, root)
@@ -226,6 +226,15 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			log.Debug("Remove stale the disklayer", "disk_root", dl.root.String())
 		}
 	}
+
+	if persisted != nil {
+		if diff, ok := tree.layers[root].(*diffLayer); ok {
+			diff.lock.Lock()
+			diff.origin = persisted
+			diff.lock.Unlock()
+		}
+	}
+
 	return nil
 }
 

From 8e6e6811718b9a27750cefbc870f234df8d8b326 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 10:57:27 +0800
Subject: [PATCH 18/22] chore: refine some code

---
 triedb/pathdb/difflayer.go | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index f70de494d8..ed2f37cbd3 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -139,11 +139,10 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
+	cache  *HashNodeCache                            // trienode cache by hash key. cache is immutable, but cache's item can be add/del.
 
 	// mutables
-	cache  *HashNodeCache // trienode cache by hash key.
-	origin *diskLayer     // The current difflayer corresponds to the underlying disklayer and is updated during cap.
-
+	origin *diskLayer   // The current difflayer corresponds to the underlying disklayer and is updated during cap.
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
 }
@@ -166,19 +165,16 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 	switch l := parent.(type) {
 	case *diskLayer:
 		dl.origin = l
+		dl.cache = &HashNodeCache{
+			cache: make(map[common.Hash]*RefTrieNode),
+		}
 	case *diffLayer:
-		dl.origin = l.origin
+		dl.origin = l.originDiskLayer()
 		dl.cache = l.cache
 	default:
 		panic("unknown parent type")
 	}
 
-	if dl.cache == nil {
-		dl.cache = &HashNodeCache{
-			cache: make(map[common.Hash]*RefTrieNode),
-		}
-	}
-
 	for _, subset := range nodes {
 		for path, n := range subset {
 			dl.memory += uint64(n.Size() + len(path))

From 7cce14360fb7291a0cdddc975ea6b2fea43f516d Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 11:36:05 +0800
Subject: [PATCH 19/22] chore: adjust some log

---
 triedb/pathdb/difflayer.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index ed2f37cbd3..3aa7221ebe 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -274,13 +274,14 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
 			// In this case, fallback to the original 128-layer recursive difflayer query path.
 			diffHashCacheSlowPathMeter.Mark(1)
-			log.Debug("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
+			log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
 			return dl.node(owner, path, hash, 0)
 		} else {
 			return blob, nil
 		}
 	}
 	diffHashCacheSlowPathMeter.Mark(1)
+	log.Info("retry difflayer due to origin is nil", "owner", owner, "path", path, "hash", hash.String())
 	return dl.node(owner, path, hash, 0)
 }
 

From 86065342e2363e5701f183bf5f1c62f61fba26d5 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 14:16:54 +0800
Subject: [PATCH 20/22] chore: refine log print

---
 triedb/pathdb/difflayer.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go
index 3aa7221ebe..fca184adfe 100644
--- a/triedb/pathdb/difflayer.go
+++ b/triedb/pathdb/difflayer.go
@@ -270,18 +270,18 @@ func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 		blob, err := persistLayer.Node(owner, path, hash)
 		if err != nil {
 			// This is a bad case with a very low probability.
-			// Reading the difflayer cache and reading the disklayer are not in the same lock,
-			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail.
+			// r/w the difflayer cache and r/w the disklayer are not in the same lock,
+			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail, eg, disklayer is stale.
 			// In this case, fallback to the original 128-layer recursive difflayer query path.
 			diffHashCacheSlowPathMeter.Mark(1)
-			log.Info("Hash map and disklayer mismatch, retry difflayer", "owner", owner, "path", path, "hash", hash.String(), "error", err)
+			log.Debug("Retry difflayer due to query origin failed", "owner", owner, "path", path, "hash", hash.String(), "error", err)
 			return dl.node(owner, path, hash, 0)
-		} else {
+		} else { // This is the fastpath.
 			return blob, nil
 		}
 	}
 	diffHashCacheSlowPathMeter.Mark(1)
-	log.Info("retry difflayer due to origin is nil", "owner", owner, "path", path, "hash", hash.String())
+	log.Debug("Retry difflayer due to origin is nil", "owner", owner, "path", path, "hash", hash.String())
 	return dl.node(owner, path, hash, 0)
 }
 

From 625f31a1e887547fe53fcdc74108d527056d7613 Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 14:25:29 +0800
Subject: [PATCH 21/22] fix: update origin workflow

---
 triedb/pathdb/layertree.go | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go
index 32bfc3b386..0577401a96 100644
--- a/triedb/pathdb/layertree.go
+++ b/triedb/pathdb/layertree.go
@@ -228,11 +228,18 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 
 	if persisted != nil {
-		if diff, ok := tree.layers[root].(*diffLayer); ok {
-			diff.lock.Lock()
-			diff.origin = persisted
-			diff.lock.Unlock()
+		var updateOriginFunc func(root common.Hash)
+		updateOriginFunc = func(root common.Hash) {
+			if diff, ok := tree.layers[root].(*diffLayer); ok {
+				diff.lock.Lock()
+				diff.origin = persisted
+				diff.lock.Unlock()
+			}
+			for _, child := range children[root] {
+				updateOriginFunc(child)
+			}
 		}
+		updateOriginFunc(persisted.root)
 	}
 
 	return nil

From 3c2d50951640e9162418b91074ba544364de59be Mon Sep 17 00:00:00 2001
From: "will@2012" <xibaow2020@qq.com>
Date: Wed, 12 Jun 2024 20:58:21 +0800
Subject: [PATCH 22/22] chore: refine log print

---
 triedb/pathdb/disklayer.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go
index 46d67011e1..d14b29e189 100644
--- a/triedb/pathdb/disklayer.go
+++ b/triedb/pathdb/disklayer.go
@@ -165,7 +165,6 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
 	// layer as stale.
 	n, err := dl.buffer.node(owner, path, hash)
 	if err != nil {
-		log.Error("Unexpected trie node in clean cache", "error", err)
 		return nil, err
 	}
 	if n != nil {