From 56ae344d42e00b030a5d4cac61293ec3d06472b4 Mon Sep 17 00:00:00 2001 From: corey Date: Wed, 18 Mar 2026 16:11:21 +0800 Subject: [PATCH 1/3] pruner: fall back to disk snapshot root when journal is missing When geth is killed uncleanly (SIGKILL before BlockChain.Stop writes the snapshot journal), prune-state fails with: WARN Loaded snapshot journal diskroot=XXX diffs=missing ERROR head doesn't match snapshot: have XXX, want YYY NewPruner now reads the persisted disk snapshot root via rawdb.ReadSnapshotRoot and retries snapshot initialisation with that root when the normal head-based init fails. Prune() then uses the disk root as the pruning target directly, bypassing the requirement for 128 in-memory diff layers that cannot exist when the journal was not written. Normal flow (clean shutdown, journal present) is unchanged. Made-with: Cursor --- core/state/pruner/pruner.go | 53 ++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 2ce5935c5..106889486 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -81,6 +81,10 @@ type Pruner struct { trieCachePath string headHeader *types.Header snaptree *snapshot.Tree + // diskRoot is set when the snapshot journal was missing and we fell back + // to the persisted disk snapshot root. Prune() uses it as the pruning + // target when no explicit root is provided. + diskRoot common.Hash } // NewPruner creates the pruner instance. @@ -90,8 +94,22 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 return nil, errors.New("Failed to load head block") } snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) + var diskRoot common.Hash if err != nil { - return nil, err // The relevant snapshot(s) might not exist + // The snapshot journal may be missing because geth was not shut down + // cleanly (SIGKILL before BlockChain.Stop could write the journal). + // Fall back: initialise the snapshot tree with the persisted disk + // snapshot root so that Prune() can still target that state. + diskRoot = rawdb.ReadSnapshotRoot(db) + if diskRoot == (common.Hash{}) { + return nil, err // No snapshot at all — nothing we can do. + } + log.Warn("Snapshot journal missing, falling back to disk snapshot root", + "diskRoot", diskRoot, "chainHead", headBlock.Root()) + snaptree, err = snapshot.New(db, trie.NewDatabase(db), 256, diskRoot, false, false, false) + if err != nil { + return nil, err + } } // Sanitize the bloom filter size if it's too small. if bloomSize < 256 { @@ -109,6 +127,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 trieCachePath: trieCachePath, headHeader: headBlock.Header(), snaptree: snaptree, + diskRoot: diskRoot, }, nil } @@ -249,18 +268,28 @@ func (p *Pruner) Prune(root common.Hash) error { // - the probability of this layer being reorg is very low var layers []snapshot.Snapshot if root == (common.Hash{}) { - // Retrieve all snapshot layers from the current HEAD. - // In theory there are 128 difflayers + 1 disk layer present, - // so 128 diff layers are expected to be returned. - layers = p.snaptree.Snapshots(p.headHeader.Root, 128, true) - if len(layers) != 128 { - // Reject if the accumulated diff layers are less than 128. It - // means in most of normal cases, there is no associated state - // with bottom-most diff layer. - return fmt.Errorf("snapshot not old enough yet: need %d more blocks", 128-len(layers)) + // When the snapshot journal was missing (unclean shutdown), we fell + // back to the persisted disk snapshot root in NewPruner. Use that + // root directly as the pruning target instead of requiring 128 diff + // layers that don't exist. + if p.diskRoot != (common.Hash{}) { + log.Info("Using disk snapshot root as pruning target (journal was missing)", + "diskRoot", p.diskRoot) + root = p.diskRoot + } else { + // Retrieve all snapshot layers from the current HEAD. + // In theory there are 128 difflayers + 1 disk layer present, + // so 128 diff layers are expected to be returned. + layers = p.snaptree.Snapshots(p.headHeader.Root, 128, true) + if len(layers) != 128 { + // Reject if the accumulated diff layers are less than 128. It + // means in most of normal cases, there is no associated state + // with bottom-most diff layer. + return fmt.Errorf("snapshot not old enough yet: need %d more blocks", 128-len(layers)) + } + // Use the bottom-most diff layer as the target + root = layers[len(layers)-1].Root() } - // Use the bottom-most diff layer as the target - root = layers[len(layers)-1].Root() } // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the From 185463cef9236648010c48fcc583fb7e910bb0bb Mon Sep 17 00:00:00 2001 From: corey Date: Wed, 18 Mar 2026 16:46:33 +0800 Subject: [PATCH 2/3] pruner: fix Cap panic on disk-layer-only tree and add generation wait log Two follow-up fixes to the journal-missing fallback (56ae344): 1. Skip snaptree.Cap(root, 0) when root is already the disk layer. Cap requires a diffLayer as its target; calling it on a disk-layer-only tree (which is exactly what the fallback produces) returns "snapshot is disk layer" and aborts after all the heavy bloom-filter and DB-sweep work is done. Guard with DiskRoot() != root. 2. Add log lines around the fallback snapshot.New() call to make it visible when snapshot generation must be resumed (async=false blocks until generation finishes, which can take hours for large state). --- core/state/pruner/pruner.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 106889486..d3f65e411 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -106,10 +106,16 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 } log.Warn("Snapshot journal missing, falling back to disk snapshot root", "diskRoot", diskRoot, "chainHead", headBlock.Root()) + // If the snapshot was mid-generation when the node was killed, New will + // resume and wait for generation to finish (async=false). This can take + // a long time for large state; the log below makes that visible. + log.Info("Loading snapshot from disk root (may wait for snapshot generation to finish)...", + "diskRoot", diskRoot) snaptree, err = snapshot.New(db, trie.NewDatabase(db), 256, diskRoot, false, false, false) if err != nil { return nil, err } + log.Info("Snapshot ready", "diskRoot", diskRoot) } // Sanitize the bloom filter size if it's too small. if bloomSize < 256 { @@ -207,8 +213,15 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta // Pruning is done, now drop the "useless" layers from the snapshot. // Firstly, flushing the target layer into the disk. After that all // diff layers below the target will all be merged into the disk. - if err := snaptree.Cap(root, 0); err != nil { - return err + // + // Skip Cap when the root is already the disk layer (no diff layers exist). + // This happens in the fallback path where the snapshot journal was missing + // and we initialised the tree directly from the persisted disk root — Cap + // would otherwise return "snapshot is disk layer" and abort needlessly. + if snaptree.DiskRoot() != root { + if err := snaptree.Cap(root, 0); err != nil { + return err + } } // Secondly, flushing the snapshot journal into the disk. All diff // layers upon are dropped silently. Eventually the entire snapshot From 83b1bd5ede437d2e3bb1f6ce01df0c0fff9d30f2 Mon Sep 17 00:00:00 2001 From: corey Date: Wed, 18 Mar 2026 17:31:54 +0800 Subject: [PATCH 3/3] pruner: rename diskRoot to snapDiskRoot to avoid confusion with diskStateRoot --- core/state/pruner/pruner.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index d3f65e411..f1f56b0fc 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -81,10 +81,10 @@ type Pruner struct { trieCachePath string headHeader *types.Header snaptree *snapshot.Tree - // diskRoot is set when the snapshot journal was missing and we fell back - // to the persisted disk snapshot root. Prune() uses it as the pruning - // target when no explicit root is provided. - diskRoot common.Hash + // snapDiskRoot is set when the snapshot journal was missing and we fell + // back to the persisted snapshot disk-layer root. Prune() uses it as the + // pruning target when no explicit root is provided. + snapDiskRoot common.Hash } // NewPruner creates the pruner instance. @@ -94,28 +94,28 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 return nil, errors.New("Failed to load head block") } snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) - var diskRoot common.Hash + var snapDiskRoot common.Hash if err != nil { // The snapshot journal may be missing because geth was not shut down // cleanly (SIGKILL before BlockChain.Stop could write the journal). // Fall back: initialise the snapshot tree with the persisted disk // snapshot root so that Prune() can still target that state. - diskRoot = rawdb.ReadSnapshotRoot(db) - if diskRoot == (common.Hash{}) { + snapDiskRoot = rawdb.ReadSnapshotRoot(db) + if snapDiskRoot == (common.Hash{}) { return nil, err // No snapshot at all — nothing we can do. } - log.Warn("Snapshot journal missing, falling back to disk snapshot root", - "diskRoot", diskRoot, "chainHead", headBlock.Root()) + log.Warn("Snapshot journal missing, falling back to snapshot disk-layer root", + "snapDiskRoot", snapDiskRoot, "chainHead", headBlock.Root()) // If the snapshot was mid-generation when the node was killed, New will // resume and wait for generation to finish (async=false). This can take // a long time for large state; the log below makes that visible. - log.Info("Loading snapshot from disk root (may wait for snapshot generation to finish)...", - "diskRoot", diskRoot) - snaptree, err = snapshot.New(db, trie.NewDatabase(db), 256, diskRoot, false, false, false) + log.Info("Loading snapshot from disk-layer root (may wait for snapshot generation to finish)...", + "snapDiskRoot", snapDiskRoot) + snaptree, err = snapshot.New(db, trie.NewDatabase(db), 256, snapDiskRoot, false, false, false) if err != nil { return nil, err } - log.Info("Snapshot ready", "diskRoot", diskRoot) + log.Info("Snapshot ready", "snapDiskRoot", snapDiskRoot) } // Sanitize the bloom filter size if it's too small. if bloomSize < 256 { @@ -133,7 +133,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint6 trieCachePath: trieCachePath, headHeader: headBlock.Header(), snaptree: snaptree, - diskRoot: diskRoot, + snapDiskRoot: snapDiskRoot, }, nil } @@ -285,10 +285,10 @@ func (p *Pruner) Prune(root common.Hash) error { // back to the persisted disk snapshot root in NewPruner. Use that // root directly as the pruning target instead of requiring 128 diff // layers that don't exist. - if p.diskRoot != (common.Hash{}) { - log.Info("Using disk snapshot root as pruning target (journal was missing)", - "diskRoot", p.diskRoot) - root = p.diskRoot + if p.snapDiskRoot != (common.Hash{}) { + log.Info("Using snapshot disk-layer root as pruning target (journal was missing)", + "snapDiskRoot", p.snapDiskRoot) + root = p.snapDiskRoot } else { // Retrieve all snapshot layers from the current HEAD. // In theory there are 128 difflayers + 1 disk layer present,