core/trie: persist TrieJournal to journal file instead of kv database

cmd/geth/main.go

@@ -103,6 +103,7 @@ var (
 		utils.TransactionHistoryFlag,
 		utils.StateHistoryFlag,
 		utils.PathDBSyncFlag,
+		utils.JournalFileFlag,
 		utils.LightServeFlag,       // deprecated
 		utils.LightIngressFlag,     // deprecated
 		utils.LightEgressFlag,      // deprecated

cmd/utils/flags.go

@@ -365,6 +365,12 @@ var (
 		Value:    false,
 		Category: flags.StateCategory,
 	}
+	JournalFileFlag = &cli.BoolFlag{
+		Name:     "journalfile",
+		Usage:    "Enable using journal file to store the TrieJournal instead of KVDB in pbss (default = false)",
+		Value:    false,
+		Category: flags.StateCategory,
+	}
 	StateHistoryFlag = &cli.Uint64Flag{
 		Name:     "history.state",
 		Usage:    "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)",
@@ -1962,6 +1968,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(PathDBSyncFlag.Name) {
 		cfg.PathSyncFlush = true
 	}
+	if ctx.IsSet(JournalFileFlag.Name) {
+		cfg.JournalFileEnabled = true
+	}
+
 	if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
 		cfg.TransactionHistory = 0
 		log.Warn("Disabled transaction unindexing for archive node")

core/blockchain.go

@@ -165,6 +165,7 @@ type CacheConfig struct {
 	StateHistory        uint64        // Number of blocks from head whose state histories are reserved.
 	StateScheme         string        // Scheme used to store ethereum states and merkle tree nodes on top
 	PathSyncFlush       bool          // Whether sync flush the trienodebuffer of pathdb to disk.
+	JournalFilePath     string
 
 	SnapshotNoBuild bool // Whether the background generation is allowed
 	SnapshotWait    bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
@@ -184,10 +185,11 @@ func (c *CacheConfig) triedbConfig() *triedb.Config {
 	}
 	if c.StateScheme == rawdb.PathScheme {
 		config.PathDB = &pathdb.Config{
-			SyncFlush:      c.PathSyncFlush,
-			StateHistory:   c.StateHistory,
-			CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
-			DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024,
+			SyncFlush:       c.PathSyncFlush,
+			StateHistory:    c.StateHistory,
+			CleanCacheSize:  c.TrieCleanLimit * 1024 * 1024,
+			DirtyCacheSize:  c.TrieDirtyLimit * 1024 * 1024,
+			JournalFilePath: c.JournalFilePath,
 		}
 	}
 	return config

core/rawdb/database.go

@@ -26,14 +26,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/olekukonko/tablewriter"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/ethdb/pebble"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/olekukonko/tablewriter"
 )
 
 // freezerdb is a database wrapper that enables freezer data retrievals.

eth/backend.go

@@ -69,6 +69,8 @@ import (
 
 const (
 	ChainDBNamespace = "eth/db/chaindata/"
+	JournalFileName  = "trie.journal"
+	ChainData        = "chaindata"
 )
 
 // Config contains the configuration options of the ETH protocol.
@@ -137,7 +139,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	}
 
 	// Assemble the Ethereum object
-	chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles,
+	chainDb, err := stack.OpenAndMergeDatabase(ChainData, config.DatabaseCache, config.DatabaseHandles,
 		config.DatabaseFreezer, config.DatabaseDiff, ChainDBNamespace, false, config.PersistDiff, config.PruneAncientData)
 	if err != nil {
 		return nil, err
@@ -250,6 +252,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion)
 		}
 	}
+	var (
+		journalFilePath string
+		path            string
+	)
+	if config.JournalFileEnabled {
+		if stack.IsSeparatedDB() {
+			path = ChainData + "/state"
+		} else {
+			path = ChainData
+		}
+		journalFilePath = stack.ResolvePath(path) + "/" + JournalFileName
+	}
 	var (
 		vmConfig = vm.Config{
 			EnablePreimageRecording: config.EnablePreimageRecording,
@@ -267,6 +281,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			StateHistory:        config.StateHistory,
 			StateScheme:         config.StateScheme,
 			PathSyncFlush:       config.PathSyncFlush,
+			JournalFilePath:     journalFilePath,
 		}
 	)
 	bcOps := make([]core.BlockChainOption, 0)

eth/ethconfig/config.go

@@ -117,8 +117,9 @@ type Config struct {
 	// State scheme represents the scheme used to store ethereum states and trie
 	// nodes on top. It can be 'hash', 'path', or none which means use the scheme
 	// consistent with persistent state.
-	StateScheme   string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
-	PathSyncFlush bool   `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
+	StateScheme        string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
+	PathSyncFlush      bool   `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
+	JournalFileEnabled bool   // Whether the TrieJournal is stored using journal file
 
 	// RequiredBlocks is a set of block number -> hash mappings which must be in the
 	// canonical chain of all remote peers. Setting the option makes geth verify the

node/node.go

@@ -77,6 +77,8 @@ const (
 
 const chainDataHandlesPercentage = 80
 
+const StateDBNamespace = "eth/db/statedata/"
+
 // New creates a new P2P node, ready for protocol registration.
 func New(conf *Config) (*Node, error) {
 	// Copy config and resolve the datadir so future changes to the current
@@ -791,7 +793,7 @@ func (n *Node) OpenAndMergeDatabase(name string, cache, handles int, freezer, di
 	// Open the separated state database if the state directory exists
 	if n.IsSeparatedDB() {
 		// Allocate half of the  handles and cache to this separate state data database
-		statediskdb, err = n.OpenDatabaseWithFreezer(name+"/state", cache/2, chainDataHandles/2, "", "eth/db/statedata/", readonly, false, false, pruneAncientData)
+		statediskdb, err = n.OpenDatabaseWithFreezer(name+"/state", cache/2, chainDataHandles/2, "", StateDBNamespace, readonly, false, false, pruneAncientData)
 		if err != nil {
 			return nil, err
 		}

triedb/pathdb/database.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"sort"
 	"strconv"
 	"sync"
@@ -96,12 +97,13 @@ type layer interface {
 
 // Config contains the settings for database.
 type Config struct {
-	SyncFlush      bool   // Flag of trienodebuffer sync flush cache to disk
-	StateHistory   uint64 // Number of recent blocks to maintain state history for
-	CleanCacheSize int    // Maximum memory allowance (in bytes) for caching clean nodes
-	DirtyCacheSize int    // Maximum memory allowance (in bytes) for caching dirty nodes
-	ReadOnly       bool   // Flag whether the database is opened in read only mode.
-	NoTries        bool
+	SyncFlush       bool   // Flag of trienodebuffer sync flush cache to disk
+	StateHistory    uint64 // Number of recent blocks to maintain state history for
+	CleanCacheSize  int    // Maximum memory allowance (in bytes) for caching clean nodes
+	DirtyCacheSize  int    // Maximum memory allowance (in bytes) for caching dirty nodes
+	ReadOnly        bool   // Flag whether the database is opened in read only mode.
+	NoTries         bool
+	JournalFilePath string
 }
 
 // sanitize checks the provided user configurations and changes anything that's
@@ -316,7 +318,7 @@ func (db *Database) Enable(root common.Hash) error {
 	// Drop the stale state journal in persistent database and
 	// reset the persistent state id back to zero.
 	batch := db.diskdb.NewBatch()
-	rawdb.DeleteTrieJournal(batch)
+	db.DeleteTrieJournal(batch)
 	rawdb.WritePersistentStateID(batch, 0)
 	if err := batch.Write(); err != nil {
 		return err
@@ -380,7 +382,7 @@ func (db *Database) Recover(root common.Hash, loader triestate.TrieLoader) error
 		// disk layer won't be accessible from outside.
 		db.tree.reset(dl)
 	}
-	rawdb.DeleteTrieJournal(db.diskdb)
+	db.DeleteTrieJournal(db.diskdb)
 	_, err := truncateFromHead(db.diskdb, db.freezer, dl.stateID())
 	if err != nil {
 		return err
@@ -524,3 +526,20 @@ func (db *Database) GetAllRooHash() [][]string {
 	data = append(data, []string{"-1", db.tree.bottom().rootHash().String()})
 	return data
 }
+
+func (db *Database) DeleteTrieJournal(writer ethdb.KeyValueWriter) error {
+	filePath := db.config.JournalFilePath
+	if len(filePath) == 0 {
+		rawdb.DeleteTrieJournal(writer)
+	} else {
+		_, err := os.Stat(filePath)
+		if os.IsNotExist(err) {
+			return err
+		}
+		errRemove := os.Remove(filePath)
+		if errRemove != nil {
+			log.Crit("Failed to remove tries journal", "journal path", filePath, "err", err)
+		}
+	}
+	return nil
+}