pbss implemention

cmd/geth/dbcmd.go

@@ -70,6 +70,7 @@ Remove blockchain and state databases`,
 			dbDumpFreezerIndex,
 			dbImportCmd,
 			dbExportCmd,
+			dbHbss2PbssCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
@@ -254,6 +255,18 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		},
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
+	dbHbss2PbssCmd = cli.Command{
+		Action:    hbss2pbss,
+		Name:      "hbss-to-pbss",
+		ArgsUsage: "<jobnum (optional)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.AncientFlag,
+		},
+		Usage:       "Convert Hash-Base to Path-Base trie node.",
+		Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`,
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -706,3 +719,38 @@ func exportChaindata(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
+
+func hbss2pbss(ctx *cli.Context) error {
+	stack, config := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	h2p, err := trie.NewHbss2Pbss(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal))
+	if err != nil {
+		return err
+	}
+
+	if ctx.NArg() > 0 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+
+	lastStateID := rawdb.ReadPersistentStateID(chaindb)
+	if lastStateID == 0 {
+		h2p.Run()
+	}
+
+	// prune hbss trie node
+	err = rawdb.PruneHashTrieNodeInDataBase(chaindb)
+	if err != nil {
+		log.Error("Prune Hash trie node in database failed", "error", err)
+		return err
+	}
+	err = h2p.Compact()
+	if err != nil {
+		log.Error("Compact trie node failed", "error", err)
+		return err
+	}
+
+	return nil
+}

cmd/geth/main.go

@@ -158,6 +158,8 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
 		utils.CatalystFlag,
+		utils.MorphZkTrieFlag,
+		utils.PathDBSyncFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/usage.go

@@ -57,6 +57,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.IdentityFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,
+			utils.MorphZkTrieFlag,
+			utils.PathDBSyncFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -851,6 +851,15 @@ var (
 		Name:  "rpc.getlogs.maxrange",
 		Usage: "Limit max fetched block range for `eth_getLogs` method",
 	}
+
+	MorphZkTrieFlag = cli.BoolFlag{
+		Name:  "morphzktrie",
+		Usage: "Use MorphZkTrie instead of ZkTrie in state",
+	}
+	PathDBSyncFlag = cli.BoolFlag{
+		Name:  "pathdb.sync",
+		Usage: "Sync flush nodes cache to disk in path schema",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1718,6 +1727,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 			cfg.EthDiscoveryURLs = SplitAndTrim(urls)
 		}
 	}
+	if ctx.GlobalIsSet(PathDBSyncFlag.Name) {
+		cfg.PathSyncFlush = true
+	}
 	// Override any default configs for hard coded networks.
 	switch {
 	case ctx.GlobalBool(MainnetFlag.Name):
@@ -1765,6 +1777,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// use morph zktrie
+		cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name)
 	case ctx.GlobalBool(MorphHoleskyFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 2810
@@ -1780,6 +1795,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// use morph zktrie
+		cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name)
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 1337

common/types.go

@@ -65,6 +65,11 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) }
 // If b is larger than len(h), b will be cropped from the left.
 func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) }
 
+// Cmp compares two hashes.
+func (h Hash) Cmp(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
 // Bytes gets the byte representation of the underlying hash.
 func (h Hash) Bytes() []byte { return h[:] }
 
@@ -226,6 +231,11 @@ func IsHexAddress(s string) bool {
 	return len(s) == 2*AddressLength && isHex(s)
 }
 
+// Cmp compares two addresses.
+func (a Address) Cmp(other Address) int {
+	return bytes.Compare(a[:], other[:])
+}
+
 // Bytes gets the string representation of the underlying address.
 func (a Address) Bytes() []byte { return a[:] }
 
@@ -433,3 +443,62 @@ func (ma *MixedcaseAddress) ValidChecksum() bool {
 func (ma *MixedcaseAddress) Original() string {
 	return ma.original
 }
+
+func ReverseBytes(b []byte) []byte {
+	o := make([]byte, len(b))
+	for i := range b {
+		o[len(b)-1-i] = b[i]
+	}
+	return o
+}
+
+func bitReverseForNibble(b byte) byte {
+	switch b {
+	case 0:
+		return 0
+	case 1:
+		return 8
+	case 2:
+		return 4
+	case 3:
+		return 12
+	case 4:
+		return 2
+	case 5:
+		return 10
+	case 6:
+		return 6
+	case 7:
+		return 14
+	case 8:
+		return 1
+	case 9:
+		return 9
+	case 10:
+		return 5
+	case 11:
+		return 13
+	case 12:
+		return 3
+	case 13:
+		return 11
+	case 14:
+		return 7
+	case 15:
+		return 15
+	default:
+		panic("unexpected input")
+	}
+}
+
+func BitReverse(inp []byte) (out []byte) {
+
+	l := len(inp)
+	out = make([]byte, l)
+
+	for i, b := range inp {
+		out[l-i-1] = bitReverseForNibble(b&15)<<4 + bitReverseForNibble(b>>4)
+	}
+
+	return
+}

core/blockchain.go

@@ -45,6 +45,8 @@ import (
 	"github.com/morph-l2/go-ethereum/metrics"
 	"github.com/morph-l2/go-ethereum/params"
 	"github.com/morph-l2/go-ethereum/trie"
+	"github.com/morph-l2/go-ethereum/triedb/hashdb"
+	"github.com/morph-l2/go-ethereum/triedb/pathdb"
 )
 
 var (
@@ -135,6 +137,9 @@ type CacheConfig struct {
 	Preimages           bool          // Whether to store preimage of trie key to the disk
 
 	SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
+
+	PathSyncFlush   bool // Whether sync flush the trienodebuffer of pathdb to disk.
+	JournalFilePath string
 }
 
 // defaultCacheConfig are the default caching values if none are specified by the
@@ -239,16 +244,37 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		log.Warn("Using fee vault address", "FeeVaultAddress", *chainConfig.Morph.FeeVaultAddress)
 	}
 
+	var hashdbConfig *hashdb.Config
+	var pathdbConfig *pathdb.Config
+	if chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled() {
+		pathdbConfig = &pathdb.Config{
+			SyncFlush:       cacheConfig.PathSyncFlush,
+			CleanCacheSize:  cacheConfig.TrieCleanLimit * 1024 * 1024,
+			DirtyCacheSize:  cacheConfig.TrieCleanLimit * 1024 * 1024,
+			JournalFilePath: cacheConfig.JournalFilePath,
+		}
+	} else {
+		if chainConfig.Morph.ZktrieEnabled() {
+			hashdbConfig = &hashdb.Config{
+				Cache:   cacheConfig.TrieCleanLimit,
+				Journal: cacheConfig.TrieCleanJournal,
+			}
+		}
+	}
+
 	bc := &BlockChain{
 		chainConfig: chainConfig,
 		cacheConfig: cacheConfig,
 		db:          db,
 		triegc:      prque.New(nil),
 		stateCache: state.NewDatabaseWithConfig(db, &trie.Config{
-			Cache:     cacheConfig.TrieCleanLimit,
-			Journal:   cacheConfig.TrieCleanJournal,
-			Preimages: cacheConfig.Preimages,
-			Zktrie:    chainConfig.Morph.ZktrieEnabled(),
+			Cache:       cacheConfig.TrieCleanLimit,
+			Journal:     cacheConfig.TrieCleanJournal,
+			Preimages:   cacheConfig.Preimages,
+			Zktrie:      chainConfig.Morph.ZktrieEnabled(),
+			MorphZkTrie: chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled(),
+			HashDB:      hashdbConfig,
+			PathDB:      pathdbConfig,
 		}),
 		quit:           make(chan struct{}),
 		chainmu:        syncx.NewClosableMutex(),
@@ -809,43 +835,50 @@ func (bc *BlockChain) Stop() {
 		}
 	}
 
-	// Ensure the state of a recent block is also stored to disk before exiting.
-	// We're writing three different states to catch different restart scenarios:
-	//  - HEAD:     So we don't need to reprocess any blocks in the general case
-	//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
-	//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
-	if !bc.cacheConfig.TrieDirtyDisabled {
-		triedb := bc.stateCache.TrieDB()
-
-		for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
-			if number := bc.CurrentBlock().NumberU64(); number > offset {
-				recent := bc.GetBlockByNumber(number - offset)
-
-				log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
-				if err := triedb.Commit(recent.Root(), true, nil); err != nil {
+	if bc.stateCache.TrieDB().Scheme() == rawdb.PathScheme {
+		// Ensure that the in-memory trie nodes are journaled to disk properly.
+		if err := bc.stateCache.TrieDB().Journal(bc.CurrentBlock().Root()); err != nil {
+			log.Info("Failed to journal in-memory trie nodes", "err", err)
+		}
+	} else {
+		// Ensure the state of a recent block is also stored to disk before exiting.
+		// We're writing three different states to catch different restart scenarios:
+		//  - HEAD:     So we don't need to reprocess any blocks in the general case
+		//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
+		//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
+		if !bc.cacheConfig.TrieDirtyDisabled {
+			triedb := bc.stateCache.TrieDB()
+
+			for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
+				if number := bc.CurrentBlock().NumberU64(); number > offset {
+					recent := bc.GetBlockByNumber(number - offset)
+
+					log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+					if err := triedb.Commit(recent.Root(), true, nil); err != nil {
+						log.Error("Failed to commit recent state trie", "err", err)
+					}
+				}
+			}
+			if snapBase != (common.Hash{}) {
+				log.Info("Writing snapshot state to disk", "root", snapBase)
+				if err := triedb.Commit(snapBase, true, nil); err != nil {
 					log.Error("Failed to commit recent state trie", "err", err)
 				}
 			}
-		}
-		if snapBase != (common.Hash{}) {
-			log.Info("Writing snapshot state to disk", "root", snapBase)
-			if err := triedb.Commit(snapBase, true, nil); err != nil {
-				log.Error("Failed to commit recent state trie", "err", err)
+			for !bc.triegc.Empty() {
+				triedb.Dereference(bc.triegc.PopItem().(common.Hash))
+			}
+			if size, _ := triedb.Size(); size != 0 {
+				log.Error("Dangling trie nodes after full cleanup")
 			}
 		}
-		for !bc.triegc.Empty() {
-			triedb.Dereference(bc.triegc.PopItem().(common.Hash))
-		}
-		if size, _ := triedb.Size(); size != 0 {
-			log.Error("Dangling trie nodes after full cleanup")
+		// Ensure all live cached entries be saved into disk, so that we can skip
+		// cache warmup when node restarts.
+		if bc.cacheConfig.TrieCleanJournal != "" {
+			triedb := bc.stateCache.TrieDB()
+			triedb.SaveCache(bc.cacheConfig.TrieCleanJournal)
 		}
 	}
-	// Ensure all live cached entries be saved into disk, so that we can skip
-	// cache warmup when node restarts.
-	if bc.cacheConfig.TrieCleanJournal != "" {
-		triedb := bc.stateCache.TrieDB()
-		triedb.SaveCache(bc.cacheConfig.TrieCleanJournal)
-	}
 	log.Info("Blockchain stopped")
 }
 

core/blockchain_l2.go

@@ -107,6 +107,10 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts []
 	if err := blockBatch.Write(); err != nil {
 		log.Crit("Failed to write block into disk", "err", err)
 	}
+
+	current := block.NumberU64()
+	origin := state.GetOriginRoot()
+
 	// Commit all cached state changes into underlying memory database.
 	root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number()))
 	if err != nil {
@@ -116,13 +120,17 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts []
 	triedb := bc.stateCache.TrieDB()
 	// If we're running an archive node, always flush
 	if bc.cacheConfig.TrieDirtyDisabled {
+		if triedb.Scheme() == rawdb.PathScheme {
+			// If node is running in path mode, skip explicit gc operation
+			// which is unnecessary in this mode.
+			return triedb.CommitState(root, origin, current, false)
+		}
 		return triedb.Commit(root, false, nil)
 	}
 	// Full but not archive node, do proper garbage collection
 	triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
 	bc.triegc.Push(root, -int64(block.NumberU64()))
 
-	current := block.NumberU64()
 	// Flush limits are not considered for the first TriesInMemory blocks.
 	if current <= TriesInMemory {
 		return nil