From d3a07a25925efbcf1e44b0b2eace84e3bcad2b9d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:16:27 +0200 Subject: [PATCH 1/8] core/state: introduce the TransitionState object --- consensus/beacon/consensus.go | 2 +- core/blockchain.go | 8 +- core/chain_makers.go | 5 +- core/genesis.go | 55 ++++++----- core/genesis_test.go | 3 +- core/rawdb/accessors_overlay.go | 30 ++++++ core/rawdb/schema.go | 8 ++ core/state/database.go | 165 ++++++++++++++++++++++++++++++-- core/state/database_history.go | 8 ++ core/state/reader.go | 4 +- core/state/statedb.go | 1 + core/verkle_witness_test.go | 6 +- params/config.go | 27 ------ 13 files changed, 247 insertions(+), 75 deletions(-) create mode 100644 core/rawdb/accessors_overlay.go diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 196cbc857ce0..e06da9157bfc 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -396,7 +396,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea return nil, errors.New("post-state tree is not available") } vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := postTrie.(*trie.VerkleTrie) + vktPostTrie, okpost := state.GetTrie().(*trie.VerkleTrie) // The witness is only attached iff both parent and current block are // using verkle tree. diff --git a/core/blockchain.go b/core/blockchain.go index d52990ec5adc..371163569f55 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -341,13 +341,7 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, if cfg == nil { cfg = DefaultConfig() } - - // Open trie database with provided config - enableVerkle, err := EnableVerkleAtGenesis(db, genesis) - if err != nil { - return nil, err - } - triedb := triedb.NewDatabase(db, cfg.triedbConfig(enableVerkle)) + triedb := triedb.NewDatabase(db, cfg.triedbConfig(genesis.IsVerkle())) // Write the supplied genesis to the database if it has not been initialized // yet. The corresponding chain config will be returned, either from the diff --git a/core/chain_makers.go b/core/chain_makers.go index b2559495a1a8..674a9d819621 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -540,8 +540,10 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine return block, b.receipts } + sdb := state.NewDatabase(trdb, nil) + for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabase(trdb, nil)) + statedb, err := state.New(parent.Root(), sdb) if err != nil { panic(err) } @@ -579,6 +581,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { db := rawdb.NewMemoryDatabase() + saveVerkleTransitionStatusAtVerlkeGenesis(db) cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) cacheConfig.SnapshotLimit = 0 triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) diff --git a/core/genesis.go b/core/genesis.go index f1a620da579d..58deafae0840 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -18,6 +18,7 @@ package core import ( "bytes" + "encoding/gob" "encoding/json" "errors" "fmt" @@ -145,6 +146,9 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { emptyRoot = types.EmptyVerkleHash } db := rawdb.NewMemoryDatabase() + if isVerkle { + saveVerkleTransitionStatusAtVerlkeGenesis(db) + } statedb, err := state.New(emptyRoot, state.NewDatabase(triedb.NewDatabase(db, config), nil)) if err != nil { return common.Hash{}, err @@ -276,6 +280,24 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { return cfg.CheckConfigForkOrder() } +// saveVerkleTransitionStatusAtVerlkeGenesis saves a conversion marker +// representing a converted state, which is used in devnets that activate +// verkle at genesis. +func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) { + saveVerkleTransitionStatus(db, common.Hash{}, &state.TransitionState{Ended: true}) +} + +func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *state.TransitionState) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(ts) + if err != nil { + log.Error("failed to encode transition state", "err", err) + return + } + rawdb.WriteVerkleTransitionState(db, root, buf.Bytes()) +} + // SetupGenesisBlock writes or updates the genesis block in db. // The block that will be used is: // @@ -299,6 +321,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if genesis != nil && genesis.Config == nil { return nil, common.Hash{}, nil, errGenesisNoConfig } + // In case of verkle-at-genesis, we need to ensure that the conversion + // markers are indicating that the conversion has completed. + if genesis != nil && genesis.Config.VerkleTime != nil && *genesis.Config.VerkleTime == genesis.Timestamp { + saveVerkleTransitionStatusAtVerlkeGenesis(db) + } // Commit the genesis if the database is empty ghash := rawdb.ReadCanonicalHash(db, 0) if (ghash == common.Hash{}) { @@ -446,7 +473,7 @@ func (g *Genesis) chainConfigOrDefault(ghash common.Hash, stored *params.ChainCo // IsVerkle indicates whether the state is already stored in a verkle // tree at genesis time. func (g *Genesis) IsVerkle() bool { - return g.Config.IsVerkleGenesis() + return g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp } // ToBlock returns the genesis block according to genesis specification. @@ -550,6 +577,9 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo if err != nil { return nil, err } + if g.IsVerkle() { + saveVerkleTransitionStatus(db, block.Root(), &state.TransitionState{Ended: true}) + } batch := db.NewBatch() rawdb.WriteGenesisStateSpec(batch, block.Hash(), blob) rawdb.WriteBlock(batch, block) @@ -572,29 +602,6 @@ func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types. return block } -// EnableVerkleAtGenesis indicates whether the verkle fork should be activated -// at genesis. This is a temporary solution only for verkle devnet testing, where -// verkle fork is activated at genesis, and the configured activation date has -// already passed. -// -// In production networks (mainnet and public testnets), verkle activation always -// occurs after the genesis block, making this function irrelevant in those cases. -func EnableVerkleAtGenesis(db ethdb.Database, genesis *Genesis) (bool, error) { - if genesis != nil { - if genesis.Config == nil { - return false, errGenesisNoConfig - } - return genesis.Config.EnableVerkleAtGenesis, nil - } - if ghash := rawdb.ReadCanonicalHash(db, 0); ghash != (common.Hash{}) { - chainCfg := rawdb.ReadChainConfig(db, ghash) - if chainCfg != nil { - return chainCfg.EnableVerkleAtGenesis, nil - } - } - return false, nil -} - // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ diff --git a/core/genesis_test.go b/core/genesis_test.go index a41dfce5783e..0077863193b8 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -287,7 +287,6 @@ func TestVerkleGenesisCommit(t *testing.T) { OsakaTime: &verkleTime, VerkleTime: &verkleTime, TerminalTotalDifficulty: big.NewInt(0), - EnableVerkleAtGenesis: true, Ethash: nil, Clique: nil, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ @@ -315,7 +314,7 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - + saveVerkleTransitionStatusAtVerlkeGenesis(db) config := *pathdb.Defaults config.NoAsyncFlush = true diff --git a/core/rawdb/accessors_overlay.go b/core/rawdb/accessors_overlay.go new file mode 100644 index 000000000000..364cc889d1ec --- /dev/null +++ b/core/rawdb/accessors_overlay.go @@ -0,0 +1,30 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +func ReadVerkleTransitionState(db ethdb.KeyValueReader, hash common.Hash) ([]byte, error) { + return db.Get(transitionStateKey(hash)) +} + +func WriteVerkleTransitionState(db ethdb.KeyValueWriter, hash common.Hash, state []byte) error { + return db.Put(transitionStateKey(hash), state) +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 388a08f2434a..72f9bd34eca3 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -158,6 +158,9 @@ var ( preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil) + + // Verkle transition information + VerkleTransitionStatePrefix = []byte("verkle-transition-state-") ) // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary @@ -397,3 +400,8 @@ func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Has binary.BigEndian.PutUint32(buf[:], blockID) return append(append(append(StateHistoryStorageBlockPrefix, addressHash.Bytes()...), storageHash.Bytes()...), buf[:]...) } + +// transitionStateKey = transitionStatusKey + hash +func transitionStateKey(hash common.Hash) []byte { + return append(VerkleTransitionStatePrefix, hash.Bytes()...) +} diff --git a/core/state/database.go b/core/state/database.go index 5fb198a6298d..d65489f4aab1 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,7 +17,10 @@ package state import ( + "bytes" + "encoding/gob" "fmt" + "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" @@ -26,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" @@ -62,6 +66,12 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree + + // SaveTransitionState saves the tree transition progress markers to the database. + SaveTransitionState(common.Hash, *TransitionState) + + // LoadTransitionState loads the tree transition progress markers from the database. + LoadTransitionState(common.Hash) *TransitionState } // Trie is a Ethereum Merkle Patricia trie. @@ -141,6 +151,50 @@ type Trie interface { IsVerkle() bool } +// TransitionState is a structure that holds the progress markers of the +// translation process. +type TransitionState struct { + CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file + Started, Ended bool + + // Mark whether the storage for an account has been processed. This is useful if the + // maximum number of leaves of the conversion is reached before the whole storage is + // processed. + StorageProcessed bool + + BaseRoot common.Hash // hash of the last read-only MPT base tree +} + +// InTransition returns true if the translation process is in progress. +func (ts *TransitionState) InTransition() bool { + return ts != nil && ts.Started && !ts.Ended +} + +// Transitioned returns true if the translation process has been completed. +func (ts *TransitionState) Transitioned() bool { + return ts != nil && ts.Ended +} + +// Copy returns a deep copy of the TransitionState object. +func (ts *TransitionState) Copy() *TransitionState { + ret := &TransitionState{ + Started: ts.Started, + Ended: ts.Ended, + CurrentSlotHash: ts.CurrentSlotHash, + CurrentPreimageOffset: ts.CurrentPreimageOffset, + StorageProcessed: ts.StorageProcessed, + } + + if ts.CurrentAccountAddress != nil { + ret.CurrentAccountAddress = &common.Address{} + copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) + } + + return ret +} + // CachingDB is an implementation of Database interface. It leverages both trie and // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. @@ -151,17 +205,21 @@ type CachingDB struct { codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] pointCache *utils.PointCache + + // Transition-specific fields + TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState] } // NewDatabase creates a state database with the provided data sources. func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { return &CachingDB{ - disk: triedb.Disk(), - triedb: triedb, - snap: snap, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), + disk: triedb.Disk(), + triedb: triedb, + snap: snap, + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + pointCache: utils.NewPointCache(pointCacheSize), + TransitionStatePerRoot: lru.NewBasicLRU[common.Hash, *TransitionState](1000), } } @@ -184,6 +242,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { readers = append(readers, newFlatReader(snap)) } } + ts := db.LoadTransitionState(stateRoot) // Configure the state reader using the path database in path mode. // This reader offers improved performance but is optional and only // partially useful if the snapshot data in path database is not @@ -196,7 +255,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } // Configure the trie reader, which is expected to be available as the // gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache, ts) if err != nil { return nil, err } @@ -223,7 +282,11 @@ func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithSta // OpenTrie opens the main account trie at a specific root hash. func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { - if db.triedb.IsVerkle() { + ts := db.LoadTransitionState(root) + if ts.InTransition() { + panic("transition isn't supported yet") + } + if ts.Transitioned() { return trie.NewVerkleTrie(root, db.triedb, db.pointCache) } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) @@ -235,10 +298,11 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { + ts := db.LoadTransitionState(stateRoot) // In the verkle case, there is only one tree. But the two-tree structure // is hardcoded in the codebase. So we need to return the same trie in this // case. - if db.triedb.IsVerkle() { + if ts.InTransition() || ts.Transitioned() { return self, nil } tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) @@ -290,3 +354,86 @@ func mustCopyTrie(t Trie) Trie { panic(fmt.Errorf("unknown trie type %T", t)) } } + +// SaveTransitionState saves the transition state to the cache and commits +// it to the database if it's not already in the cache. +func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) { + if ts == nil { + panic("nil transition state") + } + + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(ts) + if err != nil { + log.Error("failed to encode transition state", "err", err) + return + } + + if !db.TransitionStatePerRoot.Contains(root) { + // Copy so that the address pointer isn't updated after + // it has been saved. + db.TransitionStatePerRoot.Add(root, ts.Copy()) + rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes()) + } else { + // Check that the state is consistent with what is in the cache, + // which is not strictly necessary but a good sanity check. Can + // be removed when the transition is stable. + cachedState, _ := db.TransitionStatePerRoot.Get(root) + if !reflect.DeepEqual(cachedState, ts) { + fmt.Println("transition state mismatch", "cached state", cachedState, "new state", ts) + panic("transition state mismatch") + } + } + + log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) +} + +func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { + // Try to get the transition state from the cache and + // the DB if it's not there. + ts, ok := db.TransitionStatePerRoot.Get(root) + if !ok { + // Not in the cache, try getting it from the DB + data, _ := rawdb.ReadVerkleTransitionState(db.TrieDB().Disk(), root) + // if err != nil && errors.Is(err, triedb.ErrNotFound) { + // log.Error("failed to read transition state", "err", err) + // return nil + // } + + // if a state could be read from the db, attempt to decode it + if len(data) > 0 { + var ( + newts TransitionState + buf = bytes.NewBuffer(data[:]) + dec = gob.NewDecoder(buf) + ) + // Decode transition state + err := dec.Decode(&newts) + if err != nil { + log.Error("failed to decode transition state", "err", err) + return nil + } + ts = &newts + } + + // Fallback that should only happen before the transition + if ts == nil { + // Initialize the first transition state, with the "ended" + // field set to true if the database was created + // as a verkle database. + log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle()) + // Start with a fresh state + ts = &TransitionState{Ended: db.triedb.IsVerkle()} + } + + db.TransitionStatePerRoot.Add(root, ts) + } + + // Copy so that the CurrentAddress pointer in the map + // doesn't get overwritten. + // db.CurrentTransitionState = ts.Copy() + + log.Debug("loaded transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) + return ts +} diff --git a/core/state/database_history.go b/core/state/database_history.go index 314c56c4708a..705c1dd0aca5 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -153,3 +153,11 @@ func (db *HistoricDB) TrieDB() *triedb.Database { func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } + +func (db *HistoricDB) LoadTransitionState(common.Hash) *TransitionState { + panic("should not be called") +} + +func (db *HistoricDB) SaveTransitionState(common.Hash, *TransitionState) { + panic("should not be called") +} diff --git a/core/state/reader.go b/core/state/reader.go index 4628f4d5dbad..7f08ef1a562a 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -233,12 +233,12 @@ type trieReader struct { // trieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *TransitionState) (*trieReader, error) { var ( tr Trie err error ) - if !db.IsVerkle() { + if !ts.Transitioned() && !ts.InTransition() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { tr, err = trie.NewVerkleTrie(root, db, cache) diff --git a/core/state/statedb.go b/core/state/statedb.go index e80588507981..f43101641329 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1305,6 +1305,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } + s.db.SaveTransitionState(ret.root, &TransitionState{Ended: true}) if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index a89672e6e529..47e3c885d9e5 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" "math/big" "slices" "testing" @@ -58,7 +59,6 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -82,7 +82,6 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -202,6 +201,9 @@ func TestProcessVerkle(t *testing.T) { t.Log("verified verkle proof, inserting blocks into the chain") + for i, b := range chain { + fmt.Printf("%d %x\n", i, b.Root()) + } endnum, err := blockchain.InsertChain(chain) if err != nil { t.Fatalf("block %d imported with error: %v", endnum, err) diff --git a/params/config.go b/params/config.go index 85619bbe222a..4e74bf2082ff 100644 --- a/params/config.go +++ b/params/config.go @@ -423,19 +423,6 @@ type ChainConfig struct { DepositContractAddress common.Address `json:"depositContractAddress,omitempty"` - // EnableVerkleAtGenesis is a flag that specifies whether the network uses - // the Verkle tree starting from the genesis block. If set to true, the - // genesis state will be committed using the Verkle tree, eliminating the - // need for any Verkle transition later. - // - // This is a temporary flag only for verkle devnet testing, where verkle is - // activated at genesis, and the configured activation date has already passed. - // - // In production networks (mainnet and public testnets), verkle activation - // always occurs after the genesis block, making this flag irrelevant in - // those cases. - EnableVerkleAtGenesis bool `json:"enableVerkleAtGenesis,omitempty"` - // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -704,20 +691,6 @@ func (c *ChainConfig) IsBPO5(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.BPO5Time, time) } -// IsVerkleGenesis checks whether the verkle fork is activated at the genesis block. -// -// Verkle mode is considered enabled if the verkle fork time is configured, -// regardless of whether the local time has surpassed the fork activation time. -// This is a temporary workaround for verkle devnet testing, where verkle is -// activated at genesis, and the configured activation date has already passed. -// -// In production networks (mainnet and public testnets), verkle activation -// always occurs after the genesis block, making this function irrelevant in -// those cases. -func (c *ChainConfig) IsVerkleGenesis() bool { - return c.EnableVerkleAtGenesis -} - // IsEIP4762 returns whether eip 4762 has been activated at given block. func (c *ChainConfig) IsEIP4762(num *big.Int, time uint64) bool { return c.IsVerkle(num, time) From 7fd51790298be62c3d7e50c4225d74c077665421 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:16:30 +0200 Subject: [PATCH 2/8] fix failing tests --- consensus/beacon/consensus.go | 19 ++++++++----------- core/genesis.go | 2 +- core/state/database.go | 6 ------ core/state/statedb.go | 1 - 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index e06da9157bfc..b09a5cb4c84e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -353,9 +353,9 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. -func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if !beacon.IsPoSHeader(header) { - return beacon.ethone.FinalizeAndAssemble(chain, header, state, body, receipts) + return beacon.ethone.FinalizeAndAssemble(chain, header, statedb, body, receipts) } shanghai := chain.Config().IsShanghai(header.Number, header.Time) if shanghai { @@ -369,10 +369,10 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea } } // Finalize and assemble the block. - beacon.Finalize(chain, header, state, body) + beacon.Finalize(chain, header, statedb, body) // Assign the final state root to header. - header.Root = state.IntermediateRoot(true) + header.Root = statedb.IntermediateRoot(true) // Assemble the final block. block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) @@ -380,23 +380,20 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // Create the block witness and attach to block. // This step needs to happen as late as possible to catch all access events. if chain.Config().IsVerkle(header.Number, header.Time) { - keys := state.AccessEvents().Keys() + //statedb.Database().SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) + keys := statedb.AccessEvents().Keys() // Open the pre-tree to prove the pre-state against parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1) if parent == nil { return nil, fmt.Errorf("nil parent header for block %d", header.Number) } - preTrie, err := state.Database().OpenTrie(parent.Root) + preTrie, err := statedb.Database().OpenTrie(parent.Root) if err != nil { return nil, fmt.Errorf("error opening pre-state tree root: %w", err) } - postTrie := state.GetTrie() - if postTrie == nil { - return nil, errors.New("post-state tree is not available") - } vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := state.GetTrie().(*trie.VerkleTrie) + vktPostTrie, okpost := statedb.GetTrie().(*trie.VerkleTrie) // The witness is only attached iff both parent and current block are // using verkle tree. diff --git a/core/genesis.go b/core/genesis.go index 58deafae0840..2ef982afc82e 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -473,7 +473,7 @@ func (g *Genesis) chainConfigOrDefault(ghash common.Hash, stored *params.ChainCo // IsVerkle indicates whether the state is already stored in a verkle // tree at genesis time. func (g *Genesis) IsVerkle() bool { - return g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp + return g != nil && g.Config != nil && g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp } // ToBlock returns the genesis block according to genesis specification. diff --git a/core/state/database.go b/core/state/database.go index d65489f4aab1..22f2accd8861 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -66,12 +66,6 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree - - // SaveTransitionState saves the tree transition progress markers to the database. - SaveTransitionState(common.Hash, *TransitionState) - - // LoadTransitionState loads the tree transition progress markers from the database. - LoadTransitionState(common.Hash) *TransitionState } // Trie is a Ethereum Merkle Patricia trie. diff --git a/core/state/statedb.go b/core/state/statedb.go index f43101641329..e80588507981 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1305,7 +1305,6 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } - s.db.SaveTransitionState(ret.root, &TransitionState{Ended: true}) if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { From 86183a83738c33ffccc0bb4e18e677b9d27e7340 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 14 Jul 2025 11:37:43 +0200 Subject: [PATCH 3/8] fix the race condition by using lru.Cache Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- core/state/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 22f2accd8861..3cb9a78d55d6 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -201,7 +201,7 @@ type CachingDB struct { pointCache *utils.PointCache // Transition-specific fields - TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState] + TransitionStatePerRoot *lru.Cache[common.Hash, *TransitionState] } // NewDatabase creates a state database with the provided data sources. @@ -213,7 +213,7 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), pointCache: utils.NewPointCache(pointCacheSize), - TransitionStatePerRoot: lru.NewBasicLRU[common.Hash, *TransitionState](1000), + TransitionStatePerRoot: lru.NewCache[common.Hash, *TransitionState](1000), } } From a483135b2b5d39d47a783c9409b44fd08e62306d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 4 Aug 2025 12:38:24 +0200 Subject: [PATCH 4/8] don't store conversion state in genesis db in this PR Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Gary Rong --- core/chain_makers.go | 1 - core/genesis.go | 32 +------------------------------- core/genesis_test.go | 2 +- core/state/database.go | 20 +++++++++----------- core/verkle_witness_test.go | 4 +++- params/config.go | 27 +++++++++++++++++++++++++++ 6 files changed, 41 insertions(+), 45 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index 674a9d819621..af55716cca3e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -581,7 +581,6 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { db := rawdb.NewMemoryDatabase() - saveVerkleTransitionStatusAtVerlkeGenesis(db) cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) cacheConfig.SnapshotLimit = 0 triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) diff --git a/core/genesis.go b/core/genesis.go index 2ef982afc82e..5bf25559447b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -18,7 +18,6 @@ package core import ( "bytes" - "encoding/gob" "encoding/json" "errors" "fmt" @@ -146,9 +145,6 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { emptyRoot = types.EmptyVerkleHash } db := rawdb.NewMemoryDatabase() - if isVerkle { - saveVerkleTransitionStatusAtVerlkeGenesis(db) - } statedb, err := state.New(emptyRoot, state.NewDatabase(triedb.NewDatabase(db, config), nil)) if err != nil { return common.Hash{}, err @@ -280,24 +276,6 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { return cfg.CheckConfigForkOrder() } -// saveVerkleTransitionStatusAtVerlkeGenesis saves a conversion marker -// representing a converted state, which is used in devnets that activate -// verkle at genesis. -func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) { - saveVerkleTransitionStatus(db, common.Hash{}, &state.TransitionState{Ended: true}) -} - -func saveVerkleTransitionStatus(db ethdb.Database, root common.Hash, ts *state.TransitionState) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - err := enc.Encode(ts) - if err != nil { - log.Error("failed to encode transition state", "err", err) - return - } - rawdb.WriteVerkleTransitionState(db, root, buf.Bytes()) -} - // SetupGenesisBlock writes or updates the genesis block in db. // The block that will be used is: // @@ -321,11 +299,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if genesis != nil && genesis.Config == nil { return nil, common.Hash{}, nil, errGenesisNoConfig } - // In case of verkle-at-genesis, we need to ensure that the conversion - // markers are indicating that the conversion has completed. - if genesis != nil && genesis.Config.VerkleTime != nil && *genesis.Config.VerkleTime == genesis.Timestamp { - saveVerkleTransitionStatusAtVerlkeGenesis(db) - } // Commit the genesis if the database is empty ghash := rawdb.ReadCanonicalHash(db, 0) if (ghash == common.Hash{}) { @@ -473,7 +446,7 @@ func (g *Genesis) chainConfigOrDefault(ghash common.Hash, stored *params.ChainCo // IsVerkle indicates whether the state is already stored in a verkle // tree at genesis time. func (g *Genesis) IsVerkle() bool { - return g != nil && g.Config != nil && g.Config.VerkleTime != nil && *g.Config.VerkleTime == g.Timestamp + return g.Config.IsVerkleGenesis() } // ToBlock returns the genesis block according to genesis specification. @@ -577,9 +550,6 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo if err != nil { return nil, err } - if g.IsVerkle() { - saveVerkleTransitionStatus(db, block.Root(), &state.TransitionState{Ended: true}) - } batch := db.NewBatch() rawdb.WriteGenesisStateSpec(batch, block.Hash(), blob) rawdb.WriteBlock(batch, block) diff --git a/core/genesis_test.go b/core/genesis_test.go index 0077863193b8..702cd3445f93 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -287,6 +287,7 @@ func TestVerkleGenesisCommit(t *testing.T) { OsakaTime: &verkleTime, VerkleTime: &verkleTime, TerminalTotalDifficulty: big.NewInt(0), + EnableVerkleAtGenesis: true, Ethash: nil, Clique: nil, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ @@ -314,7 +315,6 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - saveVerkleTransitionStatusAtVerlkeGenesis(db) config := *pathdb.Defaults config.NoAsyncFlush = true diff --git a/core/state/database.go b/core/state/database.go index 3cb9a78d55d6..ea24b4f5cc6b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -276,12 +276,14 @@ func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithSta // OpenTrie opens the main account trie at a specific root hash. func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { - ts := db.LoadTransitionState(root) - if ts.InTransition() { - panic("transition isn't supported yet") - } - if ts.Transitioned() { - return trie.NewVerkleTrie(root, db.triedb, db.pointCache) + if db.triedb.IsVerkle() { + ts := db.LoadTransitionState(root) + if ts.InTransition() { + panic("transition isn't supported yet") + } + if ts.Transitioned() { + return trie.NewVerkleTrie(root, db.triedb, db.pointCache) + } } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) if err != nil { @@ -292,11 +294,7 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { - ts := db.LoadTransitionState(stateRoot) - // In the verkle case, there is only one tree. But the two-tree structure - // is hardcoded in the codebase. So we need to return the same trie in this - // case. - if ts.InTransition() || ts.Transitioned() { + if db.triedb.IsVerkle() { return self, nil } tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 47e3c885d9e5..e200bf7f5099 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -59,6 +59,7 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, + EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -82,6 +83,7 @@ var ( ShanghaiTime: u64(0), VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, + EnableVerkleAtGenesis: true, BlobScheduleConfig: ¶ms.BlobScheduleConfig{ Verkle: params.DefaultPragueBlobConfig, }, @@ -209,7 +211,7 @@ func TestProcessVerkle(t *testing.T) { t.Fatalf("block %d imported with error: %v", endnum, err) } - for i := 0; i < 2; i++ { + for i := range 2 { b := blockchain.GetBlockByNumber(uint64(i) + 1) if b == nil { t.Fatalf("expected block %d to be present in chain", i+1) diff --git a/params/config.go b/params/config.go index 4e74bf2082ff..85619bbe222a 100644 --- a/params/config.go +++ b/params/config.go @@ -423,6 +423,19 @@ type ChainConfig struct { DepositContractAddress common.Address `json:"depositContractAddress,omitempty"` + // EnableVerkleAtGenesis is a flag that specifies whether the network uses + // the Verkle tree starting from the genesis block. If set to true, the + // genesis state will be committed using the Verkle tree, eliminating the + // need for any Verkle transition later. + // + // This is a temporary flag only for verkle devnet testing, where verkle is + // activated at genesis, and the configured activation date has already passed. + // + // In production networks (mainnet and public testnets), verkle activation + // always occurs after the genesis block, making this flag irrelevant in + // those cases. + EnableVerkleAtGenesis bool `json:"enableVerkleAtGenesis,omitempty"` + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -691,6 +704,20 @@ func (c *ChainConfig) IsBPO5(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.BPO5Time, time) } +// IsVerkleGenesis checks whether the verkle fork is activated at the genesis block. +// +// Verkle mode is considered enabled if the verkle fork time is configured, +// regardless of whether the local time has surpassed the fork activation time. +// This is a temporary workaround for verkle devnet testing, where verkle is +// activated at genesis, and the configured activation date has already passed. +// +// In production networks (mainnet and public testnets), verkle activation +// always occurs after the genesis block, making this function irrelevant in +// those cases. +func (c *ChainConfig) IsVerkleGenesis() bool { + return c.EnableVerkleAtGenesis +} + // IsEIP4762 returns whether eip 4762 has been activated at given block. func (c *ChainConfig) IsEIP4762(num *big.Int, time uint64) bool { return c.IsVerkle(num, time) From f0f51d1ef458c1593e908741180615ad83fd07eb Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:01:47 +0200 Subject: [PATCH 5/8] move transition state to its own file Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Gary Rong --- core/genesis_test.go | 1 + core/state/database.go | 57 +++++----------------------------- core/state/database_history.go | 5 +-- core/state/reader.go | 3 +- 4 files changed, 13 insertions(+), 53 deletions(-) diff --git a/core/genesis_test.go b/core/genesis_test.go index 702cd3445f93..a41dfce5783e 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -315,6 +315,7 @@ func TestVerkleGenesisCommit(t *testing.T) { } db := rawdb.NewMemoryDatabase() + config := *pathdb.Defaults config.NoAsyncFlush = true diff --git a/core/state/database.go b/core/state/database.go index ea24b4f5cc6b..187dc79bce6c 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" @@ -145,50 +146,6 @@ type Trie interface { IsVerkle() bool } -// TransitionState is a structure that holds the progress markers of the -// translation process. -type TransitionState struct { - CurrentAccountAddress *common.Address // addresss of the last translated account - CurrentSlotHash common.Hash // hash of the last translated storage slot - CurrentPreimageOffset int64 // next byte to read from the preimage file - Started, Ended bool - - // Mark whether the storage for an account has been processed. This is useful if the - // maximum number of leaves of the conversion is reached before the whole storage is - // processed. - StorageProcessed bool - - BaseRoot common.Hash // hash of the last read-only MPT base tree -} - -// InTransition returns true if the translation process is in progress. -func (ts *TransitionState) InTransition() bool { - return ts != nil && ts.Started && !ts.Ended -} - -// Transitioned returns true if the translation process has been completed. -func (ts *TransitionState) Transitioned() bool { - return ts != nil && ts.Ended -} - -// Copy returns a deep copy of the TransitionState object. -func (ts *TransitionState) Copy() *TransitionState { - ret := &TransitionState{ - Started: ts.Started, - Ended: ts.Ended, - CurrentSlotHash: ts.CurrentSlotHash, - CurrentPreimageOffset: ts.CurrentPreimageOffset, - StorageProcessed: ts.StorageProcessed, - } - - if ts.CurrentAccountAddress != nil { - ret.CurrentAccountAddress = &common.Address{} - copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) - } - - return ret -} - // CachingDB is an implementation of Database interface. It leverages both trie and // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. @@ -201,7 +158,7 @@ type CachingDB struct { pointCache *utils.PointCache // Transition-specific fields - TransitionStatePerRoot *lru.Cache[common.Hash, *TransitionState] + TransitionStatePerRoot *lru.Cache[common.Hash, *overlay.TransitionState] } // NewDatabase creates a state database with the provided data sources. @@ -213,7 +170,7 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), pointCache: utils.NewPointCache(pointCacheSize), - TransitionStatePerRoot: lru.NewCache[common.Hash, *TransitionState](1000), + TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), } } @@ -349,7 +306,7 @@ func mustCopyTrie(t Trie) Trie { // SaveTransitionState saves the transition state to the cache and commits // it to the database if it's not already in the cache. -func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) { +func (db *CachingDB) SaveTransitionState(root common.Hash, ts *overlay.TransitionState) { if ts == nil { panic("nil transition state") } @@ -381,7 +338,7 @@ func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) } -func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { +func (db *CachingDB) LoadTransitionState(root common.Hash) *overlay.TransitionState { // Try to get the transition state from the cache and // the DB if it's not there. ts, ok := db.TransitionStatePerRoot.Get(root) @@ -396,7 +353,7 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { // if a state could be read from the db, attempt to decode it if len(data) > 0 { var ( - newts TransitionState + newts overlay.TransitionState buf = bytes.NewBuffer(data[:]) dec = gob.NewDecoder(buf) ) @@ -416,7 +373,7 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState { // as a verkle database. log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle()) // Start with a fresh state - ts = &TransitionState{Ended: db.triedb.IsVerkle()} + ts = &overlay.TransitionState{Ended: db.triedb.IsVerkle()} } db.TransitionStatePerRoot.Add(root, ts) diff --git a/core/state/database_history.go b/core/state/database_history.go index 705c1dd0aca5..88b893695e89 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -154,10 +155,10 @@ func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } -func (db *HistoricDB) LoadTransitionState(common.Hash) *TransitionState { +func (db *HistoricDB) LoadTransitionState(common.Hash) *overlay.TransitionState { panic("should not be called") } -func (db *HistoricDB) SaveTransitionState(common.Hash, *TransitionState) { +func (db *HistoricDB) SaveTransitionState(common.Hash, *overlay.TransitionState) { panic("should not be called") } diff --git a/core/state/reader.go b/core/state/reader.go index 7f08ef1a562a..d9c846cdc059 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -233,7 +234,7 @@ type trieReader struct { // trieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *TransitionState) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *overlay.TransitionState) (*trieReader, error) { var ( tr Trie err error From 0f37c99d3457791f97995b08c2f6d53f4a5b37c6 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:38:51 +0200 Subject: [PATCH 6/8] make LoadTransitionState cacheless and move it to the overlay pkg Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Gary Rong --- core/overlay/state_transition.go | 107 +++++++++++++++++++++++++++++++ core/state/database.go | 91 +------------------------- core/state/database_history.go | 9 --- 3 files changed, 109 insertions(+), 98 deletions(-) create mode 100644 core/overlay/state_transition.go diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go new file mode 100644 index 000000000000..8cc1b4baaf01 --- /dev/null +++ b/core/overlay/state_transition.go @@ -0,0 +1,107 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package overlay + +import ( + "bytes" + "encoding/gob" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +// TransitionState is a structure that holds the progress markers of the +// translation process. +type TransitionState struct { + CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file + Started, Ended bool + + // Mark whether the storage for an account has been processed. This is useful if the + // maximum number of leaves of the conversion is reached before the whole storage is + // processed. + StorageProcessed bool + + BaseRoot common.Hash // hash of the last read-only MPT base tree +} + +// InTransition returns true if the translation process is in progress. +func (ts *TransitionState) InTransition() bool { + return ts != nil && ts.Started && !ts.Ended +} + +// Transitioned returns true if the translation process has been completed. +func (ts *TransitionState) Transitioned() bool { + return ts != nil && ts.Ended +} + +// Copy returns a deep copy of the TransitionState object. +func (ts *TransitionState) Copy() *TransitionState { + ret := &TransitionState{ + Started: ts.Started, + Ended: ts.Ended, + CurrentSlotHash: ts.CurrentSlotHash, + CurrentPreimageOffset: ts.CurrentPreimageOffset, + StorageProcessed: ts.StorageProcessed, + } + + if ts.CurrentAccountAddress != nil { + ret.CurrentAccountAddress = &common.Address{} + copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) + } + + return ret +} + +// LoadTransitionState retrieves the Verkle transition state associated with +// the given state root hash from the database. +func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle bool) *TransitionState { + var ts *TransitionState + + data, _ := rawdb.ReadVerkleTransitionState(db, root) + + // if a state could be read from the db, attempt to decode it + if len(data) > 0 { + var ( + newts TransitionState + buf = bytes.NewBuffer(data[:]) + dec = gob.NewDecoder(buf) + ) + // Decode transition state + err := dec.Decode(&newts) + if err != nil { + log.Error("failed to decode transition state", "err", err) + return nil + } + ts = &newts + } + + // Fallback that should only happen before the transition + if ts == nil { + // Initialize the first transition state, with the "ended" + // field set to true if the database was created + // as a verkle database. + log.Debug("no transition state found, starting fresh", "is verkle", db) + // Start with a fresh state + ts = &TransitionState{Ended: isVerkle} + } + + return ts +} diff --git a/core/state/database.go b/core/state/database.go index 187dc79bce6c..a12209bea340 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,10 +17,7 @@ package state import ( - "bytes" - "encoding/gob" "fmt" - "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" @@ -30,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/utils" @@ -193,7 +189,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { readers = append(readers, newFlatReader(snap)) } } - ts := db.LoadTransitionState(stateRoot) + ts := overlay.LoadTransitionState(db.TrieDB().Disk(), stateRoot, db.triedb.IsVerkle()) // Configure the state reader using the path database in path mode. // This reader offers improved performance but is optional and only // partially useful if the snapshot data in path database is not @@ -234,7 +230,7 @@ func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithSta // OpenTrie opens the main account trie at a specific root hash. func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.triedb.IsVerkle() { - ts := db.LoadTransitionState(root) + ts := overlay.LoadTransitionState(db.TrieDB().Disk(), root, db.triedb.IsVerkle()) if ts.InTransition() { panic("transition isn't supported yet") } @@ -303,86 +299,3 @@ func mustCopyTrie(t Trie) Trie { panic(fmt.Errorf("unknown trie type %T", t)) } } - -// SaveTransitionState saves the transition state to the cache and commits -// it to the database if it's not already in the cache. -func (db *CachingDB) SaveTransitionState(root common.Hash, ts *overlay.TransitionState) { - if ts == nil { - panic("nil transition state") - } - - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - err := enc.Encode(ts) - if err != nil { - log.Error("failed to encode transition state", "err", err) - return - } - - if !db.TransitionStatePerRoot.Contains(root) { - // Copy so that the address pointer isn't updated after - // it has been saved. - db.TransitionStatePerRoot.Add(root, ts.Copy()) - rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes()) - } else { - // Check that the state is consistent with what is in the cache, - // which is not strictly necessary but a good sanity check. Can - // be removed when the transition is stable. - cachedState, _ := db.TransitionStatePerRoot.Get(root) - if !reflect.DeepEqual(cachedState, ts) { - fmt.Println("transition state mismatch", "cached state", cachedState, "new state", ts) - panic("transition state mismatch") - } - } - - log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) -} - -func (db *CachingDB) LoadTransitionState(root common.Hash) *overlay.TransitionState { - // Try to get the transition state from the cache and - // the DB if it's not there. - ts, ok := db.TransitionStatePerRoot.Get(root) - if !ok { - // Not in the cache, try getting it from the DB - data, _ := rawdb.ReadVerkleTransitionState(db.TrieDB().Disk(), root) - // if err != nil && errors.Is(err, triedb.ErrNotFound) { - // log.Error("failed to read transition state", "err", err) - // return nil - // } - - // if a state could be read from the db, attempt to decode it - if len(data) > 0 { - var ( - newts overlay.TransitionState - buf = bytes.NewBuffer(data[:]) - dec = gob.NewDecoder(buf) - ) - // Decode transition state - err := dec.Decode(&newts) - if err != nil { - log.Error("failed to decode transition state", "err", err) - return nil - } - ts = &newts - } - - // Fallback that should only happen before the transition - if ts == nil { - // Initialize the first transition state, with the "ended" - // field set to true if the database was created - // as a verkle database. - log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle()) - // Start with a fresh state - ts = &overlay.TransitionState{Ended: db.triedb.IsVerkle()} - } - - db.TransitionStatePerRoot.Add(root, ts) - } - - // Copy so that the CurrentAddress pointer in the map - // doesn't get overwritten. - // db.CurrentTransitionState = ts.Copy() - - log.Debug("loaded transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started) - return ts -} diff --git a/core/state/database_history.go b/core/state/database_history.go index 88b893695e89..314c56c4708a 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -21,7 +21,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -154,11 +153,3 @@ func (db *HistoricDB) TrieDB() *triedb.Database { func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } - -func (db *HistoricDB) LoadTransitionState(common.Hash) *overlay.TransitionState { - panic("should not be called") -} - -func (db *HistoricDB) SaveTransitionState(common.Hash, *overlay.TransitionState) { - panic("should not be called") -} From 24f848dac88b670ad66c2265a5e96b499f0d3e0c Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 4 Aug 2025 17:28:58 +0200 Subject: [PATCH 7/8] revert more changes to simplify this PR Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- core/blockchain.go | 8 +++++++- core/genesis.go | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 371163569f55..d52990ec5adc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -341,7 +341,13 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, if cfg == nil { cfg = DefaultConfig() } - triedb := triedb.NewDatabase(db, cfg.triedbConfig(genesis.IsVerkle())) + + // Open trie database with provided config + enableVerkle, err := EnableVerkleAtGenesis(db, genesis) + if err != nil { + return nil, err + } + triedb := triedb.NewDatabase(db, cfg.triedbConfig(enableVerkle)) // Write the supplied genesis to the database if it has not been initialized // yet. The corresponding chain config will be returned, either from the diff --git a/core/genesis.go b/core/genesis.go index 5bf25559447b..f1a620da579d 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -572,6 +572,29 @@ func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types. return block } +// EnableVerkleAtGenesis indicates whether the verkle fork should be activated +// at genesis. This is a temporary solution only for verkle devnet testing, where +// verkle fork is activated at genesis, and the configured activation date has +// already passed. +// +// In production networks (mainnet and public testnets), verkle activation always +// occurs after the genesis block, making this function irrelevant in those cases. +func EnableVerkleAtGenesis(db ethdb.Database, genesis *Genesis) (bool, error) { + if genesis != nil { + if genesis.Config == nil { + return false, errGenesisNoConfig + } + return genesis.Config.EnableVerkleAtGenesis, nil + } + if ghash := rawdb.ReadCanonicalHash(db, 0); ghash != (common.Hash{}) { + chainCfg := rawdb.ReadChainConfig(db, ghash) + if chainCfg != nil { + return chainCfg.EnableVerkleAtGenesis, nil + } + } + return false, nil +} + // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ From 320ea13898e9355db2301e9b22696574fd9a075f Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 5 Aug 2025 09:28:48 +0800 Subject: [PATCH 8/8] core, consensus: avoid transition state read in reader construction --- consensus/beacon/consensus.go | 19 +++++++++++-------- core/overlay/state_transition.go | 8 +++----- core/rawdb/database.go | 2 +- core/state/database.go | 3 +-- core/state/reader.go | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index b09a5cb4c84e..196cbc857ce0 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -353,9 +353,9 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types. // FinalizeAndAssemble implements consensus.Engine, setting the final state and // assembling the block. -func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { +func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, state *state.StateDB, body *types.Body, receipts []*types.Receipt) (*types.Block, error) { if !beacon.IsPoSHeader(header) { - return beacon.ethone.FinalizeAndAssemble(chain, header, statedb, body, receipts) + return beacon.ethone.FinalizeAndAssemble(chain, header, state, body, receipts) } shanghai := chain.Config().IsShanghai(header.Number, header.Time) if shanghai { @@ -369,10 +369,10 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea } } // Finalize and assemble the block. - beacon.Finalize(chain, header, statedb, body) + beacon.Finalize(chain, header, state, body) // Assign the final state root to header. - header.Root = statedb.IntermediateRoot(true) + header.Root = state.IntermediateRoot(true) // Assemble the final block. block := types.NewBlock(header, body, receipts, trie.NewStackTrie(nil)) @@ -380,20 +380,23 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // Create the block witness and attach to block. // This step needs to happen as late as possible to catch all access events. if chain.Config().IsVerkle(header.Number, header.Time) { - //statedb.Database().SaveTransitionState(header.Root, &state.TransitionState{Ended: true}) - keys := statedb.AccessEvents().Keys() + keys := state.AccessEvents().Keys() // Open the pre-tree to prove the pre-state against parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1) if parent == nil { return nil, fmt.Errorf("nil parent header for block %d", header.Number) } - preTrie, err := statedb.Database().OpenTrie(parent.Root) + preTrie, err := state.Database().OpenTrie(parent.Root) if err != nil { return nil, fmt.Errorf("error opening pre-state tree root: %w", err) } + postTrie := state.GetTrie() + if postTrie == nil { + return nil, errors.New("post-state tree is not available") + } vktPreTrie, okpre := preTrie.(*trie.VerkleTrie) - vktPostTrie, okpost := statedb.GetTrie().(*trie.VerkleTrie) + vktPostTrie, okpost := postTrie.(*trie.VerkleTrie) // The witness is only attached iff both parent and current block are // using verkle tree. diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index 8cc1b4baaf01..90b5c9431a43 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -61,12 +61,10 @@ func (ts *TransitionState) Copy() *TransitionState { CurrentPreimageOffset: ts.CurrentPreimageOffset, StorageProcessed: ts.StorageProcessed, } - if ts.CurrentAccountAddress != nil { - ret.CurrentAccountAddress = &common.Address{} - copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:]) + addr := *ts.CurrentAccountAddress + ret.CurrentAccountAddress = &addr } - return ret } @@ -99,9 +97,9 @@ func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle boo // field set to true if the database was created // as a verkle database. log.Debug("no transition state found, starting fresh", "is verkle", db) + // Start with a fresh state ts = &TransitionState{Ended: isVerkle} } - return ts } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 9681c39c582e..be0e8973cfa4 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -599,7 +599,7 @@ var knownMetadataKeys = [][]byte{ snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, - filterMapsRangeKey, headStateHistoryIndexKey, + filterMapsRangeKey, headStateHistoryIndexKey, VerkleTransitionStatePrefix, } // printChainMetadata prints out chain metadata to stderr. diff --git a/core/state/database.go b/core/state/database.go index a12209bea340..b46e5d500d64 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -189,7 +189,6 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { readers = append(readers, newFlatReader(snap)) } } - ts := overlay.LoadTransitionState(db.TrieDB().Disk(), stateRoot, db.triedb.IsVerkle()) // Configure the state reader using the path database in path mode. // This reader offers improved performance but is optional and only // partially useful if the snapshot data in path database is not @@ -202,7 +201,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } // Configure the trie reader, which is expected to be available as the // gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache, ts) + tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) if err != nil { return nil, err } diff --git a/core/state/reader.go b/core/state/reader.go index d9c846cdc059..f56a1bfae1e9 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -234,14 +233,15 @@ type trieReader struct { // trieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *overlay.TransitionState) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { var ( tr Trie err error ) - if !ts.Transitioned() && !ts.InTransition() { + if !db.IsVerkle() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { + // TODO @gballet determine the trie type (verkle or overlay) by transition state tr, err = trie.NewVerkleTrie(root, db, cache) } if err != nil {