From a4a4881b6715df82f57975d1531866bceb8f2b10 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 1 May 2021 11:46:33 +0200 Subject: [PATCH 1/5] cmd/geth, eth, core: snapshot dump + unify with trie dump --- cmd/geth/chaincmd.go | 98 +++++++++++++++++++++++++++++--------------- cmd/geth/snapshot.go | 90 ++++++++++++++++++++++++++++++++++++++++ cmd/utils/flags.go | 7 +++- core/state/dump.go | 64 +++++++++++++++++++---------- eth/api.go | 20 +++++++-- 5 files changed, 218 insertions(+), 61 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d00b4bc1f61b..ff6b08432ed2 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -18,7 +18,9 @@ package main import ( "encoding/json" + "errors" "fmt" + "github.com/ethereum/go-ethereum/common/hexutil" "os" "runtime" "strconv" @@ -31,8 +33,10 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -152,20 +156,20 @@ The export-preimages command export hash preimages to an RLP encoded stream`, Action: utils.MigrateFlags(dump), Name: "dump", Usage: "Dump a specific block from storage", - ArgsUsage: "[ | ]...", + ArgsUsage: "[? | ]", Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.SyncModeFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.IncludeIncompletesFlag, + utils.StartKeyFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` -The arguments are interpreted as block numbers or hashes. -Use "ethereum dump 0" to dump the genesis block.`, +This command dumps out the state for a given block (or latest, if none provided). +`, } ) @@ -373,47 +377,73 @@ func exportPreimages(ctx *cli.Context) error { return nil } -func dump(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - +func parseDumpOptions(ctx *cli.Context, stack *node.Node) (*state.DumpOptions, ethdb.Database, common.Hash, error) { db := utils.MakeChainDatabase(ctx, stack, true) - for _, arg := range ctx.Args() { - var header *types.Header + var header *types.Header + if ctx.NArg() > 1 { + return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + } else if ctx.NArg() == 1 { + arg := ctx.Args().First() if hashish(arg) { hash := common.HexToHash(arg) - number := rawdb.ReadHeaderNumber(db, hash) - if number != nil { + if number := rawdb.ReadHeaderNumber(db, hash); number != nil { header = rawdb.ReadHeader(db, hash, *number) + } else { + return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, _ := strconv.Atoi(arg) - hash := rawdb.ReadCanonicalHash(db, uint64(number)) - if hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) - } - } - if header == nil { - fmt.Println("{}") - utils.Fatalf("block not found") - } else { - state, err := state.New(header.Root, state.NewDatabase(db), nil) + number, err := strconv.Atoi(arg) if err != nil { - utils.Fatalf("could not create new state: %v", err) + return nil, nil, common.Hash{}, err } - excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name) - excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name) - includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name) - if ctx.Bool(utils.IterativeOutputFlag.Name) { - state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout)) + if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, uint64(number)) } else { - if includeMissing { - fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" + - " otherwise the accounts will overwrite each other in the resulting mapping.") - } - fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false)) + return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } } + } else { + // Use latest + header = rawdb.ReadHeadHeader(db) + } + if header == nil { + return nil, nil, common.Hash{}, errors.New("no head block found") + } + var opts = &state.DumpOptions{ + SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), + SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name), + Start: common.FromHex(ctx.String(utils.StartKeyFlag.Name)), + } + log.Info("Dump options", "block", header.Number, + "hash", header.Hash().Hex(), + "code", !opts.SkipCode, + "storage", !opts.SkipStorage, + "start", hexutil.Encode(opts.Start)) + return opts, db, header.Root, nil +} + +func dump(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + opts, db, root, err := parseDumpOptions(ctx, stack) + if err != nil { + return err + } + state, err := state.New(root, state.NewDatabase(db), nil) + if err != nil { + return err + } + if ctx.Bool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(opts, json.NewEncoder(os.Stdout)) + } else { + if opts.OnlyWithAddresses { + fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+ + " otherwise the accounts will overwrite each other in the resulting mapping.") + return fmt.Errorf("incompatible options") + } + fmt.Println(string(state.Dump(opts))) } return nil } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1af458af207e..c3190c899011 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,7 +18,9 @@ package main import ( "bytes" + "encoding/json" "errors" + "os" "time" "github.com/ethereum/go-ethereum/cmd/utils" @@ -142,6 +144,30 @@ verification. The default checking target is the HEAD state. It's basically iden to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. +`, + }, + { + Name: "dump", + Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", + ArgsUsage: "[? | ]", + Action: utils.MigrateFlags(dumpState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, + utils.StartKeyFlag, + }, + Description: ` +This command is semantically equivalent to 'geth dump', but uses the snapshots +as the backend data source, making this command a lot faster. + +The argument is interpreted as block number or hash. If none is provided, the latest +block is used. `, }, }, @@ -430,3 +456,67 @@ func parseRoot(input string) (common.Hash, error) { } return h, nil } + +func dumpState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + opts, db, root, err := parseDumpOptions(ctx, stack) + if err != nil { + return err + } + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) + if err != nil { + return err + } + accIt, err := snaptree.AccountIterator(root, common.BytesToHash(opts.Start)) + if err != nil { + return err + } + defer accIt.Release() + log.Info("Snapshot iteration started", "root", root) + var ( + start = time.Now() + logged = time.Now() + accounts uint64 + ) + enc := json.NewEncoder(os.Stdout) + enc.Encode(struct { + Root common.Hash `json:"root"` + }{root}) + for accIt.Next() { + account, err := snapshot.FullAccount(accIt.Account()) + if err != nil { + return err + } + da := &state.DumpAccount{ + Balance: account.Balance.String(), + Nonce: account.Nonce, + Root: common.Bytes2Hex(account.Root), + CodeHash: common.Bytes2Hex(account.CodeHash), + SecureKey: accIt.Hash().Bytes(), + } + if !opts.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { + da.Code = common.Bytes2Hex(rawdb.ReadCode(db, common.BytesToHash(account.CodeHash))) + } + if !opts.SkipStorage { + stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) + if err != nil { + return err + } + for stIt.Next() { + da.Storage[stIt.Hash()] = common.Bytes2Hex(stIt.Slot()) + } + } + enc.Encode(da) + accounts++ + if time.Since(logged) > 10*time.Second { + log.Info("Snapshot iteration in progress", "at", accIt.Hash(), "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + log.Info("Snapshot iteration complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5c0bba4655f1..03c7f72e8393 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -184,7 +184,7 @@ var ( Name: "exitwhensynced", Usage: "Exits after block synchronisation completes", } - IterativeOutputFlag = cli.BoolFlag{ + IterativeOutputFlag = cli.BoolTFlag{ Name: "iterative", Usage: "Print streaming JSON iteratively, delimited by newlines", } @@ -200,6 +200,11 @@ var ( Name: "nocode", Usage: "Exclude contract code (save db lookups)", } + StartKeyFlag = cli.StringFlag{ + Name: "start", + Usage: "Start position", + Value: "0x00", + } defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", diff --git a/core/state/dump.go b/core/state/dump.go index b25da714fd11..e4c617baa742 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -19,6 +19,7 @@ package state import ( "encoding/json" "fmt" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -27,6 +28,14 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +type DumpOptions struct { + SkipCode bool + SkipStorage bool + OnlyWithAddresses bool + Start []byte + Max uint64 +} + // DumpCollector interface which the state trie calls during iteration type DumpCollector interface { // OnRoot is called with the state root @@ -111,38 +120,42 @@ func (d iterativeDump) OnRoot(root common.Hash) { }{root}) } -func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) { - missingPreimages := 0 +func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey []byte) { + var ( + missingPreimages int + accounts uint64 + start = time.Now() + logged = time.Now() + ) c.OnRoot(s.trie.Hash()) - - var count int - it := trie.NewIterator(s.trie.NodeIterator(start)) + it := trie.NewIterator(s.trie.NodeIterator(opts.Start)) for it.Next() { var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } account := DumpAccount{ - Balance: data.Balance.String(), - Nonce: data.Nonce, - Root: common.Bytes2Hex(data.Root[:]), - CodeHash: common.Bytes2Hex(data.CodeHash), + Balance: data.Balance.String(), + Nonce: data.Nonce, + Root: common.Bytes2Hex(data.Root[:]), + CodeHash: common.Bytes2Hex(data.CodeHash), + SecureKey: it.Key, } addrBytes := s.trie.GetKey(it.Key) if addrBytes == nil { // Preimage missing missingPreimages++ - if excludeMissingPreimages { + if opts.OnlyWithAddresses { continue } account.SecureKey = it.Key } addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) - if !excludeCode { + if !opts.SkipCode { account.Code = common.Bytes2Hex(obj.Code(s.db)) } - if !excludeStorage { + if !opts.SkipStorage { account.Storage = make(map[common.Hash]string) storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) for storageIt.Next() { @@ -155,8 +168,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, } } c.OnAccount(addr, account) - count++ - if maxResults > 0 && count >= maxResults { + accounts++ + if time.Since(logged) > 10*time.Second { + log.Info("Trie iteration in progress", "at", it.Key, "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + if opts.Max > 0 && accounts >= opts.Max { if it.Next() { nextKey = it.Key } @@ -166,22 +184,24 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } + log.Info("Trie iteration complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) return nextKey } // RawDump returns the entire state an a single large object -func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump { +func (s *StateDB) RawDump(opts *DumpOptions) Dump { dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } - s.DumpToCollector(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) + s.DumpToCollector(dump, opts) return *dump } // Dump returns a JSON string representing the entire state as a single json-object -func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte { - dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages) +func (s *StateDB) Dump(opts *DumpOptions) []byte { + dump := s.RawDump(opts) json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("Dump err", err) @@ -190,15 +210,15 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool } // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout -func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { - s.DumpToCollector(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) +func (s *StateDB) IterativeDump(opts *DumpOptions, output *json.Encoder) { + s.DumpToCollector(iterativeDump{output}, opts) } // IteratorDump dumps out a batch of accounts starts with the given start key -func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump { +func (s *StateDB) IteratorDump(opts *DumpOptions) IteratorDump { iterator := &IteratorDump{ Accounts: make(map[common.Address]DumpAccount), } - iterator.Next = s.DumpToCollector(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults) + iterator.Next = s.DumpToCollector(iterator, opts) return *iterator } diff --git a/eth/api.go b/eth/api.go index 7387459c94e0..08ccdaf5f193 100644 --- a/eth/api.go +++ b/eth/api.go @@ -264,12 +264,17 @@ func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { // DumpBlock retrieves the entire state of the database at a given block. func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { + opts := &state.DumpOptions{ + OnlyWithAddresses: true, + Start: nil, + Max: AccountRangeMaxResults, // Sanity limit over RPC + } if blockNr == rpc.PendingBlockNumber { // If we're dumping the pending state, we need to request // both the pending block as well as the pending state from // the miner and operate on those _, stateDb := api.eth.miner.Pending() - return stateDb.RawDump(false, false, true), nil + return stateDb.RawDump(opts), nil } var block *types.Block if blockNr == rpc.LatestBlockNumber { @@ -284,7 +289,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error if err != nil { return state.Dump{}, err } - return stateDb.RawDump(false, false, true), nil + return stateDb.RawDump(opts), nil } // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over @@ -386,10 +391,17 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta return state.IteratorDump{}, errors.New("either block number or block hash must be specified") } + opts := &state.DumpOptions{ + SkipCode: nocode, + SkipStorage: nostorage, + OnlyWithAddresses: !incompletes, + Start: start, + Max: uint64(maxResults), + } if maxResults > AccountRangeMaxResults || maxResults <= 0 { - maxResults = AccountRangeMaxResults + opts.Max = AccountRangeMaxResults } - return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil + return stateDb.IteratorDump(opts), nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. From f75b65c11641922f4d33b264456c374c853f1a5e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 1 May 2021 21:20:54 +0200 Subject: [PATCH 2/5] cmd/evm: dump API fixes --- cmd/evm/internal/t8ntool/transition.go | 4 ++-- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index fedcd1243569..73928b1e6a2e 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -212,14 +212,14 @@ func Main(ctx *cli.Context) error { // Iterate over all the tests, run them and aggregate the results // Run the test and aggregate the result - state, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) + s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { return err } body, _ := rlp.EncodeToBytes(txs) // Dump the excution result collector := make(Alloc) - state.DumpToCollector(collector, false, false, false, nil, -1) + s.DumpToCollector(collector, &state.DumpOptions{}) return dispatchOutput(ctx, baseDir, result, collector, body) } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 4063767cb8f4..e70f4263d69a 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -270,7 +270,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump(false, false, true))) + fmt.Println(string(statedb.Dump(&state.DumpOptions{}))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index c4df936c75ff..be00972152ba 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -98,16 +98,16 @@ func stateTestCmd(ctx *cli.Context) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - _, state, err := test.Run(st, cfg, false) + _, s, err := test.Run(st, cfg, false) // print state root for evmlab tracing - if ctx.GlobalBool(MachineFlag.Name) && state != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + if ctx.GlobalBool(MachineFlag.Name) && s != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() - if ctx.GlobalBool(DumpFlag.Name) && state != nil { - dump := state.RawDump(false, false, true) + if ctx.GlobalBool(DumpFlag.Name) && s != nil { + dump := s.RawDump(&state.DumpOptions{}) result.State = &dump } } From c934474fdc9b96e75e47e8995100c7dc4b1efab9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 1 May 2021 22:29:43 +0200 Subject: [PATCH 3/5] cmd/geth, core, eth: fix some remaining errors --- cmd/geth/chaincmd.go | 2 +- core/state/state_test.go | 11 +++++++---- eth/api_test.go | 23 +++++++++++++++++------ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ff6b08432ed2..05b51c9af889 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ethereum/go-ethereum/common/hexutil" "os" "runtime" "strconv" @@ -29,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" diff --git a/core/state/state_test.go b/core/state/state_test.go index 956653146647..9db1b424b423 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -57,7 +57,7 @@ func TestDump(t *testing.T) { s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie - got := string(s.state.Dump(false, false, true)) + got := string(s.state.Dump(&DumpOptions{})) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { @@ -65,20 +65,23 @@ func TestDump(t *testing.T) { "balance": "22", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d" }, "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62" }, "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303" + "code": "03030303030303", + "key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1" } } }` diff --git a/eth/api_test.go b/eth/api_test.go index b44eed40bcd2..4f2a106cea18 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -34,7 +34,13 @@ import ( var dumper = spew.ConfigState{Indent: " "} func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { - result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum) + result := statedb.IteratorDump(&state.DumpOptions{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: false, + Start: start.Bytes(), + Max: uint64(requestedNum), + }) if len(result.Accounts) != expectedNum { t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) @@ -131,12 +137,17 @@ func TestEmptyAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) - state, _ = state.New(common.Hash{}, statedb, nil) + statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + st, _ = state.New(common.Hash{}, statedb, nil) ) - state.Commit(true) - state.IntermediateRoot(true) - results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults) + st.Commit(true) + st.IntermediateRoot(true) + results := st.IteratorDump(&state.DumpOptions{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: true, + Max: uint64(AccountRangeMaxResults), + }) if bytes.Equal(results.Next, (common.Hash{}).Bytes()) { t.Fatalf("Empty results should not return a second page") } From c9c4a25b95240b0cb709b859015e2ddbc411b840 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 11 May 2021 12:48:56 +0200 Subject: [PATCH 4/5] cmd/evm: dump - add limit, support address startkey, address review concerns --- cmd/evm/internal/t8ntool/transition.go | 2 +- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- cmd/geth/chaincmd.go | 41 ++++++++++++++++++-------- cmd/geth/snapshot.go | 12 +++++--- cmd/utils/flags.go | 9 ++++-- core/state/dump.go | 22 +++++++------- core/state/state_test.go | 2 +- eth/api.go | 4 +-- eth/api_test.go | 4 +-- 10 files changed, 63 insertions(+), 37 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 73928b1e6a2e..efddd03b129d 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -219,7 +219,7 @@ func Main(ctx *cli.Context) error { body, _ := rlp.EncodeToBytes(txs) // Dump the excution result collector := make(Alloc) - s.DumpToCollector(collector, &state.DumpOptions{}) + s.DumpToCollector(collector, &state.DumpConfig{}) return dispatchOutput(ctx, baseDir, result, collector, body) } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index e70f4263d69a..9edd6bfbb69e 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -270,7 +270,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump(&state.DumpOptions{}))) + fmt.Println(string(statedb.Dump(&state.DumpConfig{}))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index be00972152ba..e12f44141820 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -107,7 +107,7 @@ func stateTestCmd(ctx *cli.Context) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if ctx.GlobalBool(DumpFlag.Name) && s != nil { - dump := s.RawDump(&state.DumpOptions{}) + dump := s.RawDump(&state.DumpConfig{}) result.State = &dump } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 05b51c9af889..a99ecb665739 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -165,6 +166,7 @@ The export-preimages command export hash preimages to an RLP encoded stream`, utils.ExcludeStorageFlag, utils.IncludeIncompletesFlag, utils.StartKeyFlag, + utils.DumpLimitFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -377,12 +379,13 @@ func exportPreimages(ctx *cli.Context) error { return nil } -func parseDumpOptions(ctx *cli.Context, stack *node.Node) (*state.DumpOptions, ethdb.Database, common.Hash, error) { +func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { db := utils.MakeChainDatabase(ctx, stack, true) var header *types.Header if ctx.NArg() > 1 { return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) - } else if ctx.NArg() == 1 { + } + if ctx.NArg() == 1 { arg := ctx.Args().First() if hashish(arg) { hash := common.HexToHash(arg) @@ -409,25 +412,39 @@ func parseDumpOptions(ctx *cli.Context, stack *node.Node) (*state.DumpOptions, e if header == nil { return nil, nil, common.Hash{}, errors.New("no head block found") } - var opts = &state.DumpOptions{ + startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) + var start common.Hash + switch len(startArg) { + case 0: // common.Hash + case 32: + start = common.BytesToHash(startArg) + case 20: + start = crypto.Keccak256Hash(startArg) + log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) + default: + return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + } + var conf = &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name), - Start: common.FromHex(ctx.String(utils.StartKeyFlag.Name)), + Start: start.Bytes(), + Max: ctx.Uint64(utils.DumpLimitFlag.Name), } log.Info("Dump options", "block", header.Number, "hash", header.Hash().Hex(), - "code", !opts.SkipCode, - "storage", !opts.SkipStorage, - "start", hexutil.Encode(opts.Start)) - return opts, db, header.Root, nil + "code", !conf.SkipCode, + "storage", !conf.SkipStorage, + "start", hexutil.Encode(conf.Start), + "limit", conf.Max) + return conf, db, header.Root, nil } func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - opts, db, root, err := parseDumpOptions(ctx, stack) + conf, db, root, err := parseDumpConfig(ctx, stack) if err != nil { return err } @@ -436,14 +453,14 @@ func dump(ctx *cli.Context) error { return err } if ctx.Bool(utils.IterativeOutputFlag.Name) { - state.IterativeDump(opts, json.NewEncoder(os.Stdout)) + state.IterativeDump(conf, json.NewEncoder(os.Stdout)) } else { - if opts.OnlyWithAddresses { + if conf.OnlyWithAddresses { fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+ " otherwise the accounts will overwrite each other in the resulting mapping.") return fmt.Errorf("incompatible options") } - fmt.Println(string(state.Dump(opts))) + fmt.Println(string(state.Dump(conf))) } return nil } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index c3190c899011..5f6d70374ce7 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -161,6 +161,7 @@ It's also usable without snapshot enabled. utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.StartKeyFlag, + utils.DumpLimitFlag, }, Description: ` This command is semantically equivalent to 'geth dump', but uses the snapshots @@ -461,7 +462,7 @@ func dumpState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - opts, db, root, err := parseDumpOptions(ctx, stack) + conf, db, root, err := parseDumpConfig(ctx, stack) if err != nil { return err } @@ -469,7 +470,7 @@ func dumpState(ctx *cli.Context) error { if err != nil { return err } - accIt, err := snaptree.AccountIterator(root, common.BytesToHash(opts.Start)) + accIt, err := snaptree.AccountIterator(root, common.BytesToHash(conf.Start)) if err != nil { return err } @@ -496,10 +497,10 @@ func dumpState(ctx *cli.Context) error { CodeHash: common.Bytes2Hex(account.CodeHash), SecureKey: accIt.Hash().Bytes(), } - if !opts.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { + if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { da.Code = common.Bytes2Hex(rawdb.ReadCode(db, common.BytesToHash(account.CodeHash))) } - if !opts.SkipStorage { + if !conf.SkipStorage { stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) if err != nil { return err @@ -515,6 +516,9 @@ func dumpState(ctx *cli.Context) error { "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } + if conf.Max > 0 && accounts >= conf.Max { + break + } } log.Info("Snapshot iteration complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 03c7f72e8393..d3fb3f2cbd1b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -202,8 +202,13 @@ var ( } StartKeyFlag = cli.StringFlag{ Name: "start", - Usage: "Start position", - Value: "0x00", + Usage: "Start position. Either a hash or address", + Value: "0x0000000000000000000000000000000000000000000000000000000000000000", + } + DumpLimitFlag = cli.Uint64Flag{ + Name: "limit", + Usage: "Max number of elements (0 = no limit)", + Value: 0, } defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = TextMarshalerFlag{ diff --git a/core/state/dump.go b/core/state/dump.go index e4c617baa742..1615b74d8e82 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -type DumpOptions struct { +type DumpConfig struct { SkipCode bool SkipStorage bool OnlyWithAddresses bool @@ -120,7 +120,7 @@ func (d iterativeDump) OnRoot(root common.Hash) { }{root}) } -func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey []byte) { +func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) { var ( missingPreimages int accounts uint64 @@ -128,7 +128,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey [ logged = time.Now() ) c.OnRoot(s.trie.Hash()) - it := trie.NewIterator(s.trie.NodeIterator(opts.Start)) + it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { @@ -145,17 +145,17 @@ func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey [ if addrBytes == nil { // Preimage missing missingPreimages++ - if opts.OnlyWithAddresses { + if conf.OnlyWithAddresses { continue } account.SecureKey = it.Key } addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) - if !opts.SkipCode { + if !conf.SkipCode { account.Code = common.Bytes2Hex(obj.Code(s.db)) } - if !opts.SkipStorage { + if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) for storageIt.Next() { @@ -174,7 +174,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey [ "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } - if opts.Max > 0 && accounts >= opts.Max { + if conf.Max > 0 && accounts >= conf.Max { if it.Next() { nextKey = it.Key } @@ -191,7 +191,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, opts *DumpOptions) (nextKey [ } // RawDump returns the entire state an a single large object -func (s *StateDB) RawDump(opts *DumpOptions) Dump { +func (s *StateDB) RawDump(opts *DumpConfig) Dump { dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } @@ -200,7 +200,7 @@ func (s *StateDB) RawDump(opts *DumpOptions) Dump { } // Dump returns a JSON string representing the entire state as a single json-object -func (s *StateDB) Dump(opts *DumpOptions) []byte { +func (s *StateDB) Dump(opts *DumpConfig) []byte { dump := s.RawDump(opts) json, err := json.MarshalIndent(dump, "", " ") if err != nil { @@ -210,12 +210,12 @@ func (s *StateDB) Dump(opts *DumpOptions) []byte { } // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout -func (s *StateDB) IterativeDump(opts *DumpOptions, output *json.Encoder) { +func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) { s.DumpToCollector(iterativeDump{output}, opts) } // IteratorDump dumps out a batch of accounts starts with the given start key -func (s *StateDB) IteratorDump(opts *DumpOptions) IteratorDump { +func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump { iterator := &IteratorDump{ Accounts: make(map[common.Address]DumpAccount), } diff --git a/core/state/state_test.go b/core/state/state_test.go index 9db1b424b423..ae279b8bd64d 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -57,7 +57,7 @@ func TestDump(t *testing.T) { s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie - got := string(s.state.Dump(&DumpOptions{})) + got := string(s.state.Dump(&DumpConfig{})) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { diff --git a/eth/api.go b/eth/api.go index 08ccdaf5f193..4f60b6388b75 100644 --- a/eth/api.go +++ b/eth/api.go @@ -264,7 +264,7 @@ func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { // DumpBlock retrieves the entire state of the database at a given block. func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { - opts := &state.DumpOptions{ + opts := &state.DumpConfig{ OnlyWithAddresses: true, Start: nil, Max: AccountRangeMaxResults, // Sanity limit over RPC @@ -391,7 +391,7 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta return state.IteratorDump{}, errors.New("either block number or block hash must be specified") } - opts := &state.DumpOptions{ + opts := &state.DumpConfig{ SkipCode: nocode, SkipStorage: nostorage, OnlyWithAddresses: !incompletes, diff --git a/eth/api_test.go b/eth/api_test.go index 4f2a106cea18..39a1d5846004 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -34,7 +34,7 @@ import ( var dumper = spew.ConfigState{Indent: " "} func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { - result := statedb.IteratorDump(&state.DumpOptions{ + result := statedb.IteratorDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, OnlyWithAddresses: false, @@ -142,7 +142,7 @@ func TestEmptyAccountRange(t *testing.T) { ) st.Commit(true) st.IntermediateRoot(true) - results := st.IteratorDump(&state.DumpOptions{ + results := st.IteratorDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, OnlyWithAddresses: true, From f1185e2a49d8474895b7fb49c1572fb9c9773ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 12 May 2021 09:52:45 +0300 Subject: [PATCH 5/5] cmd, core/state, eth: minor polishes, fix snap dump crash, unify format --- cmd/evm/internal/t8ntool/transition.go | 5 ++--- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- cmd/geth/chaincmd.go | 9 +++------ cmd/geth/snapshot.go | 17 +++++++++------- core/state/dump.go | 28 +++++++++++++++++--------- core/state/state_test.go | 16 +++++++-------- eth/api.go | 1 - 8 files changed, 44 insertions(+), 36 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index efddd03b129d..22cd0dd851ee 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -219,9 +219,8 @@ func Main(ctx *cli.Context) error { body, _ := rlp.EncodeToBytes(txs) // Dump the excution result collector := make(Alloc) - s.DumpToCollector(collector, &state.DumpConfig{}) + s.DumpToCollector(collector, nil) return dispatchOutput(ctx, baseDir, result, collector, body) - } // txWithKey is a helper-struct, to allow us to use the types.Transaction along with @@ -303,7 +302,7 @@ func (g Alloc) OnAccount(addr common.Address, dumpAccount state.DumpAccount) { } } genesisAccount := core.GenesisAccount{ - Code: common.FromHex(dumpAccount.Code), + Code: dumpAccount.Code, Storage: storage, Balance: balance, Nonce: dumpAccount.Nonce, diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 9edd6bfbb69e..2d890ef1a230 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -270,7 +270,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump(&state.DumpConfig{}))) + fmt.Println(string(statedb.Dump(nil))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index e12f44141820..d8bc4eae8001 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -107,7 +107,7 @@ func stateTestCmd(ctx *cli.Context) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if ctx.GlobalBool(DumpFlag.Name) && s != nil { - dump := s.RawDump(&state.DumpConfig{}) + dump := s.RawDump(nil) result.State = &dump } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index a99ecb665739..b9bd88e21362 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -431,12 +431,9 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth Start: start.Bytes(), Max: ctx.Uint64(utils.DumpLimitFlag.Name), } - log.Info("Dump options", "block", header.Number, - "hash", header.Hash().Hex(), - "code", !conf.SkipCode, - "storage", !conf.SkipStorage, - "start", hexutil.Encode(conf.Start), - "limit", conf.Max) + log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), + "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, + "start", hexutil.Encode(conf.Start), "limit", conf.Max) return conf, db, header.Root, nil } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 5f6d70374ce7..35d027fb1648 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -475,7 +475,8 @@ func dumpState(ctx *cli.Context) error { return err } defer accIt.Release() - log.Info("Snapshot iteration started", "root", root) + + log.Info("Snapshot dumping started", "root", root) var ( start = time.Now() logged = time.Now() @@ -493,14 +494,16 @@ func dumpState(ctx *cli.Context) error { da := &state.DumpAccount{ Balance: account.Balance.String(), Nonce: account.Nonce, - Root: common.Bytes2Hex(account.Root), - CodeHash: common.Bytes2Hex(account.CodeHash), + Root: account.Root, + CodeHash: account.CodeHash, SecureKey: accIt.Hash().Bytes(), } if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { - da.Code = common.Bytes2Hex(rawdb.ReadCode(db, common.BytesToHash(account.CodeHash))) + da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) } if !conf.SkipStorage { + da.Storage = make(map[common.Hash]string) + stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) if err != nil { return err @@ -511,8 +514,8 @@ func dumpState(ctx *cli.Context) error { } enc.Encode(da) accounts++ - if time.Since(logged) > 10*time.Second { - log.Info("Snapshot iteration in progress", "at", accIt.Hash(), "accounts", accounts, + if time.Since(logged) > 8*time.Second { + log.Info("Snapshot dumping in progress", "at", accIt.Hash(), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } @@ -520,7 +523,7 @@ func dumpState(ctx *cli.Context) error { break } } - log.Info("Snapshot iteration complete", "accounts", accounts, + log.Info("Snapshot dumping complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nil } diff --git a/core/state/dump.go b/core/state/dump.go index 1615b74d8e82..00faa4ed6a6b 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -28,6 +28,8 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// DumpConfig is a set of options to control what portions of the statewill be +// iterated and collected. type DumpConfig struct { SkipCode bool SkipStorage bool @@ -48,9 +50,9 @@ type DumpCollector interface { type DumpAccount struct { Balance string `json:"balance"` Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code,omitempty"` + Root hexutil.Bytes `json:"root"` + CodeHash hexutil.Bytes `json:"codeHash"` + Code hexutil.Bytes `json:"code,omitempty"` Storage map[common.Hash]string `json:"storage,omitempty"` Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key @@ -120,14 +122,22 @@ func (d iterativeDump) OnRoot(root common.Hash) { }{root}) } +// DumpToCollector iterates the state according to the given options and inserts +// the items into a collector for aggregation or serialization. func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) { + // Sanitize the input to allow nil configs + if conf == nil { + conf = new(DumpConfig) + } var ( missingPreimages int accounts uint64 start = time.Now() logged = time.Now() ) + log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) + it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { var data Account @@ -137,8 +147,8 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] account := DumpAccount{ Balance: data.Balance.String(), Nonce: data.Nonce, - Root: common.Bytes2Hex(data.Root[:]), - CodeHash: common.Bytes2Hex(data.CodeHash), + Root: data.Root[:], + CodeHash: data.CodeHash, SecureKey: it.Key, } addrBytes := s.trie.GetKey(it.Key) @@ -153,7 +163,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) if !conf.SkipCode { - account.Code = common.Bytes2Hex(obj.Code(s.db)) + account.Code = obj.Code(s.db) } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) @@ -169,8 +179,8 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } c.OnAccount(addr, account) accounts++ - if time.Since(logged) > 10*time.Second { - log.Info("Trie iteration in progress", "at", it.Key, "accounts", accounts, + if time.Since(logged) > 8*time.Second { + log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } @@ -184,7 +194,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } - log.Info("Trie iteration complete", "accounts", accounts, + log.Info("Trie dumping complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nextKey diff --git a/core/state/state_test.go b/core/state/state_test.go index ae279b8bd64d..0a55d7781fd1 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -57,30 +57,30 @@ func TestDump(t *testing.T) { s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie - got := string(s.state.Dump(&DumpConfig{})) + got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { "0x0000000000000000000000000000000000000001": { "balance": "22", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", "key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d" }, "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", "key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62" }, "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303", + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0x87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", + "code": "0x03030303030303", "key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1" } } diff --git a/eth/api.go b/eth/api.go index 4f60b6388b75..6a22c9e41680 100644 --- a/eth/api.go +++ b/eth/api.go @@ -266,7 +266,6 @@ func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { opts := &state.DumpConfig{ OnlyWithAddresses: true, - Start: nil, Max: AccountRangeMaxResults, // Sanity limit over RPC } if blockNr == rpc.PendingBlockNumber {