diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6fb52aaa5..e122be5f5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -106,6 +106,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, @@ -263,7 +264,7 @@ func importChain(ctx *cli.Context) error { utils.Fatalf("This command requires an argument.") } // Start metrics export if enabled - utils.SetupMetrics(ctx) + utils.SetupMetrics(ctx, makeMetricsConfig(ctx)) // Start system runtime metrics collection go metrics.CollectProcessMetrics(3 * time.Second) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 9851e217e..fc92afbf1 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -252,6 +252,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) } + if ctx.GlobalIsSet(utils.MetricsInfluxDBIntervalFlag.Name) { + cfg.Metrics.InfluxDBInterval = ctx.GlobalDuration(utils.MetricsInfluxDBIntervalFlag.Name) + } if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { cfg.Metrics.EnableInfluxDBV2 = ctx.GlobalBool(utils.MetricsEnableInfluxDBV2Flag.Name) } @@ -266,6 +269,21 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { } } +// makeMetricsConfig returns a metrics.Config populated from the TOML config +// file (when --config is provided) and any metrics-related CLI flags. It is +// used by callsites that need the resolved metrics configuration before the +// full node (makeConfigNode) has been initialised. +func makeMetricsConfig(ctx *cli.Context) metrics.Config { + cfg := gethConfig{Metrics: metrics.DefaultConfig} + if file := ctx.GlobalString(configFileFlag.Name); file != "" { + if err := loadConfig(file, &cfg); err != nil { + utils.Fatalf("%v", err) + } + } + applyMetricConfig(ctx, &cfg) + return cfg.Metrics +} + func deprecated(field string) bool { switch field { case "ethconfig.Config.EVMInterpreter": diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 44af809e1..a72071d8f 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -41,6 +41,34 @@ import ( "github.com/morph-l2/go-ethereum/trie" ) +var ( + inspectTrieTopFlag = cli.IntFlag{ + Name: "top", + Usage: "Number of storage tries to include in the top-N ranking (must be > 0)", + Value: 10, + } + inspectTrieExcludeStorageFlag = cli.BoolFlag{ + Name: "exclude-storage", + Usage: "Skip walking per-account storage tries", + } + inspectTrieOutputFileFlag = cli.StringFlag{ + Name: "output", + Usage: "Write the report as JSON to the given file (default: stdout summary)", + } + inspectTrieDumpPathFlag = cli.StringFlag{ + Name: "dump-path", + Usage: "Path for the pass-1 trie dump file (default: /trie-dump.bin)", + } + inspectTrieSummarizeFlag = cli.StringFlag{ + Name: "summarize", + Usage: "Summarize an existing trie dump file (skips trie traversal)", + } + inspectTrieContractFlag = cli.StringFlag{ + Name: "contract", + Usage: "Inspect a single contract's storage footprint by 0x address", + } +) + var ( removedbCommand = cli.Command{ Action: utils.MigrateFlags(removeDB), @@ -61,6 +89,7 @@ Remove blockchain and state databases`, Category: "DATABASE COMMANDS", Subcommands: []cli.Command{ dbInspectCmd, + dbInspectTrieCmd, dbStatCmd, dbCompactCmd, dbGetCmd, @@ -72,6 +101,52 @@ Remove blockchain and state databases`, dbExportCmd, }, } + dbInspectTrieCmd = cli.Command{ + Action: utils.MigrateFlags(inspectTrie), + Name: "inspect-trie", + ArgsUsage: "[blocknum|latest|snapshot]", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.SepoliaFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.MorphFlag, + utils.MorphHoleskyFlag, + utils.MorphHoodiFlag, + inspectTrieTopFlag, + inspectTrieExcludeStorageFlag, + inspectTrieOutputFileFlag, + inspectTrieDumpPathFlag, + inspectTrieSummarizeFlag, + inspectTrieContractFlag, + }, + Usage: "Walk an MPT state trie and report structural metrics", + Description: `This command walks the MPT-encoded account trie for the given target and, +unless --exclude-storage is set, every non-empty storage trie. The walk +is a two-pass operation: pass 1 streams per-storage-trie records into +--dump-path (default /trie-dump.bin), pass 2 summarizes the +dump and prints tables to stdout (or writes JSON via --output). + +The argument selects the state to inspect: + + latest the current head (default if omitted) + the canonical block at the given decimal height + snapshot the state root recorded by the snapshot layer + +Alternative modes: + + --summarize skip the walk and re-summarize an existing dump + --contract 0xADDR inspect a single contract's storage trie + snap + view, bypassing the top-N account-trie scan + +This command only supports MPT mode. It refuses to run against ZKTrie- +encoded history (pre-JadeFork on morph mainnet/Holesky); target a block +after the JadeFork activation time or run on an MPT-native chain.`, + } dbInspectCmd = cli.Command{ Action: utils.MigrateFlags(inspect), Name: "inspect", @@ -357,6 +432,178 @@ func inspect(ctx *cli.Context) error { return rawdb.InspectDatabase(db, prefix, start) } +// inspectTrie is the handler for `geth db inspect-trie`. It dispatches to +// one of three upstream-aligned modes depending on the flags supplied: +// +// - --summarize : skip the trie walk and re-analyze an existing +// pass-1 dump. Useful to regenerate a report cheaply after a long +// walk, or to share dumps across machines for offline inspection. +// - --contract 0xADDR: run trie.InspectContract against the resolved +// state root to compare the trie and snapshot views for one +// contract. +// - otherwise: run the two-pass inspector (pass 1 writes a dump, pass +// 2 produces the summary) via trie.Inspect. +// +// The positional argument selects the state root for the trie-walk and +// contract modes: +// +// latest (default) head block's state root +// canonical block at decimal height +// snapshot state root recorded by the snapshot layer +// +// ZKTrie-encoded morph history is refused via trie.ErrUnsupportedTrieFormat. +// This matches upstream's "MPT only" contract since the inspector does +// not understand morph's zkTrie encoding. +func inspectTrie(ctx *cli.Context) error { + topN := ctx.Int(inspectTrieTopFlag.Name) + if err := validateInspectTrieTopN(topN); err != nil { + return err + } + + // Mode 1: summarize an existing dump. The trie database is not + // needed since we only read the binary dump. + if summarizePath := ctx.String(inspectTrieSummarizeFlag.Name); summarizePath != "" { + if ctx.NArg() > 0 { + return fmt.Errorf("block argument is not supported with --%s", inspectTrieSummarizeFlag.Name) + } + cfg := &trie.InspectConfig{ + TopN: topN, + Path: ctx.String(inspectTrieOutputFileFlag.Name), + DumpPath: summarizePath, + } + log.Info("Summarizing trie dump", "path", summarizePath, "top", topN) + return trie.Summarize(summarizePath, cfg) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + root, blockNumber, blockTime, blockMetaKnown, err := resolveInspectTarget(ctx, db) + if err != nil { + return err + } + + // Detect ZKTrie mode: when the chain uses ZKTrie format we must be + // able to confirm the target block is past JadeFork (where the trie + // switched to MPT). If block metadata is unknown we cannot make that + // determination, so we treat unknown metadata as unsafe and refuse. + chainConfig := rawdb.ReadChainConfig(db, rawdb.ReadCanonicalHash(db, 0)) + if chainConfig != nil && chainConfig.Morph.UseZktrie && (!blockMetaKnown || !chainConfig.IsJadeFork(blockTime)) { + return fmt.Errorf("%w (block %d time %d predates or cannot confirm JadeFork)", + trie.ErrUnsupportedTrieFormat, blockNumber, blockTime) + } + + triedb := trie.NewDatabase(db) + + // Mode 2: single-contract inspection. The result is printed to + // stdout; --output/--dump-path are ignored in this mode because + // InspectContract emits a fixed report directly. + if contractArg := ctx.String(inspectTrieContractFlag.Name); contractArg != "" { + if !common.IsHexAddress(contractArg) { + return fmt.Errorf("invalid --%s value %q: not an address", inspectTrieContractFlag.Name, contractArg) + } + address := common.HexToAddress(contractArg) + log.Info("Inspecting contract", "address", address, "root", root, "block", blockNumber) + return trie.InspectContract(triedb, db, root, address) + } + + // Mode 3: full two-pass trie inspection. + dumpPath := ctx.String(inspectTrieDumpPathFlag.Name) + if dumpPath == "" { + dumpPath = stack.ResolvePath("trie-dump.bin") + } + cfg := &trie.InspectConfig{ + NoStorage: ctx.Bool(inspectTrieExcludeStorageFlag.Name), + TopN: topN, + Path: ctx.String(inspectTrieOutputFileFlag.Name), + DumpPath: dumpPath, + } + log.Info("Inspecting trie", + "root", root, + "block", blockNumber, + "time", blockTime, + "excludeStorage", cfg.NoStorage, + "top", topN, + "dump", cfg.DumpPath, + ) + start := time.Now() + if err := trie.Inspect(triedb, root, cfg); err != nil { + return err + } + log.Info("Trie inspection complete", "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + +// resolveInspectTarget picks a state root and best-effort block metadata +// from the positional argument. For the "snapshot" keyword it first tries to +// recover canonical block metadata from the snapshot root; if no matching +// header is available the block metadata is returned as unknown. +func resolveInspectTarget(ctx *cli.Context, db ethdb.Database) (common.Hash, uint64, uint64, bool, error) { + if ctx.NArg() > 1 { + return common.Hash{}, 0, 0, false, fmt.Errorf("max 1 argument: %v", ctx.Command.ArgsUsage) + } + + arg := "latest" + if ctx.NArg() == 1 { + arg = ctx.Args().Get(0) + } + + if arg == "snapshot" { + return resolveSnapshotInspectTarget(db) + } + + if arg == "latest" { + header := rawdb.ReadHeadHeader(db) + if header == nil { + return common.Hash{}, 0, 0, false, errors.New("head header not found; database may be empty") + } + return header.Root, header.Number.Uint64(), header.Time, true, nil + } + + n, err := strconv.ParseUint(arg, 10, 64) + if err != nil { + return common.Hash{}, 0, 0, false, fmt.Errorf("invalid block argument %q: %w", arg, err) + } + hash := rawdb.ReadCanonicalHash(db, n) + if hash == (common.Hash{}) { + return common.Hash{}, 0, 0, false, fmt.Errorf("canonical hash for block %d not found", n) + } + header := rawdb.ReadHeader(db, hash, n) + if header == nil { + return common.Hash{}, 0, 0, false, fmt.Errorf("header for block %d / %x not found", n, hash) + } + return header.Root, header.Number.Uint64(), header.Time, true, nil +} + +func resolveSnapshotInspectTarget(db ethdb.Database) (common.Hash, uint64, uint64, bool, error) { + root := rawdb.ReadSnapshotRoot(db) + if root == (common.Hash{}) { + return common.Hash{}, 0, 0, false, errors.New("snapshot root not found in database") + } + if header := rawdb.ReadHeadHeader(db); header != nil && header.Root == root { + return root, header.Number.Uint64(), header.Time, true, nil + } + if number := rawdb.ReadSnapshotRecoveryNumber(db); number != nil { + hash := rawdb.ReadCanonicalHash(db, *number) + if hash != (common.Hash{}) { + if header := rawdb.ReadHeader(db, hash, *number); header != nil && header.Root == root { + return root, header.Number.Uint64(), header.Time, true, nil + } + } + } + return root, 0, 0, false, nil +} + +func validateInspectTrieTopN(topN int) error { + if topN <= 0 { + return fmt.Errorf("invalid --%s value %d (must be > 0)", inspectTrieTopFlag.Name, topN) + } + return nil +} + func showLeveldbStats(db ethdb.Stater) { if stats, err := db.Stat("leveldb.stats"); err != nil { log.Warn("Failed to read database stats", "error", err) diff --git a/cmd/geth/dbcmd_test.go b/cmd/geth/dbcmd_test.go new file mode 100644 index 000000000..400482252 --- /dev/null +++ b/cmd/geth/dbcmd_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "flag" + "math/big" + "testing" + + "gopkg.in/urfave/cli.v1" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/ethdb" +) + +func newInspectTrieContext(t *testing.T, args ...string) *cli.Context { + t.Helper() + + set := flag.NewFlagSet("inspect-trie-test", flag.ContinueOnError) + if err := set.Parse(args); err != nil { + t.Fatalf("parse args: %v", err) + } + return cli.NewContext(nil, set, nil) +} + +func writeCanonicalHeader(db ethdb.KeyValueWriter, header *types.Header) { + rawdb.WriteHeader(db, header) + rawdb.WriteCanonicalHash(db, header.Hash(), header.Number.Uint64()) +} + +func TestResolveInspectTargetSnapshotUsesHeadMetadata(t *testing.T) { + db := rawdb.NewMemoryDatabase() + root := common.HexToHash("0x1111") + header := &types.Header{ + Number: big.NewInt(42), + Time: 123456, + Root: root, + } + writeCanonicalHeader(db, header) + rawdb.WriteHeadHeaderHash(db, header.Hash()) + rawdb.WriteSnapshotRoot(db, root) + + gotRoot, gotNumber, gotTime, gotKnown, err := resolveInspectTarget(newInspectTrieContext(t, "snapshot"), db) + if err != nil { + t.Fatalf("resolveInspectTarget returned error: %v", err) + } + if gotRoot != root || gotNumber != header.Number.Uint64() || gotTime != header.Time || !gotKnown { + t.Fatalf("unexpected snapshot target: root=%x number=%d time=%d known=%v", gotRoot, gotNumber, gotTime, gotKnown) + } +} + +func TestResolveInspectTargetSnapshotUsesRecoveryMetadata(t *testing.T) { + db := rawdb.NewMemoryDatabase() + snapshotRoot := common.HexToHash("0x2222") + recoveryHeader := &types.Header{ + Number: big.NewInt(12), + Time: 98765, + Root: snapshotRoot, + } + headHeader := &types.Header{ + Number: big.NewInt(15), + Time: 99999, + Root: common.HexToHash("0x3333"), + } + writeCanonicalHeader(db, recoveryHeader) + writeCanonicalHeader(db, headHeader) + rawdb.WriteHeadHeaderHash(db, headHeader.Hash()) + rawdb.WriteSnapshotRoot(db, snapshotRoot) + rawdb.WriteSnapshotRecoveryNumber(db, recoveryHeader.Number.Uint64()) + + gotRoot, gotNumber, gotTime, gotKnown, err := resolveInspectTarget(newInspectTrieContext(t, "snapshot"), db) + if err != nil { + t.Fatalf("resolveInspectTarget returned error: %v", err) + } + if gotRoot != snapshotRoot || gotNumber != recoveryHeader.Number.Uint64() || gotTime != recoveryHeader.Time || !gotKnown { + t.Fatalf("unexpected recovery snapshot target: root=%x number=%d time=%d known=%v", gotRoot, gotNumber, gotTime, gotKnown) + } +} + +func TestResolveInspectTargetSnapshotUnknownMetadata(t *testing.T) { + db := rawdb.NewMemoryDatabase() + snapshotRoot := common.HexToHash("0x4444") + headHeader := &types.Header{ + Number: big.NewInt(8), + Time: 55555, + Root: common.HexToHash("0x5555"), + } + writeCanonicalHeader(db, headHeader) + rawdb.WriteHeadHeaderHash(db, headHeader.Hash()) + rawdb.WriteSnapshotRoot(db, snapshotRoot) + + gotRoot, gotNumber, gotTime, gotKnown, err := resolveInspectTarget(newInspectTrieContext(t, "snapshot"), db) + if err != nil { + t.Fatalf("resolveInspectTarget returned error: %v", err) + } + if gotRoot != snapshotRoot || gotNumber != 0 || gotTime != 0 || gotKnown { + t.Fatalf("unexpected unknown snapshot target: root=%x number=%d time=%d known=%v", gotRoot, gotNumber, gotTime, gotKnown) + } +} + +func TestValidateInspectTrieTopN(t *testing.T) { + tests := []struct { + name string + topN int + wantErr bool + }{ + {name: "positive", topN: 10}, + {name: "zero_rejected", topN: 0, wantErr: true}, + {name: "negative_rejected", topN: -1, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateInspectTrieTopN(tt.topN) + if tt.wantErr && err == nil { + t.Fatalf("expected error for topN=%d", tt.topN) + } + if !tt.wantErr && err != nil { + t.Fatalf("unexpected error for topN=%d: %v", tt.topN, err) + } + }) + } +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 8d4e6228d..6d80617f1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -195,6 +195,7 @@ var ( utils.RPCGlobalTxFeeCapFlag, utils.AllowUnprotectedTxs, utils.MaxBlockRangeFlag, + utils.RPCRangeLimitFlag, utils.BatchRequestLimit, utils.BatchResponseMaxSize, } @@ -210,6 +211,7 @@ var ( utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsEnableInfluxDBV2Flag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, @@ -315,7 +317,7 @@ func prepare(ctx *cli.Context) { } // Start metrics export if enabled - utils.SetupMetrics(ctx) + utils.SetupMetrics(ctx, makeMetricsConfig(ctx)) // Start system runtime metrics collection go metrics.CollectProcessMetrics(3 * time.Second) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 297e672f9..1fcffae59 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -32,7 +32,6 @@ import ( "strings" "text/tabwriter" "text/template" - "time" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" @@ -865,6 +864,12 @@ var ( Value: metrics.DefaultConfig.InfluxDBOrganization, } + MetricsInfluxDBIntervalFlag = cli.DurationFlag{ + Name: "metrics.influxdb.interval", + Usage: "Interval between metrics reports to InfluxDB (with time unit, e.g. 10s)", + Value: metrics.DefaultConfig.InfluxDBInterval, + } + CatalystFlag = cli.BoolFlag{ Name: "catalyst", Usage: "Catalyst mode (eth2 integration testing)", @@ -875,6 +880,14 @@ var ( Name: "rpc.getlogs.maxrange", Usage: "Limit max fetched block range for `eth_getLogs` method", } + // RPCRangeLimitFlag is an alias for MaxBlockRangeFlag that aligns with + // upstream go-ethereum PR #33163. Using either flag caps the block range + // (end - begin + 1) allowed in eth_getLogs; -1 and 0 both mean unlimited + // so existing deployments that rely on the default -1 keep working. + RPCRangeLimitFlag = cli.Int64Flag{ + Name: "rpc.rangelimit", + Usage: "Limit the maximum block range (end - begin + 1) allowed in range queries such as `eth_getLogs` (alias of --rpc.getlogs.maxrange; -1 or 0 = unlimited)", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1563,10 +1576,33 @@ func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { } func setMaxBlockRange(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(MaxBlockRangeFlag.Name) { - cfg.MaxBlockRange = ctx.GlobalInt64(MaxBlockRangeFlag.Name) - } else { - cfg.MaxBlockRange = -1 + // Resolution order: --rpc.rangelimit takes precedence if this helper is + // called directly with both aliases set. SetEthConfig rejects that + // combination for real CLI usage. When neither flag is set, normalize the + // zero value to unlimited without overwriting config-file/default values. + var ( + rangelimitSet = ctx.GlobalIsSet(RPCRangeLimitFlag.Name) + maxrangeSet = ctx.GlobalIsSet(MaxBlockRangeFlag.Name) + ) + switch { + case rangelimitSet: + v := ctx.GlobalInt64(RPCRangeLimitFlag.Name) + if v <= 0 { + cfg.MaxBlockRange = -1 + } else { + cfg.MaxBlockRange = v + } + case maxrangeSet: + v := ctx.GlobalInt64(MaxBlockRangeFlag.Name) + if v <= 0 { + cfg.MaxBlockRange = -1 + } else { + cfg.MaxBlockRange = v + } + default: + if cfg.MaxBlockRange == 0 { + cfg.MaxBlockRange = -1 + } } } @@ -1617,6 +1653,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag, MorphFlag, MorphHoleskyFlag, MorphHoodiFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer + CheckExclusive(ctx, RPCRangeLimitFlag, MaxBlockRangeFlag) if ctx.GlobalString(GCModeFlag.Name) == GCModeArchive && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { ctx.GlobalSet(TxLookupLimitFlag.Name, "0") log.Warn("Disable transaction unindexing for archive node") @@ -1991,7 +2028,7 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf return filterSystem } -func SetupMetrics(ctx *cli.Context) { +func SetupMetrics(ctx *cli.Context, cfg metrics.Config) { if metrics.Enabled { log.Info("Enabling metrics collection") @@ -2026,20 +2063,30 @@ func SetupMetrics(ctx *cli.Context) { token = ctx.GlobalString(MetricsInfluxDBTokenFlag.Name) bucket = ctx.GlobalString(MetricsInfluxDBBucketFlag.Name) organization = ctx.GlobalString(MetricsInfluxDBOrganizationFlag.Name) + + interval = cfg.InfluxDBInterval ) + if ctx.GlobalIsSet(MetricsInfluxDBIntervalFlag.Name) { + interval = ctx.GlobalDuration(MetricsInfluxDBIntervalFlag.Name) + } + if enableExport || enableExportV2 { + if err := metrics.ValidateInfluxDBInterval(interval); err != nil { + Fatalf("%v", err) + } + } if enableExport { tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) - log.Info("Enabling metrics export to InfluxDB") + log.Info("Enabling metrics export to InfluxDB", "interval", interval) - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, interval, endpoint, database, username, password, "geth.", tagsMap) } else if enableExportV2 { tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) - log.Info("Enabling metrics export to InfluxDB (v2)") + log.Info("Enabling metrics export to InfluxDB (v2)", "interval", interval) - go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) + go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, interval, endpoint, token, bucket, organization, "geth.", tagsMap) } if ctx.GlobalIsSet(MetricsHTTPFlag.Name) { diff --git a/cmd/utils/rangelimit_test.go b/cmd/utils/rangelimit_test.go new file mode 100644 index 000000000..d80d3c518 --- /dev/null +++ b/cmd/utils/rangelimit_test.go @@ -0,0 +1,96 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import ( + "flag" + "testing" + + "gopkg.in/urfave/cli.v1" + + "github.com/morph-l2/go-ethereum/eth/ethconfig" +) + +// newFlagContext builds a minimal *cli.Context that exposes both +// MaxBlockRangeFlag and RPCRangeLimitFlag, parsing the provided args in the +// same way urfave/cli.v1 does during command-line dispatch. +func newFlagContext(t *testing.T, args []string) *cli.Context { + t.Helper() + set := flag.NewFlagSet("test", flag.ContinueOnError) + MaxBlockRangeFlag.Apply(set) + RPCRangeLimitFlag.Apply(set) + if err := set.Parse(args); err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return cli.NewContext(nil, set, nil) +} + +// TestSetMaxBlockRangeResolution verifies the alias semantics between the +// legacy --rpc.getlogs.maxrange flag and the upstream-aligned +// --rpc.rangelimit flag introduced for P3-4(B). The rules are: +// +// 1. neither flag set -> MaxBlockRange = -1 (preserve historical default) +// 2. only legacy flag -> MaxBlockRange mirrors the user-supplied value +// verbatim (including -1 which is the documented "unlimited" sentinel) +// 3. only new flag -> non-positive values are coerced to -1, positive +// values are stored as-is. This matches upstream's "0 = unlimited" +// semantic while keeping morph's int64 field shape intact. +// 4. both flags set -> --rpc.rangelimit wins so operators can opt into +// the upstream name without scrubbing their existing launch scripts. +func TestSetMaxBlockRangeResolution(t *testing.T) { + tests := []struct { + name string + args []string + want int64 + }{ + {"default_unlimited", nil, -1}, + {"legacy_explicit_unlimited", []string{"-rpc.getlogs.maxrange", "-1"}, -1}, + {"legacy_bounded", []string{"-rpc.getlogs.maxrange", "1000"}, 1000}, + {"new_flag_zero_is_unlimited", []string{"-rpc.rangelimit", "0"}, -1}, + {"new_flag_negative_is_unlimited", []string{"-rpc.rangelimit", "-5"}, -1}, + {"new_flag_bounded", []string{"-rpc.rangelimit", "500"}, 500}, + {"both_set_new_wins_over_legacy", []string{ + "-rpc.getlogs.maxrange", "999", + "-rpc.rangelimit", "500", + }, 500}, + {"both_set_new_unlimited_wins", []string{ + "-rpc.getlogs.maxrange", "999", + "-rpc.rangelimit", "0", + }, -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := newFlagContext(t, tt.args) + cfg := ðconfig.Config{} + setMaxBlockRange(ctx, cfg) + if cfg.MaxBlockRange != tt.want { + t.Fatalf("MaxBlockRange = %d, want %d (args=%v)", cfg.MaxBlockRange, tt.want, tt.args) + } + }) + } +} + +func TestSetMaxBlockRangePreservesPreconfiguredValue(t *testing.T) { + ctx := newFlagContext(t, nil) + cfg := ðconfig.Config{MaxBlockRange: 123} + + setMaxBlockRange(ctx, cfg) + + if cfg.MaxBlockRange != 123 { + t.Fatalf("MaxBlockRange = %d, want preconfigured value 123", cfg.MaxBlockRange) + } +} diff --git a/core/tx_pool.go b/core/tx_pool.go index 76f96024e..e9599e739 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -953,8 +953,11 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) - // Successful promotion, bump the heartbeat - pool.beats[from] = time.Now() + // Successful replacement. If the sender already has queued non-executable + // transactions, bump the heartbeat to give them more time before lifetime + // eviction. Otherwise leave the beats map untouched so pending-only + // senders do not retain stale heartbeats. + pool.bumpBeats(from) return old != nil, nil } // New transaction isn't replacing a pending one, push into queue @@ -1011,8 +1014,13 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction, local boo pool.all.Add(tx, local) pool.priced.Put(tx, local) } - // If we never record the heartbeat, do it right now. - if _, exist := pool.beats[from]; !exist { + // Refresh the heartbeat only for external activity (addAll=true). + // Internal reshuffles such as demoteUnexecutables and removeTx pass + // addAll=false; they must not extend an existing queue lifetime, but + // still need a baseline timestamp if they just created the queue. + if addAll { + pool.beats[from] = time.Now() + } else if _, ok := pool.beats[from]; !ok { pool.beats[from] = time.Now() } return old != nil, nil @@ -1061,11 +1069,28 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) - // Successful promotion, bump the heartbeat - pool.beats[addr] = time.Now() + // Successful promotion, bump the heartbeat only if the sender still has + // queued non-executable transactions. The caller (promoteExecutables) + // will clear pool.beats[addr] once the future queue drains, so we must + // not resurrect a heartbeat for an account that has nothing left in the + // queue. + pool.bumpBeats(addr) return true } +// bumpBeats refreshes the heartbeat for addr, but only if an entry already +// exists. This mirrors upstream go-ethereum PR #33704: heartbeats track the +// age of queued (non-executable) transactions; writing the beat for an +// account without queued entries would pollute the Lifetime bookkeeping and +// keep stale entries alive across reorg/truncation boundaries. +// +// Note: this method assumes the pool lock is held by the caller. +func (pool *TxPool) bumpBeats(addr common.Address) { + if _, ok := pool.beats[addr]; ok { + pool.beats[addr] = time.Now() + } +} + // AddLocals enqueues a batch of transactions into the pool if they are valid, marking the // senders as a local ones, ensuring they go around the local pricing constraints. // diff --git a/core/tx_pool_heartbeat_test.go b/core/tx_pool_heartbeat_test.go new file mode 100644 index 000000000..707c9ccf0 --- /dev/null +++ b/core/tx_pool_heartbeat_test.go @@ -0,0 +1,268 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "math/big" + "testing" + "time" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/tracing" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/crypto" +) + +// The tests in this file exercise the heartbeat semantics introduced in +// P3-6 (upstream PR #33704). The invariants are: +// +// 1. bumpBeats is a no-op when the account has no existing heartbeat. +// 2. bumpBeats refreshes the timestamp when one already exists. +// 3. enqueueTx refreshes the heartbeat for external enqueues (addAll=true) +// so long-lived but still-active queues are not evicted by Lifetime; +// internal reshuffles (addAll=false) must initialize missing heartbeats +// without resetting existing ones. +// 4. A pending-only replace via add() does not resurrect a heartbeat for +// an account that has nothing queued. + +func TestTxPoolBumpBeatsNoop(t *testing.T) { + pool, _ := setupTxPool() + defer pool.Stop() + + addr := common.HexToAddress("0xdeadbeef") + pool.mu.Lock() + defer pool.mu.Unlock() + + pool.bumpBeats(addr) + if _, ok := pool.beats[addr]; ok { + t.Fatalf("bumpBeats inserted a heartbeat for an unknown account") + } +} + +func TestTxPoolBumpBeatsUpdate(t *testing.T) { + pool, _ := setupTxPool() + defer pool.Stop() + + addr := common.HexToAddress("0xcafebabe") + pool.mu.Lock() + defer pool.mu.Unlock() + + past := time.Now().Add(-1 * time.Hour) + pool.beats[addr] = past + + pool.bumpBeats(addr) + got, ok := pool.beats[addr] + if !ok { + t.Fatalf("heartbeat was removed instead of refreshed") + } + if !got.After(past) { + t.Fatalf("heartbeat not refreshed: got %v, want after %v", got, past) + } +} + +// TestTxPoolEnqueueAlwaysBumpsBeats verifies that repeat non-executable +// enqueues from the same sender refresh the heartbeat, preventing premature +// Lifetime eviction of long-lived-but-still-active queues. This matches the +// post-#33704 semantics for the enqueueTx path. +func TestTxPoolEnqueueAlwaysBumpsBeats(t *testing.T) { + pool, key := setupTxPool() + defer pool.Stop() + + from := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(from, big.NewInt(1_000_000_000_000_000_000), tracing.BalanceChangeUnspecified) + + // First enqueue: nonce 2 (non-executable, current state nonce is 0). + if err := pool.addRemoteSync(transaction(2, 100000, key)); err != nil { + t.Fatalf("first enqueue failed: %v", err) + } + + pool.mu.RLock() + first, ok := pool.beats[from] + pool.mu.RUnlock() + if !ok { + t.Fatalf("heartbeat missing after first enqueue") + } + + // Clamp the heartbeat to the past so the second enqueue produces a + // distinctly larger timestamp without needing a real sleep. + pool.mu.Lock() + pool.beats[from] = first.Add(-time.Hour) + pool.mu.Unlock() + + // Second enqueue: nonce 3 (also non-executable). + if err := pool.addRemoteSync(transaction(3, 100000, key)); err != nil { + t.Fatalf("second enqueue failed: %v", err) + } + + pool.mu.RLock() + second, ok := pool.beats[from] + pool.mu.RUnlock() + if !ok { + t.Fatalf("heartbeat missing after second enqueue") + } + if !second.After(first) { + t.Fatalf("second enqueue did not refresh heartbeat: first=%v second=%v", first, second) + } +} + +// TestTxPoolPendingReplaceDoesNotSpawnBeats verifies that the add() pending +// replacement path no longer unconditionally writes to beats. A sender +// whose transactions only ever reach the pending list (never the queue) +// must not accumulate a heartbeat entry, otherwise Lifetime eviction would +// track the wrong cohort. +func TestTxPoolPendingReplaceDoesNotSpawnBeats(t *testing.T) { + pool, key := setupTxPool() + defer pool.Stop() + + from := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(from, big.NewInt(1_000_000_000_000_000_000), tracing.BalanceChangeUnspecified) + + // Insert an executable tx (nonce 0) so it lands directly in pending. + if err := pool.addRemoteSync(transaction(0, 100000, key)); err != nil { + t.Fatalf("initial pending insert failed: %v", err) + } + + // Sanity: the queue for this sender is empty, so beats should be + // untouched. If this invariant ever changes in morph, this test + // serves as a clear signal. + pool.mu.RLock() + _, hadBeats := pool.beats[from] + pool.mu.RUnlock() + if hadBeats { + t.Fatalf("beats populated after pure pending insertion; was expecting queue-only bookkeeping") + } + + // Replace the pending tx with a higher-priced one of the same nonce + // to exercise the L957 path. + replacement := pricedTransaction(0, 100000, big.NewInt(2), key) + if err := pool.addRemoteSync(replacement); err != nil { + t.Fatalf("replacement insert failed: %v", err) + } + + pool.mu.RLock() + _, beatsExist := pool.beats[from] + pool.mu.RUnlock() + if beatsExist { + t.Fatalf("pending replace path resurrected a heartbeat for queue-less sender") + } + + // Verify the expected state: pending has the replacement, queue is empty. + pool.mu.RLock() + pendingList := pool.pending[from] + queueList := pool.queue[from] + pool.mu.RUnlock() + if pendingList == nil || pendingList.Len() != 1 { + t.Fatalf("pending list shape unexpected after replace: %+v", pendingList) + } + if queueList != nil && queueList.Len() > 0 { + t.Fatalf("queue should remain empty, got %d entries", queueList.Len()) + } + flat := pendingList.txs.Flatten() + if len(flat) != 1 { + t.Fatalf("expected 1 pending tx, got %d", len(flat)) + } + if got, want := flat[0].Hash(), replacement.Hash(); got != want { + t.Fatalf("pending head mismatch: got %x want %x", got, want) + } +} + +func TestTxPoolInternalEnqueueInitializesMissingBeat(t *testing.T) { + pool, key := setupTxPool() + defer pool.Stop() + + from := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(from, big.NewInt(1_000_000_000_000_000_000), tracing.BalanceChangeUnspecified) + + tx0 := transaction(0, 100000, key) + tx1 := transaction(1, 100000, key) + tx2 := transaction(2, 100000, key) + for _, tx := range []*types.Transaction{tx0, tx1, tx2} { + if err := pool.addRemoteSync(tx); err != nil { + t.Fatalf("add executable tx %d failed: %v", tx.Nonce(), err) + } + } + + pool.mu.RLock() + _, hadBeat := pool.beats[from] + pool.mu.RUnlock() + if hadBeat { + t.Fatalf("pending-only account unexpectedly had heartbeat before demotion") + } + + pool.RemoveTx(tx0.Hash(), true) + + pool.mu.RLock() + beat, ok := pool.beats[from] + queue := pool.queue[from] + pool.mu.RUnlock() + if !ok { + t.Fatalf("internal enqueue created queue without heartbeat") + } + if beat.IsZero() { + t.Fatalf("internal enqueue initialized zero heartbeat") + } + if queue == nil || queue.Len() != 2 { + t.Fatalf("expected two demoted queued transactions, got queue=%+v", queue) + } +} + +func TestTxPoolDemoteUnexecutablesInitializesMissingBeat(t *testing.T) { + pool, key := setupTxPool() + defer pool.Stop() + + from := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(from, big.NewInt(1_000_000_000_000_000_000), tracing.BalanceChangeUnspecified) + + tx0 := transaction(0, 200000, key) + tx1 := transaction(1, 100000, key) + tx2 := transaction(2, 100000, key) + for _, tx := range []*types.Transaction{tx0, tx1, tx2} { + if err := pool.addRemoteSync(tx); err != nil { + t.Fatalf("add executable tx %d failed: %v", tx.Nonce(), err) + } + } + + pool.mu.RLock() + _, hadBeat := pool.beats[from] + pool.mu.RUnlock() + if hadBeat { + t.Fatalf("pending-only account unexpectedly had heartbeat before demotion") + } + + pool.mu.Lock() + pool.currentState.SetBalance(from, big.NewInt(150000), tracing.BalanceChangeUnspecified) + pool.demoteUnexecutables() + pool.mu.Unlock() + + pool.mu.RLock() + beat, ok := pool.beats[from] + queue := pool.queue[from] + pending := pool.pending[from] + pool.mu.RUnlock() + if !ok { + t.Fatalf("demoteUnexecutables created queue without heartbeat") + } + if beat.IsZero() { + t.Fatalf("demoteUnexecutables initialized zero heartbeat") + } + if queue == nil || queue.Len() != 2 { + t.Fatalf("expected two invalidated queued transactions, got queue=%+v", queue) + } + if pending != nil && pending.Len() != 0 { + t.Fatalf("expected pending list to be empty after demotion, got %d", pending.Len()) + } +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1fcd600ec..480899068 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -996,6 +996,67 @@ func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.A return res[:], state.Error() } +// maxGetStorageSlots caps the total number of storage slots that +// eth_getStorageValues will read per request. The value mirrors upstream +// go-ethereum PR #32591 and prevents a single oversized RPC call from +// pinning a state snapshot for an unbounded amount of time. +const maxGetStorageSlots = 1024 + +// GetStorageValues returns multiple storage slot values for multiple accounts +// at the given block, following upstream go-ethereum PR #32591. Compared to +// issuing many eth_getStorageAt calls, this batches the state snapshot +// lookup so every slot is read against the same consistent view. +// +// The returned map preserves the input per-address slot ordering; slots that +// are not set resolve to the all-zero hash. +// +// Errors: +// - empty request is rejected with a parameter error, +// - total requested slot count must not exceed maxGetStorageSlots. +func (s *PublicBlockChainAPI) GetStorageValues(ctx context.Context, requests map[common.Address][]common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (map[common.Address][]hexutil.Bytes, error) { + if len(requests) == 0 { + return nil, &invalidParamsError{message: "empty request"} + } + // Validate empty slot lists first so this error is always reported before + // the slot-count limit, regardless of map iteration order. + for _, keys := range requests { + if len(keys) == 0 { + return nil, &invalidParamsError{message: "address with empty slot list"} + } + } + var totalSlots int + for _, keys := range requests { + totalSlots += len(keys) + if totalSlots > maxGetStorageSlots { + return nil, &clientLimitExceededError{message: fmt.Sprintf("too many slots (max %d)", maxGetStorageSlots)} + } + } + + state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + + result := make(map[common.Address][]hexutil.Bytes, len(requests)) + for addr, keys := range requests { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + vals := make([]hexutil.Bytes, len(keys)) + for i, key := range keys { + v := state.GetState(addr, key) + vals[i] = v[:] + } + if err := state.Error(); err != nil { + return nil, err + } + result[addr] = vals + } + return result, nil +} + // GetBlockReceipts returns the block receipts for the given block hash or number or tag. func (s *PublicBlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) { block, err := s.b.BlockByNumberOrHash(ctx, blockNrOrHash) @@ -1524,6 +1585,7 @@ func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Bloc type RPCTransaction struct { BlockHash *common.Hash `json:"blockHash"` BlockNumber *hexutil.Big `json:"blockNumber"` + BlockTimestamp *hexutil.Uint64 `json:"blockTimestamp"` From common.Address `json:"from"` Gas hexutil.Uint64 `json:"gas"` GasPrice *hexutil.Big `json:"gasPrice"` @@ -1579,6 +1641,7 @@ func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber if blockHash != (common.Hash{}) { result.BlockHash = &blockHash result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) + result.BlockTimestamp = (*hexutil.Uint64)(&blockTime) result.TransactionIndex = (*hexutil.Uint64)(&index) } switch tx.Type() { diff --git a/internal/ethapi/api_blocktimestamp_test.go b/internal/ethapi/api_blocktimestamp_test.go new file mode 100644 index 000000000..3b79a68f8 --- /dev/null +++ b/internal/ethapi/api_blocktimestamp_test.go @@ -0,0 +1,115 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "encoding/json" + "math/big" + "strings" + "testing" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/crypto" + "github.com/morph-l2/go-ethereum/params" +) + +// signTestLegacyTx produces a signed legacy transaction using a freshly +// generated key. Only the signature shape matters for the fields exercised +// by these tests (Sender recovery, etc.); the concrete sender address is +// irrelevant. +func signTestLegacyTx(t *testing.T) *types.Transaction { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + inner := &types.LegacyTx{ + Nonce: 0, + To: &common.Address{0xaa}, + Value: big.NewInt(1), + Gas: 21000, + GasPrice: big.NewInt(1000000000), + } + signer := types.MakeSigner(params.TestChainConfig, big.NewInt(1), 0) + tx, err := types.SignNewTx(key, signer, inner) + if err != nil { + t.Fatalf("sign: %v", err) + } + return tx +} + +// TestRPCTransactionBlockTimestamp verifies that NewRPCTransaction populates +// the BlockTimestamp field with the supplied block timestamp when the +// transaction has been mined (blockHash is non-zero), matching upstream +// go-ethereum PR #33709. Pending transactions (zero blockHash) should expose +// the field as null so eth_getTransactionByHash returns a stable shape and +// clients can distinguish pending from confirmed txs. +func TestRPCTransactionBlockTimestamp(t *testing.T) { + tx := signTestLegacyTx(t) + + t.Run("mined", func(t *testing.T) { + const ( + blockNumber = uint64(10) + blockTime = uint64(1234567890) + index = uint64(2) + ) + blockHash := common.HexToHash("0xabc") + + got := NewRPCTransaction(tx, blockHash, blockNumber, blockTime, index, nil, params.TestChainConfig) + + if got.BlockTimestamp == nil { + t.Fatalf("BlockTimestamp is nil for mined tx; want %d", blockTime) + } + if uint64(*got.BlockTimestamp) != blockTime { + t.Fatalf("BlockTimestamp = %d, want %d", uint64(*got.BlockTimestamp), blockTime) + } + if got.BlockNumber == nil || got.BlockNumber.ToInt().Uint64() != blockNumber { + t.Fatalf("BlockNumber not populated for mined tx: %v", got.BlockNumber) + } + + raw, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(raw), `"blockTimestamp":"0x499602d2"`) { + t.Fatalf("blockTimestamp not in JSON: %s", string(raw)) + } + }) + + t.Run("pending", func(t *testing.T) { + got := NewRPCTransaction(tx, common.Hash{}, 0, 0, 0, nil, params.TestChainConfig) + + if got.BlockTimestamp != nil { + t.Fatalf("BlockTimestamp should be nil for pending tx; got %d", uint64(*got.BlockTimestamp)) + } + if got.BlockHash != nil { + t.Fatalf("BlockHash should be nil for pending tx; got %v", got.BlockHash) + } + if got.BlockNumber != nil { + t.Fatalf("BlockNumber should be nil for pending tx; got %v", got.BlockNumber) + } + + raw, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(raw), `"blockTimestamp":null`) { + t.Fatalf("expected blockTimestamp:null for pending tx: %s", string(raw)) + } + }) +} diff --git a/internal/ethapi/api_getstoragevalues_test.go b/internal/ethapi/api_getstoragevalues_test.go new file mode 100644 index 000000000..f17f221ff --- /dev/null +++ b/internal/ethapi/api_getstoragevalues_test.go @@ -0,0 +1,204 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "context" + "math/big" + "strings" + "testing" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/state" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/rpc" +) + +func assertRPCErrorCode(t *testing.T, err error, want int) { + t.Helper() + + rpcErr, ok := err.(rpc.Error) + if !ok { + t.Fatalf("expected rpc.Error, got %T (%v)", err, err) + } + if got := rpcErr.ErrorCode(); got != want { + t.Fatalf("unexpected rpc error code: got %d want %d", got, want) + } +} + +// storageValuesBackend is a minimal Backend that only implements +// StateAndHeaderByNumberOrHash so the GetStorageValues method can be +// exercised without spinning up a full eth backend. Every other method is +// delegated to an embedded nil interface and will panic if invoked. +type storageValuesBackend struct { + Backend + state *state.StateDB +} + +func (b *storageValuesBackend) StateAndHeaderByNumberOrHash(ctx context.Context, _ rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + return b.state, &types.Header{Number: big.NewInt(0)}, nil +} + +func newStorageValuesAPI(t *testing.T) (*PublicBlockChainAPI, *state.StateDB) { + t.Helper() + statedb, err := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + if err != nil { + t.Fatalf("new statedb: %v", err) + } + return &PublicBlockChainAPI{b: &storageValuesBackend{state: statedb}}, statedb +} + +// TestGetStorageValuesHappyPath mirrors upstream PR #32591's test: a single +// RPC call reads multiple slots across multiple accounts, with the per- +// address slot order preserved in the response. +func TestGetStorageValuesHappyPath(t *testing.T) { + api, statedb := newStorageValuesAPI(t) + + var ( + addr1 = common.HexToAddress("0x1111") + addr2 = common.HexToAddress("0x2222") + slot0 = common.Hash{} + slot1 = common.BigToHash(big.NewInt(1)) + slot2 = common.BigToHash(big.NewInt(2)) + val0 = common.BigToHash(big.NewInt(42)) + val1 = common.BigToHash(big.NewInt(100)) + val2 = common.BigToHash(big.NewInt(200)) + ) + statedb.SetState(addr1, slot0, val0) + statedb.SetState(addr1, slot1, val1) + statedb.SetState(addr2, slot2, val2) + + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + result, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: {slot0, slot1}, + addr2: {slot2}, + }, blockRef) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 addresses in result, got %d", len(result)) + } + if got := common.BytesToHash(result[addr1][0]); got != val0 { + t.Errorf("addr1 slot0: want %x, got %x", val0, got) + } + if got := common.BytesToHash(result[addr1][1]); got != val1 { + t.Errorf("addr1 slot1: want %x, got %x", val1, got) + } + if got := common.BytesToHash(result[addr2][0]); got != val2 { + t.Errorf("addr2 slot2: want %x, got %x", val2, got) + } +} + +// TestGetStorageValuesMissingSlotReturnsZero documents that unset slots +// resolve to the all-zero hash, matching eth_getStorageAt's existing +// behavior. +func TestGetStorageValuesMissingSlotReturnsZero(t *testing.T) { + api, _ := newStorageValuesAPI(t) + + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + result, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + common.HexToAddress("0xabc"): {common.HexToHash("0xff")}, + }, blockRef) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := common.BytesToHash(result[common.HexToAddress("0xabc")][0]); got != (common.Hash{}) { + t.Errorf("unset slot: want zero, got %x", got) + } +} + +// TestGetStorageValuesEmptyRequestRejected ensures we never hit the state +// snapshot lookup when the caller supplies an empty request map, which is +// an obvious parameter error. +func TestGetStorageValuesEmptyRequestRejected(t *testing.T) { + api, _ := newStorageValuesAPI(t) + + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + _, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{}, blockRef) + if err == nil { + t.Fatal("expected error for empty request") + } + if !strings.Contains(err.Error(), "empty request") { + t.Fatalf("unexpected error message: %v", err) + } + assertRPCErrorCode(t, err, errCodeInvalidParams) +} + +// TestGetStorageValuesEmptyPerAddressRejected covers the subtle case where +// the top-level map is non-empty but every slot list is empty. The total +// slot count is zero, which is equivalent to an empty request. +func TestGetStorageValuesEmptyPerAddressRejected(t *testing.T) { + api, _ := newStorageValuesAPI(t) + + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + _, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + common.HexToAddress("0x1"): {}, + common.HexToAddress("0x2"): {}, + }, blockRef) + if err == nil { + t.Fatal("expected error for zero total slots") + } + if !strings.Contains(err.Error(), "empty request") { + t.Fatalf("unexpected error message: %v", err) + } + assertRPCErrorCode(t, err, errCodeInvalidParams) +} + +// TestGetStorageValuesExceedsLimit ensures that requests with more than +// maxGetStorageSlots keys are rejected before any state read occurs. +func TestGetStorageValuesExceedsLimit(t *testing.T) { + api, _ := newStorageValuesAPI(t) + + tooMany := make([]common.Hash, maxGetStorageSlots+1) + for i := range tooMany { + tooMany[i] = common.BigToHash(big.NewInt(int64(i))) + } + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + _, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + common.HexToAddress("0xdeadbeef"): tooMany, + }, blockRef) + if err == nil { + t.Fatal("expected error for request exceeding slot limit") + } + if !strings.Contains(err.Error(), "too many slots") { + t.Fatalf("unexpected error message: %v", err) + } + assertRPCErrorCode(t, err, errCodeClientLimitExceeded) +} + +// TestGetStorageValuesLimitBoundary ensures that the maximum allowed slot +// count is accepted, guarding against off-by-one regressions. +func TestGetStorageValuesLimitBoundary(t *testing.T) { + api, _ := newStorageValuesAPI(t) + + maxKeys := make([]common.Hash, maxGetStorageSlots) + for i := range maxKeys { + maxKeys[i] = common.BigToHash(big.NewInt(int64(i))) + } + blockRef := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + result, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + common.HexToAddress("0xabc"): maxKeys, + }, blockRef) + if err != nil { + t.Fatalf("expected success at slot limit, got: %v", err) + } + if len(result[common.HexToAddress("0xabc")]) != maxGetStorageSlots { + t.Fatalf("expected %d slots, got %d", maxGetStorageSlots, len(result[common.HexToAddress("0xabc")])) + } +} diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go new file mode 100644 index 000000000..d236fda2f --- /dev/null +++ b/internal/ethapi/errors.go @@ -0,0 +1,35 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +const ( + errCodeInvalidParams = -32602 + errCodeClientLimitExceeded = -38026 +) + +// invalidParamsError and clientLimitExceededError mirror the JSON-RPC +// semantics used by upstream go-ethereum for public RPC parameter and limit +// checks. The rpc package only requires Error() plus ErrorCode(). +type invalidParamsError struct{ message string } + +func (e *invalidParamsError) Error() string { return e.message } +func (e *invalidParamsError) ErrorCode() int { return errCodeInvalidParams } + +type clientLimitExceededError struct{ message string } + +func (e *clientLimitExceededError) Error() string { return e.message } +func (e *clientLimitExceededError) ErrorCode() int { return errCodeClientLimitExceeded } diff --git a/internal/tablewriter/tablewriter.go b/internal/tablewriter/tablewriter.go new file mode 100644 index 000000000..e98f6c19f --- /dev/null +++ b/internal/tablewriter/tablewriter.go @@ -0,0 +1,173 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package tablewriter is a minimal, self-contained stand-in for the +// third-party tablewriter library used by upstream go-ethereum, trimmed to +// the subset exercised by morph's CLI reporters. The public API mirrors +// upstream so call sites can be ported verbatim, but the rendering is +// backed by a small fixed-padding renderer for alignment. +// +// This package is intentionally naive: it performs light validation at +// Render() time rather than buffering per-row diagnostics, and it panics +// only for programmer errors (nil writer). Operators receive readable +// errors for common misuse (missing headers, mismatched column counts). +package tablewriter + +import ( + "errors" + "fmt" + "io" + "strings" + "unicode/utf8" +) + +// Table accumulates headers, data rows, and an optional footer, and +// renders them as an aligned text table. +// +// A Table is not safe for concurrent use by multiple goroutines. +type Table struct { + out io.Writer + header []string + footer []string + rows [][]string +} + +// NewWriter returns a new Table that writes to w. Subsequent calls to +// SetHeader / AppendBulk / SetFooter populate the table; Render emits the +// aligned output to w in one pass. +func NewWriter(w io.Writer) *Table { + if w == nil { + panic("tablewriter: nil writer") + } + return &Table{out: w} +} + +// SetHeader records the column headers. The first header also fixes the +// expected column count for subsequent rows and the optional footer. +func (t *Table) SetHeader(header []string) { + t.header = append(t.header[:0], header...) +} + +// SetFooter records the footer row shown after all data rows. Pass a nil +// slice to clear a previously set footer. +func (t *Table) SetFooter(footer []string) { + if footer == nil { + t.footer = nil + return + } + t.footer = append(t.footer[:0], footer...) +} + +// AppendBulk appends one or more data rows to the table. +// +// Each row must match the header column count; the check is deferred to +// Render so partially-built tables still round-trip through inspection. +// The input slice is deep-copied so later mutations by the caller do not +// affect the stored rows (matching the defensive behaviour of SetHeader and +// SetFooter). +func (t *Table) AppendBulk(rows [][]string) { + snapshot := make([][]string, len(rows)) + for i, row := range rows { + r := make([]string, len(row)) + copy(r, row) + snapshot[i] = r + } + t.rows = append(t.rows, snapshot...) +} + +// Render validates the accumulated state and writes the aligned table to +// the configured writer. Render returns an error when: +// +// - no header has been set, +// - any row has a different number of columns than the header, +// - the footer (when set) has a different number of columns. +// +// Rendering is idempotent: call it multiple times to emit the same table +// repeatedly. +func (t *Table) Render() error { + if len(t.header) == 0 { + return errors.New("tablewriter: no header configured") + } + cols := len(t.header) + for i, row := range t.rows { + if len(row) != cols { + return fmt.Errorf("tablewriter: row %d has %d columns, want %d", i, len(row), cols) + } + } + if len(t.footer) > 0 && len(t.footer) != cols { + return fmt.Errorf("tablewriter: footer has %d columns, want %d", len(t.footer), cols) + } + + // Compute per-column max widths (in runes) across header, rows and + // footer so the separator aligns with the actual column content. + w := make([]int, cols) + measure := func(cells []string) { + for i, c := range cells { + if n := utf8.RuneCountInString(c); n > w[i] { + w[i] = n + } + } + } + measure(t.header) + for _, row := range t.rows { + measure(row) + } + if len(t.footer) > 0 { + measure(t.footer) + } + const padding = 2 + sepParts := make([]string, cols) + for i, n := range w { + if i < cols-1 { + n += padding + } + sepParts[i] = strings.Repeat("─", n) + } + sep := strings.Join(sepParts, "") + + var ret error + writeString := func(s string) { + if ret != nil { + return + } + _, ret = io.WriteString(t.out, s) + } + writeLine := func(s string) { + writeString(s) + writeString("\n") + } + writeRow := func(cells []string) { + for i, cell := range cells { + writeString(cell) + if i < cols-1 { + pad := w[i] - utf8.RuneCountInString(cell) + padding + writeString(strings.Repeat(" ", pad)) + } + } + writeString("\n") + } + + writeRow(t.header) + writeLine(sep) + for _, row := range t.rows { + writeRow(row) + } + if len(t.footer) > 0 { + writeLine(sep) + writeRow(t.footer) + } + return ret +} diff --git a/internal/tablewriter/tablewriter_test.go b/internal/tablewriter/tablewriter_test.go new file mode 100644 index 000000000..ee7c4a81d --- /dev/null +++ b/internal/tablewriter/tablewriter_test.go @@ -0,0 +1,145 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tablewriter + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +var errRenderSink = errors.New("render sink failed") + +type errWriter struct{} + +func (errWriter) Write([]byte) (int, error) { + return 0, errRenderSink +} + +func TestTableWriterHappyPath(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A", "B", "C"}) + table.AppendBulk([][]string{ + {"x", "y", "z"}, + {"1", "2", "3"}, + }) + table.SetFooter([]string{"Total", "3", ""}) + if err := table.Render(); err != nil { + t.Fatalf("render: %v", err) + } + out := buf.String() + if !strings.Contains(out, "A") || !strings.Contains(out, "Total") { + t.Fatalf("output missing header/footer rows: %q", out) + } +} + +func TestTableWriterMissingHeader(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.AppendBulk([][]string{{"x", "y", "z"}}) + if err := table.Render(); err == nil { + t.Fatal("expected error for missing header") + } +} + +func TestTableWriterMismatchedRow(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A", "B", "C"}) + table.AppendBulk([][]string{{"x", "y"}}) + if err := table.Render(); err == nil { + t.Fatal("expected error for row column mismatch") + } +} + +func TestTableWriterMismatchedFooter(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A", "B"}) + table.SetFooter([]string{"only one"}) + if err := table.Render(); err == nil { + t.Fatal("expected error for footer column mismatch") + } +} + +func TestTableWriterAppendBulkCumulative(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A"}) + table.AppendBulk([][]string{{"one"}}) + table.AppendBulk([][]string{{"two"}}) + if err := table.Render(); err != nil { + t.Fatalf("render: %v", err) + } + out := buf.String() + if !strings.Contains(out, "one") || !strings.Contains(out, "two") { + t.Fatalf("AppendBulk must be cumulative, got %q", out) + } +} + +func TestTableWriterSeparatorCoversPadding(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A", "B"}) + table.AppendBulk([][]string{{"x", "y"}}) + if err := table.Render(); err != nil { + t.Fatalf("render: %v", err) + } + + want := "A B\n────\nx y\n" + if got := buf.String(); got != want { + t.Fatalf("output mismatch:\ngot %q\nwant %q", got, want) + } +} + +func TestTableWriterNilWriterPanics(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for nil writer") + } + }() + NewWriter(nil) +} + +func TestTableWriterRenderIdempotent(t *testing.T) { + var buf bytes.Buffer + table := NewWriter(&buf) + table.SetHeader([]string{"A"}) + table.AppendBulk([][]string{{"x"}}) + if err := table.Render(); err != nil { + t.Fatalf("first render: %v", err) + } + first := buf.String() + buf.Reset() + if err := table.Render(); err != nil { + t.Fatalf("second render: %v", err) + } + if buf.String() != first { + t.Fatalf("render not idempotent: first=%q second=%q", first, buf.String()) + } +} + +func TestTableWriterReturnsWriterError(t *testing.T) { + table := NewWriter(errWriter{}) + table.SetHeader([]string{"A"}) + table.AppendBulk([][]string{{"x"}}) + if err := table.Render(); !errors.Is(err, errRenderSink) { + t.Fatalf("Render error = %v, want %v", err, errRenderSink) + } +} diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 51cc9cfba..3dbeef0a4 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -598,6 +598,12 @@ web3._extend({ call: 'eth_getBlockReceipts', params: 1, }), + new web3._extend.Method({ + name: 'getStorageValues', + call: 'eth_getStorageValues', + params: 2, + inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter] + }), ], properties: [ new web3._extend.Property({ diff --git a/metrics/config.go b/metrics/config.go index 2eb09fb48..1fc21e51e 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -16,18 +16,24 @@ package metrics +import ( + "fmt" + "time" +) + // Config contains the configuration for the metric collection. type Config struct { - Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:",omitempty"` - HTTP string `toml:",omitempty"` - Port int `toml:",omitempty"` - EnableInfluxDB bool `toml:",omitempty"` - InfluxDBEndpoint string `toml:",omitempty"` - InfluxDBDatabase string `toml:",omitempty"` - InfluxDBUsername string `toml:",omitempty"` - InfluxDBPassword string `toml:",omitempty"` - InfluxDBTags string `toml:",omitempty"` + Enabled bool `toml:",omitempty"` + EnabledExpensive bool `toml:",omitempty"` + HTTP string `toml:",omitempty"` + Port int `toml:",omitempty"` + EnableInfluxDB bool `toml:",omitempty"` + InfluxDBEndpoint string `toml:",omitempty"` + InfluxDBDatabase string `toml:",omitempty"` + InfluxDBUsername string `toml:",omitempty"` + InfluxDBPassword string `toml:",omitempty"` + InfluxDBTags string `toml:",omitempty"` + InfluxDBInterval time.Duration `toml:",omitempty"` EnableInfluxDBV2 bool `toml:",omitempty"` InfluxDBToken string `toml:",omitempty"` @@ -47,6 +53,7 @@ var DefaultConfig = Config{ InfluxDBUsername: "test", InfluxDBPassword: "test", InfluxDBTags: "host=localhost", + InfluxDBInterval: 10 * time.Second, // influxdbv2-specific flags EnableInfluxDBV2: false, @@ -54,3 +61,12 @@ var DefaultConfig = Config{ InfluxDBBucket: "geth", InfluxDBOrganization: "geth", } + +// ValidateInfluxDBInterval rejects non-positive reporting intervals before the +// exporter reaches time.Tick, which would otherwise panic at runtime. +func ValidateInfluxDBInterval(interval time.Duration) error { + if interval <= 0 { + return fmt.Errorf("invalid InfluxDB interval %v (must be > 0)", interval) + } + return nil +} diff --git a/metrics/config_test.go b/metrics/config_test.go new file mode 100644 index 000000000..26e2cd9a4 --- /dev/null +++ b/metrics/config_test.go @@ -0,0 +1,69 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +import ( + "testing" + "time" +) + +// TestDefaultInfluxDBInterval verifies that DefaultConfig.InfluxDBInterval +// defaults to 10 seconds, matching the previously hard-coded behavior in +// SetupMetrics. This guards against accidental default changes that would +// silently alter operators' metrics cadence. See upstream PR #33767. +func TestDefaultInfluxDBInterval(t *testing.T) { + if got, want := DefaultConfig.InfluxDBInterval, 10*time.Second; got != want { + t.Fatalf("DefaultConfig.InfluxDBInterval = %v, want %v", got, want) + } +} + +// TestConfigInfluxDBIntervalCustom verifies that a user-provided interval is +// preserved verbatim on a Config value and does not leak global state. +func TestConfigInfluxDBIntervalCustom(t *testing.T) { + cfg := DefaultConfig + cfg.InfluxDBInterval = 5 * time.Second + if cfg.InfluxDBInterval != 5*time.Second { + t.Fatalf("custom interval not preserved: got %v", cfg.InfluxDBInterval) + } + // The global default must remain unchanged. + if DefaultConfig.InfluxDBInterval != 10*time.Second { + t.Fatalf("DefaultConfig was mutated through local copy: got %v", DefaultConfig.InfluxDBInterval) + } +} + +func TestValidateInfluxDBInterval(t *testing.T) { + tests := []struct { + name string + interval time.Duration + wantErr bool + }{ + {name: "positive", interval: 5 * time.Second}, + {name: "zero_rejected", interval: 0, wantErr: true}, + {name: "negative_rejected", interval: -1 * time.Second, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateInfluxDBInterval(tt.interval) + if tt.wantErr && err == nil { + t.Fatalf("expected error for interval=%v", tt.interval) + } + if !tt.wantErr && err != nil { + t.Fatalf("unexpected error for interval=%v: %v", tt.interval, err) + } + }) + } +} diff --git a/trie/inspect.go b/trie/inspect.go new file mode 100644 index 000000000..136bf577c --- /dev/null +++ b/trie/inspect.go @@ -0,0 +1,1033 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package trie — inspect.go is a port of upstream go-ethereum PR #28892 +// adapted to morph's v1.10.26-era trie package and ZKTrie-aware state. +// +// The inspector walks a Merkle Patricia Trie in two passes: +// +// 1. Pass 1 (Inspect) walks the account trie, classifies every node by +// type and depth into a LevelStats, streams one fixed-size record per +// storage trie to a dump file on disk, and records the account trie +// itself as a sentinel record (owner = zero hash) at the end. +// +// 2. Pass 2 (Summarize) reads the dump file back and produces the final +// textual / JSON report, including three top-N rankings (by max +// depth, total nodes, and value-node count). +// +// InspectContract runs the inspector against a single contract's storage +// trie and additionally compares against the snapshot view so operators +// can reason about both representations side-by-side. +// +// ZKTrie-encoded state is explicitly out of scope: the caller (typically +// the geth CLI) is responsible for checking the chain config and +// refusing to invoke Inspect / InspectContract against pre-JadeFork +// morph history. The ErrUnsupportedTrieFormat constant is provided for +// that purpose. +package trie + +import ( + "bufio" + "bytes" + "cmp" + "container/heap" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/crypto" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/internal/tablewriter" + "github.com/morph-l2/go-ethereum/log" + "github.com/morph-l2/go-ethereum/rlp" +) + +// Dump / runtime constants. The record size is derived from the number of +// tracked trie levels so changing trieStatLevels stays in sync. +const ( + // inspectDumpRecordSize is the number of bytes occupied by one dump + // record: 32 bytes of owner hash followed by (3 uint32 counters + 1 + // uint64 byte-size) per level. + inspectDumpRecordSize = 32 + trieStatLevels*(3*4+8) + // inspectDefaultTopN matches upstream for consistent reporting. + inspectDefaultTopN = 10 + // inspectParallelism bounds the number of concurrent subtree walkers + // spawned by the inspector. The value is chosen to saturate an NVMe + // disk under random reads without overwhelming the operator's node. + inspectParallelism = int64(16) + // inspectDefaultDumpName is the dump file the inspector creates when + // the caller does not supply an explicit path. + inspectDefaultDumpName = "trie-dump.bin" +) + +// ErrUnsupportedTrieFormat is returned when a caller attempts to inspect a +// state that is not stored in MPT format (e.g. morph pre-JadeFork ZKTrie +// state). The inspector itself does not sniff the database; callers must +// detect this condition and surface the error to the operator. +var ErrUnsupportedTrieFormat = errors.New("inspect-trie: state format not supported (zktrie mode not supported; only MPT is implemented)") + +// InspectConfig is the set of options shared between the pass-1 inspector +// and the pass-2 summarizer. Field names and defaults match upstream so +// the CLI surface stays consistent. +type InspectConfig struct { + // NoStorage, when true, skips the per-account storage trie walk. + // The sentinel record for the account trie is still emitted. + NoStorage bool + + // TopN sets the size of each top-N list produced by Summarize. Zero + // or negative values fall back to inspectDefaultTopN. + TopN int + + // Path, when non-empty, causes Summarize to serialize the final + // report to that path as indented JSON. When empty the report is + // printed to stdout instead. + Path string + + // DumpPath is the pass-1 dump file location. When empty the default + // inspectDefaultDumpName is used. + DumpPath string +} + +// normalizeInspectConfig fills in defaults on a (possibly nil) config so +// every downstream consumer works from a fully populated value. +func normalizeInspectConfig(config *InspectConfig) *InspectConfig { + if config == nil { + config = &InspectConfig{} + } + if config.TopN <= 0 { + config.TopN = inspectDefaultTopN + } + if config.DumpPath == "" { + config.DumpPath = inspectDefaultDumpName + } + return config +} + +// inspector coordinates the pass-1 walk across multiple goroutines. The +// zero value is not usable; see Inspect for the correct construction +// sequence. +type inspector struct { + triedb *Database + root common.Hash + + config *InspectConfig + accountStat *LevelStats + + sem *semaphore.Weighted + + // Pass-1 dump file state. + dumpMu sync.Mutex + dumpBuf *bufio.Writer + dumpFile *os.File + storageRecordsWritten atomic.Uint64 + + errMu sync.Mutex + err error +} + +type inspectTrieKind uint8 + +const ( + inspectUnknownTrie inspectTrieKind = iota + inspectAccountTrie + inspectStorageTrie +) + +// Inspect walks the trie at root, records per-level node statistics, and +// streams one record per storage trie to disk. After the walk completes +// the file is finalized and Summarize is invoked to produce the report. +func Inspect(triedb *Database, root common.Hash, config *InspectConfig) error { + trie, err := New(root, triedb) + if err != nil { + return fmt.Errorf("fail to open trie %s: %w", root, err) + } + config = normalizeInspectConfig(config) + + dumpFile, err := os.OpenFile(config.DumpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to create trie dump %s: %w", config.DumpPath, err) + } + in := inspector{ + triedb: triedb, + root: root, + config: config, + accountStat: NewLevelStats(), + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriterSize(dumpFile, 1<<20), + dumpFile: dumpFile, + } + + // Emit periodic progress lines so large-state scans do not look + // hung. The reporter shuts down via done when the walk completes. + start := time.Now() + done := make(chan struct{}) + progressStopped := make(chan struct{}) + var stopProgress sync.Once + stopProgressReporter := func() { + stopProgress.Do(func() { + close(done) + <-progressStopped + }) + } + go func() { + defer close(progressStopped) + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + accountNodes := in.accountStat.TotalNodes() + storageRecords := in.storageRecordsWritten.Load() + log.Info("Inspecting trie", + "accountNodes", accountNodes, + "storageRecords", storageRecords, + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + defer stopProgressReporter() + + in.recordRootSize(root, in.accountStat) + in.inspect(trie, trie.root, 0, []byte{}, in.accountStat, inspectAccountTrie) + + // inspect is synchronous: it waits for every spawned goroutine in + // its subtree before returning, so no extra wait is needed here. + + // Sentinel record: zero owner hash marks the account trie stats. + in.writeDumpRecord(common.Hash{}, in.accountStat) + if err := in.closeDump(); err != nil { + in.setError(err) + } + + if err := in.getError(); err != nil { + stopProgressReporter() + return err + } + stopProgressReporter() + return Summarize(config.DumpPath, config) +} + +// InspectContract inspects a single contract's storage footprint. It +// reports the snapshot slot count / byte size and the storage-trie +// per-depth node distribution, running both paths in parallel so the +// wall-clock cost is bounded by the slower of the two. +func InspectContract(triedb *Database, db ethdb.Database, stateRoot common.Hash, address common.Address) error { + accountHash := crypto.Keccak256Hash(address.Bytes()) + + accountTrie, err := New(stateRoot, triedb) + if err != nil { + return fmt.Errorf("failed to open account trie: %w", err) + } + accountRLP, err := accountTrie.TryGet(crypto.Keccak256(address.Bytes())) + if err != nil { + return fmt.Errorf("failed to read account: %w", err) + } + if accountRLP == nil { + return fmt.Errorf("account not found: %s", address) + } + var account types.StateAccount + if err := rlp.DecodeBytes(accountRLP, &account); err != nil { + return fmt.Errorf("failed to decode account: %w", err) + } + if account.Root == (common.Hash{}) || account.Root == emptyRoot { + return fmt.Errorf("account %s has no storage", address) + } + + // Look up account snapshot (may be absent on morph; handled below). + accountData := rawdb.ReadAccountSnapshot(db, accountHash) + + var ( + snapSlots atomic.Uint64 + snapSize atomic.Uint64 + g errgroup.Group + start = time.Now() + ) + + // Goroutine 1: iterate the snapshot storage prefix for this account + // to compute slot count and raw byte size. + g.Go(func() error { + prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...) + it := db.NewIterator(prefix, nil) + defer it.Release() + + for it.Next() { + if !bytes.HasPrefix(it.Key(), prefix) { + break + } + snapSlots.Add(1) + snapSize.Add(uint64(len(it.Key()) + len(it.Value()))) + } + return it.Error() + }) + + // Goroutine 2: walk the storage trie with an inspector instance + // configured to drop dump writes (the output is never persisted + // from InspectContract). + storageStat := NewLevelStats() + g.Go(func() error { + storage, err := New(account.Root, triedb) + if err != nil { + return fmt.Errorf("failed to open storage trie: %w", err) + } + in := &inspector{ + triedb: triedb, + root: stateRoot, + config: &InspectConfig{NoStorage: true}, + accountStat: NewLevelStats(), + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriter(io.Discard), + } + in.recordRootSize(account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat, inspectStorageTrie) + return in.getError() + }) + + // Lightweight progress reporter for long scans. + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + log.Info("Inspecting contract", + "snapSlots", snapSlots.Load(), + "trieNodes", storageStat.TotalNodes(), + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + defer close(done) + + if err := g.Wait(); err != nil { + return err + } + + // Display results. + fmt.Printf("\n=== Contract Inspection: %s ===\n", address) + fmt.Printf("Account hash: %s\n\n", accountHash) + + if len(accountData) == 0 { + fmt.Println("Account snapshot: not found") + } else { + fmt.Printf("Account snapshot: %s\n", common.StorageSize(len(accountData))) + } + + fmt.Printf("Snapshot storage: %d slots (%s)\n", + snapSlots.Load(), common.StorageSize(snapSize.Load())) + + var trieTotal, trieSize uint64 + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Short", "Full", "Value", "Nodes", "Size"}) + for i := 0; i < trieStatLevels; i++ { + short, full, value, size := storageStat.level[i].load() + trieTotal += short + full + value + trieSize += size + total := short + full + value + if total == 0 && size == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(short), + fmt.Sprint(full), + fmt.Sprint(value), + fmt.Sprint(total), + common.StorageSize(size).String(), + }}) + } + fmt.Printf("Storage trie: %d nodes (%s)\n", trieTotal, common.StorageSize(trieSize)) + fmt.Println("\nStorage Trie Depth Distribution:") + if err := table.Render(); err != nil { + return err + } + fmt.Print(b.String()) + return nil +} + +// recordRootSize accounts the on-disk size of the root node into stat at +// depth 0. Root nodes, being topmost, are frequently resolved via the +// database directly; we ask the database for the raw blob and record its +// length. Missing nodes are logged but not fatal — the inspector keeps +// going so partial reports still appear in the summary. +func (in *inspector) recordRootSize(root common.Hash, stat *LevelStats) { + if root == (common.Hash{}) || root == emptyRoot { + return + } + blob, err := in.triedb.Node(root) + if err != nil || len(blob) == 0 { + log.Error("Failed to read trie root for size accounting", "root", root, "err", err) + return + } + stat.addSize(0, uint64(len(blob))) +} + +// closeDump flushes and closes the pass-1 dump file, returning the +// aggregated error if either step fails. +func (in *inspector) closeDump() error { + var ret error + if in.dumpBuf != nil { + if err := in.dumpBuf.Flush(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to flush trie dump %s: %w", in.config.DumpPath, err)) + } + } + if in.dumpFile != nil { + if err := in.dumpFile.Close(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to close trie dump %s: %w", in.config.DumpPath, err)) + } + } + return ret +} + +func (in *inspector) setError(err error) { + if err == nil { + return + } + in.errMu.Lock() + defer in.errMu.Unlock() + in.err = errors.Join(in.err, err) +} + +func (in *inspector) getError() error { + in.errMu.Lock() + defer in.errMu.Unlock() + return in.err +} + +func (in *inspector) hasError() bool { + return in.getError() != nil +} + +// trySpawn attempts to run fn in a new goroutine bounded by the +// inspector's semaphore. When a slot is available the goroutine starts +// and is tracked via wg so the caller can wait before observing any +// state fn writes. Returns false (without starting anything) when no +// slot is currently available — the caller then runs fn inline. +func (in *inspector) trySpawn(wg *sync.WaitGroup, fn func()) bool { + if !in.sem.TryAcquire(1) { + return false + } + wg.Add(1) + go func() { + defer in.sem.Release(1) + defer wg.Done() + fn() + }() + return true +} + +// writeDumpRecord serializes stats for owner into the pass-1 dump file. +// The zero owner hash is reserved for the account trie sentinel record; +// any other value is treated as a storage trie owner. +func (in *inspector) writeDumpRecord(owner common.Hash, s *LevelStats) { + if in.hasError() { + return + } + var buf [inspectDumpRecordSize]byte + copy(buf[:32], owner[:]) + + off := 32 + for i := 0; i < trieStatLevels; i++ { + short := s.level[i].short.Load() + if short > math.MaxUint32 { + in.setError(fmt.Errorf("dump record overflow: level %d short counter %d exceeds uint32 max", i, short)) + return + } + full := s.level[i].full.Load() + if full > math.MaxUint32 { + in.setError(fmt.Errorf("dump record overflow: level %d full counter %d exceeds uint32 max", i, full)) + return + } + value := s.level[i].value.Load() + if value > math.MaxUint32 { + in.setError(fmt.Errorf("dump record overflow: level %d value counter %d exceeds uint32 max", i, value)) + return + } + binary.LittleEndian.PutUint32(buf[off:], uint32(short)) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(full)) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(value)) + off += 4 + binary.LittleEndian.PutUint64(buf[off:], s.level[i].size.Load()) + off += 8 + } + in.dumpMu.Lock() + _, err := in.dumpBuf.Write(buf[:]) + in.dumpMu.Unlock() + if err != nil { + in.setError(fmt.Errorf("failed writing trie dump record: %w", err)) + return + } + if owner != (common.Hash{}) { + in.storageRecordsWritten.Add(1) + } +} + +// inspect walks the subtree rooted at n and records its statistics into +// stat. It may spawn goroutines for children when the semaphore allows, +// but always waits for them before returning so stat is fully populated +// by the time the caller observes it. +func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *LevelStats, kind inspectTrieKind) { + if n == nil || height >= trieStatLevels { + return + } + + // Goroutines spawned at this level are tracked locally so we block + // on them before recording n itself. This preserves the invariant + // "stat is complete on return", which downstream writeDumpRecord + // calls rely on. + var wg sync.WaitGroup + + switch n := (n).(type) { + case *shortNode: + nextPath := slices.Concat(path, n.Key) + in.inspect(trie, n.Val, height+1, nextPath, stat, kind) + case *fullNode: + for idx, child := range n.Children { + if child == nil { + continue + } + childPath := slices.Concat(path, []byte{byte(idx)}) + childNode := child + if in.trySpawn(&wg, func() { + in.inspect(trie, childNode, height+1, childPath, stat, kind) + }) { + continue + } + in.inspect(trie, childNode, height+1, childPath, stat, kind) + } + case hashNode: + // Resolve the raw node via the shared database; count its + // on-disk byte size against the current level before decoding + // and recursing. This mirrors upstream's reader.Node() path. + hash := common.BytesToHash(n) + blob, err := in.triedb.Node(hash) + if err != nil { + log.Error("Failed to resolve HashNode", "err", err, "trie", trie.Hash(), "height", height+1, "path", path) + return + } + stat.addSize(height, uint64(len(blob))) + resolved := mustDecodeNode(n, blob) + in.inspect(trie, resolved, height, path, stat, kind) + // Return early: hash nodes are a transparent indirection and + // must not be counted twice at this depth. + return + case valueNode: + if kind != inspectAccountTrie || in.config.NoStorage || !hasTerm(path) { + break + } + var account types.StateAccount + if err := rlp.Decode(bytes.NewReader(n), &account); err != nil { + break + } + if account.Root == (common.Hash{}) || account.Root == emptyRoot { + break + } + owner := common.BytesToHash(hexToKeybytes(path)) + storage, err := New(account.Root, in.triedb) + if err != nil { + log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) + break + } + storageStat := NewLevelStats() + run := func() { + in.recordRootSize(account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat, inspectStorageTrie) + in.writeDumpRecord(owner, storageStat) + } + if in.trySpawn(&wg, run) { + break + } + run() + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } + + wg.Wait() + + // Record stats for the current node once the subtree (and any + // spawned siblings) have been fully accounted for. + stat.add(n, height) +} + +// Summarize is pass 2: read the dump file, aggregate every storage-trie +// record, compute the top-N rankings, and emit either stdout tables or a +// JSON blob depending on InspectConfig.Path. +func Summarize(dumpPath string, config *InspectConfig) error { + config = normalizeInspectConfig(config) + if dumpPath == "" { + dumpPath = config.DumpPath + } + if dumpPath == "" { + return errors.New("missing dump path") + } + file, err := os.Open(dumpPath) + if err != nil { + return fmt.Errorf("failed to open trie dump %s: %w", dumpPath, err) + } + defer file.Close() + + if info, err := file.Stat(); err == nil { + if info.Size()%inspectDumpRecordSize != 0 { + return fmt.Errorf("invalid trie dump size %d (not a multiple of %d)", info.Size(), inspectDumpRecordSize) + } + } + + depthTop := newStorageStatsTopN(config.TopN, compareStorageStatsByDepth) + totalTop := newStorageStatsTopN(config.TopN, compareStorageStatsByTotal) + valueTop := newStorageStatsTopN(config.TopN, compareStorageStatsByValue) + + summary := &inspectSummary{} + reader := bufio.NewReaderSize(file, 1<<20) + var buf [inspectDumpRecordSize]byte + + for { + _, err := io.ReadFull(reader, buf[:]) + if errors.Is(err, io.EOF) { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("truncated trie dump %s", dumpPath) + } + if err != nil { + return fmt.Errorf("failed reading trie dump %s: %w", dumpPath, err) + } + + record := decodeDumpRecord(buf[:]) + snapshot := newStorageStats(record.Owner, record.Levels) + if record.Owner == (common.Hash{}) { + summary.Account = snapshot + continue + } + summary.StorageCount++ + summary.DepthHistogram[snapshot.MaxDepth]++ + for i := 0; i < trieStatLevels; i++ { + summary.StorageLevels[i].Short += record.Levels[i].Short + summary.StorageLevels[i].Full += record.Levels[i].Full + summary.StorageLevels[i].Value += record.Levels[i].Value + summary.StorageLevels[i].Size += record.Levels[i].Size + } + depthTop.TryInsert(snapshot) + totalTop.TryInsert(snapshot) + valueTop.TryInsert(snapshot) + } + if summary.Account == nil { + return fmt.Errorf("dump file %s does not contain the account trie sentinel record", dumpPath) + } + for i := 0; i < trieStatLevels; i++ { + summary.StorageTotals.Short += summary.StorageLevels[i].Short + summary.StorageTotals.Full += summary.StorageLevels[i].Full + summary.StorageTotals.Value += summary.StorageLevels[i].Value + summary.StorageTotals.Size += summary.StorageLevels[i].Size + } + summary.TopByDepth = depthTop.Sorted() + summary.TopByTotalNodes = totalTop.Sorted() + summary.TopByValueNodes = valueTop.Sorted() + + if config.Path != "" { + return summary.writeJSON(config.Path) + } + summary.display() + return nil +} + +// ----------------------------------------------------------------------- +// Internal dump-record types and accessors +// ----------------------------------------------------------------------- + +type dumpRecord struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel +} + +func decodeDumpRecord(raw []byte) dumpRecord { + var ( + record dumpRecord + off = 32 + ) + copy(record.Owner[:], raw[:32]) + for i := 0; i < trieStatLevels; i++ { + record.Levels[i] = jsonLevel{ + Short: uint64(binary.LittleEndian.Uint32(raw[off:])), + Full: uint64(binary.LittleEndian.Uint32(raw[off+4:])), + Value: uint64(binary.LittleEndian.Uint32(raw[off+8:])), + Size: binary.LittleEndian.Uint64(raw[off+12:]), + } + off += 20 + } + return record +} + +type storageStats struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel + Summary jsonLevel + MaxDepth int + TotalNodes uint64 + TotalSize uint64 +} + +func newStorageStats(owner common.Hash, levels [trieStatLevels]jsonLevel) *storageStats { + snapshot := &storageStats{Owner: owner, Levels: levels} + for i := 0; i < trieStatLevels; i++ { + level := levels[i] + if level.Short != 0 || level.Full != 0 || level.Value != 0 { + snapshot.MaxDepth = i + } + snapshot.Summary.Short += level.Short + snapshot.Summary.Full += level.Full + snapshot.Summary.Value += level.Value + snapshot.Summary.Size += level.Size + } + snapshot.TotalNodes = snapshot.Summary.Short + snapshot.Summary.Full + snapshot.Summary.Value + snapshot.TotalSize = snapshot.Summary.Size + return snapshot +} + +func trimLevels(levels [trieStatLevels]jsonLevel) []jsonLevel { + n := len(levels) + for n > 0 && levels[n-1] == (jsonLevel{}) { + n-- + } + return levels[:n] +} + +func (s *storageStats) MarshalJSON() ([]byte, error) { + type jsonStorageSnapshot struct { + Owner common.Hash `json:"Owner"` + MaxDepth int `json:"MaxDepth"` + TotalNodes uint64 `json:"TotalNodes"` + TotalSize uint64 `json:"TotalSize"` + ValueNodes uint64 `json:"ValueNodes"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + return json.Marshal(jsonStorageSnapshot{ + Owner: s.Owner, + MaxDepth: s.MaxDepth, + TotalNodes: s.TotalNodes, + TotalSize: s.TotalSize, + ValueNodes: s.Summary.Value, + Levels: trimLevels(s.Levels), + Summary: s.Summary, + }) +} + +func (s *storageStats) toLevelStats() *LevelStats { + stats := NewLevelStats() + for i := 0; i < trieStatLevels; i++ { + stats.level[i].short.Store(s.Levels[i].Short) + stats.level[i].full.Store(s.Levels[i].Full) + stats.level[i].value.Store(s.Levels[i].Value) + stats.level[i].size.Store(s.Levels[i].Size) + } + return stats +} + +// ----------------------------------------------------------------------- +// Top-N bookkeeping +// ----------------------------------------------------------------------- + +type storageStatsCompare func(a, b *storageStats) int + +type storageStatsTopN struct { + limit int + cmp storageStatsCompare + heap storageStatsHeap +} + +type storageStatsHeap struct { + items []*storageStats + cmp storageStatsCompare +} + +func (h storageStatsHeap) Len() int { return len(h.items) } + +func (h storageStatsHeap) Less(i, j int) bool { + // Keep the weakest entry at the root (min-heap semantics) so we + // can evict the smallest in O(log N) when a bigger one arrives. + return h.cmp(h.items[i], h.items[j]) < 0 +} + +func (h storageStatsHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] } + +func (h *storageStatsHeap) Push(x any) { + h.items = append(h.items, x.(*storageStats)) +} + +func (h *storageStatsHeap) Pop() any { + item := h.items[len(h.items)-1] + h.items = h.items[:len(h.items)-1] + return item +} + +func newStorageStatsTopN(limit int, cmp storageStatsCompare) *storageStatsTopN { + h := storageStatsHeap{cmp: cmp} + heap.Init(&h) + return &storageStatsTopN{limit: limit, cmp: cmp, heap: h} +} + +func (t *storageStatsTopN) TryInsert(item *storageStats) { + if t.limit <= 0 { + return + } + if t.heap.Len() < t.limit { + heap.Push(&t.heap, item) + return + } + if t.cmp(item, t.heap.items[0]) <= 0 { + return + } + heap.Pop(&t.heap) + heap.Push(&t.heap, item) +} + +func (t *storageStatsTopN) Sorted() []*storageStats { + out := append([]*storageStats(nil), t.heap.items...) + sort.Slice(out, func(i, j int) bool { return t.cmp(out[i], out[j]) > 0 }) + return out +} + +// The three ordering functions below share the same tie-breaking chain +// (secondary by MaxDepth / TotalNodes / value count and finally by owner +// hash for stability). Adjusting any one should be mirrored in the other +// two so ranking output stays consistent. + +func compareStorageStatsByDepth(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByTotal(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByValue(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.Summary.Value, b.Summary.Value), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +// ----------------------------------------------------------------------- +// Summary report assembly / rendering +// ----------------------------------------------------------------------- + +type inspectSummary struct { + Account *storageStats + StorageCount uint64 + StorageTotals jsonLevel + StorageLevels [trieStatLevels]jsonLevel + DepthHistogram [trieStatLevels]uint64 + TopByDepth []*storageStats + TopByTotalNodes []*storageStats + TopByValueNodes []*storageStats +} + +func (s *inspectSummary) display() { + s.displayCombinedDepthTable() + s.Account.toLevelStats().display("Accounts trie") + fmt.Println("Storage trie aggregate summary") + fmt.Printf("Total storage tries: %d\n", s.StorageCount) + totalNodes := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + fmt.Printf("Total nodes: %d\n", totalNodes) + fmt.Printf("Total size: %s\n", common.StorageSize(s.StorageTotals.Size)) + fmt.Printf(" Short nodes: %d\n", s.StorageTotals.Short) + fmt.Printf(" Full nodes: %d\n", s.StorageTotals.Full) + fmt.Printf(" Value nodes: %d\n", s.StorageTotals.Value) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Max Depth", "Storage Tries"}) + for i, count := range s.DepthHistogram { + table.AppendBulk([][]string{{fmt.Sprint(i), fmt.Sprint(count)}}) + } + _ = table.Render() + fmt.Print(b.String()) + fmt.Println() + + s.displayTop("Top storage tries by max depth", s.TopByDepth) + s.displayTop("Top storage tries by total node count", s.TopByTotalNodes) + s.displayTop("Top storage tries by value (slot) count", s.TopByValueNodes) +} + +func (s *inspectSummary) displayCombinedDepthTable() { + accountTotal := s.Account.Summary.Short + s.Account.Summary.Full + s.Account.Summary.Value + storageTotal := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + accountTotalSize := s.Account.Summary.Size + storageTotalSize := s.StorageTotals.Size + + fmt.Println("Trie Depth Distribution") + fmt.Printf("Account Trie: %d nodes (%s)\n", accountTotal, common.StorageSize(accountTotalSize)) + fmt.Printf("Storage Tries: %d nodes (%s) across %d tries\n", storageTotal, common.StorageSize(storageTotalSize), s.StorageCount) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Account Nodes", "Account Size", "Storage Nodes", "Storage Size"}) + for i := 0; i < trieStatLevels; i++ { + accountNodes := s.Account.Levels[i].Short + s.Account.Levels[i].Full + s.Account.Levels[i].Value + accountSize := s.Account.Levels[i].Size + storageNodes := s.StorageLevels[i].Short + s.StorageLevels[i].Full + s.StorageLevels[i].Value + storageSize := s.StorageLevels[i].Size + if accountNodes == 0 && storageNodes == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(accountNodes), + common.StorageSize(accountSize).String(), + fmt.Sprint(storageNodes), + common.StorageSize(storageSize).String(), + }}) + } + _ = table.Render() + fmt.Print(b.String()) + fmt.Println() +} + +func (s *inspectSummary) displayTop(title string, list []*storageStats) { + fmt.Println(title) + if len(list) == 0 { + fmt.Println("No storage tries found") + fmt.Println() + return + } + for i, item := range list { + fmt.Printf("%d: %s\n", i+1, item.Owner) + item.toLevelStats().display("storage trie") + } +} + +func (s *inspectSummary) MarshalJSON() ([]byte, error) { + type jsonAccountTrie struct { + Name string `json:"Name"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + type jsonStorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } + type jsonInspectSummary struct { + AccountTrie jsonAccountTrie `json:"AccountTrie"` + StorageSummary jsonStorageSummary `json:"StorageSummary"` + TopByDepth []*storageStats `json:"TopByDepth"` + TopByTotalNodes []*storageStats `json:"TopByTotalNodes"` + TopByValueNodes []*storageStats `json:"TopByValueNodes"` + } + return json.Marshal(jsonInspectSummary{ + AccountTrie: jsonAccountTrie{ + Name: "account trie", + Levels: trimLevels(s.Account.Levels), + Summary: s.Account.Summary, + }, + StorageSummary: jsonStorageSummary{ + TotalStorageTries: s.StorageCount, + Totals: s.StorageTotals, + Levels: trimLevels(s.StorageLevels), + DepthHistogram: s.DepthHistogram, + }, + TopByDepth: s.TopByDepth, + TopByTotalNodes: s.TopByTotalNodes, + TopByValueNodes: s.TopByValueNodes, + }) +} + +func (s *inspectSummary) writeJSON(path string) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + return enc.Encode(s) +} + +// display prints a per-level breakdown for the statistics collected by +// the given LevelStats. Title is truncated when long so console tables +// stay readable even for storage trie owners (32-byte hex strings). +func (s *LevelStats) display(title string) { + if len(title) > 32 { + title = title[0:8] + "..." + title[len(title)-8:] + } + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{title, "Level", "Short Nodes", "Full Node", "Value Node"}) + + total := &stat{} + for i := range s.level { + if s.level[i].empty() { + continue + } + short, full, value, _ := s.level[i].load() + table.AppendBulk([][]string{{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}}) + total.add(&s.level[i]) + } + short, full, value, _ := total.load() + table.SetFooter([]string{"Total", "", fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) + _ = table.Render() + fmt.Print(b.String()) + fmt.Println("Max depth", s.MaxDepth()) + fmt.Println() +} + +// jsonLevel is the per-level record laid out in dump files and serialized +// into JSON summaries. Fields are exported so the binary and textual +// representations share the same layout. +type jsonLevel struct { + Short uint64 + Full uint64 + Value uint64 + Size uint64 +} diff --git a/trie/inspect_test.go b/trie/inspect_test.go new file mode 100644 index 000000000..0907e70e7 --- /dev/null +++ b/trie/inspect_test.go @@ -0,0 +1,552 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "encoding/json" + "math/big" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/crypto" + "github.com/morph-l2/go-ethereum/crypto/codehash" + "github.com/morph-l2/go-ethereum/ethdb/memorydb" + "github.com/morph-l2/go-ethereum/rlp" +) + +// makeInspectFixture synthesises an account trie with `size` accounts. +// When withStorage is true every account gets its own small storage trie +// (random slot count between 1 and 256) so the inspector has something +// to count in both the account and storage passes. Accounts also carry a +// random-sized balance to exercise the size-on-disk accounting. +// +// The returned trie is a plain MPT (not SecureTrie): keys are pre-hashed +// so callers can look up accounts via `crypto.Keccak256(address)` — the +// convention that Inspect/InspectContract follow internally. +func makeInspectFixture(t *testing.T, size int, withStorage bool) (*Database, common.Hash) { + t.Helper() + db := NewDatabase(memorydb.New()) + + random := rand.New(rand.NewSource(0)) + accTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open account trie: %v", err) + } + for i := 0; i < size; i++ { + var addr [20]byte + random.Read(addr[:]) + accountKey := crypto.Keccak256(addr[:]) + + var storageRoot common.Hash + if withStorage { + storageTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open storage trie: %v", err) + } + slots := int(random.Uint32()%256 + 1) + for j := 0; j < slots; j++ { + k := make([]byte, 32) + v := make([]byte, 32) + random.Read(k) + random.Read(v) + storageTrie.Update(k, v) + } + var committed common.Hash + committed, _, err = storageTrie.Commit(nil) + if err != nil { + t.Fatalf("commit storage trie: %v", err) + } + storageRoot = committed + if err := db.Commit(storageRoot, false, nil); err != nil { + t.Fatalf("flush storage trie: %v", err) + } + } else { + storageRoot = emptyRoot + } + + // Balance uses a random byte length so the RLP payload varies. + numBytes := random.Uint32() % 33 + balanceBytes := make([]byte, numBytes) + random.Read(balanceBytes) + balance := new(big.Int).SetBytes(balanceBytes) + + acc := types.StateAccount{ + Nonce: uint64(random.Int63()), + Balance: balance, + Root: storageRoot, + KeccakCodeHash: codehash.EmptyKeccakCodeHash.Bytes(), + } + enc, err := rlp.EncodeToBytes(&acc) + if err != nil { + t.Fatalf("encode account: %v", err) + } + accTrie.Update(accountKey, enc) + } + root, _, err := accTrie.Commit(nil) + if err != nil { + t.Fatalf("commit account trie: %v", err) + } + if err := db.Commit(root, false, nil); err != nil { + t.Fatalf("flush account trie: %v", err) + } + return db, root +} + +// TestInspectRoundTripsDump walks a small account trie with storage, +// writes the pass-1 dump to a temp directory, then independently runs +// Summarize against the same dump and checks that both produce identical +// JSON summaries. This is the upstream parity invariant: Inspect and +// Summarize must agree on every aggregate. +func TestInspectRoundTripsDump(t *testing.T) { + db, root := makeInspectFixture(t, 11, true) + + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + inspectJSON := filepath.Join(tempDir, "trie-summary.json") + reanalysisJSON := filepath.Join(tempDir, "trie-summary-reanalysis.json") + + if err := Inspect(db, root, &InspectConfig{ + TopN: 1, + DumpPath: dumpPath, + Path: inspectJSON, + }); err != nil { + t.Fatalf("inspect failed: %v", err) + } + if err := Summarize(dumpPath, &InspectConfig{ + TopN: 1, + Path: reanalysisJSON, + }); err != nil { + t.Fatalf("summarize failed: %v", err) + } + + inspectOut := loadInspectJSON(t, inspectJSON) + reanalysisOut := loadInspectJSON(t, reanalysisJSON) + + if len(inspectOut.StorageSummary.Levels) == 0 { + t.Fatal("expected StorageSummary.Levels to be populated") + } + if inspectOut.AccountTrie.Summary.Size == 0 { + t.Fatal("expected account trie size summary to be populated") + } + if inspectOut.StorageSummary.Totals.Size == 0 { + t.Fatal("expected storage trie size summary to be populated") + } + if !reflect.DeepEqual(inspectOut.AccountTrie, reanalysisOut.AccountTrie) { + t.Fatal("account trie summary mismatch between inspect and summarize") + } + if !reflect.DeepEqual(inspectOut.StorageSummary, reanalysisOut.StorageSummary) { + t.Fatal("storage summary mismatch between inspect and summarize") + } + + assertStorageTotalsMatchLevels(t, inspectOut) + assertStorageTotalsMatchLevels(t, reanalysisOut) + assertAccountTotalsMatchLevels(t, inspectOut.AccountTrie) + assertAccountTotalsMatchLevels(t, reanalysisOut.AccountTrie) + + var histogramTotal uint64 + for _, count := range inspectOut.StorageSummary.DepthHistogram { + histogramTotal += count + } + if histogramTotal != inspectOut.StorageSummary.TotalStorageTries { + t.Fatalf("depth histogram total %d does not match total storage tries %d", + histogramTotal, inspectOut.StorageSummary.TotalStorageTries) + } +} + +// TestInspectNoStorageSkipsWalk confirms the NoStorage option short- +// circuits per-account storage walks while still producing an account- +// trie report. +func TestInspectNoStorageSkipsWalk(t *testing.T) { + db, root := makeInspectFixture(t, 5, true) + + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + jsonPath := filepath.Join(tempDir, "trie-summary.json") + + if err := Inspect(db, root, &InspectConfig{ + NoStorage: true, + TopN: 3, + DumpPath: dumpPath, + Path: jsonPath, + }); err != nil { + t.Fatalf("inspect failed: %v", err) + } + + out := loadInspectJSON(t, jsonPath) + if out.StorageSummary.TotalStorageTries != 0 { + t.Fatalf("expected 0 storage tries with NoStorage, got %d", out.StorageSummary.TotalStorageTries) + } + if out.StorageSummary.Totals.Size != 0 { + t.Fatalf("expected zero storage size with NoStorage, got %d", out.StorageSummary.Totals.Size) + } + if out.AccountTrie.Summary.Size == 0 { + t.Fatal("account trie size should still be populated") + } +} + +// TestInspectEmptyRootEmitsAccountSentinel verifies that running Inspect +// against an empty trie emits exactly one (account) record with no +// storage entries — guarding against regressions where an early-return +// path forgets to write the sentinel. +func TestInspectEmptyRootEmitsAccountSentinel(t *testing.T) { + db := NewDatabase(memorydb.New()) + + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + jsonPath := filepath.Join(tempDir, "trie-summary.json") + + // emptyRoot corresponds to an empty MPT; `New` accepts it directly. + if err := Inspect(db, emptyRoot, &InspectConfig{ + TopN: 3, + DumpPath: dumpPath, + Path: jsonPath, + }); err != nil { + t.Fatalf("inspect empty trie: %v", err) + } + + info, err := os.Stat(dumpPath) + if err != nil { + t.Fatalf("stat dump: %v", err) + } + if info.Size() != inspectDumpRecordSize { + t.Fatalf("expected exactly one sentinel record (%d bytes), got %d", inspectDumpRecordSize, info.Size()) + } + out := loadInspectJSON(t, jsonPath) + if out.StorageSummary.TotalStorageTries != 0 { + t.Fatalf("expected 0 storage tries for empty trie, got %d", out.StorageSummary.TotalStorageTries) + } +} + +// TestInspectRejectsMissingRoot ensures that asking the inspector for a +// root whose nodes are not in the database surfaces an error rather than +// silently producing an empty report. +func TestInspectRejectsMissingRoot(t *testing.T) { + db := NewDatabase(memorydb.New()) + tempDir := t.TempDir() + + unknown := common.HexToHash("0x9999999999999999999999999999999999999999999999999999999999999999") + err := Inspect(db, unknown, &InspectConfig{ + TopN: 1, + DumpPath: filepath.Join(tempDir, "trie-dump.bin"), + }) + if err == nil { + t.Fatal("expected error for unknown root, got nil") + } +} + +// TestSummarizeRejectsTruncatedDump ensures Summarize fails fast on a +// dump that is not a multiple of the record size. +func TestSummarizeRejectsTruncatedDump(t *testing.T) { + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "truncated.bin") + if err := os.WriteFile(dumpPath, []byte{1, 2, 3}, 0o644); err != nil { + t.Fatalf("write truncated dump: %v", err) + } + err := Summarize(dumpPath, &InspectConfig{TopN: 1}) + if err == nil { + t.Fatal("expected error for truncated dump") + } +} + +// TestSummarizeRejectsMissingAccountSentinel ensures Summarize refuses +// dumps that contain only storage records (no account sentinel). +func TestSummarizeRejectsMissingAccountSentinel(t *testing.T) { + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "no-sentinel.bin") + + // One record with a non-zero owner (storage) and no account + // sentinel. The record itself is all zeros except for the owner. + var raw [inspectDumpRecordSize]byte + raw[0] = 0x11 + if err := os.WriteFile(dumpPath, raw[:], 0o644); err != nil { + t.Fatalf("write dump: %v", err) + } + err := Summarize(dumpPath, &InspectConfig{TopN: 1}) + if err == nil { + t.Fatal("expected error for dump without account sentinel") + } +} + +// TestInspectContract inspects a single contract with populated storage +// and snapshot data, mirroring the upstream exercise of InspectContract. +func TestInspectContract(t *testing.T) { + diskdb := rawdb.NewMemoryDatabase() + db := NewDatabase(diskdb) + + address := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + accountHash := crypto.Keccak256Hash(address.Bytes()) + + storageTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open storage trie: %v", err) + } + storageSlots := make(map[common.Hash][]byte) + for i := 0; i < 10; i++ { + k := crypto.Keccak256Hash([]byte{byte(i)}) + v := []byte{byte(i + 1)} + storageTrie.Update(k.Bytes(), v) + storageSlots[k] = v + } + storageRoot, _, err := storageTrie.Commit(nil) + if err != nil { + t.Fatalf("commit storage trie: %v", err) + } + if err := db.Commit(storageRoot, false, nil); err != nil { + t.Fatalf("flush storage trie: %v", err) + } + + account := types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(1000), + Root: storageRoot, + KeccakCodeHash: codehash.EmptyKeccakCodeHash.Bytes(), + } + accountRLP, err := rlp.EncodeToBytes(&account) + if err != nil { + t.Fatalf("encode account: %v", err) + } + + // Plain Trie: keys are stored by their hashed form (matching state db + // layout), which is what InspectContract reads via + // accountTrie.TryGet(crypto.Keccak256(address)). + accountTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open account trie: %v", err) + } + accountTrie.Update(crypto.Keccak256(address.Bytes()), accountRLP) + stateRoot, _, err := accountTrie.Commit(nil) + if err != nil { + t.Fatalf("commit account trie: %v", err) + } + if err := db.Commit(stateRoot, false, nil); err != nil { + t.Fatalf("flush account trie: %v", err) + } + + // Populate snapshot entries so InspectContract can exercise the + // snapshot accounting code path as well. + rawdb.WriteAccountSnapshot(diskdb, accountHash, accountRLP) + for k, v := range storageSlots { + rawdb.WriteStorageSnapshot(diskdb, accountHash, k, v) + } + + if err := InspectContract(db, diskdb, stateRoot, address); err != nil { + t.Fatalf("InspectContract failed: %v", err) + } +} + +// TestInspectContractRejectsMissingAccount asserts that InspectContract +// refuses to operate on an address that is absent from the account trie, +// instead of silently producing an empty snapshot. +func TestInspectContractRejectsMissingAccount(t *testing.T) { + diskdb := rawdb.NewMemoryDatabase() + db := NewDatabase(diskdb) + // A valid but empty state root; any address lookup returns nil. + err := InspectContract(db, diskdb, emptyRoot, common.HexToAddress("0xaabb")) + if err == nil { + t.Fatal("expected error for missing account") + } +} + +// TestInspectContractRejectsStorageless asserts that InspectContract +// refuses to operate on accounts that exist but have no storage trie. +func TestInspectContractRejectsStorageless(t *testing.T) { + diskdb := rawdb.NewMemoryDatabase() + db := NewDatabase(diskdb) + + address := common.HexToAddress("0xcafebabe0000000000000000000000000000dead") + account := types.StateAccount{ + Nonce: 0, + Balance: big.NewInt(0), + Root: emptyRoot, + KeccakCodeHash: codehash.EmptyKeccakCodeHash.Bytes(), + } + enc, err := rlp.EncodeToBytes(&account) + if err != nil { + t.Fatalf("encode account: %v", err) + } + accTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open account trie: %v", err) + } + accTrie.Update(crypto.Keccak256(address.Bytes()), enc) + root, _, err := accTrie.Commit(nil) + if err != nil { + t.Fatalf("commit: %v", err) + } + if err := db.Commit(root, false, nil); err != nil { + t.Fatalf("flush: %v", err) + } + + err = InspectContract(db, diskdb, root, address) + if err == nil { + t.Fatal("expected error for storageless account") + } +} + +func TestInspectDoesNotExpandStorageValuesAsAccounts(t *testing.T) { + db := NewDatabase(memorydb.New()) + + childTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open child trie: %v", err) + } + childTrie.Update(crypto.Keccak256([]byte("child-slot")), []byte{0x01}) + childRoot, _, err := childTrie.Commit(nil) + if err != nil { + t.Fatalf("commit child trie: %v", err) + } + if err := db.Commit(childRoot, false, nil); err != nil { + t.Fatalf("flush child trie: %v", err) + } + + decoyAccount := types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(1), + Root: childRoot, + KeccakCodeHash: codehash.EmptyKeccakCodeHash.Bytes(), + } + decoyValue, err := rlp.EncodeToBytes(&decoyAccount) + if err != nil { + t.Fatalf("encode decoy account: %v", err) + } + + storageTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open storage trie: %v", err) + } + storageTrie.Update(crypto.Keccak256([]byte("slot")), decoyValue) + storageRoot, _, err := storageTrie.Commit(nil) + if err != nil { + t.Fatalf("commit storage trie: %v", err) + } + if err := db.Commit(storageRoot, false, nil); err != nil { + t.Fatalf("flush storage trie: %v", err) + } + + account := types.StateAccount{ + Nonce: 1, + Balance: big.NewInt(1), + Root: storageRoot, + KeccakCodeHash: codehash.EmptyKeccakCodeHash.Bytes(), + } + accountValue, err := rlp.EncodeToBytes(&account) + if err != nil { + t.Fatalf("encode account: %v", err) + } + accountTrie, err := New(common.Hash{}, db) + if err != nil { + t.Fatalf("open account trie: %v", err) + } + accountTrie.Update(crypto.Keccak256([]byte("account")), accountValue) + stateRoot, _, err := accountTrie.Commit(nil) + if err != nil { + t.Fatalf("commit account trie: %v", err) + } + if err := db.Commit(stateRoot, false, nil); err != nil { + t.Fatalf("flush account trie: %v", err) + } + + tempDir := t.TempDir() + jsonPath := filepath.Join(tempDir, "trie-summary.json") + if err := Inspect(db, stateRoot, &InspectConfig{ + TopN: 10, + DumpPath: filepath.Join(tempDir, "trie-dump.bin"), + Path: jsonPath, + }); err != nil { + t.Fatalf("inspect failed: %v", err) + } + + out := loadInspectJSON(t, jsonPath) + if out.StorageSummary.TotalStorageTries != 1 { + t.Fatalf("expected exactly one real storage trie, got %d", out.StorageSummary.TotalStorageTries) + } +} + +func TestInspectSkipsDepthsOutsideLevelStats(t *testing.T) { + stats := NewLevelStats() + in := &inspector{} + + in.inspect(nil, valueNode{0x01}, trieStatLevels, nil, stats, inspectStorageTrie) + + if nodes := stats.TotalNodes(); nodes != 0 { + t.Fatalf("recorded %d nodes past tracked depth, want 0", nodes) + } +} + +// ----------------------------------------------------------------------- +// JSON shape helpers +// ----------------------------------------------------------------------- + +// inspectJSONOutput mirrors the shape of inspectSummary's MarshalJSON +// output — using storageStats for AccountTrie avoids a parallel type +// definition since only Levels and Summary are populated by inspect. +type inspectJSONOutput struct { + AccountTrie storageStats `json:"AccountTrie"` + StorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } `json:"StorageSummary"` +} + +func loadInspectJSON(t *testing.T, path string) inspectJSONOutput { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + var out inspectJSONOutput + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("failed to decode %s: %v", path, err) + } + return out +} + +func assertStorageTotalsMatchLevels(t *testing.T, out inspectJSONOutput) { + t.Helper() + var fromLevels jsonLevel + for _, level := range out.StorageSummary.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels != out.StorageSummary.Totals { + t.Fatalf("storage totals mismatch: levels=%+v totals=%+v", fromLevels, out.StorageSummary.Totals) + } +} + +func assertAccountTotalsMatchLevels(t *testing.T, account storageStats) { + t.Helper() + var fromLevels jsonLevel + for _, level := range account.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels != account.Summary { + t.Fatalf("account totals mismatch: levels=%+v totals=%+v", fromLevels, account.Summary) + } +} diff --git a/trie/levelstats.go b/trie/levelstats.go new file mode 100644 index 000000000..9ea83c182 --- /dev/null +++ b/trie/levelstats.go @@ -0,0 +1,141 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "sync/atomic" +) + +// trieStatLevels is the maximum number of levels LevelStats tracks. While +// a secure MPT with 32-byte keys can theoretically reach depth 64, every +// production trie observed in practice finishes well before depth 16. +// Keeping the fixed bound matches upstream go-ethereum PR #28892 so that +// on-disk dump records have a stable size and layout. +const trieStatLevels = 16 + +// LevelStats tracks the type and count of trie nodes at each level in a +// Merkle Patricia Trie. It is safe for concurrent use: every counter is an +// atomic value so multiple walker goroutines can record nodes without +// external synchronization. +// +// Level indexing follows the walker's notion of "depth from the root" and +// must not exceed trieStatLevels-1; attempts to record nodes at depth >= +// trieStatLevels trigger a panic. Callers that operate on arbitrary trie +// data should pre-validate depth before invoking add / AddLeaf. +type LevelStats struct { + level [trieStatLevels]stat +} + +// NewLevelStats creates an empty trie statistics collector. +func NewLevelStats() *LevelStats { + return &LevelStats{} +} + +// MaxDepth iterates each level and finds the deepest level with at least +// one trie node. Returns zero for an empty LevelStats. +func (s *LevelStats) MaxDepth() int { + depth := 0 + for i := range s.level { + if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { + depth = i + } + } + return depth +} + +// TotalNodes returns the total number of nodes across all levels and node +// types. Hash nodes are not counted — only their resolved underlying nodes +// contribute. +func (s *LevelStats) TotalNodes() uint64 { + var total uint64 + for i := range s.level { + total += s.level[i].short.Load() + s.level[i].full.Load() + s.level[i].value.Load() + } + return total +} + +// add increases the node count by one for the specified node type and +// depth. hashNodes do not have their own slot: the caller is expected to +// resolve them and record the resulting short/full/value node instead. +func (s *LevelStats) add(n node, depth uint32) { + d := int(depth) + switch (n).(type) { + case *shortNode: + s.level[d].short.Add(1) + case *fullNode: + s.level[d].full.Add(1) + case valueNode: + s.level[d].value.Add(1) + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +// addSize increases the raw byte-size tally at the specified depth. It is +// used to account for the encoded size of resolved hash-referenced nodes, +// which the walker learns on lookup but whose node type is recorded +// separately via add() once decoded. +func (s *LevelStats) addSize(depth uint32, size uint64) { + s.level[depth].size.Add(size) +} + +// AddLeaf records a leaf observation at the given depth. The leaf bucket +// reuses the value-node counter so stateless witness stats and inspect- +// trie share the same storage layout. Panics when depth is outside +// [0, trieStatLevels-1]. +func (s *LevelStats) AddLeaf(depth int) { + s.level[depth].value.Add(1) +} + +// LeafDepths returns leaf counts grouped by depth. The returned array is a +// snapshot; subsequent updates to LevelStats do not affect it. +func (s *LevelStats) LeafDepths() [trieStatLevels]int64 { + var leaves [trieStatLevels]int64 + for i := range s.level { + leaves[i] = int64(s.level[i].value.Load()) + } + return leaves +} + +// stat is a specific level's count of each node type plus raw byte-size. +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 + size atomic.Uint64 +} + +// empty returns whether there are any trie nodes at the level. +func (s *stat) empty() bool { + return s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 && s.size.Load() == 0 +} + +// load atomically reads every field of the stat. +func (s *stat) load() (uint64, uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load(), s.size.Load() +} + +// add folds other into s in place and returns s. Used when aggregating +// per-level totals into a grand total. +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + s.size.Add(other.size.Load()) + return s +} diff --git a/trie/levelstats_test.go b/trie/levelstats_test.go new file mode 100644 index 000000000..4a213f3cf --- /dev/null +++ b/trie/levelstats_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "testing" + +func TestLevelStatsAddLeafDepthBounds(t *testing.T) { + stats := NewLevelStats() + stats.AddLeaf(15) + + if got := stats.LeafDepths()[15]; got != 1 { + t.Fatalf("leaf count at depth 15 = %d, want 1", got) + } + if got := stats.MaxDepth(); got != 15 { + t.Fatalf("MaxDepth = %d, want 15", got) + } + if got := stats.TotalNodes(); got != 1 { + t.Fatalf("TotalNodes = %d, want 1", got) + } +} + +func TestLevelStatsAddLeafPanicsOnDepth16(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + NewLevelStats().AddLeaf(16) +} + +func TestLevelStatsAddNodeClassification(t *testing.T) { + stats := NewLevelStats() + stats.add(&shortNode{}, 3) + stats.add(&fullNode{}, 4) + stats.add(valueNode{0x01}, 5) + + short, full, value, _ := stats.level[3].load() + if short != 1 || full != 0 || value != 0 { + t.Fatalf("level 3 = short=%d full=%d value=%d, want short=1", short, full, value) + } + _, full, _, _ = stats.level[4].load() + if full != 1 { + t.Fatalf("level 4 full = %d, want 1", full) + } + _, _, value, _ = stats.level[5].load() + if value != 1 { + t.Fatalf("level 5 value = %d, want 1", value) + } + if got := stats.TotalNodes(); got != 3 { + t.Fatalf("TotalNodes = %d, want 3", got) + } +} + +func TestLevelStatsAddSizeAccumulates(t *testing.T) { + stats := NewLevelStats() + stats.addSize(2, 100) + stats.addSize(2, 50) + _, _, _, size := stats.level[2].load() + if size != 150 { + t.Fatalf("size accumulation = %d, want 150", size) + } +} + +func TestLevelStatsStatEmpty(t *testing.T) { + var s stat + if !s.empty() { + t.Fatalf("fresh stat should be empty") + } + s.short.Add(1) + if s.empty() { + t.Fatalf("stat with nonzero short must not be empty") + } +} + +func TestLevelStatsStatAdd(t *testing.T) { + var a, b stat + a.short.Store(1) + a.full.Store(2) + a.value.Store(3) + a.size.Store(4) + b.short.Store(10) + b.full.Store(20) + b.value.Store(30) + b.size.Store(40) + + sum := (&stat{}).add(&a).add(&b) + s, f, v, sz := sum.load() + if s != 11 || f != 22 || v != 33 || sz != 44 { + t.Fatalf("stat add: got %d/%d/%d/%d, want 11/22/33/44", s, f, v, sz) + } +}