diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60a0dfb6ed..dbdc21c098 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Golang-ci install if: runner.os == 'Linux' - run: make lintci-deps + run: make lint-deps - name: Lint if: runner.os == 'Linux' diff --git a/.golangci.yml b/.golangci.yml index da69967d34..7c75529f61 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,7 @@ # This file configures github.com/golangci/golangci-lint. version: '2' run: + go: 1.25.5 tests: true linters: default: none @@ -34,6 +35,9 @@ linters: # - makezero # false positives # - nilerr # several intentional settings: + govet: + disable: + - buildtag staticcheck: checks: # disable Quickfixes diff --git a/.travis.yml b/.travis.yml index e882aa3b75..f71e23ca79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ jobs: os: linux arch: amd64 dist: focal - go: 1.24.x + go: 1.25.x env: - docker services: @@ -51,7 +51,7 @@ jobs: os: linux dist: focal sudo: required - go: 1.24.x + go: 1.25.x env: - azure-linux git: @@ -141,7 +141,7 @@ jobs: os: linux arch: amd64 dist: focal - go: 1.24.x + go: 1.25.x script: - travis_wait 45 go run build/ci.go test $TEST_PACKAGES @@ -166,7 +166,7 @@ jobs: if: type = cron || (type = push && tag ~= /^v[0-9]/) os: linux dist: focal - go: 1.24.x + go: 1.25.x env: - ubuntu-ppa git: @@ -182,7 +182,7 @@ jobs: if: type = cron os: linux dist: focal - go: 1.24.x + go: 1.25.x env: - azure-purge git: @@ -195,7 +195,7 @@ jobs: if: type = cron os: linux dist: focal - go: 1.24.x + go: 1.25.x env: - racetests script: diff --git a/Dockerfile b/Dockerfile index d335d81b5b..e14bf93611 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # ─── BUILDER STAGE ─────────────────────────────────────────────────────────────── -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder ARG BOR_DIR=/var/lib/bor/ ENV BOR_DIR=$BOR_DIR @@ -36,4 +36,4 @@ COPY --from=builder ${BOR_DIR}/build/bin/bor /usr/bin/ EXPOSE 8545 8546 8547 30303 30303/udp -ENTRYPOINT ["bor"] \ No newline at end of file +ENTRYPOINT ["bor"] diff --git a/Dockerfile.alltools b/Dockerfile.alltools index d28928b7a4..d32767a0ff 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,5 +1,5 @@ # Build Geth in a stock Go builder container -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder RUN apk add --no-cache make gcc musl-dev linux-headers git diff --git a/Makefile b/Makefile index d310942d00..50d345f9d2 100644 --- a/Makefile +++ b/Makefile @@ -82,9 +82,9 @@ escape: lint: @./build/bin/golangci-lint run --config ./.golangci.yml -lintci-deps: +lint-deps: rm -f ./build/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v2.1.5 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v2.6.2 .PHONY: vulncheck diff --git a/accounts/abi/bind/v2/dep_tree_test.go b/accounts/abi/bind/v2/dep_tree_test.go index e686e3fec4..ec4759183a 100644 --- a/accounts/abi/bind/v2/dep_tree_test.go +++ b/accounts/abi/bind/v2/dep_tree_test.go @@ -329,7 +329,7 @@ func TestContractLinking(t *testing.T) { map[rune]struct{}{}, }, // two contracts ('a' and 'f') share some dependencies. contract 'a' is marked as an override. expect that any of - // its depdencies that aren't shared with 'f' are not deployed. + // its dependencies that aren't shared with 'f' are not deployed. linkTestCaseInput{map[rune][]rune{ 'a': {'b', 'c', 'd', 'e'}, 'f': {'g', 'c', 'd', 'h'}}, diff --git a/accounts/abi/bind/v2/internal/contracts/db/contract.sol b/accounts/abi/bind/v2/internal/contracts/db/contract.sol index f24aa8d381..640436182a 100644 --- a/accounts/abi/bind/v2/internal/contracts/db/contract.sol +++ b/accounts/abi/bind/v2/internal/contracts/db/contract.sol @@ -25,7 +25,7 @@ contract DB { if (v == 0) { return _keys.length; } - // Check if a key is being overriden + // Check if a key is being overridden if (_store[k] == 0) { _keys.push(k); _stats.inserts++; diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 7d70282603..bcbd2f3925 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "slices" "sort" @@ -881,9 +882,27 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header // Ensure that the difficulty corresponds to the turn-ness of the signer if !c.fakeDiff { - difficulty := Difficulty(snap.ValidatorSet, signer) - if header.Difficulty.Uint64() != difficulty { - return &WrongDifficultyError{number, difficulty, header.Difficulty.Uint64(), signer.Bytes()} + expected := Difficulty(snap.ValidatorSet, signer) + // range check: difficulty must fit in uint64 (no high bits allowed). + if header.Difficulty == nil || !header.Difficulty.IsUint64() { + // reject the block. + return &WrongDifficultyError{ + Number: header.Number.Uint64(), + Expected: expected, + Actual: math.MaxUint64, // invalid sentinel + Signer: signer.Bytes(), + } + } + + // value check, now it's safe to use Uint64(). + actual := header.Difficulty.Uint64() + if actual != expected { + return &WrongDifficultyError{ + Number: header.Number.Uint64(), + Expected: expected, + Actual: actual, + Signer: signer.Bytes(), + } } } diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index fdd09e222d..c2ceedda6e 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -2,10 +2,13 @@ package bor import ( "context" + "errors" + "math" "math/big" "testing" "time" + "github.com/ethereum/go-ethereum/crypto" "github.com/holiman/uint256" "github.com/stretchr/testify/require" @@ -732,3 +735,88 @@ func TestCustomBlockTimeBackwardCompatibility(t *testing.T) { require.True(t, header.ActualTime.IsZero(), "ActualTime should not be set when blockTime is 0") }) } + +func TestVerifySealRejectsOversizedDifficulty(t *testing.T) { + t.Parallel() + + // real key so ecrecover works + privKey, err := crypto.GenerateKey() + require.NoError(t, err) + + signerAddr := crypto.PubkeyToAddress(privKey.PublicKey) + + sp := &fakeSpanner{ + vals: []*valset.Validator{ + {Address: signerAddr, VotingPower: 1}, + }, + } + + borCfg := ¶ms.BorConfig{ + Sprint: map[string]uint64{"0": 64}, + Period: map[string]uint64{"0": 2}, + } + + // devFake=false, we need real signatures for the sake of this test + chain, b := newChainAndBorForTest(t, sp, borCfg, false, common.Address{}) + + parent := chain.HeaderChain().GetHeaderByNumber(0) + require.NotNil(t, parent) + + header := &types.Header{ + ParentHash: parent.Hash(), + Number: big.NewInt(1), + Time: parent.Time + borCfg.Period["0"], + } + + // Build snapshot so we can compute the expected difficulty + snap, err := b.snapshot(chain.HeaderChain(), header, []*types.Header{parent}, true) + require.NoError(t, err) + require.NotNil(t, snap) + + expected := Difficulty(snap.ValidatorSet, signerAddr) + + // Craft a huge difficulty whose low 64 bits match the expected + hugeDiff := new(big.Int).Add( + new(big.Int).SetUint64(expected), + new(big.Int).Lsh(big.NewInt(1), 64), + ) + header.Difficulty = hugeDiff + + // 32 bytes vanity + 65 bytes for the signature + header.Extra = make([]byte, 32+65) + + // Compute the seal hash over the header + sigHash := SealHash(header, borCfg) + + // Sign the seal hash + sig, err := crypto.Sign(sigHash.Bytes(), privKey) + require.NoError(t, err) + require.Len(t, sig, 65) + + // Put the signature in the last 65 bytes of Extra + copy(header.Extra[len(header.Extra)-65:], sig) + + // verify the seal: we expect the difficulty validation to reject it + err = b.verifySeal(chain.HeaderChain(), header, []*types.Header{parent}) + if err == nil { + t.Fatalf("expected verifySeal to reject oversized difficulty, got nil") + } + + var diffErr *WrongDifficultyError + ok := errors.As(err, &diffErr) + if !ok { + t.Fatalf("expected WrongDifficultyError, got %T (%v)", err, err) + } + if diffErr.Number != header.Number.Uint64() { + t.Fatalf("unexpected Number in WrongDifficultyError: got %d, want %d", + diffErr.Number, header.Number.Uint64()) + } + if diffErr.Expected != expected { + t.Fatalf("unexpected Expected in WrongDifficultyError: got %d, want %d", + diffErr.Expected, expected) + } + if diffErr.Actual != math.MaxUint64 { + t.Fatalf("unexpected Actual in WrongDifficultyError: got %d, want %d", + diffErr.Actual, uint64(math.MaxUint64)) + } +} diff --git a/consensus/bor/heimdallgrpc/client.go b/consensus/bor/heimdallgrpc/client.go index 38443fd8a4..5b5c958254 100644 --- a/consensus/bor/heimdallgrpc/client.go +++ b/consensus/bor/heimdallgrpc/client.go @@ -2,11 +2,15 @@ package heimdallgrpc import ( "context" + "crypto/tls" + "net" + "net/url" "strings" "time" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "github.com/ethereum/go-ethereum/consensus/bor/heimdall" @@ -34,25 +38,95 @@ type HeimdallGRPCClient struct { milestoneQueryClient milestoneTypes.QueryClient } +// NewHeimdallGRPCClient creates a new Heimdall gRPC client with appropriate credentials +// based on the provided address scheme. func NewHeimdallGRPCClient(grpcAddress string, heimdallURL string, timeout time.Duration) *HeimdallGRPCClient { - grpcAddress = removePrefix(grpcAddress) + addr := grpcAddress + var dialOpts []grpc.DialOption - opts := []grpcRetry.CallOption{ + // URL mode + if strings.Contains(grpcAddress, "://") { + // Decide credentials and normalized address based on the provided scheme + u, err := url.Parse(grpcAddress) + if err != nil { + log.Crit("Invalid Heimdall gRPC URL", "url", grpcAddress, "err", err) + } + + switch u.Scheme { + case "https": + // Remote secure connection + addr = u.Host + if addr == "" { + log.Crit("Invalid Heimdall gRPC https URL", "url", grpcAddress) + } + + tlsCfg := &tls.Config{ + ServerName: strings.Split(addr, ":")[0], + MinVersion: tls.VersionTLS12, + } + dialOpts = append(dialOpts, + grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), + ) + + case "http": + // plaintext only allowed for local host + addr = u.Host + if !isLocalhost(addr) { + log.Crit("Refusing insecure non-local Heimdall gRPC over http; use https or localhost only", + "addr", addr) + } + dialOpts = append(dialOpts, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + + case "unix": + // support unix://path for on-box Heimdall nodes + path := u.Path + if path == "" { + log.Crit("Invalid unix Heimdall gRPC URL", "url", grpcAddress) + } + addr = "unix://" + path + dialOpts = append(dialOpts, + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return net.DialTimeout("unix", strings.TrimPrefix(addr, "unix://"), timeout) + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + + default: + log.Crit("Unsupported Heimdall gRPC URL scheme", "url", grpcAddress, "scheme", u.Scheme) + } + } else { + addr = grpcAddress + // No scheme provided, treat as host:port, but only allow if local + if !isLocalhost(addr) { + log.Crit("Refusing insecure non-local Heimdall gRPC without scheme; use https://host:port", + "addr", addr) + } + dialOpts = append(dialOpts, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + } + + // Retry options + retryOpts := []grpcRetry.CallOption{ grpcRetry.WithMax(10000), grpcRetry.WithBackoff(grpcRetry.BackoffLinear(5 * time.Second)), grpcRetry.WithCodes(codes.Internal, codes.Unavailable, codes.Aborted, codes.NotFound), } - conn, err := grpc.NewClient(grpcAddress, - grpc.WithStreamInterceptor(grpcRetry.StreamClientInterceptor(opts...)), - grpc.WithUnaryInterceptor(grpcRetry.UnaryClientInterceptor(opts...)), - grpc.WithTransportCredentials(insecure.NewCredentials()), + dialOpts = append(dialOpts, + grpc.WithStreamInterceptor(grpcRetry.StreamClientInterceptor(retryOpts...)), + grpc.WithUnaryInterceptor(grpcRetry.UnaryClientInterceptor(retryOpts...)), ) + + // dial using address and dialOpts + conn, err := grpc.NewClient(addr, dialOpts...) if err != nil { - log.Crit("Failed to connect to Heimdall gRPC", "error", err) + log.Crit("Failed to connect to Heimdall gRPC", "addr", addr, "error", err) } - log.Info("Connected to Heimdall gRPC server", "grpcAddress", grpcAddress) + log.Info("Connected to Heimdall gRPC server", "grpcAddress", grpcAddress, "dialAddr", addr) return &HeimdallGRPCClient{ conn: conn, @@ -69,14 +143,19 @@ func (h *HeimdallGRPCClient) Close() { h.conn.Close() } -// removePrefix removes the http:// or https:// prefix from the address, if present. -func removePrefix(address string) string { - if strings.HasPrefix(address, "http://") || strings.HasPrefix(address, "https://") { - return address[strings.Index(address, "//")+2:] - } - return address -} - func (h *HeimdallGRPCClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return h.client.FetchStatus(ctx) } + +// isLocalhost returns true if host/port refers to localhost/loopback. +func isLocalhost(hostport string) bool { + host, _, err := net.SplitHostPort(hostport) + if err != nil { + host = hostport + } + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/core/blockchain.go b/core/blockchain.go index d0e93c3eea..71493f09f1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -742,10 +742,12 @@ func (bc *BlockChain) ProcessBlock(block *types.Block, parent *types.Header, wit parallel bool } - // temporary disabled block STM because witness not work on parallel. - // TODO: how to enable witness on parallel state processing - bc.parallelProcessor = nil - bc.enforceParallelProcessor = false + // Only disable Parallel Processor for witness producers + // TODO: work on enabling witness production for parallel processor + if witness != nil { + bc.parallelProcessor = nil + bc.enforceParallelProcessor = false + } var resultChanLen int = 2 if bc.enforceParallelProcessor { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 7b4a307930..3bb7919f1f 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -125,6 +125,8 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, instanceLock: lock, } + freezer.offset.Store(offset) + // Create the tables. for name, config := range tables { table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, config, readonly) @@ -154,6 +156,10 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, return nil, err } + // Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to + // represent the absolute number of blocks already frozen. + freezer.frozen.Add(offset) + // Create the write batch. freezer.writeBatch = newFreezerBatch(freezer) diff --git a/core/types/block.go b/core/types/block.go index 6f07abd4a3..11f14bed8a 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -173,8 +173,8 @@ func (h *Header) SanityCheck() error { } if h.Difficulty != nil { - if diffLen := h.Difficulty.BitLen(); diffLen > 80 { - return fmt.Errorf("too large block difficulty: bitlen %d", diffLen) + if diffLen := h.Difficulty.BitLen(); diffLen > 64 { + return fmt.Errorf("too large block difficulty: bitlen %d (must be <= 64)", diffLen) } } diff --git a/core/types/block_test.go b/core/types/block_test.go index 821996dbf4..236bbbe30b 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -669,3 +669,12 @@ func TestHeaderGetActualTime(t *testing.T) { } }) } + +func TestHeaderSanityRejectsBitlenOver64(t *testing.T) { + h := &Header{ + Difficulty: new(big.Int).Lsh(big.NewInt(1), 64), // bitlen=65 + } + if err := h.SanityCheck(); err == nil { + t.Fatalf("expected sanity check to reject difficulty bitlen > 64") + } +} diff --git a/docs/cli/backfill-statesync-txs.md b/docs/cli/backfill-statesync-txs.md new file mode 100644 index 0000000000..0b33ae4c1e --- /dev/null +++ b/docs/cli/backfill-statesync-txs.md @@ -0,0 +1,11 @@ +# Backfill StateSync Txs + +The ```backfill-statesync-txs``` command receives a trusted file containing statesync txs and events from a time period and backfill it into the database. It walks over the block period checking any missing data and backfilling them. It writes just over KV database, which means the data which were supposed to already be on ancient will be now always at ancient. + +## Options + +- ```backfill-file```: Path of the file containing the backfill data + +- ```datadir```: Path of the data directory to store information + +- ```keystore```: Path of the data directory to store keys \ No newline at end of file diff --git a/docs/cli/server.md b/docs/cli/server.md index d1b58bac89..439dca0431 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -114,6 +114,10 @@ The ```bor server``` command runs the Bor client. - ```witness.fastforwardthreshold```: Minimum necessary distance between local header and chain tip to trigger fast forward (default: 6400) +- ```witness.parallelstatelessimport```: Enable parallel stateless block import (default: false) + +- ```witness.parallelstatelessimportworkers```: Number of workers to use for parallel stateless import (0 = GOMAXPROCS) (default: 0) + - ```witness.producewitnesses```: Produce witnesses while syncing (default: false) - ```witness.syncwithwitnesses```: Sync blocks with witnesses (default: false) @@ -134,12 +138,20 @@ The ```bor server``` command runs the Bor client. - ```cache```: Megabytes of memory allocated to internal caching (default: 1024) +- ```cache.addresscachesizes```: Address-specific cache sizes for biased caching in MB (format: address=sizeMB,address=sizeMB, e.g. 0x1234...=1024,0x5678...=512) + - ```cache.blocklogs```: Size (in number of blocks) of the log cache for filtering (default: 32) - ```cache.database```: Percentage of cache memory allowance to use for database io (default: 50) - ```cache.gc```: Percentage of cache memory allowance to use for trie pruning (default: 25) +- ```cache.godebug```: Set GODEBUG variables for runtime debugging (e.g. 'gctrace=1,gcpacertrace=1') + +- ```cache.gogc```: Set GOGC percentage for garbage collection trigger (default: 100) (default: 100) + +- ```cache.gomemlimit```: Set GOMEMLIMIT for the runtime (e.g. '34GB', '34359738368'). Empty means no limit + - ```cache.noprefetch```: Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data) (default: false) - ```cache.preimages```: Enable recording the SHA3/keccak preimages of trie keys (default: false) @@ -284,9 +296,11 @@ The ```bor server``` command runs the Bor client. ### Sealer Options +- ```allow-gas-tip-override```: Allows block producers to override the mining gas tip (default: false) + - ```mine```: Enable mining (default: false) -- ```allow-gas-tip-override```: Allows block producers to override the mining gas tip (default: false) +- ```miner.blocktime```: The block time defined by the miner. Needs to be larger or equal to the consensus block time. If not set (default = 0), the miner will use the consensus block time. (default: 0s) - ```miner.etherbase```: Public address for block mining rewards diff --git a/eth/bor_checkpoint_verifier.go b/eth/bor_checkpoint_verifier.go index 0f63c2a451..f077eda7a7 100644 --- a/eth/bor_checkpoint_verifier.go +++ b/eth/bor_checkpoint_verifier.go @@ -37,7 +37,7 @@ var ( // errEndBlock is returned when we're unable to fetch the tip confirmation block locally. errTipConfirmationBlock = errors.New("failed to get tip confirmation block") - // rewindLengthMeter for collecting info about the length of chain rewinded + // rewindLengthMeter for collecting info about the length of chain rewound rewindLengthMeter = metrics.NewRegisteredMeter("chain/autorewind/length", nil) ) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 6a6464925c..e758d8df87 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -84,7 +84,7 @@ const ( minTrienodeHealThrottle = 1 // maxTrienodeHealThrottle is the maximum divisor for throttling trie node - // heal requests to avoid overloading the local node and exessively expanding + // heal requests to avoid overloading the local node and excessively expanding // the state trie bedth wise. maxTrienodeHealThrottle = maxTrieRequestCount diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index da7d9c8d31..8df468886b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -87,6 +87,40 @@ type Database struct { liveIterGauge *metrics.Gauge // Gauge for tracking the number of live database iterators levelsGauge []*metrics.Gauge // Gauge for tracking the number of tables in levels + // Read and Write Amplification metrics + readAmpGauge *metrics.GaugeFloat64 // Gauge for tracking read amplification + levelWriteAmpGauge []*metrics.GaugeFloat64 // Gauge for tracking write amplification per level + totalWriteAmpGauge *metrics.GaugeFloat64 // Gauge for tracking total write amplification + + // Detailed I/O tracking metrics + walBytesWrittenMeter *metrics.Meter // Bytes written to WAL + walFileCountGauge *metrics.Gauge // Number of WAL files + sstBytesReadMeter *metrics.Meter // Bytes read from SST files (compaction input) + sstBytesWrittenMeter *metrics.Meter // Bytes written to SST files (compaction output) + flushBytesWrittenMeter *metrics.Meter // Bytes written during memtable flush + + // Per-level size tracking + levelSizeGauge []*metrics.Gauge // Size of each level in bytes + levelScoreGauge []*metrics.Gauge // Compaction score per level (>1 means needs compaction) + + // Detailed WAL metrics + walSizeGauge *metrics.Gauge // Current WAL size + walPhysicalSizeGauge *metrics.Gauge // Physical WAL size on disk + walObsoleteSizeGauge *metrics.Gauge // Obsolete WAL data size + + // Snapshot metrics + snapshotCountGauge *metrics.Gauge // Number of snapshots + + // Keys metrics for understanding data distribution + keysCountGauge []*metrics.Gauge // Number of keys per level + + // Calculated amplification metrics + calcWriteAmpGauge *metrics.GaugeFloat64 // Calculated write amplification: total physical writes / logical user data + calcReadAmpGauge *metrics.GaugeFloat64 // Calculated read amplification (same as readamp) + calcSpaceAmpGauge *metrics.GaugeFloat64 // Calculated space amplification: disk/size / actual data size + walWriteAmpGauge *metrics.GaugeFloat64 // WAL write amplification: WAL physical / logical data + actualDataSizeGauge *metrics.Gauge // Actual user data size (from live SST files) + quitLock sync.RWMutex // Mutex protecting the quit channel and the closed flag quitChan chan chan error // Quit channel to stop the metrics collection before closing the database closed bool // keep track of whether we're Closed @@ -164,6 +198,26 @@ func (d *Database) onWriteStallEnd() { d.writeDelayStartTime = time.Time{} } +// Track SST file operations +func (d *Database) onTableCreated(info pebble.TableCreateInfo) { + metrics.GetOrRegisterMeter(d.namespace+"file/sst/created", nil).Mark(1) + d.log.Debug("SST file created", "reason", info.Reason, "fileNum", info.FileNum) +} + +func (d *Database) onTableDeleted(info pebble.TableDeleteInfo) { + metrics.GetOrRegisterMeter(d.namespace+"file/sst/deleted", nil).Mark(1) +} + +// Track WAL (.log) file operations +func (d *Database) onWALCreated(info pebble.WALCreateInfo) { + metrics.GetOrRegisterMeter(d.namespace+"file/wal/created", nil).Mark(1) + d.log.Debug("WAL file created", "fileNum", info.FileNum, "recycled", info.RecycledFileNum) +} + +func (d *Database) onWALDeleted(info pebble.WALDeleteInfo) { + metrics.GetOrRegisterMeter(d.namespace+"file/wal/deleted", nil).Mark(1) +} + // panicLogger is just a noop logger to disable Pebble's internal logger. // // TODO(karalabe): Remove when Pebble sets this as the default. @@ -277,6 +331,10 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( CompactionEnd: db.onCompactionEnd, WriteStallBegin: db.onWriteStallBegin, WriteStallEnd: db.onWriteStallEnd, + TableCreated: db.onTableCreated, + TableDeleted: db.onTableDeleted, + WALCreated: db.onWALCreated, + WALDeleted: db.onWALDeleted, }, Logger: panicLogger{}, // TODO(karalabe): Delete when this is upstreamed in Pebble @@ -338,6 +396,32 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( db.liveCompSizeGauge = metrics.GetOrRegisterGauge(namespace+"compact/live/size", nil) db.liveIterGauge = metrics.GetOrRegisterGauge(namespace+"iter/count", nil) + // Register read and write amplification metrics + db.readAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"readamp", nil) + db.totalWriteAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"writeamp/total", nil) + + // Register detailed I/O tracking metrics + db.walBytesWrittenMeter = metrics.GetOrRegisterMeter(namespace+"wal/bytes", nil) + db.walFileCountGauge = metrics.GetOrRegisterGauge(namespace+"wal/files", nil) + db.sstBytesReadMeter = metrics.GetOrRegisterMeter(namespace+"sst/read", nil) + db.sstBytesWrittenMeter = metrics.GetOrRegisterMeter(namespace+"sst/written", nil) + db.flushBytesWrittenMeter = metrics.GetOrRegisterMeter(namespace+"flush/bytes", nil) + + // WAL size metrics + db.walSizeGauge = metrics.GetOrRegisterGauge(namespace+"wal/size", nil) + db.walPhysicalSizeGauge = metrics.GetOrRegisterGauge(namespace+"wal/physicalsize", nil) + db.walObsoleteSizeGauge = metrics.GetOrRegisterGauge(namespace+"wal/obsoletesize", nil) + + // Snapshot metrics + db.snapshotCountGauge = metrics.GetOrRegisterGauge(namespace+"snapshots/count", nil) + + // Calculated amplification metrics + db.calcWriteAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"amplification/write/calculated", nil) + db.calcReadAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"amplification/read/calculated", nil) + db.calcSpaceAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"amplification/space/calculated", nil) + db.walWriteAmpGauge = metrics.GetOrRegisterGaugeFloat64(namespace+"amplification/wal", nil) + db.actualDataSizeGauge = metrics.GetOrRegisterGauge(namespace+"disk/actualsize", nil) + // Start up the metrics gathering and return go db.meter(metricsGatheringInterval, namespace) return db, nil @@ -540,7 +624,9 @@ func (d *Database) meter(refresh time.Duration, namespace string) { compWrites [2]int64 compReads [2]int64 - nWrites [2]int64 + nWrites [2]int64 + flushBytes [2]int64 // Add tracking for flush bytes + walWrites [2]int64 // Track WAL writes separately writeDelayTimes [2]int64 writeDelayCounts [2]int64 @@ -566,18 +652,35 @@ func (d *Database) meter(refresh time.Duration, namespace string) { writeDelayCounts[i%2] = writeDelayCount compTimes[i%2] = compTime + var totalFlushBytes int64 for _, levelMetrics := range stats.Levels { - nWrite += int64(levelMetrics.BytesCompacted) - nWrite += int64(levelMetrics.BytesFlushed) + // Don't add to nWrite yet - we'll calculate physical writes separately compWrite += int64(levelMetrics.BytesCompacted) compRead += int64(levelMetrics.BytesRead) + totalFlushBytes += int64(levelMetrics.BytesFlushed) + } + + // Track both logical and physical WAL metrics + walLogicalWrites := int64(stats.WAL.BytesWritten) + walPhysicalSize := int64(stats.WAL.PhysicalSize) + + // Calculate physical writes including WAL overhead + // For nWrite, we need to account for physical WAL overhead + // Use the ratio of physical/logical for current WAL as a multiplier + walOverheadRatio := 1.0 + if stats.WAL.BytesWritten > 0 { + walOverheadRatio = float64(walPhysicalSize) / float64(stats.WAL.BytesWritten) } - nWrite += int64(stats.WAL.BytesWritten) + // Estimate physical writes as: SST writes + (logical WAL * overhead ratio) + // This gives us a better approximation of actual disk I/O + nWrite = compWrite + totalFlushBytes + int64(float64(walLogicalWrites)*walOverheadRatio) compWrites[i%2] = compWrite compReads[i%2] = compRead nWrites[i%2] = nWrite + walWrites[i%2] = walLogicalWrites + flushBytes[i%2] = totalFlushBytes d.writeDelayNMeter.Mark(writeDelayCounts[i%2] - writeDelayCounts[(i-1)%2]) d.writeDelayMeter.Mark(writeDelayTimes[i%2] - writeDelayTimes[(i-1)%2]) @@ -616,14 +719,85 @@ func (d *Database) meter(refresh time.Duration, namespace string) { d.filterHitGauge.Update(stats.Filter.Hits) d.filterMissGauge.Update(stats.Filter.Misses) + // Update read amplification metric + // ReadAmp returns the current read amplification of the database + d.readAmpGauge.Update(float64(stats.ReadAmp())) + + // Track detailed I/O metrics + var ( + totalSSTBytesRead int64 + totalSSTBytesWritten int64 + ) + + // Calculate and update write amplification metrics per level for i, level := range stats.Levels { // Append metrics for additional layers if i >= len(d.levelsGauge) { d.levelsGauge = append(d.levelsGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("tables/level%v", i), nil)) + d.levelWriteAmpGauge = append(d.levelWriteAmpGauge, metrics.GetOrRegisterGaugeFloat64(namespace+fmt.Sprintf("writeamp/level%v", i), nil)) + d.levelSizeGauge = append(d.levelSizeGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("size/level%v", i), nil)) + d.levelScoreGauge = append(d.levelScoreGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("score/level%v", i), nil)) + d.keysCountGauge = append(d.keysCountGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("keys/level%v", i), nil)) } d.levelsGauge[i].Update(level.NumFiles) + + // Update write amplification for this level + writeAmp := level.WriteAmp() + d.levelWriteAmpGauge[i].Update(writeAmp) + + // Update level size + d.levelSizeGauge[i].Update(level.Size) + + // Update compaction score (>1.0 means level needs compaction) + d.levelScoreGauge[i].Update(int64(level.Score * 1000)) // Multiply by 1000 for precision + + // Update keys count per level + d.keysCountGauge[i].Update(level.NumFiles) // Approximate by file count + + // Accumulate I/O stats (these are cumulative from Pebble) + totalSSTBytesRead += int64(level.BytesRead) + totalSSTBytesWritten += int64(level.BytesCompacted) + } + // Update I/O meters (mark only the delta since last measurement) + if i > 1 { + deltaRead := totalSSTBytesRead - compReads[(i-1)%2] + deltaWrite := totalSSTBytesWritten - compWrites[(i-1)%2] + deltaWAL := walWrites[i%2] - walWrites[(i-1)%2] + deltaFlush := flushBytes[i%2] - flushBytes[(i-1)%2] + + // Only mark positive deltas to avoid negative values + if deltaRead > 0 { + d.sstBytesReadMeter.Mark(deltaRead) + } + if deltaWrite > 0 { + d.sstBytesWrittenMeter.Mark(deltaWrite) + } + // Track WAL logical writes (the actual application data) + if deltaWAL > 0 { + d.walBytesWrittenMeter.Mark(deltaWAL) + } + if deltaFlush > 0 { + d.flushBytesWrittenMeter.Mark(deltaFlush) + } } + // Calculate total write amplification using Pebble's built-in method + totalMetrics := stats.Total() + totalWriteAmp := totalMetrics.WriteAmp() + d.totalWriteAmpGauge.Update(totalWriteAmp) + + // Update WAL metrics + d.walFileCountGauge.Update(stats.WAL.Files) + d.walSizeGauge.Update(int64(stats.WAL.Size)) + d.walPhysicalSizeGauge.Update(int64(stats.WAL.PhysicalSize)) + d.walObsoleteSizeGauge.Update(int64(stats.WAL.ObsoletePhysicalSize)) + + // Update snapshot count + d.snapshotCountGauge.Update(int64(stats.Snapshots.Count)) + + // Calculate and update custom amplification metrics + d.updateCalculatedAmplifications(stats) + // Sleep a bit, then repeat the stats collection select { case errc = <-d.quitChan: @@ -797,3 +971,101 @@ func (iter *pebbleIterator) Release() { iter.released = true } } + +// updateCalculatedAmplifications calculates and updates custom amplification metrics +func (d *Database) updateCalculatedAmplifications(stats *pebble.Metrics) { + // Calculate Write Amplification for the database + calcWriteAmp := d.calculateDatabaseWriteAmp(stats) + if calcWriteAmp >= 0 { + d.calcWriteAmpGauge.Update(calcWriteAmp) + } + + // Calculate WAL Write Amplification + walWriteAmp := d.calculateWALWriteAmp(stats) + if walWriteAmp >= 0 { + d.walWriteAmpGauge.Update(walWriteAmp) + } + + // Calculate Read Amplification (same as Pebble's built-in metric) + // This represents how many levels/sublevels need to be checked for a read + readAmp := float64(stats.ReadAmp()) + d.calcReadAmpGauge.Update(readAmp) + + // Calculate Space Amplification: Total disk space / Actual user data size + // This represents how much extra space is used compared to the logical data size + diskSpaceUsed := int64(stats.DiskSpaceUsage()) + + // Calculate actual user data size (sum of all live SST file sizes) + // This excludes obsolete files, WAL files, and internal metadata + // level.Size is the CURRENT live size, which already accounts for deleted files + var actualDataSize int64 + for _, level := range stats.Levels { + actualDataSize += level.Size + } + + d.actualDataSizeGauge.Update(actualDataSize) + + if actualDataSize > 0 { + // Space Amp = Total disk usage / Live data size + // A value of 1.0 means no amplification (ideal) + // A value of 2.0 means using 2x the space of actual data + spaceAmp := float64(diskSpaceUsed) / float64(actualDataSize) + d.calcSpaceAmpGauge.Update(spaceAmp) + } +} + +// calculateDatabaseWriteAmp calculates the write amplification for database writes. +func (d *Database) calculateDatabaseWriteAmp(stats *pebble.Metrics) float64 { + var totalBytesIn uint64 + for _, level := range stats.Levels { + totalBytesIn += level.BytesIn + } + + if totalBytesIn == 0 { + return -1 + } + + // Calculate SST writes (cumulative) + var totalSSTWrites uint64 + for _, level := range stats.Levels { + totalSSTWrites += level.BytesFlushed + level.BytesCompacted + } + + // WAL.BytesWritten is the cumulative physical bytes written to .log files + // This already includes: + // - Record headers and checksums + // - Batching overhead + // - fsync/sync overhead + // + // But it does NOT include: + // - Block alignment padding + // - Pre-allocated space + // - Recycled file space + // + // The ratio BytesWritten/BytesIn gives us the WAL encoding overhead + walBytesWritten := stats.WAL.BytesWritten + + // Calculate total physical writes + totalPhysicalWrites := walBytesWritten + totalSSTWrites + + // Write Amplification = Total physical writes / Logical user input + writeAmp := float64(totalPhysicalWrites) / float64(totalBytesIn) + + return writeAmp +} + +// calculateWALWriteAmp calculates WAL-specific write amplification +func (d *Database) calculateWALWriteAmp(stats *pebble.Metrics) float64 { + if stats.WAL.BytesIn == 0 { + return -1 + } + + // WAL write amplification = Physical writes / Logical application writes + // This captures the overhead from: + // - WAL record format (headers, checksums) + // - Batching (multiple logical writes in one physical write) + // - Sync overhead + walWriteAmp := float64(stats.WAL.BytesWritten) / float64(stats.WAL.BytesIn) + + return walWriteAmp +} diff --git a/go.mod b/go.mod index 1f6d8f19c5..92691d6a48 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/ethereum/go-ethereum // Note: Change the go image version in Dockerfile if you change this. -go 1.24.9 +go 1.25.5 require ( github.com/0xPolygon/crand v1.0.3 - github.com/0xPolygon/heimdall-v2 v0.3.1 + github.com/0xPolygon/heimdall-v2 v0.4.5 github.com/0xPolygon/polyproto v0.0.7 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/BurntSushi/toml v1.4.0 @@ -20,7 +20,7 @@ require ( github.com/cespare/cp v1.1.1 github.com/cloudflare/cloudflare-go v0.114.0 github.com/cockroachdb/pebble v1.1.5 - github.com/cometbft/cometbft v0.38.17 + github.com/cometbft/cometbft v0.38.19 github.com/consensys/gnark-crypto v0.18.1 github.com/cosmos/cosmos-sdk v0.50.14 github.com/cosmos/gogoproto v1.7.0 @@ -28,7 +28,7 @@ require ( github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set/v2 v2.6.0 - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20240516125602-ccbae20bcec2 github.com/emirpasic/gods v1.18.1 @@ -149,7 +149,7 @@ require ( github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -165,7 +165,7 @@ require ( github.com/pion/transport/v3 v3.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_golang v1.21.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -311,7 +311,7 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spf13/viper v1.19.0 // indirect github.com/streadway/amqp v1.1.0 // indirect @@ -335,7 +335,7 @@ require ( google.golang.org/api v0.171.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect - gotest.tools/v3 v3.5.1 // indirect + gotest.tools/v3 v3.5.2 // indirect nhooyr.io/websocket v1.8.7 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) @@ -355,9 +355,10 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 - github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.0-polygon + github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.4-polygon + github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon github.com/cosmos/cosmos-sdk => github.com/0xPolygon/cosmos-sdk v0.2.6-polygon - github.com/ethereum/go-ethereum => github.com/0xPolygon/bor/v2 v2.2.11 + github.com/ethereum/go-ethereum => github.com/0xPolygon/bor v1.14.14-0.20251125190736-ff906a05db96 github.com/ethereum/go-ethereum/common/math => github.com/0xPolygon/bor/common/math v1.5.5 go.mongodb.org/mongo-driver => go.mongodb.org/mongo-driver v1.14.0 ) diff --git a/go.sum b/go.sum index 61a1f82043..3cc33f6712 100644 --- a/go.sum +++ b/go.sum @@ -68,16 +68,18 @@ cosmossdk.io/x/tx v0.13.7/go.mod h1:V6DImnwJMTq5qFjeGWpXNiT/fjgE4HtmclRmTqRVM3w= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= -github.com/0xPolygon/cometbft v0.3.0-polygon h1:PBbdyZ0e2VWePQFBgIGw3HxgN1jbjvEF05nVOvnvfQk= -github.com/0xPolygon/cometbft v0.3.0-polygon/go.mod h1:KSgxJRpXbKIBsogWNOmJxt3e78N0HXhAuLi6fPHcteE= +github.com/0xPolygon/cometbft v0.3.4-polygon h1:510h62swAMUtvuesb/9t+sgMpAu9HehvtaPruh676mw= +github.com/0xPolygon/cometbft v0.3.4-polygon/go.mod h1:ZFIXbBrjS2AlVL5UFIgnShMe3b2dso9M0yHLN+mBg4Y= +github.com/0xPolygon/cometbft-db v0.14.1-polygon h1:sMlEPISgW0Wm9bC3bnLVPiPnyZ9EOuWJxoAV8ujrN3o= +github.com/0xPolygon/cometbft-db v0.14.1-polygon/go.mod h1:KHP1YghilyGV/xjD5DP3+2hyigWx0WTp9X+0Gnx0RxQ= github.com/0xPolygon/cosmos-sdk v0.2.6-polygon h1:jLi8SR8WbUp3kXwELxwcnLEd2gvVbGXHHggBn/bjM6Q= github.com/0xPolygon/cosmos-sdk v0.2.6-polygon/go.mod h1:Z5578vP+IFgxslttbNf24pSGkM9wfDn9Ce8LlGQpLpk= github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/rADWexzB4FJckdO/DnUlk+s= github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.3.1 h1:+OxOuAdTT6J8FmFjTj9owOqW0uy7qzejQg7UOQe8B4o= -github.com/0xPolygon/heimdall-v2 v0.3.1/go.mod h1:qyFF/Nefmn4sBIKMtTYbQVEgdpTTWQv+03AydRKvaKs= +github.com/0xPolygon/heimdall-v2 v0.4.5 h1:kCGDI0Ar7TTsLKJ/8ftRq9ah/JuC47gho4JdWckWJAM= +github.com/0xPolygon/heimdall-v2 v0.4.5/go.mod h1:PoZaNVZtTspBtrXuL3ji+km2Et6HhqhosOs6Zd0gmmk= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= @@ -287,12 +289,10 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/cometbft/cometbft-db v0.14.1 h1:SxoamPghqICBAIcGpleHbmoPqy+crij/++eZz3DlerQ= -github.com/cometbft/cometbft-db v0.14.1/go.mod h1:KHP1YghilyGV/xjD5DP3+2hyigWx0WTp9X+0Gnx0RxQ= github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= -github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -318,7 +318,6 @@ github.com/cosmos/ledger-cosmos-go v0.13.3 h1:7ehuBGuyIytsXbd4MP43mLeoN2LTOEnk5n github.com/cosmos/ledger-cosmos-go v0.13.3/go.mod h1:HENcEP+VtahZFw38HZ3+LS3Iv5XV6svsnkk9vdJtLr8= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crate-crypto/go-eth-kzg v1.3.0 h1:05GrhASN9kDAidaFJOda6A4BEvgvuXbazXg/0E3OOdI= @@ -339,10 +338,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= @@ -781,8 +780,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -930,8 +929,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= -github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= +github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -994,8 +993,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= +github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1094,8 +1093,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -1751,8 +1750,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/miner/worker.go b/miner/worker.go index a2ceed37a4..6315c8de60 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -689,7 +689,7 @@ func (w *worker) mainLoop() { tcount := w.current.tcount - w.commitTransactions(w.current, plainTxs, blobTxs, nil, new(uint256.Int)) + w.commitTransactions(w.current, plainTxs, blobTxs, nil) stopFn() // Only update the snapshot if any new transactons were added @@ -959,11 +959,12 @@ func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]* } env.txs = append(env.txs, tx) env.receipts = append(env.receipts, receipt) + env.tcount++ return receipt.Logs, nil } -func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32, minTip *uint256.Int) error { +func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { defer func(t0 time.Time) { commitTransactionsTimer.Update(time.Since(t0)) }(time.Now()) @@ -1072,11 +1073,6 @@ mainloop: continue } - // If we don't receive enough tip for the next transaction, skip the account - if ptip.Cmp(minTip) < 0 { - log.Trace("Not enough tip for transaction", "hash", ltx.Hash, "tip", ptip, "needed", minTip) - break // If the next-best is too low, surely no better will be available - } // Transaction seems to fit, pull it up from the pool tx := ltx.Resolve() if tx == nil { @@ -1155,7 +1151,6 @@ mainloop: case errors.Is(err, nil): // Everything ok, collect the logs and shift in the next transaction from the same account coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ if EnableMVHashMap && w.IsRunning() { env.depsMVFullWriteList = append(env.depsMVFullWriteList, env.state.MVFullWriteList()) @@ -1455,8 +1450,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err if len(prioPlainTxs) > 0 || len(prioBlobTxs) > 0 { plainTxs := newTransactionsByPriceAndNonce(env.signer, prioPlainTxs, env.header.BaseFee, &w.interruptBlockBuilding) blobTxs := newTransactionsByPriceAndNonce(env.signer, prioBlobTxs, env.header.BaseFee, &w.interruptBlockBuilding) - - if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt, new(uint256.Int)); err != nil { + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } @@ -1466,7 +1460,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err blobTxs := newTransactionsByPriceAndNonce(env.signer, normalBlobTxs, env.header.BaseFee, &w.interruptBlockBuilding) txHeapInitTimer.Update(time.Since(heapInitTime)) - if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt, new(uint256.Int)); err != nil { + if err := w.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } diff --git a/node/api.go b/node/api.go index 52144fc589..d767fba985 100644 --- a/node/api.go +++ b/node/api.go @@ -52,7 +52,7 @@ func (n *Node) apis() []rpc.API { } // adminAPI is the collection of administrative API methods exposed over -// both secure and unsecure RPC channels. +// both secure and insecure RPC channels. type adminAPI struct { node *Node // Node interfaced by this API } diff --git a/params/version.go b/params/version.go index ca157b66db..013e045f8c 100644 --- a/params/version.go +++ b/params/version.go @@ -25,7 +25,7 @@ import ( const ( VersionMajor = 2 // Major version component of the current release VersionMinor = 5 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release + VersionPatch = 3 // Patch version component of the current release VersionMeta = "" // Version metadata to append to the version string )