Merge pull request #2388 from bnb-chain/develop

merge develop to master for several 4844 bug fix.

accounts/external/backend.go

@@ -163,7 +163,7 @@ func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, d
 		hexutil.Encode(data)); err != nil {
 		return nil, err
 	}
-	// If V is on 27/28-form, convert to to 0/1 for Clique and Parlia
+	// If V is on 27/28-form, convert to 0/1 for Clique and Parlia
 	if (mimeType == accounts.MimetypeClique || mimeType == accounts.MimetypeParlia) && (res[64] == 27 || res[64] == 28) {
 		res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique and Parlia use
 	}

cmd/geth/snapshot.go

@@ -92,7 +92,7 @@ WARNING: it's only supported in hash mode(--state.scheme=hash)".
 geth offline prune-block for block data in ancientdb.
 The amount of blocks expected for remaining after prune can be specified via block-amount-reserved in this command,
 will prune and only remain the specified amount of old block data in ancientdb.
-the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb 
+the brief workflow is to backup the number of this specified amount blocks backward in original ancientdb
 into new ancient_backup, then delete the original ancientdb dir and rename the ancient_backup to original one for replacement,
 finally assemble the statedb and new ancientDb together.
 The purpose of doing it is because the block data will be moved into the ancient store when it

consensus/parlia/parlia.go

@@ -599,9 +599,7 @@ func (p *Parlia) verifyHeader(chain consensus.ChainHeaderReader, header *types.H
 		switch {
 		case header.ParentBeaconRoot != nil:
 			return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot)
-		// types.EmptyWithdrawalsHash represents a empty value when EIP-4895 enabled,
-		// here, EIP-4895 still be disabled, value expected to be `types.EmptyWithdrawalsHash` is only to feet the demand of rlp encode/decode
-		case header.WithdrawalsHash == nil || *header.WithdrawalsHash != types.EmptyWithdrawalsHash:
+		case !header.EmptyWithdrawalsHash():
 			return errors.New("header has wrong WithdrawalsHash")
 		}
 		if err := eip4844.VerifyEIP4844Header(parent, header); err != nil {

core/block_validator.go

@@ -96,15 +96,15 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 		},
 		func() error {
 			// Withdrawals are present after the Shanghai fork.
-			if !header.EmptyWithdrawalsHash() {
+			if header.WithdrawalsHash != nil {
 				// Withdrawals list must be present in body after Shanghai.
 				if block.Withdrawals() == nil {
 					return errors.New("missing withdrawals in block body")
 				}
 				if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash {
 					return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash)
 				}
-			} else if len(block.Withdrawals()) != 0 { // Withdrawals turn into empty from nil when BlockBody has Sidecars
+			} else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars
 				// Withdrawals are not allowed prior to shanghai fork
 				return errors.New("withdrawals present in block body")
 			}

core/chain_makers.go

@@ -372,6 +372,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			if err != nil {
 				panic(err)
 			}
+			if block.Header().EmptyWithdrawalsHash() {
+				block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
+			}
 			if config.IsCancun(block.Number(), block.Time()) {
 				for _, s := range b.sidecars {
 					s.BlockNumber = block.Number()

core/data_availability_test.go

@@ -1,9 +1,12 @@
 package core
 
 import (
+	"crypto/rand"
 	"math/big"
 	"testing"
 
+	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
+	gokzg4844 "github.com/crate-crypto/go-kzg-4844"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
@@ -247,6 +250,49 @@ func TestCheckDataAvailableInBatch(t *testing.T) {
 	}
 }
 
+func BenchmarkEmptySidecarDAChecking(b *testing.B) {
+	hr := NewMockDAHeaderReader(params.ParliaTestChainConfig)
+	block := types.NewBlockWithHeader(&types.Header{
+		Number: big.NewInt(1),
+	}).WithBody(types.Transactions{
+		createMockDATx(hr.Config(), emptySidecar()),
+		createMockDATx(hr.Config(), emptySidecar()),
+		createMockDATx(hr.Config(), emptySidecar()),
+		createMockDATx(hr.Config(), emptySidecar()),
+		createMockDATx(hr.Config(), emptySidecar()),
+		createMockDATx(hr.Config(), emptySidecar()),
+	}, nil)
+	block = block.WithSidecars(collectBlobsFromTxs(block.Header(), block.Transactions()))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		IsDataAvailable(hr, block)
+	}
+}
+
+func BenchmarkRandomSidecarDAChecking(b *testing.B) {
+	hr := NewMockDAHeaderReader(params.ParliaTestChainConfig)
+	const count = 10
+	blocks := make([]*types.Block, count)
+	for i := 0; i < len(blocks); i++ {
+		block := types.NewBlockWithHeader(&types.Header{
+			Number: big.NewInt(1),
+		}).WithBody(types.Transactions{
+			createMockDATx(hr.Config(), randomSidecar()),
+			createMockDATx(hr.Config(), randomSidecar()),
+			createMockDATx(hr.Config(), randomSidecar()),
+			createMockDATx(hr.Config(), randomSidecar()),
+			createMockDATx(hr.Config(), randomSidecar()),
+			createMockDATx(hr.Config(), randomSidecar()),
+		}, nil)
+		block = block.WithSidecars(collectBlobsFromTxs(block.Header(), block.Transactions()))
+		blocks[i] = block
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		IsDataAvailable(hr, blocks[i%count])
+	}
+}
+
 func collectBlobsFromTxs(header *types.Header, txs types.Transactions) types.BlobSidecars {
 	sidecars := make(types.BlobSidecars, 0, len(txs))
 	for i, tx := range txs {
@@ -348,3 +394,43 @@ func createMockDATx(config *params.ChainConfig, sidecar *types.BlobTxSidecar) *t
 	}
 	return types.NewTx(tx)
 }
+
+func randFieldElement() [32]byte {
+	bytes := make([]byte, 32)
+	_, err := rand.Read(bytes)
+	if err != nil {
+		panic("failed to get random field element")
+	}
+	var r fr.Element
+	r.SetBytes(bytes)
+
+	return gokzg4844.SerializeScalar(r)
+}
+
+func randBlob() kzg4844.Blob {
+	var blob kzg4844.Blob
+	for i := 0; i < len(blob); i += gokzg4844.SerializedScalarSize {
+		fieldElementBytes := randFieldElement()
+		copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:])
+	}
+	return blob
+}
+
+func randomSidecar() *types.BlobTxSidecar {
+	blob := randBlob()
+	commitment, _ := kzg4844.BlobToCommitment(blob)
+	proof, _ := kzg4844.ComputeBlobProof(blob, commitment)
+	return &types.BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{blob},
+		Commitments: []kzg4844.Commitment{commitment},
+		Proofs:      []kzg4844.Proof{proof},
+	}
+}
+
+func emptySidecar() *types.BlobTxSidecar {
+	return &types.BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{emptyBlob},
+		Commitments: []kzg4844.Commitment{emptyBlobCommit},
+		Proofs:      []kzg4844.Proof{emptyBlobProof},
+	}
+}

core/rawdb/accessors_chain.go

@@ -808,7 +808,7 @@ func WriteAncientBlocksWithBlobs(db ethdb.AncientWriter, blocks []*types.Block,
 			break
 		}
 	}
-	log.Info("WriteAncientBlocks", "startAt", blocks[0].Number(), "cancunIndex", cancunIndex, "len", len(blocks))
+	log.Debug("WriteAncientBlocks", "startAt", blocks[0].Number(), "cancunIndex", cancunIndex, "len", len(blocks))
 
 	var (
 		tdSum   = new(big.Int).Set(td)

core/rawdb/chain_freezer.go

@@ -17,6 +17,7 @@
 package rawdb
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"sync"
@@ -42,6 +43,10 @@ const (
 	freezerBatchLimit = 30000
 )
 
+var (
+	missFreezerEnvErr = errors.New("missing freezer env error")
+)
+
 // chainFreezer is a wrapper of freezer with additional chain freezing feature.
 // The background thread will keep moving ancient chain segments from key-value
 // database to flat files for saving space on live database.
@@ -119,6 +124,20 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 				return
 			}
 		}
+
+		// check freezer env first, it must wait a while when the env is necessary
+		err := f.checkFreezerEnv()
+		if err == missFreezerEnvErr {
+			log.Warn("Freezer need related env, may wait for a while", "err", err)
+			backoff = true
+			continue
+		}
+		if err != nil {
+			log.Error("Freezer check FreezerEnv err", "err", err)
+			backoff = true
+			continue
+		}
+
 		// Retrieve the freezing threshold.
 		hash := ReadHeadBlockHash(nfdb)
 		if hash == (common.Hash{}) {
@@ -278,7 +297,7 @@ func (f *chainFreezer) tryPruneBlobAncientTable(env *ethdb.FreezerEnv, num uint6
 		log.Error("Cannot prune blob ancient", "block", num, "expectTail", expectTail, "err", err)
 		return
 	}
-	log.Info("Chain freezer prune useless blobs, now ancient data is", "from", expectTail, "to", num, "cost", common.PrettyDuration(time.Since(start)))
+	log.Debug("Chain freezer prune useless blobs, now ancient data is", "from", expectTail, "to", num, "cost", common.PrettyDuration(time.Since(start)))
 }
 
 func getBlobExtraReserveFromEnv(env *ethdb.FreezerEnv) uint64 {
@@ -290,7 +309,7 @@ func getBlobExtraReserveFromEnv(env *ethdb.FreezerEnv) uint64 {
 
 func (f *chainFreezer) freezeRangeWithBlobs(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) {
 	defer func() {
-		log.Info("freezeRangeWithBlobs", "from", number, "to", limit, "err", err)
+		log.Debug("freezeRangeWithBlobs", "from", number, "to", limit, "err", err)
 	}()
 	lastHash := ReadCanonicalHash(nfdb, limit)
 	if lastHash == (common.Hash{}) {
@@ -413,6 +432,21 @@ func (f *chainFreezer) SetupFreezerEnv(env *ethdb.FreezerEnv) error {
 	return nil
 }
 
+func (f *chainFreezer) checkFreezerEnv() error {
+	_, exist := f.freezeEnv.Load().(*ethdb.FreezerEnv)
+	if exist {
+		return nil
+	}
+	blobFrozen, err := f.TableAncients(ChainFreezerBlobSidecarTable)
+	if err != nil {
+		return err
+	}
+	if blobFrozen > 0 {
+		return missFreezerEnvErr
+	}
+	return nil
+}
+
 func isCancun(env *ethdb.FreezerEnv, num *big.Int, time uint64) bool {
 	if env == nil || env.ChainCfg == nil {
 		return false

core/rawdb/freezer.go

@@ -765,7 +765,7 @@ func (f *Freezer) ResetTable(kind string, startAt uint64, onlyEmpty bool) error
 	f.frozen.Add(f.offset)
 	f.tail.Add(f.offset)
 	f.writeBatch = newFreezerBatch(f)
-	log.Info("Reset Table", "kind", kind, "tail", f.tables[kind].itemHidden.Load(), "frozen", f.tables[kind].items.Load())
+	log.Debug("Reset Table", "kind", kind, "tail", f.tables[kind].itemHidden.Load(), "frozen", f.tables[kind].items.Load())
 	return nil
 }
 

core/types/block.go

@@ -193,9 +193,9 @@ func (h *Header) EmptyReceipts() bool {
 	return h.ReceiptHash == EmptyReceiptsHash
 }
 
-// EmptyWithdrawalsHash returns true if there are no WithdrawalsHash for this header/block.
+// EmptyWithdrawalsHash returns true if the WithdrawalsHash is EmptyWithdrawalsHash.
 func (h *Header) EmptyWithdrawalsHash() bool {
-	return h.WithdrawalsHash == nil || *h.WithdrawalsHash == EmptyWithdrawalsHash
+	return h.WithdrawalsHash != nil && *h.WithdrawalsHash == EmptyWithdrawalsHash
 }
 
 // Body is a simple (mutable, non-safe) data container for storing and moving

eth/downloader/queue.go

@@ -81,7 +81,7 @@ func newFetchResult(header *types.Header, fastSync bool, pid string) *fetchResul
 	}
 	if !header.EmptyBody() {
 		item.pending.Store(item.pending.Load() | (1 << bodyType))
-	} else if !header.EmptyWithdrawalsHash() {
+	} else if header.WithdrawalsHash != nil {
 		item.Withdrawals = make(types.Withdrawals, 0)
 	}
 	if fastSync && !header.EmptyReceipts() {
@@ -788,9 +788,9 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH
 		if uncleListHashes[index] != header.UncleHash {
 			return errInvalidBody
 		}
-		if header.EmptyWithdrawalsHash() {
+		if header.WithdrawalsHash == nil {
 			// nil hash means that withdrawals should not be present in body
-			if len(withdrawalLists[index]) != 0 {
+			if withdrawalLists[index] != nil {
 				return errInvalidBody
 			}
 		} else { // non-nil hash: body must have withdrawals

eth/fetcher/block_fetcher.go

@@ -576,7 +576,7 @@ func (f *BlockFetcher) loop() {
 					select {
 					case res := <-resCh:
 						res.Done <- nil
-						// Ignoring withdrawals here, since the block fetcher is not used post-merge.
+						// Ignoring withdrawals here, will set it to empty later if EmptyWithdrawalsHash in header.
 						txs, uncles, _, sidecars := res.Res.(*eth.BlockBodiesResponse).Unpack()
 						f.FilterBodies(peer, txs, uncles, sidecars, time.Now())
 
@@ -639,6 +639,9 @@ func (f *BlockFetcher) loop() {
 							log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash())
 
 							block := types.NewBlockWithHeader(header)
+							if block.Header().EmptyWithdrawalsHash() {
+								block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
+							}
 							block.ReceivedAt = task.time
 
 							complete = append(complete, block)
@@ -723,6 +726,9 @@ func (f *BlockFetcher) loop() {
 						matched = true
 						if f.getBlock(hash) == nil {
 							block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i])
+							if block.Header().EmptyWithdrawalsHash() {
+								block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
+							}
 							block = block.WithSidecars(task.sidecars[i])
 							block.ReceivedAt = task.time
 							blocks = append(blocks, block)

eth/protocols/eth/handlers.go

@@ -366,8 +366,7 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error {
 		for i, body := range res.BlockBodiesResponse {
 			txsHashes[i] = types.DeriveSha(types.Transactions(body.Transactions), hasher)
 			uncleHashes[i] = types.CalcUncleHash(body.Uncles)
-			// Withdrawals may be not nil, but a empty value when Sidecars not empty
-			if len(body.Withdrawals) > 0 {
+			if body.Withdrawals != nil {
 				withdrawalHashes[i] = types.DeriveSha(types.Withdrawals(body.Withdrawals), hasher)
 			}
 		}

graphql/graphql.go

@@ -1053,7 +1053,7 @@ func (b *Block) Withdrawals(ctx context.Context) (*[]*Withdrawal, error) {
 		return nil, err
 	}
 	// Pre-shanghai blocks
-	if block.Header().EmptyWithdrawalsHash() {
+	if block.Header().WithdrawalsHash == nil {
 		return nil, nil
 	}
 	ret := make([]*Withdrawal, 0, len(block.Withdrawals()))

internal/ethapi/api.go

@@ -1018,7 +1018,7 @@ func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.B
 		return nil, nil
 	}
 	blobSidecars, err := s.b.GetBlobSidecars(ctx, header.Hash())
-	if err != nil || len(blobSidecars) == 0 {
+	if err != nil || blobSidecars == nil {
 		return nil, nil
 	}
 	result := make([]map[string]interface{}, len(blobSidecars))
@@ -1040,7 +1040,7 @@ func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.
 		return nil, nil
 	}
 	blobSidecars, err := s.b.GetBlobSidecars(ctx, blockHash)
-	if err != nil || len(blobSidecars) == 0 {
+	if err != nil || blobSidecars == nil || len(blobSidecars) == 0 {
 		return nil, nil
 	}
 	for _, sidecar := range blobSidecars {

internal/ethapi/testdata/eth_getBlobSidecars-block-with-blob-tx.json

@@ -1 +1 @@
-null
+[]

miner/worker.go

@@ -1367,6 +1367,10 @@ func (w *worker) commit(env *environment, interval func(), update bool, start ti
 		// env.receipts = receipts
 		finalizeBlockTimer.UpdateSince(finalizeStart)
 
+		if block.Header().EmptyWithdrawalsHash() {
+			block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
+		}
+
 		// If Cancun enabled, sidecars can't be nil then.
 		if w.chainConfig.IsCancun(env.header.Number, env.header.Time) && env.sidecars == nil {
 			env.sidecars = make(types.BlobSidecars, 0)