Merge branch 'tag v1.5.8' into develop

CHANGELOG.md

@@ -1,4 +1,19 @@
 # Changelog
+## v1.5.8
+### FEATURE
+* [\#2955](https://github.com/bnb-chain/bsc/pull/2955) pbs: enable GreedyMergeTx by default
+
+### BUGFIX
+* [\#2967](https://github.com/bnb-chain/bsc/pull/2967) fix: gas compare in bid simulator
+
+### IMPROVEMENT
+* [\#2951](https://github.com/bnb-chain/bsc/pull/2951) bump golang.org/x/net from 0.34.0 to 0.36.0
+* [\#0000](https://github.com/bnb-chain/bsc/pull/0000) golang: upgrade toolchain to v1.23.0 (commit:3be156eec)
+* [\#2957](https://github.com/bnb-chain/bsc/pull/2957) miner: stop GreedyMergeTx before worker picking bids
+* [\#2959](https://github.com/bnb-chain/bsc/pull/2959) pbs: fix a inaccurate bid result log
+* [\#2971](https://github.com/bnb-chain/bsc/pull/2971) mev: no interrupt if it is too later
+* [\#2974](https://github.com/bnb-chain/bsc/pull/2974) miner: add metrics for bid simulation
+
 ## v1.5.7
 v1.5.7 conduct small upstream code merge to follow the latest pectra hard fork and apply some bug fix. There are two PR for the code merge:
 * [\#2897](https://github.com/bnb-chain/bsc/pull/2897) upstream: merge tag 'geth-v1.15.1' into bsc-develop

core/types/bid.go

@@ -172,10 +172,19 @@ type Bid struct {
 	GasUsed      uint64
 	GasFee       *big.Int
 	BuilderFee   *big.Int
+	committed    bool // whether the bid has been committed to simulate or not
 
 	rawBid RawBid
 }
 
+func (b *Bid) Commit() {
+	b.committed = true
+}
+
+func (b *Bid) IsCommitted() bool {
+	return b.committed
+}
+
 // Hash returns the bid hash.
 func (b *Bid) Hash() common.Hash {
 	return b.rawBid.Hash()
@@ -192,6 +201,7 @@ type BidIssue struct {
 type MevParams struct {
 	ValidatorCommission   uint64 // 100 means 1%
 	BidSimulationLeftOver time.Duration
+	NoInterruptLeftOver   time.Duration
 	GasCeil               uint64
 	GasPrice              *big.Int // Minimum avg gas price for bid block
 	BuilderFeeCeil        *big.Int

core/vote/vote_pool.go

@@ -361,7 +361,7 @@ func (pool *VotePool) basicVerify(vote *types.VoteEnvelope, headNumber uint64, m
 
 	// Check duplicate voteMessage firstly.
 	if pool.receivedVotes.Contains(voteHash) {
-		log.Debug("Vote pool already contained the same vote", "voteHash", voteHash)
+		log.Trace("Vote pool already contained the same vote", "voteHash", voteHash)
 		return false
 	}
 

go.mod

@@ -2,8 +2,6 @@ module github.com/ethereum/go-ethereum
 
 go 1.23.0
 
-toolchain go1.23.7
-
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
 	github.com/Microsoft/go-winio v0.6.2

miner/bid_simulator.go

@@ -112,8 +112,9 @@ type bidSimulator struct {
 	pendingMu sync.RWMutex
 	pending   map[uint64]map[common.Address]map[common.Hash]struct{} // blockNumber -> builder -> bidHash -> struct{}
 
-	bestBidMu sync.RWMutex
-	bestBid   map[common.Hash]*BidRuntime // prevBlockHash -> bidRuntime
+	bestBidMu    sync.RWMutex
+	bestBid      map[common.Hash]*BidRuntime // prevBlockHash -> bidRuntime
+	bestBidToRun map[common.Hash]*types.Bid  // prevBlockHash -> *types.Bid
 
 	simBidMu      sync.RWMutex
 	simulatingBid map[common.Hash]*BidRuntime // prevBlockHash -> bidRuntime, in the process of simulation
@@ -144,6 +145,7 @@ func newBidSimulator(
 		newBidCh:      make(chan newBidPackage, 100),
 		pending:       make(map[uint64]map[common.Address]map[common.Hash]struct{}),
 		bestBid:       make(map[common.Hash]*BidRuntime),
+		bestBidToRun:  make(map[common.Hash]*types.Bid),
 		simulatingBid: make(map[common.Hash]*BidRuntime),
 	}
 
@@ -256,6 +258,7 @@ func (b *bidSimulator) ExistBuilder(builder common.Address) bool {
 	return ok
 }
 
+// best bid here is based on packedBlockReward after the bid is simulated
 func (b *bidSimulator) SetBestBid(prevBlockHash common.Hash, bid *BidRuntime) {
 	b.bestBidMu.Lock()
 	defer b.bestBidMu.Unlock()
@@ -276,6 +279,34 @@ func (b *bidSimulator) GetBestBid(prevBlockHash common.Hash) *BidRuntime {
 	return b.bestBid[prevBlockHash]
 }
 
+// best bid to run is based on bid's expectedBlockReward before the bid is simulated
+func (b *bidSimulator) SetBestBidToRun(prevBlockHash common.Hash, bid *types.Bid) {
+	b.bestBidMu.Lock()
+	defer b.bestBidMu.Unlock()
+
+	b.bestBidToRun[prevBlockHash] = bid
+}
+
+// in case the bid is invalid(invalid GasUsed,Reward,GasPrice...), remove it.
+func (b *bidSimulator) DelBestBidToRun(prevBlockHash common.Hash, delBid *types.Bid) {
+	b.bestBidMu.Lock()
+	defer b.bestBidMu.Unlock()
+	cur := b.bestBidToRun[prevBlockHash]
+	if cur == nil || delBid == nil {
+		return
+	}
+	if cur.Hash() == delBid.Hash() {
+		delete(b.bestBidToRun, prevBlockHash)
+	}
+}
+
+func (b *bidSimulator) GetBestBidToRun(prevBlockHash common.Hash) *types.Bid {
+	b.bestBidMu.RLock()
+	defer b.bestBidMu.RUnlock()
+
+	return b.bestBidToRun[prevBlockHash]
+}
+
 func (b *bidSimulator) SetSimulatingBid(prevBlockHash common.Hash, bid *BidRuntime) {
 	b.simBidMu.Lock()
 	defer b.simBidMu.Unlock()
@@ -319,6 +350,15 @@ func (b *bidSimulator) mainLoop() {
 	}
 }
 
+func (b *bidSimulator) canBeInterrupted(targetTime uint64) bool {
+	if targetTime == 0 {
+		// invalid targetTime, disable the interrupt check
+		return true
+	}
+	left := time.Until(time.UnixMilli(int64(targetTime)))
+	return left >= b.config.NoInterruptLeftOver
+}
+
 func (b *bidSimulator) newBidLoop() {
 	var (
 		interruptCh chan int32
@@ -332,6 +372,7 @@ func (b *bidSimulator) newBidLoop() {
 			close(interruptCh)
 		}
 		interruptCh = make(chan int32, 1)
+		bidRuntime.bid.Commit()
 		select {
 		case b.simBidCh <- &simBidReq{interruptCh: interruptCh, bid: bidRuntime}:
 			log.Debug("BidSimulator: commit", "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex())
@@ -360,27 +401,61 @@ func (b *bidSimulator) newBidLoop() {
 			}
 
 			var replyErr error
-			// simulatingBid will be nil if there is no bid in simulation, compare with the bestBid instead
-			if simulatingBid := b.GetSimulatingBid(newBid.bid.ParentHash); simulatingBid != nil {
-				// simulatingBid always better than bestBid, so only compare with simulatingBid if a simulatingBid exists
-				if bidRuntime.isExpectedBetterThan(simulatingBid) {
-					commit(commitInterruptBetterBid, bidRuntime)
+			toCommit := true
+			bestBidToRun := b.GetBestBidToRun(newBid.bid.ParentHash)
+			if bestBidToRun != nil {
+				bestBidRuntime, _ := newBidRuntime(bestBidToRun, b.config.ValidatorCommission)
+				if bidRuntime.isExpectedBetterThan(bestBidRuntime) {
+					// new bid has better expectedBlockReward, use bidRuntime
+					log.Debug("new bid has better expectedBlockReward",
+						"builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().TerminalString())
+				} else if !bestBidToRun.IsCommitted() {
+					// bestBidToRun is not committed yet, this newBid will trigger bestBidToRun to commit
+					bidRuntime = bestBidRuntime
+					replyErr = genDiscardedReply(bidRuntime)
+					log.Debug("discard new bid and to simulate the non-committed bestBidToRun",
+						"builder", bestBidToRun.Builder, "bidHash", bestBidToRun.Hash().TerminalString())
 				} else {
-					replyErr = genDiscardedReply(simulatingBid)
+					// new bid will be discarded, as it is useless now.
+					toCommit = false
+					replyErr = genDiscardedReply(bestBidRuntime)
+					log.Debug("new bid will be discarded", "builder", bestBidToRun.Builder,
+						"bidHash", bestBidToRun.Hash().TerminalString())
 				}
-			} else {
-				// bestBid is nil means the bid is the first bid, otherwise the bid should compare with the bestBid
-				if bestBid := b.GetBestBid(newBid.bid.ParentHash); bestBid == nil ||
-					bidRuntime.isExpectedBetterThan(bestBid) {
-					commit(commitInterruptBetterBid, bidRuntime)
+			}
+
+			if toCommit {
+				b.SetBestBidToRun(bidRuntime.bid.ParentHash, bidRuntime.bid)
+				// try to commit the new bid
+				// but if there is a simulating bid and with a short time left, don't interrupt it
+				if simulatingBid := b.GetSimulatingBid(newBid.bid.ParentHash); simulatingBid != nil {
+					parentHeader := b.chain.GetHeaderByHash(newBid.bid.ParentHash)
+					blockInterval := b.getBlockInterval(parentHeader)
+					blockTime := parentHeader.MilliTimestamp() + blockInterval
+					left := time.Until(time.UnixMilli(int64(blockTime)))
+					if b.canBeInterrupted(blockTime) {
+						log.Debug("simulate in progress, interrupt",
+							"blockTime", blockTime, "left", left.Milliseconds(),
+							"NoInterruptLeftOver", b.config.NoInterruptLeftOver.Milliseconds(),
+							"builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().TerminalString())
+						commit(commitInterruptBetterBid, bidRuntime)
+					} else {
+						log.Debug("simulate in progress, no interrupt",
+							"blockTime", blockTime, "left", left.Milliseconds(),
+							"NoInterruptLeftOver", b.config.NoInterruptLeftOver.Milliseconds(),
+							"builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().TerminalString())
+						if newBid.bid.Hash() == bidRuntime.bid.Hash() {
+							replyErr = fmt.Errorf("bid is pending as no enough time to interrupt, left:%d, NoInterruptLeftOver:%d",
+								left.Milliseconds(), b.config.NoInterruptLeftOver.Milliseconds())
+						}
+					}
 				} else {
-					replyErr = genDiscardedReply(bestBid)
+					commit(commitInterruptBetterBid, bidRuntime)
 				}
 			}
 
 			if newBid.feedback != nil {
 				newBid.feedback <- replyErr
-
 				log.Info("[BID ARRIVED]",
 					"block", newBid.bid.BlockNumber,
 					"builder", newBid.bid.Builder,
@@ -398,16 +473,24 @@ func (b *bidSimulator) newBidLoop() {
 	}
 }
 
-func (b *bidSimulator) bidBetterBefore(parentHash common.Hash) time.Time {
-	parentHeader := b.chain.GetHeaderByHash(parentHash)
+// get block interval for current block by using parent header
+func (b *bidSimulator) getBlockInterval(parentHeader *types.Header) uint64 {
+	if parentHeader == nil {
+		return 1500 // lorentzBlockInterval
+	}
 	parlia, _ := b.engine.(*parlia.Parlia)
 	// only `Number` and `ParentHash` are used when `BlockInterval`
-	tmpHeader := &types.Header{ParentHash: parentHash, Number: new(big.Int).Add(parentHeader.Number, common.Big1)}
+	tmpHeader := &types.Header{ParentHash: parentHeader.Hash(), Number: new(big.Int).Add(parentHeader.Number, common.Big1)}
 	blockInterval, err := parlia.BlockInterval(b.chain, tmpHeader)
 	if err != nil {
 		log.Debug("failed to get BlockInterval when bidBetterBefore")
 	}
-	return bidutil.BidBetterBefore(parentHeader, blockInterval, b.delayLeftOver, b.config.BidSimulationLeftOver)
+	return blockInterval
+}
+
+func (b *bidSimulator) bidBetterBefore(parentHash common.Hash) time.Time {
+	parentHeader := b.chain.GetHeaderByHash(parentHash)
+	return bidutil.BidBetterBefore(parentHeader, b.getBlockInterval(parentHeader), b.delayLeftOver, b.config.BidSimulationLeftOver)
 }
 
 func (b *bidSimulator) clearLoop() {
@@ -427,6 +510,12 @@ func (b *bidSimulator) clearLoop() {
 				delete(b.bestBid, k)
 			}
 		}
+		delete(b.bestBidToRun, parentHash)
+		for k, v := range b.bestBidToRun {
+			if v.BlockNumber <= blockNumber-b.chain.TriesInMemory() {
+				delete(b.bestBidToRun, k)
+			}
+		}
 		b.bestBidMu.Unlock()
 
 		b.simBidMu.Lock()
@@ -527,13 +616,15 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 
 	// ensure simulation exited then start next simulation
 	b.SetSimulatingBid(parentHash, bidRuntime)
+	bestBidOnStart := b.GetBestBid(parentHash)
 
 	defer func(simStart time.Time) {
 		logCtx := []any{
 			"blockNumber", blockNumber,
 			"parentHash", parentHash,
 			"builder", builder,
 			"gasUsed", bidRuntime.bid.GasUsed,
+			"simElapsed", time.Since(simStart),
 		}
 
 		if bidRuntime.env != nil {
@@ -565,9 +656,12 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 
 			select {
 			case b.newBidCh <- newBidPackage{bid: bidRuntime.bid}:
-				log.Debug("BidSimulator: recommit", "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex())
+				log.Debug("BidSimulator: recommit", "builder", bidRuntime.bid.Builder,
+					"bidHash", bidRuntime.bid.Hash().Hex(), "simElapsed", bidRuntime.duration)
 			default:
 			}
+		} else {
+			b.DelBestBidToRun(parentHash, bidRuntime.bid)
 		}
 	}(startTS)
 
@@ -689,14 +783,17 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 
 	// if enable greedy merge, fill bid env with transactions from mempool
 	if b.config.GreedyMergeTx {
-		delay := b.engine.Delay(b.chain, bidRuntime.env.header, &b.delayLeftOver)
+		endingBidsExtra := 20 * time.Millisecond // Add a buffer to ensure ending bids before `delayLeftOver`
+		minTimeLeftForEndingBids := b.delayLeftOver + endingBidsExtra
+		delay := b.engine.Delay(b.chain, bidRuntime.env.header, &minTimeLeftForEndingBids)
 		if delay != nil && *delay > 0 {
 			bidTxsSet := mapset.NewThreadUnsafeSetWithSize[common.Hash](len(bidRuntime.bid.Txs))
 			for _, tx := range bidRuntime.bid.Txs {
 				bidTxsSet.Add(tx.Hash())
 			}
-
-			fillErr := b.bidWorker.fillTransactions(interruptCh, bidRuntime.env, nil, bidTxsSet)
+			stopTimer := time.NewTimer(*delay)
+			defer stopTimer.Stop()
+			fillErr := b.bidWorker.fillTransactions(interruptCh, bidRuntime.env, stopTimer, bidTxsSet)
 			log.Trace("BidSimulator: greedy merge stopped", "block", bidRuntime.env.header.Number,
 				"builder", bidRuntime.bid.Builder, "tx count", bidRuntime.env.tcount-bidTxLen+1, "err", fillErr)
 
@@ -725,7 +822,12 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 
 	bestBid := b.GetBestBid(parentHash)
 	if bestBid == nil {
-		log.Info("[BID RESULT]", "win", "true[first]", "builder", bidRuntime.bid.Builder, "hash", bidRuntime.bid.Hash().TerminalString())
+		winResult := "true[first]"
+		if bestBidOnStart != nil {
+			// new block was imported, so the bestBidOnStart was cleared, the bid will be stale and useless.
+			winResult = "false[stale]"
+		}
+		log.Info("[BID RESULT]", "win", winResult, "builder", bidRuntime.bid.Builder, "hash", bidRuntime.bid.Hash().TerminalString())
 		b.SetBestBid(bidRuntime.bid.ParentHash, bidRuntime)
 		success = true
 		return

miner/miner_mev.go

@@ -99,6 +99,7 @@ func (miner *Miner) MevParams() *types.MevParams {
 	return &types.MevParams{
 		ValidatorCommission:   miner.worker.config.Mev.ValidatorCommission,
 		BidSimulationLeftOver: miner.worker.config.Mev.BidSimulationLeftOver,
+		NoInterruptLeftOver:   miner.worker.config.Mev.NoInterruptLeftOver,
 		GasCeil:               miner.worker.config.GasCeil,
 		GasPrice:              miner.worker.config.GasPrice,
 		BuilderFeeCeil:        builderFeeCeil,

miner/minerconfig/config.go

@@ -75,6 +75,7 @@ type MevConfig struct {
 	Builders              []BuilderConfig // The list of builders
 	ValidatorCommission   uint64          // 100 means the validator claims 1% from block reward
 	BidSimulationLeftOver time.Duration
+	NoInterruptLeftOver   time.Duration
 }
 
 var DefaultMevConfig = MevConfig{
@@ -84,4 +85,5 @@ var DefaultMevConfig = MevConfig{
 	Builders:              nil,
 	ValidatorCommission:   100,
 	BidSimulationLeftOver: 50 * time.Millisecond,
+	NoInterruptLeftOver:   400 * time.Millisecond,
 }