pevm: support delay gas fee calculation & Uts; (bnb-chain#11)

* pevm: support delay gas fee calculation;
txdag: check gas fee receiver;
tests: support PEVM+TxDAG UTs;

* txdag: skip some cost time operation;
tests: fix some broken UTs;

---------

Co-authored-by: galaio <[email protected]>

cmd/evm/blockrunner.go

@@ -92,7 +92,7 @@ func blockTestCmd(ctx *cli.Context) error {
 					fmt.Println(string(state.Dump(nil)))
 				}
 			}
-		}); err != nil {
+		}, "", true); err != nil {
 			return fmt.Errorf("test %v: %w", name, err)
 		}
 	}

core/blockchain.go

@@ -2837,7 +2837,7 @@ func (bc *BlockChain) TxDAGEnabled() bool {
 	return bc.enableTxDAG
 }
 
-func (bc *BlockChain) EnableTxDAGGeneration(output string) {
+func (bc *BlockChain) SetupTxDAGGeneration(output string) {
 	bc.enableTxDAG = true
 	if len(output) == 0 {
 		return
@@ -2846,7 +2846,7 @@ func (bc *BlockChain) EnableTxDAGGeneration(output string) {
 	var err error
 	bc.txDAGMapping, err = readTxDAGMappingFromFile(output)
 	if err != nil {
-		log.Error("read TxDAG err", err)
+		log.Error("read TxDAG err", "err", err)
 	}
 
 	// write handler

core/parallel_state_processor.go

@@ -50,6 +50,7 @@ type ParallelStateProcessor struct {
 	targetStage2Count     int // when executed txNUM reach it, enter stage2 RT confirm
 	nextStage2TxIndex     int
 	disableStealTx        bool
+	delayGasFee           bool // it is provided by TxDAG
 }
 
 func NewParallelStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine, parallelNum int) *ParallelStateProcessor {
@@ -187,6 +188,8 @@ func (p *ParallelStateProcessor) resetState(txNum int, statedb *state.StateDB) {
 // 3. TODO(galaio) it need to schedule the slow dep tx path properly;
 // 4. TODO(galaio) it is unfriendly for cross slot deps, maybe we can delay dispatch when tx cross in slots, it may increase PEVM parallelism;
 func (p *ParallelStateProcessor) doStaticDispatchV2(txReqs []*ParallelTxRequest, txDAG types.TxDAG) {
+	p.disableStealTx = false
+	p.delayGasFee = false
 	// only support PlainTxDAG dispatch now.
 	if txDAG == nil || txDAG.Type() != types.PlainTxDAGType {
 		p.doStaticDispatch(txReqs)
@@ -213,6 +216,7 @@ func (p *ParallelStateProcessor) doStaticDispatchV2(txReqs []*ParallelTxRequest,
 
 	// it's unnecessary to enable slot steal mechanism, opt the steal mechanism later;
 	p.disableStealTx = true
+	p.delayGasFee = true
 }
 
 // Benefits of StaticDispatch:
@@ -331,7 +335,7 @@ func (p *ParallelStateProcessor) executeInSlot(slotIndex int, txReq *ParallelTxR
 
 	slotDB.SetTxContext(txReq.tx.Hash(), txReq.txIndex)
 
-	evm, result, err := applyTransactionStageExecution(txReq.msg, gpSlot, slotDB, vmenv)
+	evm, result, err := applyTransactionStageExecution(txReq.msg, gpSlot, slotDB, vmenv, p.delayGasFee)
 	txResult := ParallelTxResult{
 		executedIndex: execNum,
 		slotIndex:     slotIndex,
@@ -660,7 +664,19 @@ func (p *ParallelStateProcessor) confirmTxResults(statedb *state.StateDB, gp *Ga
 	}
 
 	resultTxIndex := result.txReq.txIndex
-
+	delayGasFee := result.result.delayFees
+	// add delayed gas fee
+	if delayGasFee != nil {
+		if delayGasFee.TipFee != nil {
+			result.slotDB.AddBalance(delayGasFee.Coinbase, delayGasFee.TipFee)
+		}
+		if delayGasFee.BaseFee != nil {
+			result.slotDB.AddBalance(params.OptimismBaseFeeRecipient, delayGasFee.BaseFee)
+		}
+		if delayGasFee.L1Fee != nil {
+			result.slotDB.AddBalance(params.OptimismL1FeeRecipient, delayGasFee.L1Fee)
+		}
+	}
 	var root []byte
 	header := result.txReq.block.Header()
 
@@ -669,7 +685,7 @@ func (p *ParallelStateProcessor) confirmTxResults(statedb *state.StateDB, gp *Ga
 	result.slotDB.FinaliseForParallel(isByzantium || isEIP158, statedb)
 
 	// merge slotDB into mainDB
-	statedb.MergeSlotDB(result.slotDB, result.receipt, resultTxIndex)
+	statedb.MergeSlotDB(result.slotDB, result.receipt, resultTxIndex, result.result.delayFees)
 
 	// Do IntermediateRoot after mergeSlotDB.
 	if !isByzantium {
@@ -887,13 +903,21 @@ func (p *ParallelStateProcessor) Process(block *types.Block, statedb *state.Stat
 	return receipts, allLogs, *usedGas, nil
 }
 
-func applyTransactionStageExecution(msg *Message, gp *GasPool, statedb *state.ParallelStateDB, evm *vm.EVM) (*vm.EVM, *ExecutionResult, error) {
+func applyTransactionStageExecution(msg *Message, gp *GasPool, statedb *state.ParallelStateDB, evm *vm.EVM, delayGasFee bool) (*vm.EVM, *ExecutionResult, error) {
 	// Create a new context to be used in the EVM environment.
 	txContext := NewEVMTxContext(msg)
 	evm.Reset(txContext, statedb)
 
 	// Apply the transaction to the current state (included in the env).
-	result, err := ApplyMessage(evm, msg, gp)
+	var (
+		result *ExecutionResult
+		err    error
+	)
+	if delayGasFee {
+		result, err = ApplyMessageDelayGasFee(evm, msg, gp)
+	} else {
+		result, err = ApplyMessage(evm, msg, gp)
+	}
 
 	if err != nil {
 		return nil, nil, err

core/state/statedb.go

@@ -60,6 +60,13 @@ type StateObjectSyncMap struct {
 	sync.Map
 }
 
+type DelayedGasFee struct {
+	BaseFee  *big.Int
+	TipFee   *big.Int
+	L1Fee    *big.Int
+	Coinbase common.Address
+}
+
 func (s *StateObjectSyncMap) LoadStateObject(addr common.Address) (*stateObject, bool) {
 	so, ok := s.Load(addr)
 	if !ok {
@@ -239,9 +246,10 @@ type StateDB struct {
 	logSize uint
 
 	// parallel EVM related
-	rwSet    *types.RWSet
-	mvStates *types.MVStates
-	es       *types.ExeStat
+	rwSet        *types.RWSet
+	mvStates     *types.MVStates
+	stat         *types.ExeStat
+	rwRecordFlag bool
 
 	// Preimages occurred seen by VM in the scope of block.
 	preimages map[common.Hash][]byte
@@ -2370,6 +2378,7 @@ func (s *StateDB) BeforeTxTransition() {
 	s.rwSet = types.NewRWSet(types.StateVersion{
 		TxIndex: s.txIndex,
 	})
+	s.rwRecordFlag = true
 }
 
 func (s *StateDB) BeginTxStat(index int) {
@@ -2379,7 +2388,9 @@ func (s *StateDB) BeginTxStat(index int) {
 	if s.mvStates == nil {
 		return
 	}
-	s.es = types.NewExeStat(index).Begin()
+	if metrics.EnabledExpensive {
+		s.stat = types.NewExeStat(index).Begin()
+	}
 }
 
 func (s *StateDB) StopTxStat(usedGas uint64) {
@@ -2390,16 +2401,16 @@ func (s *StateDB) StopTxStat(usedGas uint64) {
 		return
 	}
 	// record stat first
-	if s.es != nil {
-		s.es.Done().WithGas(usedGas).WithRead(len(s.rwSet.ReadSet()))
+	if metrics.EnabledExpensive && s.stat != nil {
+		s.stat.Done().WithGas(usedGas).WithRead(len(s.rwSet.ReadSet()))
 	}
 }
 
 func (s *StateDB) RecordRead(key types.RWKey, val interface{}) {
 	if s.isParallel && s.parallel.isSlotDB {
 		return
 	}
-	if s.mvStates == nil || s.rwSet == nil {
+	if !s.rwRecordFlag {
 		return
 	}
 	// TODO: read from MVStates, record with ver
@@ -2412,7 +2423,7 @@ func (s *StateDB) RecordWrite(key types.RWKey, val interface{}) {
 	if s.isParallel && s.parallel.isSlotDB {
 		return
 	}
-	if s.mvStates == nil || s.rwSet == nil {
+	if !s.rwRecordFlag {
 		return
 	}
 	s.rwSet.RecordWrite(key, val)
@@ -2424,13 +2435,14 @@ func (s *StateDB) ResetMVStates(txCount int) {
 	}
 	s.mvStates = types.NewMVStates(txCount)
 	s.rwSet = nil
+	s.rwRecordFlag = false
 }
 
 func (s *StateDB) FinaliseRWSet() error {
 	if s.isParallel && s.parallel.isSlotDB {
 		return nil
 	}
-	if s.mvStates == nil || s.rwSet == nil {
+	if !s.rwRecordFlag {
 		return nil
 	}
 	if metrics.EnabledExpensive {
@@ -2468,7 +2480,8 @@ func (s *StateDB) FinaliseRWSet() error {
 		}
 	}
 
-	return s.mvStates.FulfillRWSet(s.rwSet, s.es)
+	s.rwRecordFlag = false
+	return s.mvStates.FulfillRWSet(s.rwSet, s.stat)
 }
 
 func (s *StateDB) queryStateObjectsDestruct(addr common.Address) (*types.StateAccount, bool) {
@@ -2498,7 +2511,7 @@ func (s *StateDB) deleteStateObjectsDestruct(addr common.Address) {
 	delete(s.stateObjectsDestruct, addr)
 }
 
-func (s *StateDB) MVStates2TxDAG() (types.TxDAG, map[int]*types.ExeStat) {
+func (s *StateDB) ResolveTxDAG(gasFeeReceivers []common.Address) (types.TxDAG, map[int]*types.ExeStat) {
 	if s.isParallel && s.parallel.isSlotDB {
 		return nil, nil
 	}
@@ -2511,7 +2524,7 @@ func (s *StateDB) MVStates2TxDAG() (types.TxDAG, map[int]*types.ExeStat) {
 		}(time.Now())
 	}
 
-	return s.mvStates.ResolveTxDAG(), s.mvStates.Stats()
+	return s.mvStates.ResolveTxDAG(gasFeeReceivers), s.mvStates.Stats()
 }
 
 func (s *StateDB) MVStates() *types.MVStates {
@@ -2600,7 +2613,7 @@ func (s *StateDB) AddrPrefetch(slotDb *ParallelStateDB) {
 // MergeSlotDB is for Parallel execution mode, when the transaction has been
 // finalized(dirty -> pending) on execution slot, the execution results should be
 // merged back to the main StateDB.
-func (s *StateDB) MergeSlotDB(slotDb *ParallelStateDB, slotReceipt *types.Receipt, txIndex int) *StateDB {
+func (s *StateDB) MergeSlotDB(slotDb *ParallelStateDB, slotReceipt *types.Receipt, txIndex int, fees *DelayedGasFee) *StateDB {
 	s.SetTxContext(slotDb.thash, slotDb.txIndex)
 
 	for s.nextRevisionId < slotDb.nextRevisionId {

core/state/statedb_test.go

@@ -1541,7 +1541,7 @@ func TestMergeSlotDB(t *testing.T) {
 	newSlotDb.SelfDestruct(addr)
 	newSlotDb.Finalise(true)
 
-	changeList := oldSlotDb.MergeSlotDB(newSlotDb, &types.Receipt{}, 0)
+	changeList := oldSlotDb.MergeSlotDB(newSlotDb, &types.Receipt{}, 0, nil)
 
 	if ok := changeList.getDeletedStateObject(addr); ok == nil || !ok.selfDestructed {
 		t.Fatalf("address should exist in StateObjectSuicided")

core/state_processor.go

@@ -22,6 +22,8 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/log"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
@@ -125,10 +127,12 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	if p.bc.enableTxDAG {
 		// TODO(galaio): append dag into block body, TxDAGPerformance will print metrics when profile is enabled
 		// compare input TxDAG when it enable in consensus
-		dag, exrStats := statedb.MVStates2TxDAG()
-		fmt.Print(types.EvaluateTxDAGPerformance(dag, exrStats))
-		//log.Info("Process result", "block", block.NumberU64(), "txDAG", dag)
-		// try write txDAG into file
+		dag, extraStats := statedb.ResolveTxDAG([]common.Address{context.Coinbase, params.OptimismBaseFeeRecipient, params.OptimismL1FeeRecipient})
+		log.Debug("Process TxDAG result", "block", block.NumberU64(), "txDAG", dag)
+		if metrics.EnabledExpensive {
+			types.EvaluateTxDAGPerformance(dag, extraStats)
+		}
+		// try to write txDAG into file
 		if p.bc.txDAGWriteCh != nil && dag != nil {
 			p.bc.txDAGWriteCh <- TxDAGOutputItem{
 				blockNumber: block.NumberU64(),

core/state_transition.go

@@ -19,6 +19,7 @@ package core
 import (
 	"errors"
 	"fmt"
+	"github.com/ethereum/go-ethereum/core/state"
 	"math"
 	"math/big"
 	"time"
@@ -40,6 +41,7 @@ type ExecutionResult struct {
 	RefundedGas uint64 // Total gas refunded after execution
 	Err         error  // Any error encountered during the execution(listed in core/vm/errors.go)
 	ReturnData  []byte // Returned data from evm(function result or data supplied with revert opcode)
+	delayFees   *state.DelayedGasFee
 }
 
 // Unwrap returns the internal evm error which allows us for further
@@ -196,6 +198,12 @@ func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, err
 	return NewStateTransition(evm, msg, gp).TransitionDb()
 }
 
+func ApplyMessageDelayGasFee(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, error) {
+	transition := NewStateTransition(evm, msg, gp)
+	transition.delayGasFee = true
+	return transition.TransitionDb()
+}
+
 // StateTransition represents a state transition.
 //
 // == The State Transitioning Model
@@ -225,6 +233,7 @@ type StateTransition struct {
 	initialGas   uint64
 	state        vm.StateDB
 	evm          *vm.EVM
+	delayGasFee  bool
 }
 
 // NewStateTransition initialises and returns a new state transition object.
@@ -540,40 +549,65 @@ func (st *StateTransition) innerTransitionDb() (*ExecutionResult, error) {
 		}, nil
 	}
 
+	var (
+		tipFee  *big.Int
+		baseFee *big.Int
+		l1Fee   *big.Int
+	)
 	effectiveTip := msg.GasPrice
 	if rules.IsLondon {
 		effectiveTip = cmath.BigMin(msg.GasTipCap, new(big.Int).Sub(msg.GasFeeCap, st.evm.Context.BaseFee))
 	}
 
+	// delay gas fee calculation, provide from TxDAG
 	if st.evm.Config.NoBaseFee && msg.GasFeeCap.Sign() == 0 && msg.GasTipCap.Sign() == 0 {
 		// Skip fee payment when NoBaseFee is set and the fee fields
 		// are 0. This avoids a negative effectiveTip being applied to
 		// the coinbase when simulating calls.
 	} else {
 		fee := new(big.Int).SetUint64(st.gasUsed())
 		fee.Mul(fee, effectiveTip)
-		st.state.AddBalance(st.evm.Context.Coinbase, fee)
+		if st.delayGasFee {
+			tipFee = fee
+		} else {
+			st.state.AddBalance(st.evm.Context.Coinbase, fee)
+		}
 	}
 
 	// Check that we are post bedrock to enable op-geth to be able to create pseudo pre-bedrock blocks (these are pre-bedrock, but don't follow l2 geth rules)
 	// Note optimismConfig will not be nil if rules.IsOptimismBedrock is true
 	if optimismConfig := st.evm.ChainConfig().Optimism; optimismConfig != nil && rules.IsOptimismBedrock && !st.msg.IsDepositTx {
-		st.state.AddBalance(params.OptimismBaseFeeRecipient, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.evm.Context.BaseFee))
+		fee := new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.evm.Context.BaseFee)
+		if st.delayGasFee {
+			baseFee = fee
+		} else {
+			st.state.AddBalance(params.OptimismBaseFeeRecipient, fee)
+		}
 		var l1Cost *big.Int
 		if st.msg.GasPrice.Cmp(big.NewInt(0)) == 0 && st.evm.ChainConfig().IsWright(st.evm.Context.Time) {
 			l1Cost = big.NewInt(0)
 		} else {
 			l1Cost = st.evm.Context.L1CostFunc(st.msg.RollupCostData, st.evm.Context.Time)
 		}
 		if l1Cost != nil {
-			st.state.AddBalance(params.OptimismL1FeeRecipient, l1Cost)
+			if st.delayGasFee {
+				l1Fee = l1Cost
+			} else {
+				st.state.AddBalance(params.OptimismL1FeeRecipient, l1Cost)
+			}
 		}
 	}
 	return &ExecutionResult{
 		UsedGas:     st.gasUsed(),
 		RefundedGas: gasRefund,
 		Err:         vmerr,
 		ReturnData:  ret,
+		delayFees: &state.DelayedGasFee{
+			TipFee:   tipFee,
+			BaseFee:  baseFee,
+			L1Fee:    l1Fee,
+			Coinbase: st.evm.Context.Coinbase,
+		},
 	}, nil
 }