op-e2e: Action testing L2 Engine Block building

op-e2e/actions/l2_engine.go

@@ -4,12 +4,15 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/beacon"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	geth "github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/tracers"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -38,7 +41,14 @@ type L2Engine struct {
 	l2Signer   types.Signer
 
 	// L2 block building data
-	// TODO proto - block building PR
+	l2BuildingHeader *types.Header             // block header that we add txs to for block building
+	l2BuildingState  *state.StateDB            // state used for block building
+	l2GasPool        *core.GasPool             // track gas used of ongoing building
+	pendingIndices   map[common.Address]uint64 // per account, how many txs from the pool were already included in the block, since the pool is lagging behind block mining.
+	l2Transactions   []*types.Transaction      // collects txs that were successfully included into current block build
+	l2Receipts       []*types.Receipt          // collect receipts of ongoing building
+	l2ForceEmpty     bool                      // when no additional txs may be processed (i.e. when sequencer drift runs out)
+	l2TxFailed       []*types.Transaction      // log of failed transactions which could not be included
 
 	payloadID beacon.PayloadID // ID of payload that is currently being built
 
@@ -103,6 +113,11 @@ func NewL2Engine(log log.Logger, genesis *core.Genesis, rollupGenesisL1 eth.Bloc
 	return eng
 }
 
+func (s *L2Engine) EthClient() *ethclient.Client {
+	cl, _ := s.node.Attach() // never errors
+	return ethclient.NewClient(cl)
+}
+
 func (e *L2Engine) RPCClient() client.RPC {
 	cl, _ := e.node.Attach() // never errors
 	return testutils.RPCErrFaker{
@@ -123,3 +138,40 @@ func (e *L2Engine) ActL2RPCFail(t Testing) {
 	}
 	e.failL2RPC = errors.New("mock L2 RPC error")
 }
+
+// ActL2IncludeTx includes the next transaction from the given address in the block that is being built
+func (e *L2Engine) ActL2IncludeTx(from common.Address) Action {
+	return func(t Testing) {
+		if e.l2BuildingHeader == nil {
+			t.InvalidAction("not currently building a block, cannot include tx from queue")
+			return
+		}
+		if e.l2ForceEmpty {
+			t.InvalidAction("cannot include any sequencer txs")
+			return
+		}
+
+		i := e.pendingIndices[from]
+		txs, q := e.eth.TxPool().ContentFrom(from)
+		if uint64(len(txs)) <= i {
+			t.Fatalf("no pending txs from %s, and have %d unprocessable queued txs from this account", from, len(q))
+		}
+		tx := txs[i]
+		if tx.Gas() > e.l2BuildingHeader.GasLimit {
+			t.Fatalf("tx consumes %d gas, more than available in L2 block %d", tx.Gas(), e.l2BuildingHeader.GasLimit)
+		}
+		if tx.Gas() > uint64(*e.l2GasPool) {
+			t.InvalidAction("action takes too much gas: %d, only have %d", tx.Gas(), uint64(*e.l2GasPool))
+			return
+		}
+		e.pendingIndices[from] = i + 1 // won't retry the tx
+		receipt, err := core.ApplyTransaction(e.l2Cfg.Config, e.l2Chain, &e.l2BuildingHeader.Coinbase,
+			e.l2GasPool, e.l2BuildingState, e.l2BuildingHeader, tx, &e.l2BuildingHeader.GasUsed, *e.l2Chain.GetVMConfig())
+		if err != nil {
+			e.l2TxFailed = append(e.l2TxFailed, tx)
+			t.Fatalf("failed to apply transaction to L1 block (tx %d): %v", len(e.l2Transactions), err)
+		}
+		e.l2Receipts = append(e.l2Receipts, receipt)
+		e.l2Transactions = append(e.l2Transactions, tx)
+	}
+}

op-e2e/actions/l2_engine_api.go

@@ -2,11 +2,19 @@ package actions
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/consensus/misc"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/trie"
+
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/beacon"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -26,12 +34,101 @@ var (
 	INVALID_TERMINAL_BLOCK = eth.PayloadStatusV1{Status: eth.ExecutionInvalid, LatestValidHash: &common.Hash{}}
 )
 
+// computePayloadId computes a pseudo-random payloadid, based on the parameters.
+func computePayloadId(headBlockHash common.Hash, params *eth.PayloadAttributes) beacon.PayloadID {
+	// Hash
+	hasher := sha256.New()
+	hasher.Write(headBlockHash[:])
+	_ = binary.Write(hasher, binary.BigEndian, params.Timestamp)
+	hasher.Write(params.PrevRandao[:])
+	hasher.Write(params.SuggestedFeeRecipient[:])
+	for _, tx := range params.Transactions {
+		_ = binary.Write(hasher, binary.BigEndian, uint64(len(tx))) // length-prefix to avoid collisions
+		hasher.Write(tx)
+	}
+	if params.NoTxPool {
+		hasher.Write([]byte{1})
+	}
+	var out beacon.PayloadID
+	copy(out[:], hasher.Sum(nil)[:8])
+	return out
+}
+
 func (ea *L2EngineAPI) startBlock(parent common.Hash, params *eth.PayloadAttributes) error {
-	return fmt.Errorf("todo")
+	if ea.l2BuildingHeader != nil {
+		ea.log.Warn("started building new block without ending previous block", "previous", ea.l2BuildingHeader, "prev_payload_id", ea.payloadID)
+	}
+
+	parentHeader := ea.l2Chain.GetHeaderByHash(parent)
+	if parentHeader == nil {
+		return fmt.Errorf("uknown parent block: %s", parent)
+	}
+	statedb, err := state.New(parentHeader.Root, state.NewDatabase(ea.l2Database), nil)
+	if err != nil {
+		return fmt.Errorf("failed to init state db around block %s (state %s): %w", parent, parentHeader.Root, err)
+	}
+
+	header := &types.Header{
+		ParentHash: parent,
+		Coinbase:   params.SuggestedFeeRecipient,
+		Difficulty: common.Big0,
+		Number:     new(big.Int).Add(parentHeader.Number, common.Big1),
+		GasLimit:   parentHeader.GasLimit,
+		Time:       uint64(params.Timestamp),
+		Extra:      nil,
+		MixDigest:  common.Hash(params.PrevRandao),
+	}
+
+	header.BaseFee = misc.CalcBaseFee(ea.l2Cfg.Config, parentHeader)
+
+	ea.l2BuildingHeader = header
+	ea.l2BuildingState = statedb
+	ea.l2Receipts = make([]*types.Receipt, 0)
+	ea.l2Transactions = make([]*types.Transaction, 0)
+	ea.pendingIndices = make(map[common.Address]uint64)
+	ea.l2ForceEmpty = params.NoTxPool
+	ea.l2GasPool = new(core.GasPool).AddGas(header.GasLimit)
+	ea.payloadID = computePayloadId(parent, params)
+
+	// pre-process the deposits
+	for i, otx := range params.Transactions {
+		var tx types.Transaction
+		if err := tx.UnmarshalBinary(otx); err != nil {
+			return fmt.Errorf("transaction %d is not valid: %v", i, err)
+		}
+
+		receipt, err := core.ApplyTransaction(ea.l2Cfg.Config, ea.l2Chain, &ea.l2BuildingHeader.Coinbase,
+			ea.l2GasPool, ea.l2BuildingState, ea.l2BuildingHeader, &tx, &ea.l2BuildingHeader.GasUsed, *ea.l2Chain.GetVMConfig())
+		if err != nil {
+			ea.l2TxFailed = append(ea.l2TxFailed, &tx)
+			return fmt.Errorf("failed to apply deposit transaction to L2 block (tx %d): %w", i, err)
+		}
+		ea.l2Receipts = append(ea.l2Receipts, receipt)
+		ea.l2Transactions = append(ea.l2Transactions, &tx)
+	}
+	return nil
 }
 
 func (ea *L2EngineAPI) endBlock() (*types.Block, error) {
-	return nil, fmt.Errorf("todo")
+	if ea.l2BuildingHeader == nil {
+		return nil, fmt.Errorf("no block is being built currently (id %s)", ea.payloadID)
+	}
+	header := ea.l2BuildingHeader
+	ea.l2BuildingHeader = nil
+
+	header.GasUsed = header.GasLimit - uint64(*ea.l2GasPool)
+	header.Root = ea.l2BuildingState.IntermediateRoot(ea.l2Cfg.Config.IsEIP158(header.Number))
+	block := types.NewBlock(header, ea.l2Transactions, nil, ea.l2Receipts, trie.NewStackTrie(nil))
+
+	// Write state changes to db
+	root, err := ea.l2BuildingState.Commit(ea.l2Cfg.Config.IsEIP158(header.Number))
+	if err != nil {
+		return nil, fmt.Errorf("l2 state write error: %v", err)
+	}
+	if err := ea.l2BuildingState.Database().TrieDB().Commit(root, false, nil); err != nil {
+		return nil, fmt.Errorf("l2 trie write error: %v", err)
+	}
+	return block, nil
 }
 
 func (ea *L2EngineAPI) GetPayloadV1(ctx context.Context, payloadId eth.PayloadID) (*eth.ExecutionPayload, error) {

op-e2e/actions/l2_engine_test.go

@@ -1,13 +1,16 @@
 package actions
 
 import (
+	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -81,6 +84,87 @@ func TestL2EngineAPI(gt *testing.T) {
 	require.Equal(t, payloadB.BlockHash, engine.l2Chain.CurrentBlock().Hash(), "now payload B is canonical")
 }
 
+func TestL2EngineAPIBlockBuilding(gt *testing.T) {
+	t := NewDefaultTesting(gt)
+	jwtPath := e2eutils.WriteDefaultJWT(t)
+	dp := e2eutils.MakeDeployParams(t, defaultRollupTestParams)
+	sd := e2eutils.Setup(t, dp, defaultAlloc)
+	log := testlog.Logger(t, log.LvlDebug)
+	genesisBlock := sd.L2Cfg.ToBlock()
+	db := rawdb.NewMemoryDatabase()
+	sd.L2Cfg.MustCommit(db)
+
+	engine := NewL2Engine(log, sd.L2Cfg, sd.RollupCfg.Genesis.L1, jwtPath)
+
+	cl := engine.EthClient()
+	signer := types.LatestSigner(sd.L2Cfg.Config)
+
+	// send a tx to the miner
+	tx := types.MustSignNewTx(dp.Secrets.Alice, signer, &types.DynamicFeeTx{
+		ChainID:   sd.L2Cfg.Config.ChainID,
+		Nonce:     0,
+		GasTipCap: big.NewInt(2 * params.GWei),
+		GasFeeCap: new(big.Int).Add(engine.l2Chain.CurrentBlock().BaseFee(), big.NewInt(2*params.GWei)),
+		Gas:       params.TxGas,
+		To:        &dp.Addresses.Bob,
+		Value:     e2eutils.Ether(2),
+	})
+	require.NoError(gt, cl.SendTransaction(t.Ctx(), tx))
+
+	buildBlock := func(includeAlice bool) {
+		parent := engine.l2Chain.CurrentBlock()
+		l2Cl, err := sources.NewEngineClient(engine.RPCClient(), log, nil, sources.EngineClientDefaultConfig(sd.RollupCfg))
+		require.NoError(t, err)
+
+		// Now let's ask the engine to build a block
+		fcRes, err := l2Cl.ForkchoiceUpdate(t.Ctx(), &eth.ForkchoiceState{
+			HeadBlockHash:      parent.Hash(),
+			SafeBlockHash:      genesisBlock.Hash(),
+			FinalizedBlockHash: genesisBlock.Hash(),
+		}, &eth.PayloadAttributes{
+			Timestamp:             eth.Uint64Quantity(parent.Time()) + 2,
+			PrevRandao:            eth.Bytes32{},
+			SuggestedFeeRecipient: common.Address{'C'},
+			Transactions:          nil,
+			NoTxPool:              false,
+		})
+		require.NoError(t, err)
+		require.Equal(t, fcRes.PayloadStatus.Status, eth.ExecutionValid)
+		require.NotNil(t, fcRes.PayloadID, "building a block now")
+
+		if includeAlice {
+			engine.ActL2IncludeTx(dp.Addresses.Alice)(t)
+		}
+
+		payload, err := l2Cl.GetPayload(t.Ctx(), *fcRes.PayloadID)
+		require.NoError(t, err)
+		require.Equal(t, parent.Hash(), payload.ParentHash, "block builds on parent block")
+
+		// apply the payload
+		status, err := l2Cl.NewPayload(t.Ctx(), payload)
+		require.NoError(t, err)
+		require.Equal(t, status.Status, eth.ExecutionValid)
+		require.Equal(t, parent.Hash(), engine.l2Chain.CurrentBlock().Hash(), "processed payloads are not immediately canonical")
+
+		// recognize the payload as canonical
+		fcRes, err = l2Cl.ForkchoiceUpdate(t.Ctx(), &eth.ForkchoiceState{
+			HeadBlockHash:      payload.BlockHash,
+			SafeBlockHash:      genesisBlock.Hash(),
+			FinalizedBlockHash: genesisBlock.Hash(),
+		}, nil)
+		require.NoError(t, err)
+		require.Equal(t, fcRes.PayloadStatus.Status, eth.ExecutionValid)
+		require.Equal(t, payload.BlockHash, engine.l2Chain.CurrentBlock().Hash(), "now payload is canonical")
+	}
+	buildBlock(false)
+	require.Zero(t, engine.l2Chain.CurrentBlock().Transactions().Len(), "no tx included")
+	buildBlock(true)
+	require.Equal(gt, 1, engine.l2Chain.CurrentBlock().Transactions().Len(), "tx from alice is included")
+	buildBlock(false)
+	require.Zero(t, engine.l2Chain.CurrentBlock().Transactions().Len(), "no tx included")
+	require.Equal(t, uint64(3), engine.l2Chain.CurrentBlock().NumberU64(), "built 3 blocks")
+}
+
 func TestL2EngineAPIFail(gt *testing.T) {
 	t := NewDefaultTesting(gt)
 	jwtPath := e2eutils.WriteDefaultJWT(t)