Allow EVM state to be flushed to disk every Nth block instead of every block

Makefile

@@ -33,7 +33,7 @@ GO_LOOM_GIT_REV = HEAD
 # Specifies the loomnetwork/transfer-gateway branch/revision to use.
 TG_GIT_REV = HEAD
 # loomnetwork/go-ethereum loomchain branch
-ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9
+ETHEREUM_GIT_REV = in-memory-statedb
 # use go-plugin we get 'timeout waiting for connection info' error
 HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961
 LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9
@@ -197,14 +197,15 @@ $(BINANCE_TGORACLE_DIR):
 	git clone -q git@github.com:loomnetwork/binance-tgoracle.git $@
 	cd $(BINANCE_TGORACLE_DIR) && git checkout master && git pull && git checkout $(BINANCE_TG_GIT_REV)
 
-validators-tool: $(TRANSFER_GATEWAY_DIR)
-	go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd
-
-deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR)
-	# Temp workaround for https://github.com/prometheus/procfs/issues/221
+$(PROMETHEUS_PROCFS_DIR):
 	git clone -q git@github.com:prometheus/procfs $(PROMETHEUS_PROCFS_DIR)
 	cd $(PROMETHEUS_PROCFS_DIR) && git checkout master && git pull && git checkout d3b299e382e6acf1baa852560d862eca4ff643c8
 
+validators-tool: $(TRANSFER_GATEWAY_DIR)
+	go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd
+
+deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR) $(PROMETHEUS_PROCFS_DIR)
+
 	go get \
 		golang.org/x/crypto/ed25519 \
 		google.golang.org/grpc \

app.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"time"
 
+	gcommon "github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/loomnetwork/go-loom/config"
 	"github.com/loomnetwork/go-loom/util"
 	"github.com/loomnetwork/loomchain/eth/utils"
@@ -39,6 +41,7 @@ type ReadOnlyState interface {
 	Config() *cctypes.Config
 	EnabledFeatures() []string
 	GetMinBuildNumber() uint64
+	GetTrieDB() *trie.Database
 }
 
 type State interface {
@@ -59,6 +62,7 @@ type StoreState struct {
 	validators      loom.ValidatorSet
 	getValidatorSet GetValidatorSet
 	config          *cctypes.Config
+	trieDB          *trie.Database
 }
 
 var _ = State(&StoreState{})
@@ -100,6 +104,11 @@ func (s *StoreState) WithOnChainConfig(config *cctypes.Config) *StoreState {
 	return s
 }
 
+func (s *StoreState) WithTrieDB(trieDB *trie.Database) *StoreState {
+	s.trieDB = trieDB
+	return s
+}
+
 func (s *StoreState) Range(prefix []byte) plugin.RangeData {
 	return s.store.Range(prefix)
 }
@@ -140,15 +149,32 @@ func (s *StoreState) Context() context.Context {
 	return s.ctx
 }
 
+func (s *StoreState) GetTrieDB() *trie.Database {
+	return s.trieDB
+}
+
 const (
 	featurePrefix = "feature"
 	MinBuildKey   = "minbuild"
 )
 
+var (
+	vmPrefix = []byte("vm")
+	rootKey  = []byte("vmroot")
+	// This is the prefix of versioning Patricia roots
+	evmRootPrefix = []byte("evmroot")
+)
+
 func featureKey(featureName string) []byte {
 	return util.PrefixKey([]byte(featurePrefix), []byte(featureName))
 }
 
+func evmRootKey(blockHeight int64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(blockHeight))
+	return util.PrefixKey(vmPrefix, []byte(evmRootPrefix), b)
+}
+
 func (s *StoreState) EnabledFeatures() []string {
 	featuresFromState := s.Range([]byte(featurePrefix))
 	enabledFeatures := make([]string, 0, len(featuresFromState))
@@ -233,6 +259,7 @@ func (s *StoreState) WithContext(ctx context.Context) State {
 		ctx:             ctx,
 		validators:      s.validators,
 		getValidatorSet: s.getValidatorSet,
+		trieDB:          s.trieDB,
 	}
 }
 
@@ -243,6 +270,7 @@ func (s *StoreState) WithPrefix(prefix []byte) State {
 		ctx:             s.ctx,
 		validators:      s.validators,
 		getValidatorSet: s.getValidatorSet,
+		trieDB:          s.trieDB,
 	}
 }
 
@@ -362,6 +390,9 @@ type Application struct {
 	config                      *cctypes.Config
 	childTxRefs                 []evmaux.ChildTxRef // links Tendermint txs to EVM txs
 	ReceiptsVersion             int32
+	TrieDB                      *trie.Database
+	FlushInterval               int64 // commit Patricia trie to disk every N blocks
+	lastSavedEVMRoot            []byte
 }
 
 var _ abci.Application = &Application{}
@@ -505,7 +536,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB
 			a.curBlockHeader,
 			a.curBlockHash,
 			a.GetValidatorSet,
-		).WithOnChainConfig(a.config)
+		).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 		contractUpkeepHandler, err := a.CreateContractUpkeepHandler(upkeepState)
 		if err != nil {
 			panic(err)
@@ -525,7 +556,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB
 		a.curBlockHeader,
 		nil,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	validatorManager, err := a.CreateValidatorManager(state)
 	if err != registry.ErrNotFound {
@@ -593,7 +624,7 @@ func (a *Application) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
 		a.curBlockHeader,
 		nil,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	validatorManager, err := a.CreateValidatorManager(state)
 	if err != registry.ErrNotFound {
@@ -648,7 +679,7 @@ func (a *Application) CheckTx(txBytes []byte) abci.ResponseCheckTx {
 		a.curBlockHeader,
 		a.curBlockHash,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	// Receipts & events generated in CheckTx must be discarded since the app state changes they
 	// reflect aren't persisted.
@@ -685,7 +716,7 @@ func (a *Application) DeliverTx(txBytes []byte) abci.ResponseDeliverTx {
 		a.curBlockHeader,
 		a.curBlockHash,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	var r abci.ResponseDeliverTx
 
@@ -718,7 +749,7 @@ func (a *Application) processTx(storeTx store.KVStoreTx, txBytes []byte, isCheck
 		a.curBlockHeader,
 		a.curBlockHash,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	receiptHandler := a.ReceiptHandlerProvider.Store()
 	defer receiptHandler.DiscardCurrentReceipt()
@@ -771,7 +802,7 @@ func (a *Application) deliverTx2(storeTx store.KVStoreTx, txBytes []byte) abci.R
 		a.curBlockHeader,
 		a.curBlockHash,
 		a.GetValidatorSet,
-	).WithOnChainConfig(a.config)
+	).WithOnChainConfig(a.config).WithTrieDB(a.TrieDB)
 
 	receiptHandler := a.ReceiptHandlerProvider.Store()
 	defer receiptHandler.DiscardCurrentReceipt()
@@ -829,6 +860,39 @@ func (a *Application) Commit() abci.ResponseCommit {
 		committedBlockCount.With(lvs...).Add(1)
 		commitBlockLatency.With(lvs...).Observe(time.Since(begin).Seconds())
 	}(time.Now())
+
+	state := NewStoreState(
+		context.Background(),
+		a.Store,
+		a.curBlockHeader,
+		a.curBlockHash,
+		a.GetValidatorSet,
+	)
+	curHeight := a.curBlockHeader.GetHeight()
+
+	flushInterval := a.FlushInterval
+	if flushInterval == 0 {
+		cfg := state.Config()
+		flushInterval = int64(cfg.GetAppStore().GetIAVLFlushInterval())
+	} else if flushInterval == -1 {
+		flushInterval = 0
+	}
+
+	// Only commit Patricia tree every N blocks
+	if flushInterval == 0 || curHeight%flushInterval == 0 {
+		evmRoot := state.Get(util.PrefixKey(vmPrefix, rootKey))
+		if len(evmRoot) > 0 && bytes.Compare(a.lastSavedEVMRoot, evmRoot) != 0 {
+			ethDB := store.NewLoomEthDB(state, nil)
+			a.TrieDB.SetDiskDB(ethDB)
+			if err := a.TrieDB.Commit(gcommon.BytesToHash(evmRoot), false); err != nil {
+				panic(err)
+			}
+			a.lastSavedEVMRoot = evmRoot
+			a.TrieDB = trie.NewDatabase(ethDB)
+			a.Store.Set(evmRootKey(curHeight), evmRoot)
+		}
+	}
+
 	appHash, _, err := a.Store.SaveVersion()
 	if err != nil {
 		panic(err)
@@ -893,5 +957,5 @@ func (a *Application) ReadOnlyState() State {
 		a.lastBlockHeader,
 		nil, // TODO: last block hash!
 		a.GetValidatorSet,
-	)
+	).WithTrieDB(a.TrieDB)
 }

cmd/loom/db/evm.go

@@ -285,7 +285,7 @@ func newDumpEVMStateFromEvmDB() *cobra.Command {
 				nil,
 			)
 
-			srcStateDB := gstate.NewDatabase(evm.NewLoomEthdb(state, nil))
+			srcStateDB := gstate.NewDatabase(store.NewLoomEthDB(state, nil))
 			srcStateDBTrie, err := srcStateDB.OpenTrie(evmRoot)
 			if err != nil {
 				fmt.Printf("cannot open trie, %s\n", evmRoot.Hex())

cmd/loom/loom.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/push"
 	"github.com/tendermint/tendermint/libs/db"
 
+	"github.com/ethereum/go-ethereum/trie"
 	kitprometheus "github.com/go-kit/kit/metrics/prometheus"
 	"github.com/gogo/protobuf/proto"
 	"github.com/loomnetwork/go-loom"
@@ -836,7 +837,7 @@ func loadApp(
 			return evm.NewLoomVm(state, eventHandler, receiptHandlerProvider.Writer(), createABM, cfg.EVMDebugEnabled), nil
 		})
 	}
-	evm.LogEthDbBatch = cfg.LogEthDbBatch
+	store.LogEthDBBatch = cfg.LogEthDbBatch
 
 	deployTxHandler := &vm.DeployTxHandler{
 		Manager:                vmManager,
@@ -1126,6 +1127,8 @@ func loadApp(
 		GetValidatorSet:             getValidatorSet,
 		EvmAuxStore:                 evmAuxStore,
 		ReceiptsVersion:             cfg.ReceiptsVersion,
+		TrieDB:                      trie.NewDatabase(nil),
+		FlushInterval:               cfg.AppStore.IAVLFlushInterval,
 	}, nil
 }
 

evm/evm_test.go

@@ -13,6 +13,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/trie"
+
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	ethvm "github.com/ethereum/go-ethereum/core/vm"
@@ -40,7 +42,9 @@ func mockState() loomchain.State {
 	header := abci.Header{}
 	header.Height = BlockHeight
 	header.Time = blockTime
-	return loomchain.NewStoreState(context.Background(), store.NewMemStore(), header, nil, nil)
+	memDB := store.NewMemStore()
+	trieDB := trie.NewDatabase(store.NewLoomEthDB(memDB, nil))
+	return loomchain.NewStoreState(context.Background(), memDB, header, nil, nil).WithTrieDB(trieDB)
 }
 
 func TestProcessDeployTx(t *testing.T) {

evm/loomevm.go

@@ -22,6 +22,7 @@ import (
 	"github.com/loomnetwork/loomchain/features"
 	"github.com/loomnetwork/loomchain/receipts"
 	"github.com/loomnetwork/loomchain/receipts/handler"
+	"github.com/loomnetwork/loomchain/store"
 	"github.com/loomnetwork/loomchain/vm"
 	"github.com/pkg/errors"
 )
@@ -38,12 +39,6 @@ type StateDB interface {
 	Commit(bool) (common.Hash, error)
 }
 
-type ethdbLogContext struct {
-	blockHeight  int64
-	contractAddr loom.Address
-	callerAddr   loom.Address
-}
-
 // TODO: this doesn't need to be exported, rename to loomEvmWithState
 type LoomEvm struct {
 	*Evm
@@ -54,21 +49,26 @@ type LoomEvm struct {
 // TODO: this doesn't need to be exported, rename to newLoomEvmWithState
 func NewLoomEvm(
 	loomState loomchain.State, accountBalanceManager AccountBalanceManager,
-	logContext *ethdbLogContext, debug bool,
+	logContext *store.EthDBLogContext, debug bool,
 ) (*LoomEvm, error) {
 	p := new(LoomEvm)
-	p.db = NewLoomEthdb(loomState, logContext)
+	p.db = store.NewLoomEthDB(loomState, logContext)
 	oldRoot, err := p.db.Get(rootKey)
 	if err != nil {
 		return nil, err
 	}
 
+	trieDB := loomState.GetTrieDB()
+	trieDB.SetDiskDB(p.db)
+	sdb := state.NewDatabase(p.db)
+	sdb.SetTrieDB(trieDB)
+
 	var abm *evmAccountBalanceManager
 	if accountBalanceManager != nil {
 		abm = newEVMAccountBalanceManager(accountBalanceManager, loomState.Block().ChainID)
-		p.sdb, err = newLoomStateDB(abm, common.BytesToHash(oldRoot), state.NewDatabase(p.db))
+		p.sdb, err = newLoomStateDB(abm, common.BytesToHash(oldRoot), sdb)
 	} else {
-		p.sdb, err = state.New(common.BytesToHash(oldRoot), state.NewDatabase(p.db))
+		p.sdb, err = state.New(common.BytesToHash(oldRoot), sdb)
 	}
 	if err != nil {
 		return nil, err
@@ -83,9 +83,9 @@ func (levm LoomEvm) Commit() (common.Hash, error) {
 	if err != nil {
 		return root, err
 	}
-	if err := levm.sdb.Database().TrieDB().Commit(root, false); err != nil {
-		return root, err
-	}
+	// if err := levm.sdb.Database().TrieDB().Commit(root, false); err != nil {
+	// 	return root, err
+	// }
 	if err := levm.db.Put(rootKey, root[:]); err != nil {
 		return root, err
 	}
@@ -146,11 +146,7 @@ func (lvm LoomVm) accountBalanceManager(readOnly bool) AccountBalanceManager {
 }
 
 func (lvm LoomVm) Create(caller loom.Address, code []byte, value *loom.BigUInt) ([]byte, loom.Address, error) {
-	logContext := &ethdbLogContext{
-		blockHeight:  lvm.state.Block().Height,
-		contractAddr: loom.Address{},
-		callerAddr:   caller,
-	}
+	logContext := store.NewEthDBLogContext(lvm.state.Block().Height, loom.Address{}, caller)
 	levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug)
 	if err != nil {
 		return nil, loom.Address{}, err
@@ -209,11 +205,7 @@ func (lvm LoomVm) Create(caller loom.Address, code []byte, value *loom.BigUInt)
 }
 
 func (lvm LoomVm) Call(caller, addr loom.Address, input []byte, value *loom.BigUInt) ([]byte, error) {
-	logContext := &ethdbLogContext{
-		blockHeight:  lvm.state.Block().Height,
-		contractAddr: addr,
-		callerAddr:   caller,
-	}
+	logContext := store.NewEthDBLogContext(lvm.state.Block().Height, addr, caller)
 	levm, err := NewLoomEvm(lvm.state, lvm.accountBalanceManager(false), logContext, lvm.debug)
 	if err != nil {
 		return nil, err