From 7ee823a95277dcc3a41ed7bce8d81caac50dd188 Mon Sep 17 00:00:00 2001 From: Huynh Minh Tri Date: Wed, 25 Dec 2024 17:12:18 +0700 Subject: [PATCH 1/4] core: add metrics monitor block and header --- core/blockchain.go | 18 ++++++++++++++++++ core/headerchain.go | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 73b68dfa7..a1f26d0a4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -55,6 +55,10 @@ import ( ) var ( + headBlockGauge = metrics.NewRegisteredGauge("chain/head/block", nil) + headHeaderGauge = metrics.NewRegisteredGauge("chain/head/header", nil) + headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil) + accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil) @@ -351,6 +355,7 @@ func (bc *BlockChain) loadLastState() error { } // Everything seems to be fine, set as the head block bc.currentBlock.Store(currentBlock) + headBlockGauge.Update(int64(currentBlock.NumberU64())) // Restore the last known head header currentHeader := currentBlock.Header() @@ -363,9 +368,12 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head fast block bc.currentFastBlock.Store(currentBlock) + headFastBlockGauge.Update(int64(currentBlock.NumberU64())) + if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) } } @@ -429,7 +437,9 @@ func (bc *BlockChain) SetHead(head uint64) error { bc.currentFastBlock.Store(bc.genesisBlock) } currentBlock := bc.CurrentBlock() + headBlockGauge.Update(int64(currentBlock.NumberU64())) currentFastBlock := bc.CurrentFastBlock() + headFastBlockGauge.Update(int64(currentFastBlock.NumberU64())) if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil { log.Crit("Failed to reset head full block", "err", err) } @@ -453,6 +463,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // If all checks out, manually set the head block bc.mu.Lock() bc.currentBlock.Store(block) + headBlockGauge.Update(int64(block.NumberU64())) bc.mu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) @@ -585,9 +596,11 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { bc.genesisBlock = genesis bc.insert(bc.genesisBlock) bc.currentBlock.Store(bc.genesisBlock) + headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentFastBlock.Store(bc.genesisBlock) + headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) return nil } @@ -685,6 +698,7 @@ func (bc *BlockChain) insert(block *types.Block) { log.Crit("Failed to insert head block hash", "err", err) } bc.currentBlock.Store(block) + headBlockGauge.Update(int64(block.NumberU64())) // save cache BlockSigners if bc.chainConfig.Posv != nil && !bc.chainConfig.IsTIPSigning(block.Number()) { @@ -702,6 +716,7 @@ func (bc *BlockChain) insert(block *types.Block) { log.Crit("Failed to insert head fast block hash", "err", err) } bc.currentFastBlock.Store(block) + headFastBlockGauge.Update(int64(block.NumberU64())) } } @@ -1013,11 +1028,13 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) bc.currentFastBlock.Store(newFastBlock) + headFastBlockGauge.Update(int64(newFastBlock.NumberU64())) WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) } if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) bc.currentBlock.Store(newBlock) + headBlockGauge.Update(int64(newBlock.NumberU64())) WriteHeadBlockHash(bc.db, newBlock.Hash()) } } @@ -1139,6 +1156,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ log.Crit("Failed to update head fast block hash", "err", err) } bc.currentFastBlock.Store(head) + headFastBlockGauge.Update(int64(head.NumberU64())) } } bc.mu.Unlock() diff --git a/core/headerchain.go b/core/headerchain.go index 2f8531741..e874774af 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -26,7 +26,7 @@ import ( "sync/atomic" "time" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/core/types" @@ -104,6 +104,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c } } hc.currentHeaderHash = hc.CurrentHeader().Hash() + headHeaderGauge.Update(hc.CurrentHeader().Number.Int64()) return hc, nil } @@ -185,6 +186,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er } hc.currentHeaderHash = hash hc.currentHeader.Store(types.CopyHeader(header)) + headHeaderGauge.Update(header.Number.Int64()) status = CanonStatTy } else { @@ -400,6 +402,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { } hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() + headHeaderGauge.Update(head.Number.Int64()) } // DeleteCallback is a callback function that is called by SetHead before @@ -438,6 +441,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { hc.currentHeader.Store(hc.genesisHeader) } hc.currentHeaderHash = hc.CurrentHeader().Hash() + headHeaderGauge.Update(hc.CurrentHeader().Number.Int64()) if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { log.Crit("Failed to reset head header hash", "err", err) From eec5de80cf7f9b26be5e9488c675d0fab2113f08 Mon Sep 17 00:00:00 2001 From: Huynh Minh Tri Date: Wed, 25 Dec 2024 17:39:45 +0700 Subject: [PATCH 2/4] core: add metrics monitor block insert/validate/execute/write timer --- core/blockchain.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a1f26d0a4..a47db7392 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -69,9 +69,13 @@ var ( storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil) storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil) - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) - CheckpointCh = make(chan int) - ErrNoGenesis = errors.New("Genesis not found in chain") + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) + + CheckpointCh = make(chan int) + ErrNoGenesis = errors.New("Genesis not found in chain") ) const ( @@ -1662,13 +1666,16 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } feeCapacity := state.GetTRC21FeeCapacityFromStateWithCache(parent.Root(), statedb) // Process block using the parent state as reference point. + t0 := time.Now() receipts, logs, usedGas, err := bc.processor.Process(block, statedb, tradingState, bc.vmConfig, feeCapacity) + t1 := time.Now() if err != nil { bc.reportBlock(block, receipts, err) return i, events, coalescedLogs, err } // Validate the state using the default validator err = bc.Validator().ValidateState(block, parent, statedb, receipts, usedGas) + t2 := time.Now() if err != nil { bc.reportBlock(block, receipts, err) return i, events, coalescedLogs, err @@ -1676,6 +1683,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty proctime := time.Since(bstart) // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, statedb, tradingState, lendingState) + t3 := time.Now() if err != nil { return i, events, coalescedLogs, err } @@ -1709,6 +1717,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } } } + blockInsertTimer.UpdateSince(bstart) + blockExecutionTimer.Update(t1.Sub(t0)) + blockValidationTimer.Update(t2.Sub(t1)) + blockWriteTimer.Update(t3.Sub(t2)) switch status { case CanonStatTy: log.Debug("Inserted new block from downloader", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), From 6b270c359bc3bfa6d5c680efbb144edf06d55521 Mon Sep 17 00:00:00 2001 From: Huynh Minh Tri Date: Thu, 26 Dec 2024 14:16:21 +0700 Subject: [PATCH 3/4] core: add metrics monitor block reorg PR #18950 --- core/blockchain.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a47db7392..781993072 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -74,6 +74,9 @@ var ( blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) + blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) + blockReorgDropMeter = metrics.NewRegisteredMeter("chain/reorg/drop", nil) + CheckpointCh = make(chan int) ErrNoGenesis = errors.New("Genesis not found in chain") ) @@ -2239,12 +2242,16 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } // Ensure the user sees large reorgs if len(oldChain) > 0 && len(newChain) > 0 { - logFn := log.Debug + logFn := log.Info + msg := "Chain reorg detected" if len(oldChain) > 63 { + msg = "Large chain reorg detected" logFn = log.Warn } - logFn("Chain split detected", "number", commonBlock.Number(), "hash", commonBlock.Hash(), + logFn(msg, "number", commonBlock.Number(), "hash", commonBlock.Hash(), "drop", len(oldChain), "dropfrom", oldChain[0].Hash(), "add", len(newChain), "addfrom", newChain[0].Hash()) + blockReorgAddMeter.Mark(int64(len(newChain))) + blockReorgDropMeter.Mark(int64(len(oldChain))) } else { log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash()) } From 637d01a5896681603ca6c001add4dc5d9321ba8a Mon Sep 17 00:00:00 2001 From: Huynh Minh Tri Date: Thu, 26 Dec 2024 16:33:13 +0700 Subject: [PATCH 4/4] core: add metrics monitor block reorg PR #21420 --- core/blockchain.go | 7 +++++-- core/tx_pool.go | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 781993072..e8a556ca7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -74,8 +74,10 @@ var ( blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) - blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) - blockReorgDropMeter = metrics.NewRegisteredMeter("chain/reorg/drop", nil) + blockReorgMeter = metrics.NewRegisteredMeter("chain/reorg/executes", nil) + blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) + blockReorgDropMeter = metrics.NewRegisteredMeter("chain/reorg/drop", nil) + blockReorgInvalidatedTx = metrics.NewRegisteredMeter("chain/reorg/invalidTx", nil) CheckpointCh = make(chan int) ErrNoGenesis = errors.New("Genesis not found in chain") @@ -2252,6 +2254,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { "drop", len(oldChain), "dropfrom", oldChain[0].Hash(), "add", len(newChain), "addfrom", newChain[0].Hash()) blockReorgAddMeter.Mark(int64(len(newChain))) blockReorgDropMeter.Mark(int64(len(oldChain))) + blockReorgMeter.Mark(1) } else { log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash()) } diff --git a/core/tx_pool.go b/core/tx_pool.go index a1028ccaa..87555408c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -19,13 +19,14 @@ package core import ( "errors" "fmt" - "github.com/tomochain/tomochain/consensus" "math" "math/big" "sort" "sync" "time" + "github.com/tomochain/tomochain/consensus" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" @@ -1238,11 +1239,14 @@ func (pool *TxPool) demoteUnexecutables() { } // If there's a gap in front, warn (should never happen) and postpone all transactions if list.Len() > 0 && list.txs.Get(nonce) == nil { + gapped := list.Cap(0) for _, tx := range list.Cap(0) { hash := tx.Hash() log.Warn("Demoting invalidated transaction", "hash", hash) pool.enqueueTx(hash, tx) } + // This might happen in a reorg, so log it to the metering + blockReorgInvalidatedTx.Mark(int64(len(gapped))) } // Delete the entire queue entry if it became empty. if list.Empty() {