diff --git a/op-service/txmgr/metrics/noop.go b/op-service/txmgr/metrics/noop.go index b08e5f20bd438..b4fbf4746a8a6 100644 --- a/op-service/txmgr/metrics/noop.go +++ b/op-service/txmgr/metrics/noop.go @@ -4,4 +4,7 @@ import "github.com/ethereum/go-ethereum/core/types" type NoopTxMetrics struct{} -func (*NoopTxMetrics) RecordL1GasFee(*types.Receipt) {} +func (*NoopTxMetrics) RecordNonce(uint64) {} +func (*NoopTxMetrics) TxConfirmed(*types.Receipt) {} +func (*NoopTxMetrics) TxPublished(error) {} +func (*NoopTxMetrics) RPCError() {} diff --git a/op-service/txmgr/metrics/tx_metrics.go b/op-service/txmgr/metrics/tx_metrics.go index 6201e13a9b3c5..7a289d23a40a2 100644 --- a/op-service/txmgr/metrics/tx_metrics.go +++ b/op-service/txmgr/metrics/tx_metrics.go @@ -1,6 +1,8 @@ package metrics import ( + "time" + "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" @@ -9,11 +11,32 @@ import ( ) type TxMetricer interface { - RecordL1GasFee(receipt *types.Receipt) + RecordNonce(nonce uint64) + TxConfirmed(*types.Receipt) + TxPublished(err error) + RPCError() } type TxMetrics struct { - TxL1GasFee prometheus.Gauge + TxL1GasFee prometheus.Gauge + currentNonce prometheus.Gauge + txConfirmed *prometheus.CounterVec + txPublished prometheus.Counter + txPublishError *prometheus.CounterVec + lastPublishTime prometheus.Gauge + lastConfirmTime prometheus.Gauge + rpcError prometheus.Counter +} + +func receiptStatusString(receipt *types.Receipt) string { + switch receipt.Status { + case types.ReceiptStatusSuccessful: + return "success" + case types.ReceiptStatusFailed: + return "failed" + default: + return "unkown_status" + } } var _ TxMetricer = (*TxMetrics)(nil) @@ -26,9 +49,72 @@ func MakeTxMetrics(ns string, factory metrics.Factory) TxMetrics { Help: "L1 gas fee for transactions in GWEI", Subsystem: "txmgr", }), + currentNonce: factory.NewGauge(prometheus.GaugeOpts{ + Namespace: ns, + Name: "current_nonce", + Help: "", + Subsystem: "txmgr", + }), + txConfirmed: factory.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Name: "tx_confirmed_count", + Help: "", + Subsystem: "txmgr", + }, []string{"status"}), + txPublished: factory.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "tx_published_count", + Help: "", + Subsystem: "txmgr", + }), + txPublishError: factory.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Name: "tx_publish_error_count", + Help: "", + Subsystem: "txmgr", + }, []string{"error"}), + lastPublishTime: factory.NewGauge(prometheus.GaugeOpts{ + Namespace: ns, + Name: "last_publish_time_unix_secs", + Help: "", + Subsystem: "txmgr", + }), + lastConfirmTime: factory.NewGauge(prometheus.GaugeOpts{ + Namespace: ns, + Name: "last_confirm_time_unix_secs", + Help: "", + Subsystem: "txmgr", + }), + rpcError: factory.NewCounter(prometheus.CounterOpts{ + Namespace: ns, + Name: "rpc_error_count", + Help: "", + Subsystem: "txmgr", + }), } } -func (t *TxMetrics) RecordL1GasFee(receipt *types.Receipt) { +func (t *TxMetrics) RecordNonce(nonce uint64) { + t.currentNonce.Set(float64(nonce)) +} + +// TxConfirmed records lots of information about the confirmed transaction +func (t *TxMetrics) TxConfirmed(receipt *types.Receipt) { + t.lastConfirmTime.Set(float64(time.Now().Unix())) + t.txConfirmed.WithLabelValues(receiptStatusString(receipt)).Inc() t.TxL1GasFee.Set(float64(receipt.EffectiveGasPrice.Uint64() * receipt.GasUsed / params.GWei)) } + +func (t *TxMetrics) TxPublished(err error) { + if err != nil { + t.txPublishError.WithLabelValues(err.Error()).Inc() + } else { + t.txPublishError.WithLabelValues("nil").Inc() // TODO: DO we want this? + t.txPublished.Inc() + t.lastPublishTime.Set(float64(time.Now().Unix())) + } +} + +func (t *TxMetrics) RPCError() { + t.rpcError.Inc() +} diff --git a/op-service/txmgr/txmgr.go b/op-service/txmgr/txmgr.go index 96cd4f73c4d75..169b5d3b0e49e 100644 --- a/op-service/txmgr/txmgr.go +++ b/op-service/txmgr/txmgr.go @@ -152,6 +152,7 @@ func (m *SimpleTxManager) Send(ctx context.Context, candidate TxCandidate) (*typ func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (*types.Transaction, error) { gasTipCap, basefee, err := m.suggestGasPriceCaps(ctx) if err != nil { + m.metr.RPCError() return nil, fmt.Errorf("failed to get gas price info: %w", err) } gasFeeCap := calcGasFeeCap(basefee, gasTipCap) @@ -161,8 +162,10 @@ func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (* defer cancel() nonce, err := m.backend.NonceAt(childCtx, candidate.From, nil) if err != nil { + m.metr.RPCError() return nil, fmt.Errorf("failed to get nonce: %w", err) } + m.metr.RecordNonce(nonce) rawTx := &types.DynamicFeeTx{ ChainID: m.chainID, @@ -241,6 +244,8 @@ func (m *SimpleTxManager) send(ctx context.Context, tx *types.Transaction) (*typ return nil, ctx.Err() case receipt := <-receiptChan: + m.metr.RecordL1GasFee(receipt) + m.metr.TxConfirmed() return receipt, nil } } @@ -257,6 +262,7 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra defer cancel() err := m.backend.SendTransaction(cCtx, tx) sendState.ProcessSendError(err) + m.metr.TxPublished(err) // Properly log & exit if there is an error if err != nil { @@ -264,6 +270,7 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra case errStringMatch(err, core.ErrNonceTooLow): log.Warn("nonce too low", "err", err) case errStringMatch(err, context.Canceled): + m.metr.RPCError() log.Warn("transaction send cancelled", "err", err) case errStringMatch(err, txpool.ErrAlreadyKnown): log.Warn("resubmitted already known transaction", "err", err) @@ -272,6 +279,7 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra case errStringMatch(err, txpool.ErrUnderpriced): log.Warn("transaction is underpriced", "err", err) default: + m.metr.RPCError() log.Error("unable to publish transaction", "err", err) } return @@ -318,9 +326,11 @@ func (m *SimpleTxManager) queryReceipt(ctx context.Context, txHash common.Hash, m.l.Trace("Transaction not yet mined", "hash", txHash) return nil } else if err != nil { + m.metr.RPCError() m.l.Info("Receipt retrieval failed", "hash", txHash, "err", err) return nil } else if receipt == nil { + m.metr.RPCError() m.l.Warn("Receipt and error are both nil", "hash", txHash) return nil } @@ -404,6 +414,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b defer cancel() tip, err := m.backend.SuggestGasTipCap(cCtx) if err != nil { + m.metr.RPCError() return nil, nil, fmt.Errorf("failed to fetch the suggested gas tip cap: %w", err) } else if tip == nil { return nil, nil, errors.New("the suggested tip was nil") @@ -412,6 +423,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b defer cancel() head, err := m.backend.HeaderByNumber(cCtx, nil) if err != nil { + m.metr.RPCError() return nil, nil, fmt.Errorf("failed to fetch the suggested basefee: %w", err) } else if head.BaseFee == nil { return nil, nil, errors.New("txmgr does not support pre-london blocks that do not have a basefee") diff --git a/op-service/txmgr/txmgr_test.go b/op-service/txmgr/txmgr_test.go index d913304899d9d..d1794414e41fc 100644 --- a/op-service/txmgr/txmgr_test.go +++ b/op-service/txmgr/txmgr_test.go @@ -691,6 +691,7 @@ func TestWaitMinedReturnsReceiptAfterFailure(t *testing.T) { name: "TEST", backend: &borkedBackend, l: testlog.Logger(t, log.LvlCrit), + metr: &metrics.NoopTxMetrics{}, } // Don't mine the tx with the default backend. The failingBackend will @@ -726,6 +727,7 @@ func doGasPriceIncrease(t *testing.T, txTipCap, txFeeCap, newTip, newBaseFee int name: "TEST", backend: &borkedBackend, l: testlog.Logger(t, log.LvlCrit), + metr: &metrics.NoopTxMetrics{}, } tx := types.NewTx(&types.DynamicFeeTx{ @@ -829,6 +831,7 @@ func TestIncreaseGasPriceNotExponential(t *testing.T) { name: "TEST", backend: &borkedBackend, l: testlog.Logger(t, log.LvlCrit), + metr: &metrics.NoopTxMetrics{}, } tx := types.NewTx(&types.DynamicFeeTx{ GasTipCap: big.NewInt(10),