Jg/extended metrics

op-service/txmgr/metrics/noop.go

@@ -4,4 +4,7 @@ import "github.com/ethereum/go-ethereum/core/types"
 
 type NoopTxMetrics struct{}
 
-func (*NoopTxMetrics) RecordL1GasFee(*types.Receipt) {}
+func (*NoopTxMetrics) RecordNonce(uint64)         {}
+func (*NoopTxMetrics) TxConfirmed(*types.Receipt) {}
+func (*NoopTxMetrics) TxPublished(error)          {}
+func (*NoopTxMetrics) RPCError()                  {}

op-service/txmgr/metrics/tx_metrics.go

@@ -1,6 +1,8 @@
 package metrics
 
 import (
+	"time"
+
 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
@@ -9,11 +11,32 @@ import (
 )
 
 type TxMetricer interface {
-	RecordL1GasFee(receipt *types.Receipt)
+	RecordNonce(nonce uint64)
+	TxConfirmed(*types.Receipt)
+	TxPublished(err error)
+	RPCError()
 }
 
 type TxMetrics struct {
-	TxL1GasFee prometheus.Gauge
+	TxL1GasFee      prometheus.Gauge
+	currentNonce    prometheus.Gauge
+	txConfirmed     *prometheus.CounterVec
+	txPublished     prometheus.Counter
+	txPublishError  *prometheus.CounterVec
+	lastPublishTime prometheus.Gauge
+	lastConfirmTime prometheus.Gauge
+	rpcError        prometheus.Counter
+}
+
+func receiptStatusString(receipt *types.Receipt) string {
+	switch receipt.Status {
+	case types.ReceiptStatusSuccessful:
+		return "success"
+	case types.ReceiptStatusFailed:
+		return "failed"
+	default:
+		return "unkown_status"
+	}
 }
 
 var _ TxMetricer = (*TxMetrics)(nil)
@@ -26,9 +49,72 @@ func MakeTxMetrics(ns string, factory metrics.Factory) TxMetrics {
 			Help:      "L1 gas fee for transactions in GWEI",
 			Subsystem: "txmgr",
 		}),
+		currentNonce: factory.NewGauge(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "current_nonce",
+			Help:      "",
+			Subsystem: "txmgr",
+		}),
+		txConfirmed: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "tx_confirmed_count",
+			Help:      "",
+			Subsystem: "txmgr",
+		}, []string{"status"}),
+		txPublished: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "tx_published_count",
+			Help:      "",
+			Subsystem: "txmgr",
+		}),
+		txPublishError: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "tx_publish_error_count",
+			Help:      "",
+			Subsystem: "txmgr",
+		}, []string{"error"}),
+		lastPublishTime: factory.NewGauge(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "last_publish_time_unix_secs",
+			Help:      "",
+			Subsystem: "txmgr",
+		}),
+		lastConfirmTime: factory.NewGauge(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "last_confirm_time_unix_secs",
+			Help:      "",
+			Subsystem: "txmgr",
+		}),
+		rpcError: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "rpc_error_count",
+			Help:      "",
+			Subsystem: "txmgr",
+		}),
 	}
 }
 
-func (t *TxMetrics) RecordL1GasFee(receipt *types.Receipt) {
+func (t *TxMetrics) RecordNonce(nonce uint64) {
+	t.currentNonce.Set(float64(nonce))
+}
+
+// TxConfirmed records lots of information about the confirmed transaction
+func (t *TxMetrics) TxConfirmed(receipt *types.Receipt) {
+	t.lastConfirmTime.Set(float64(time.Now().Unix()))
+	t.txConfirmed.WithLabelValues(receiptStatusString(receipt)).Inc()
 	t.TxL1GasFee.Set(float64(receipt.EffectiveGasPrice.Uint64() * receipt.GasUsed / params.GWei))
 }
+
+func (t *TxMetrics) TxPublished(err error) {
+	if err != nil {
+		t.txPublishError.WithLabelValues(err.Error()).Inc()
+	} else {
+		t.txPublishError.WithLabelValues("nil").Inc() // TODO: DO we want this?
+		t.txPublished.Inc()
+		t.lastPublishTime.Set(float64(time.Now().Unix()))
+	}
+}
+
+func (t *TxMetrics) RPCError() {
+	t.rpcError.Inc()
+}

op-service/txmgr/txmgr.go

@@ -152,6 +152,7 @@ func (m *SimpleTxManager) Send(ctx context.Context, candidate TxCandidate) (*typ
 func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (*types.Transaction, error) {
 	gasTipCap, basefee, err := m.suggestGasPriceCaps(ctx)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, fmt.Errorf("failed to get gas price info: %w", err)
 	}
 	gasFeeCap := calcGasFeeCap(basefee, gasTipCap)
@@ -161,8 +162,10 @@ func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (*
 	defer cancel()
 	nonce, err := m.backend.NonceAt(childCtx, candidate.From, nil)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, fmt.Errorf("failed to get nonce: %w", err)
 	}
+	m.metr.RecordNonce(nonce)
 
 	rawTx := &types.DynamicFeeTx{
 		ChainID:   m.chainID,
@@ -241,6 +244,8 @@ func (m *SimpleTxManager) send(ctx context.Context, tx *types.Transaction) (*typ
 			return nil, ctx.Err()
 
 		case receipt := <-receiptChan:
+			m.metr.RecordL1GasFee(receipt)
+			m.metr.TxConfirmed()
 			return receipt, nil
 		}
 	}
@@ -257,13 +262,15 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra
 	defer cancel()
 	err := m.backend.SendTransaction(cCtx, tx)
 	sendState.ProcessSendError(err)
+	m.metr.TxPublished(err)
 
 	// Properly log & exit if there is an error
 	if err != nil {
 		switch {
 		case errStringMatch(err, core.ErrNonceTooLow):
 			log.Warn("nonce too low", "err", err)
 		case errStringMatch(err, context.Canceled):
+			m.metr.RPCError()
 			log.Warn("transaction send cancelled", "err", err)
 		case errStringMatch(err, txpool.ErrAlreadyKnown):
 			log.Warn("resubmitted already known transaction", "err", err)
@@ -272,6 +279,7 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra
 		case errStringMatch(err, txpool.ErrUnderpriced):
 			log.Warn("transaction is underpriced", "err", err)
 		default:
+			m.metr.RPCError()
 			log.Error("unable to publish transaction", "err", err)
 		}
 		return
@@ -318,9 +326,11 @@ func (m *SimpleTxManager) queryReceipt(ctx context.Context, txHash common.Hash,
 		m.l.Trace("Transaction not yet mined", "hash", txHash)
 		return nil
 	} else if err != nil {
+		m.metr.RPCError()
 		m.l.Info("Receipt retrieval failed", "hash", txHash, "err", err)
 		return nil
 	} else if receipt == nil {
+		m.metr.RPCError()
 		m.l.Warn("Receipt and error are both nil", "hash", txHash)
 		return nil
 	}
@@ -404,6 +414,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b
 	defer cancel()
 	tip, err := m.backend.SuggestGasTipCap(cCtx)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, nil, fmt.Errorf("failed to fetch the suggested gas tip cap: %w", err)
 	} else if tip == nil {
 		return nil, nil, errors.New("the suggested tip was nil")
@@ -412,6 +423,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b
 	defer cancel()
 	head, err := m.backend.HeaderByNumber(cCtx, nil)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, nil, fmt.Errorf("failed to fetch the suggested basefee: %w", err)
 	} else if head.BaseFee == nil {
 		return nil, nil, errors.New("txmgr does not support pre-london blocks that do not have a basefee")

op-service/txmgr/txmgr_test.go

@@ -691,6 +691,7 @@ func TestWaitMinedReturnsReceiptAfterFailure(t *testing.T) {
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 
 	// Don't mine the tx with the default backend. The failingBackend will
@@ -726,6 +727,7 @@ func doGasPriceIncrease(t *testing.T, txTipCap, txFeeCap, newTip, newBaseFee int
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 
 	tx := types.NewTx(&types.DynamicFeeTx{
@@ -829,6 +831,7 @@ func TestIncreaseGasPriceNotExponential(t *testing.T) {
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 	tx := types.NewTx(&types.DynamicFeeTx{
 		GasTipCap: big.NewInt(10),