diff --git a/.changeset/eighty-squids-impress.md b/.changeset/eighty-squids-impress.md new file mode 100644 index 0000000000000..dafb7150503d2 --- /dev/null +++ b/.changeset/eighty-squids-impress.md @@ -0,0 +1,5 @@ +--- +'@eth-optimism/teleportr': patch +--- + +Add metrics for balances diff --git a/go/teleportr/drivers/disburser/driver.go b/go/teleportr/drivers/disburser/driver.go index 117e86147124d..f938df6ad2ac8 100644 --- a/go/teleportr/drivers/disburser/driver.go +++ b/go/teleportr/drivers/disburser/driver.go @@ -139,6 +139,9 @@ func (d *Driver) ClearPendingTx( func (d *Driver) GetBatchBlockRange( ctx context.Context) (*big.Int, *big.Int, error) { + // Update balance metrics on each iteration. + d.updateBalanceMetrics(ctx) + // Clear the current deposit IDs from any prior iteration. d.currentDepositIDs = nil @@ -317,7 +320,7 @@ func (d *Driver) SendTransaction( txHash := tx.Hash() startID := d.currentDepositIDs[0] - endID := d.currentDepositIDs[len(d.currentDepositIDs) - 1] + 1 + endID := d.currentDepositIDs[len(d.currentDepositIDs)-1] + 1 // Record the pending transaction hash so that we can recover if we crash // after publishing. @@ -430,10 +433,10 @@ func (d *Driver) processPendingTxs(ctx context.Context) error { "blockTimestamp", blockTimestamp) } - d.metrics.SuccessfulDisbursements.Set(float64(successfulDisbursements)) - d.metrics.FailedDisbursements.Set(float64(failedDisbursements)) + d.metrics.SuccessfulDisbursements.Add(float64(successfulDisbursements)) + d.metrics.FailedDisbursements.Add(float64(failedDisbursements)) d.metrics.FailedDatabaseMethods.With(DBMethodUpsertDisbursement). - Set(float64(failedUpserts)) + Inc() // We have completed our post-processing once all of the disbursements are // written without failures. @@ -620,7 +623,7 @@ func (d *Driver) logDatabaseContractMismatch( log.Warn("Recorded disbursements behind contract", "last_disbursement_id", *lastDisbursementID, "contract_next_id", contractNextID) - d.metrics.DepositIDMismatch.Set(1.0) + d.metrics.DepositIDMismatch.Inc() // The last recorded disbursement is ahead of what the contract believes. // This should NEVER happen unless the sequencer blows up and loses @@ -643,7 +646,7 @@ func (d *Driver) logDatabaseContractMismatch( log.Warn("Recorded disbursements behind contract", "last_disbursement_id", nil, "contract_next_id", contractNextID) - d.metrics.DepositIDMismatch.Set(1.0) + d.metrics.DepositIDMismatch.Inc() // Database and contract indicate we have not done a disbursement. default: @@ -654,10 +657,9 @@ func (d *Driver) logDatabaseContractMismatch( func (d *Driver) upsertDeposits(deposits []db.Deposit, end uint64) error { err := d.cfg.Database.UpsertDeposits(deposits, end) if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodUpsertDeposits).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodUpsertDeposits).Inc() return err } - d.metrics.FailedDatabaseMethods.With(DBMethodUpsertDeposits).Set(0.0) return nil } @@ -666,59 +668,70 @@ func (d *Driver) confirmedDeposits(blockNumber uint64) ([]db.Deposit, error) { blockNumber, d.cfg.NumConfirmations, ) if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodConfirmedDeposits).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodConfirmedDeposits).Inc() return nil, err } - d.metrics.FailedDatabaseMethods.With(DBMethodConfirmedDeposits).Set(0.0) return confirmedDeposits, nil } func (d *Driver) lastProcessedBlock() (*uint64, error) { lastProcessedBlock, err := d.cfg.Database.LastProcessedBlock() if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodLastProcessedBlock).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodLastProcessedBlock).Inc() return nil, err } - d.metrics.FailedDatabaseMethods.With(DBMethodLastProcessedBlock).Set(0.0) return lastProcessedBlock, nil } func (d *Driver) upsertPendingTx(pendingTx db.PendingTx) error { err := d.cfg.Database.UpsertPendingTx(pendingTx) if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodUpsertPendingTx).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodUpsertPendingTx).Inc() return err } - d.metrics.FailedDatabaseMethods.With(DBMethodUpsertPendingTx).Set(0.0) return nil } func (d *Driver) listPendingTxs() ([]db.PendingTx, error) { pendingTxs, err := d.cfg.Database.ListPendingTxs() if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodListPendingTxs).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodListPendingTxs).Inc() return nil, err } - d.metrics.FailedDatabaseMethods.With(DBMethodListPendingTxs).Set(0.0) return pendingTxs, nil } func (d *Driver) latestDisbursementID() (*uint64, error) { lastDisbursementID, err := d.cfg.Database.LatestDisbursementID() if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodLatestDisbursementID).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodLatestDisbursementID).Inc() return nil, err } - d.metrics.FailedDatabaseMethods.With(DBMethodLatestDisbursementID).Set(0.0) return lastDisbursementID, nil } func (d *Driver) deletePendingTx(startID, endID uint64) error { err := d.cfg.Database.DeletePendingTx(startID, endID) if err != nil { - d.metrics.FailedDatabaseMethods.With(DBMethodDeletePendingTx).Set(1.0) + d.metrics.FailedDatabaseMethods.With(DBMethodDeletePendingTx).Inc() return err } - d.metrics.FailedDatabaseMethods.With(DBMethodDeletePendingTx).Set(0.0) return nil } + +func (d *Driver) updateBalanceMetrics(ctx context.Context) { + disburserBal, err := d.cfg.L2Client.BalanceAt(ctx, d.walletAddr, nil) + if err != nil { + log.Error("Error getting disburser wallet balance", "err", err) + disburserBal = big.NewInt(0) + } + + depositBal, err := d.cfg.L1Client.BalanceAt(ctx, d.cfg.DepositAddr, nil) + if err != nil { + log.Error("Error getting deposit contract balance", "err", err) + depositBal = big.NewInt(0) + } + + d.metrics.DisburserBalance.Set(float64(disburserBal.Uint64())) + d.metrics.DepositContractBalance.Set(float64(depositBal.Uint64())) +} diff --git a/go/teleportr/drivers/disburser/metrics.go b/go/teleportr/drivers/disburser/metrics.go index 328c3969b3e87..f109c899a64ea 100644 --- a/go/teleportr/drivers/disburser/metrics.go +++ b/go/teleportr/drivers/disburser/metrics.go @@ -41,7 +41,7 @@ type Metrics struct { // FailedDatabaseMethods tracks the number of database failures for each // known database method. - FailedDatabaseMethods *prometheus.GaugeVec + FailedDatabaseMethods *prometheus.CounterVec // DepositIDMismatch tracks whether or not our database is in sync with the // disrburser contract. 1 means in sync, 0 means out of sync. @@ -53,11 +53,11 @@ type Metrics struct { // SuccessfulDisbursements tracks the number of disbursements that emit a // success event from a given tx. - SuccessfulDisbursements prometheus.Gauge + SuccessfulDisbursements prometheus.Counter // FailedDisbursements tracks the number of disbursements that emit a failed // event from a given tx. - FailedDisbursements prometheus.Gauge + FailedDisbursements prometheus.Counter // PostgresLastDisbursedID tracks the latest disbursement id in postgres. PostgresLastDisbursedID prometheus.Gauge @@ -65,6 +65,12 @@ type Metrics struct { // ContractNextDisbursementID tracks the next disbursement id expected by // the disburser contract. ContractNextDisbursementID prometheus.Gauge + + // DisburserBalance tracks Teleportr's disburser account balance. + DisburserBalance prometheus.Gauge + + // DepositContractBalance tracks Teleportr's deposit contract balance. + DepositContractBalance prometheus.Gauge } // NewMetrics initializes a new, extended metrics object. @@ -72,7 +78,7 @@ func NewMetrics(subsystem string) *Metrics { base := metrics.NewBase(subsystem, "") return &Metrics{ Base: base, - FailedDatabaseMethods: promauto.NewGaugeVec(prometheus.GaugeOpts{ + FailedDatabaseMethods: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "failed_database_operations", Help: "Tracks the number of database failures", Subsystem: base.SubsystemName(), @@ -111,5 +117,15 @@ func NewMetrics(subsystem string) *Metrics { Help: "Next disbursement id expected by the disburser contract", Subsystem: base.SubsystemName(), }), + DisburserBalance: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "disburser_balance", + Help: "Balance in Wei of Teleportr's disburser wallet", + Subsystem: base.SubsystemName(), + }), + DepositContractBalance: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "deposit_contract_balance", + Help: "Balance in Wei of Teleportr's deposit contract", + Subsystem: base.SubsystemName(), + }), } }