chore: add metrics and logs to debug db replica usage

go/pkg/db/database.go

@@ -74,6 +74,9 @@ func New(config Config) (*database, error) {
 			db:   read,
 			mode: "ro",
 		}
+		config.Logger.Info("database configured with separate read replica")
+	} else {
+		config.Logger.Info("database configured without separate read replica, using primary for reads")
 	}
 
 	return &database{

go/pkg/db/replica.go

@@ -5,8 +5,10 @@ package db
 import (
 	"context"
 	"database/sql"
+	"time"
 
 	"github.com/unkeyed/unkey/go/pkg/otel/tracing"
+	"github.com/unkeyed/unkey/go/pkg/prometheus/metrics"
 	"go.opentelemetry.io/otel/attribute"
 )
 
@@ -29,7 +31,22 @@ func (r *Replica) ExecContext(ctx context.Context, query string, args ...interfa
 		attribute.String("mode", r.mode),
 		attribute.String("query", query),
 	)
-	return r.db.ExecContext(ctx, query, args...)
+
+	// Track metrics
+	start := time.Now()
+	result, err := r.db.ExecContext(ctx, query, args...)
+
+	// Record latency and operation count
+	duration := time.Since(start).Seconds()
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+
+	metrics.DatabaseOperationLatency.WithLabelValues(r.mode, "exec", status).Observe(duration)
+	metrics.DatabaseOperationTotal.WithLabelValues(r.mode, "exec", status).Inc()
+
+	return result, err
 }
 
 // PrepareContext prepares a SQL statement for later execution.
@@ -40,7 +57,22 @@ func (r *Replica) PrepareContext(ctx context.Context, query string) (*sql.Stmt,
 		attribute.String("mode", r.mode),
 		attribute.String("query", query),
 	)
-	return r.db.PrepareContext(ctx, query) // nolint:sqlclosecheck
+
+	// Track metrics
+	start := time.Now()
+	stmt, err := r.db.PrepareContext(ctx, query)
+
+	// Record latency and operation count
+	duration := time.Since(start).Seconds()
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+
+	metrics.DatabaseOperationLatency.WithLabelValues(r.mode, "prepare", status).Observe(duration)
+	metrics.DatabaseOperationTotal.WithLabelValues(r.mode, "prepare", status).Inc()
+
+	return stmt, err // nolint:sqlclosecheck
 }
 
 // QueryContext executes a SQL query that returns rows.
@@ -51,8 +83,22 @@ func (r *Replica) QueryContext(ctx context.Context, query string, args ...interf
 		attribute.String("mode", r.mode),
 		attribute.String("query", query),
 	)
-	return r.db.QueryContext(ctx, query, args...) // nolint:sqlclosecheck
 
+	// Track metrics
+	start := time.Now()
+	rows, err := r.db.QueryContext(ctx, query, args...)
+
+	// Record latency and operation count
+	duration := time.Since(start).Seconds()
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+
+	metrics.DatabaseOperationLatency.WithLabelValues(r.mode, "query", status).Observe(duration)
+	metrics.DatabaseOperationTotal.WithLabelValues(r.mode, "query", status).Inc()
+
+	return rows, err // nolint:sqlclosecheck
 }
 
 // QueryRowContext executes a SQL query that returns a single row.
@@ -63,7 +109,20 @@ func (r *Replica) QueryRowContext(ctx context.Context, query string, args ...int
 		attribute.String("mode", r.mode),
 		attribute.String("query", query),
 	)
-	return r.db.QueryRowContext(ctx, query, args...)
+
+	// Track metrics
+	start := time.Now()
+	row := r.db.QueryRowContext(ctx, query, args...)
+
+	// Record latency and operation count
+	duration := time.Since(start).Seconds()
+	// QueryRowContext doesn't return an error, but we can still track timing
+	status := "success"
+
+	metrics.DatabaseOperationLatency.WithLabelValues(r.mode, "query_row", status).Observe(duration)
+	metrics.DatabaseOperationTotal.WithLabelValues(r.mode, "query_row", status).Inc()
+
+	return row
 }
 
 // Begin starts a transaction and returns it.
@@ -73,5 +132,19 @@ func (r *Replica) Begin(ctx context.Context) (*sql.Tx, error) {
 	defer span.End()
 	span.SetAttributes(attribute.String("mode", r.mode))
 
-	return r.db.BeginTx(ctx, nil)
+	// Track metrics
+	start := time.Now()
+	tx, err := r.db.BeginTx(ctx, nil)
+
+	// Record latency and operation count
+	duration := time.Since(start).Seconds()
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+
+	metrics.DatabaseOperationLatency.WithLabelValues(r.mode, "begin", status).Observe(duration)
+	metrics.DatabaseOperationTotal.WithLabelValues(r.mode, "begin", status).Inc()
+
+	return tx, err
 }

go/pkg/prometheus/metrics/database.go

@@ -0,0 +1,51 @@
+/*
+Package metrics provides Prometheus metric collectors for monitoring application performance.
+
+This file contains database-related metrics for tracking database operation performance,
+usage patterns, and latency across primary and read-only replicas.
+*/
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// DatabaseOperationLatency tracks database operation latencies as a histogram,
+	// labeled by replica type (rw/ro), operation type, and success status.
+	// This collector uses predefined buckets optimized for typical database operation latencies.
+	//
+	// Example usage:
+	//   timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
+	//       metrics.DatabaseOperationLatency.WithLabelValues("rw", "exec", "success").Observe(v)
+	//   }))
+	//   defer timer.ObserveDuration()
+	DatabaseOperationLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem:   "database",
+			Name:        "operation_latency_seconds",
+			Help:        "Histogram of database operation latencies in seconds.",
+			Buckets:     latencyBuckets,
+			ConstLabels: constLabels,
+		},
+		[]string{"replica", "operation", "status"},
+	)
+
+	// DatabaseOperationTotal tracks the total number of database operations,
+	// labeled by replica type (rw/ro), operation type, and success status.
+	// Use this counter to monitor database traffic patterns and error rates.
+	//
+	// Example usage:
+	//   metrics.DatabaseOperationTotal.WithLabelValues("rw", "exec", "success").Inc()
+	//   metrics.DatabaseOperationTotal.WithLabelValues("ro", "query", "error").Inc()
+	DatabaseOperationTotal = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem:   "database",
+			Name:        "operations_total",
+			Help:        "Total number of database operations processed.",
+			ConstLabels: constLabels,
+		},
+		[]string{"replica", "operation", "status"},
+	)
+)