fix: panics not being catched

go/apps/api/routes/register.go

@@ -58,13 +58,15 @@ func Register(srv *zen.Server, svc *Services) {
 	withMetrics := zen.WithMetrics(svc.ClickHouse)
 
 	withLogging := zen.WithLogging(svc.Logger)
+	withPanicRecovery := zen.WithPanicRecovery(svc.Logger)
 	withErrorHandling := zen.WithErrorHandling(svc.Logger)
 	withValidation := zen.WithValidation(svc.Validator)
 
 	defaultMiddlewares := []zen.Middleware{
 		withTracing,
 		withMetrics,
 		withLogging,
+		withPanicRecovery,
 		withErrorHandling,
 		withValidation,
 	}
@@ -78,6 +80,7 @@ func Register(srv *zen.Server, svc *Services) {
 		chproxyMiddlewares := []zen.Middleware{
 			withMetrics,
 			withLogging,
+			withPanicRecovery,
 			withErrorHandling,
 		}
 
@@ -507,6 +510,7 @@ func Register(srv *zen.Server, svc *Services) {
 		withTracing,
 		withMetrics,
 		withLogging,
+		withPanicRecovery,
 		withErrorHandling,
 	}, &reference.Handler{
 		Logger: svc.Logger,
@@ -515,6 +519,7 @@ func Register(srv *zen.Server, svc *Services) {
 		withTracing,
 		withMetrics,
 		withLogging,
+		withPanicRecovery,
 		withErrorHandling,
 	}, &openapi.Handler{
 		Logger: svc.Logger,

go/internal/services/keys/get.go

@@ -90,7 +90,7 @@ func (s *service) Get(ctx context.Context, sess *zen.Session, rawKey string) (*K
 		return &KeyVerifier{
 			Status:  StatusNotFound,
 			message: "key does not exist",
-		}, nil, nil
+		}, emptyLog, nil
 	}
 
 	// ForWorkspace set but that doesn't exist

go/pkg/prometheus/metrics/panic.go

@@ -0,0 +1,20 @@
+/*
+Package metrics provides Prometheus metric collectors for monitoring application performance.
+
+This file contains a metric for tracking panics across http handlers.
+*/
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	PanicsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "unkey",
+		Subsystem: "handler",
+		Name:      "panics_total",
+		Help:      "Counter to track panics across http handlers",
+	}, []string{"caller", "path"})
+)

go/pkg/zen/middleware_panic_recovery.go

@@ -0,0 +1,62 @@
+package zen
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"runtime/debug"
+
+	"github.com/unkeyed/unkey/go/apps/api/openapi"
+	"github.com/unkeyed/unkey/go/pkg/codes"
+	"github.com/unkeyed/unkey/go/pkg/fault"
+	"github.com/unkeyed/unkey/go/pkg/otel/logging"
+	"github.com/unkeyed/unkey/go/pkg/prometheus/metrics"
+)
+
+// WithPanicRecovery returns middleware that recovers from panics and converts them
+// into appropriate HTTP error responses.
+func WithPanicRecovery(logger logging.Logger) Middleware {
+	return func(next HandleFunc) HandleFunc {
+		return func(ctx context.Context, s *Session) (err error) {
+			defer func() {
+				if r := recover(); r != nil {
+					// Get stack trace
+					stack := debug.Stack()
+
+					// Log the panic with stack trace
+					logger.Error("panic recovered",
+						"panic", fmt.Sprintf("%v", r),
+						"requestId", s.RequestID(),
+						"method", s.r.Method,
+						"path", s.r.URL.Path,
+						"stack", string(stack),
+					)
+
+					metrics.PanicsTotal.WithLabelValues("", s.r.URL.Path).Inc()
+
+					// Convert panic to an error
+					panicErr := fault.New("Internal Server Error",
+						fault.Code(codes.App.Internal.UnexpectedError.URN()),
+						fault.Internal(fmt.Sprintf("panic: %v", r)),
+						fault.Public("An unexpected error occurred while processing your request."),
+					)
+
+					// Return internal server error
+					err = s.JSON(http.StatusInternalServerError, openapi.InternalServerErrorResponse{
+						Meta: openapi.Meta{
+							RequestId: s.RequestID(),
+						},
+						Error: openapi.BaseError{
+							Title:  "Internal Server Error",
+							Type:   codes.App.Internal.UnexpectedError.DocsURL(),
+							Detail: fault.UserFacingMessage(panicErr),
+							Status: http.StatusInternalServerError,
+						},
+					})
+				}
+			}()
+
+			return next(ctx, s)
+		}
+	}
+}