feat(backend): Prometheus /metrics endpoint + request middleware

packages/backend/package.json

@@ -24,6 +24,7 @@
         "express": "^5.2.1",
         "express-rate-limit": "^8.5.1",
         "express-session": "^1.19.0",
+        "prom-client": "^15.1.3",
         "session-file-store": "^1.5.0"
     },
     "devDependencies": {
@@ -38,8 +39,8 @@
         "jest": "^30.4.2",
         "supertest": "^7.0.0",
         "ts-jest": "^29.4.9",
-        "tsx": "^4.21.0",
         "tsup": "^8.5.1",
+        "tsx": "^4.21.0",
         "typescript": "^6.0.3"
     }
 }

packages/backend/src/middleware/index.ts

@@ -5,6 +5,7 @@ import { existsSync } from 'node:fs'
 import path from 'path'
 import { setupSessionMiddleware } from './session'
 import { requestLogger } from './requestLogger'
+import { metricsMiddleware } from './metrics'
 import { getFrontendOrigins } from '../utils/frontendOrigin'
 
 export function setupMiddleware(app: Express): void {
@@ -59,6 +60,7 @@ export function setupMiddleware(app: Express): void {
     )
 
     app.use(requestLogger)
+    app.use(metricsMiddleware)
     app.use(express.json())
     app.use(express.urlencoded({ extended: true }))
     app.use(cookieParser())

packages/backend/src/middleware/metrics.ts

@@ -0,0 +1,49 @@
+import type { Request, Response, NextFunction } from 'express'
+import {
+    httpRequestsTotal,
+    httpRequestDurationSeconds,
+    httpServerErrorsTotal,
+} from '../utils/prometheus'
+
+/**
+ * Express middleware that records request count and duration to the
+ * Prometheus registry. Uses Express's resolved `req.route.path` template
+ * (e.g. `/api/guilds/:guildId`) as the label so cardinality stays bounded;
+ * if no route is matched (404, static files), the literal string `unmatched`
+ * is used instead of the raw URL.
+ */
+export function metricsMiddleware(
+    req: Request,
+    res: Response,
+    next: NextFunction,
+): void {
+    const start = process.hrtime.bigint()
+
+    res.on('finish', () => {
+        const elapsedSeconds =
+            Number(process.hrtime.bigint() - start) / 1_000_000_000
+        const method = req.method
+        const route = resolveRoute(req)
+        const status = String(res.statusCode)
+        httpRequestsTotal.inc({ method, route, status })
+        httpRequestDurationSeconds.observe(
+            { method, route, status },
+            elapsedSeconds,
+        )
+        if (res.statusCode >= 500) {
+            httpServerErrorsTotal.inc({ method, route })
+        }
+    })
+
+    next()
+}
+
+function resolveRoute(req: Request): string {
+    const route = req.route as { path?: string } | undefined
+    if (route?.path && typeof route.path === 'string') {
+        return route.path
+    }
+    // baseUrl is set when middleware is mounted under a prefix (e.g. /api).
+    if (req.baseUrl) return `${req.baseUrl}/unmatched`
+    return 'unmatched'
+}

packages/backend/src/routes/index.ts

@@ -26,6 +26,7 @@ import { requireAdmin } from '../middleware/requireAdmin'
 import { requireGuildModuleAccess } from '../middleware/guildAccess'
 import { errorHandler } from '../middleware/errorHandler'
 import { setupHealthRoutes } from './health'
+import { setupMetricsRoute } from './metrics'
 import { setupStatsRoutes } from './stats'
 import { setupInviteRoute } from './invite'
 
@@ -84,6 +85,7 @@ const routeSetups = [
 export function setupRoutes(app: Express): void {
     setupInviteRoute(app)
     setupHealthRoutes(app)
+    setupMetricsRoute(app)
     setupStatsRoutes(app)
     setupInternalNotifyRoutes(app)
     setupWebhookPublicRoutes(app)

packages/backend/src/routes/metrics.ts

@@ -0,0 +1,30 @@
+import type { Express, Request, Response } from 'express'
+import { errorLog } from '@lucky/shared/utils'
+import { metricsContentType, renderMetrics } from '../utils/prometheus'
+
+/**
+ * Mount the /metrics endpoint that Prometheus scrapes.
+ *
+ * Intentionally unauthenticated and outside the /api rate limiter so
+ * the scraper can pull at its configured cadence. Restrict access at the
+ * network layer (Docker network, ingress allowlist).
+ *
+ * Set METRICS_DISABLED=true to skip the mount entirely.
+ */
+export function setupMetricsRoute(app: Express): void {
+    if (process.env.METRICS_DISABLED === 'true') return
+    app.get('/metrics', (_req: Request, res: Response) => {
+        renderMetrics()
+            .then((body) => {
+                res.set('content-type', metricsContentType)
+                res.status(200).send(body)
+            })
+            .catch((error: unknown) => {
+                errorLog({
+                    message: 'metrics route: failed to render',
+                    error,
+                })
+                res.status(500).send('internal error')
+            })
+    })
+}

packages/backend/src/utils/prometheus.ts

@@ -0,0 +1,57 @@
+import {
+    Registry,
+    collectDefaultMetrics,
+    Counter,
+    Histogram,
+} from 'prom-client'
+
+/**
+ * Shared Prometheus registry for the backend. Mounted at GET /metrics.
+ */
+export const registry = new Registry()
+
+registry.setDefaultLabels({ service: 'lucky-backend' })
+collectDefaultMetrics({ register: registry })
+
+/**
+ * Counter: total HTTP requests served, labelled by method, route template
+ * (NOT raw path — we use `req.route.path` so cardinality stays bounded),
+ * and status code.
+ */
+export const httpRequestsTotal = new Counter<'method' | 'route' | 'status'>({
+    name: 'lucky_backend_http_requests_total',
+    help: 'Count of HTTP requests handled by the backend, labelled by method, route template, and status code.',
+    labelNames: ['method', 'route', 'status'],
+    registers: [registry],
+})
+
+/**
+ * Histogram: HTTP request duration in seconds. Buckets tuned for typical
+ * API workloads (sub-50ms to ~5s p99).
+ */
+export const httpRequestDurationSeconds = new Histogram<
+    'method' | 'route' | 'status'
+>({
+    name: 'lucky_backend_http_request_duration_seconds',
+    help: 'HTTP request duration in seconds, observed at response finish.',
+    labelNames: ['method', 'route', 'status'],
+    buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+    registers: [registry],
+})
+
+/**
+ * Counter: HTTP errors (status >= 500) classified by method and route.
+ * Separate from the total counter for easier alerting.
+ */
+export const httpServerErrorsTotal = new Counter<'method' | 'route'>({
+    name: 'lucky_backend_http_server_errors_total',
+    help: 'Count of HTTP 5xx responses, labelled by method and route template.',
+    labelNames: ['method', 'route'],
+    registers: [registry],
+})
+
+export async function renderMetrics(): Promise<string> {
+    return registry.metrics()
+}
+
+export const metricsContentType: string = registry.contentType

packages/backend/tests/unit/middleware/metrics.test.ts

@@ -0,0 +1,118 @@
+import { describe, test, expect, beforeEach, jest } from '@jest/globals'
+import { EventEmitter } from 'events'
+import { metricsMiddleware } from '../../../src/middleware/metrics'
+import {
+    registry,
+    httpRequestsTotal,
+    httpRequestDurationSeconds,
+    httpServerErrorsTotal,
+} from '../../../src/utils/prometheus'
+
+function createReq(
+    overrides: Partial<{
+        method: string
+        route: { path: string } | undefined
+        baseUrl: string
+    }> = {},
+): Parameters<typeof metricsMiddleware>[0] {
+    return {
+        method: 'GET',
+        route: { path: '/api/guilds/:guildId' },
+        baseUrl: '',
+        ...overrides,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } as any
+}
+
+function createRes(statusCode: number) {
+    const emitter = new EventEmitter()
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return Object.assign(emitter, { statusCode }) as any
+}
+
+describe('metricsMiddleware', () => {
+    beforeEach(() => {
+        // Reset counters between tests so call counts don't bleed.
+        httpRequestsTotal.reset()
+        httpRequestDurationSeconds.reset()
+        httpServerErrorsTotal.reset()
+    })
+
+    test('calls next immediately', () => {
+        const req = createReq()
+        const res = createRes(200)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        expect(next).toHaveBeenCalledTimes(1)
+    })
+
+    test('records count + duration on response finish with route template label', async () => {
+        const req = createReq()
+        const res = createRes(200)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        res.emit('finish')
+
+        const text = await registry.metrics()
+        expect(text).toMatch(
+            /lucky_backend_http_requests_total\{[^}]*method="GET"[^}]*route="\/api\/guilds\/:guildId"[^}]*status="200"[^}]*\}\s+1/,
+        )
+        expect(text).toContain('lucky_backend_http_request_duration_seconds_bucket')
+        expect(text).toMatch(
+            /lucky_backend_http_request_duration_seconds_count\{[^}]*route="\/api\/guilds\/:guildId"[^}]*\}\s+1/,
+        )
+    })
+
+    test('increments server-errors counter for 5xx responses', async () => {
+        const req = createReq({ method: 'POST' })
+        const res = createRes(503)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        res.emit('finish')
+
+        const text = await registry.metrics()
+        expect(text).toMatch(
+            /lucky_backend_http_server_errors_total\{[^}]*method="POST"[^}]*route="\/api\/guilds\/:guildId"[^}]*\}\s+1/,
+        )
+    })
+
+    test('does NOT increment server-errors counter for 4xx responses', async () => {
+        const req = createReq()
+        const res = createRes(404)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        res.emit('finish')
+
+        const text = await registry.metrics()
+        // No 5xx error line should appear at all for this test.
+        expect(text).not.toMatch(
+            /lucky_backend_http_server_errors_total\{[^}]*\}\s+[1-9]/,
+        )
+    })
+
+    test('uses "unmatched" label when no route is resolved', async () => {
+        const req = createReq({ route: undefined })
+        const res = createRes(404)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        res.emit('finish')
+
+        const text = await registry.metrics()
+        expect(text).toMatch(
+            /lucky_backend_http_requests_total\{[^}]*route="unmatched"[^}]*\}\s+1/,
+        )
+    })
+
+    test('uses "<baseUrl>/unmatched" when middleware is sub-mounted', async () => {
+        const req = createReq({ route: undefined, baseUrl: '/api' })
+        const res = createRes(404)
+        const next = jest.fn()
+        metricsMiddleware(req, res, next)
+        res.emit('finish')
+
+        const text = await registry.metrics()
+        expect(text).toMatch(
+            /lucky_backend_http_requests_total\{[^}]*route="\/api\/unmatched"[^}]*\}\s+1/,
+        )
+    })
+})