From 7d42e6b7aee1bb2f8861f55e11ccd7974cfd0f00 Mon Sep 17 00:00:00 2001
From: liuke <1262946533@qq.com>
Date: Fri, 16 Oct 2026 11:36:53 +0800
Subject: [PATCH 1/4] [Fix] metrics: record HTTP request metrics in tokenizer
 workers

The track-response middleware was registered from
`_setup_and_run_http_server`, which runs in the parent process. With
`--tokenizer-worker-num > 1` uvicorn spawns the HTTP workers and each of them
re-imports `http_server`, so the app they actually serve never had the
middleware. `sglang:http_requests_total`, `sglang:http_responses_total`,
`sglang:http_requests_active` and `sglang:routing_keys_active` were therefore
missing from `/metrics` whenever more than one tokenizer worker was used.

Register the middleware while the module is imported instead, so that every
worker picks it up. The metrics themselves are created in a separate step,
`init_http_request_metrics`, called from the lifespan once the process has
published its config: prometheus_client fixes its value backend when it is
first imported, and SGLang only sets `PROMETHEUS_MULTIPROC_DIR` in
`_set_envs_and_config`, which runs after `http_server` has been imported.
Creating them at import time would leave them in the in-memory registry that
the `/metrics` endpoint never reads, breaking single-tokenizer mode.

Because the middleware now runs inside each worker, also expose that worker's
in-flight request count as `sglang:http_worker_active_requests{worker_id}`.
---
 python/sglang/srt/entrypoints/http_server.py  |  17 ++-
 python/sglang/srt/utils/common.py             | 127 +++++++++++++-----
 .../test_http_request_metrics.py              |  82 +++++++++++
 3 files changed, 188 insertions(+), 38 deletions(-)
 create mode 100644 test/registered/unit/observability/test_http_request_metrics.py

diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index 7165a5ca8c3c..115ec2a4ad96 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -184,6 +184,7 @@
     build_server_info,
     delete_directory,
     get_bool_env_var,
+    init_http_request_metrics,
     is_mps,
     kill_process_tree,
     set_uvicorn_logging_configs,
@@ -292,16 +293,21 @@ async def lifespan(fast_api_app: FastAPI):
         server_args = fast_api_app.server_args
         warmup_thread_kwargs = fast_api_app.warmup_thread_kwargs
         thread_label = "Tokenizer"
+        tokenizer_worker_id = None
     else:
         # Initialize multi-tokenizer support for worker processes
         server_args = await init_multi_tokenizer()
         warmup_thread_kwargs = dict(server_args=server_args)
-        thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
+        tokenizer_worker_id = _global_state.tokenizer_manager.worker_id
+        thread_label = f"MultiTokenizer-{tokenizer_worker_id}"
 
     # Add prometheus middleware
     if get_observability().enable_metrics:
         add_prometheus_middleware(app)
         enable_func_timer()
+        init_http_request_metrics(
+            worker_id=None if tokenizer_worker_id is None else str(tokenizer_worker_id)
+        )
 
     # Init tracing
     if get_observability().enable_trace:
@@ -499,6 +505,12 @@ async def custom_handler(request: Request):
 
     app.add_middleware(RequestDecompressionMiddleware)
 
+# Registered while this module is imported, so that every tokenizer worker
+# process gets it. With --tokenizer-worker-num > 1 uvicorn spawns workers that
+# re-import this module; a middleware added afterwards in the parent process
+# never reaches the app those workers actually serve.
+add_prometheus_track_response_middleware(app)
+
 # Include routers
 from sglang.srt.entrypoints.v1_loads import router as v1_loads_router
 
@@ -2648,9 +2660,6 @@ def _setup_and_run_http_server(
     if tokenizer_manager is not None:
         tokenizer_manager._subprocess_watchdog = subprocess_watchdog
 
-    if get_observability().enable_metrics:
-        add_prometheus_track_response_middleware(app)
-
     # Pass additional arguments to the lifespan function.
     # They will be used for additional initialization setups.
     if get_serving().tokenizer_worker_num == 1:
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index a66496264a77..71c0e088543a 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -2848,41 +2848,87 @@ def dec(self, key: str):
                 self._gauge.dec()
 
 
-def add_prometheus_track_response_middleware(
-    app, extra_labels: Optional[Dict[str, str]] = None
-):
-    from prometheus_client import Counter, Gauge
+class _HttpRequestMetrics:
+    """The prometheus objects behind the HTTP request metrics of one process."""
 
-    extra_labels = extra_labels or {}
-    extra_label_names = list(extra_labels.keys())
+    def __init__(self, extra_labels: Dict[str, str], worker_id: Optional[str]):
+        from prometheus_client import Counter, Gauge
 
-    http_request_counter = Counter(
-        name="sglang:http_requests_total",
-        documentation="Total number of HTTP requests by endpoint and method",
-        labelnames=extra_label_names + ["endpoint", "method"],
-    )
+        extra_label_names = list(extra_labels.keys())
+        self.extra_labels = extra_labels
 
-    http_response_counter = Counter(
-        name="sglang:http_responses_total",
-        documentation="Total number of HTTP responses by endpoint and status code",
-        labelnames=extra_label_names + ["endpoint", "status_code", "method"],
-    )
+        self.request_counter = Counter(
+            name="sglang:http_requests_total",
+            documentation="Total number of HTTP requests by endpoint and method",
+            labelnames=extra_label_names + ["endpoint", "method"],
+        )
 
-    http_requests_active = Gauge(
-        name="sglang:http_requests_active",
-        documentation="Number of currently active HTTP requests",
-        labelnames=extra_label_names + ["endpoint", "method"],
-        multiprocess_mode="livesum",
-    )
+        self.response_counter = Counter(
+            name="sglang:http_responses_total",
+            documentation="Total number of HTTP responses by endpoint and status code",
+            labelnames=extra_label_names + ["endpoint", "status_code", "method"],
+        )
 
-    routing_keys_active = RefCountedGauge(
-        Gauge(
-            name="sglang:routing_keys_active",
-            documentation="Number of unique routing keys with active requests",
+        self.requests_active = Gauge(
+            name="sglang:http_requests_active",
+            documentation="Number of currently active HTTP requests",
+            labelnames=extra_label_names + ["endpoint", "method"],
             multiprocess_mode="livesum",
         )
-    )
 
+        self.routing_keys_active = RefCountedGauge(
+            Gauge(
+                name="sglang:routing_keys_active",
+                documentation="Number of unique routing keys with active requests",
+                multiprocess_mode="livesum",
+            )
+        )
+
+        self.worker_active = (
+            Gauge(
+                name="sglang:http_worker_active_requests",
+                documentation=(
+                    "Number of currently active HTTP requests handled by this "
+                    "tokenizer worker"
+                ),
+                labelnames=["worker_id"],
+                multiprocess_mode="livesum",
+            ).labels(worker_id=worker_id)
+            if worker_id is not None
+            else None
+        )
+
+
+_http_request_metrics: Optional[_HttpRequestMetrics] = None
+
+
+def init_http_request_metrics(
+    *, extra_labels: Optional[Dict[str, str]] = None, worker_id: Optional[str] = None
+) -> None:
+    """Create this process's HTTP request metrics.
+
+    Called once the process has published its config rather than at import:
+    importing prometheus_client fixes its value backend from
+    ``PROMETHEUS_MULTIPROC_DIR``, which SGLang only sets in
+    ``_set_envs_and_config`` -- after this module has already been imported.
+    Creating the metrics earlier would leave them in the in-memory registry,
+    where the ``/metrics`` endpoint (which reads the multiprocess directory)
+    never sees them.
+    """
+    global _http_request_metrics
+    _http_request_metrics = _HttpRequestMetrics(extra_labels or {}, worker_id)
+
+
+def add_prometheus_track_response_middleware(app):
+    """Track per-endpoint HTTP request metrics.
+
+    Registering the middleware and creating the metrics (see
+    ``init_http_request_metrics``) are deliberately separate, because the two
+    have to happen at different times: the middleware must be in place while
+    this module is imported so that every tokenizer worker process gets it,
+    while the metrics can only be created once the process has published its
+    config.
+    """
     # Fix: replace BaseHTTPMiddleware's call_next with a pure ASGI version
     # that passes `receive` through, so request.is_disconnected() keeps working.
     from sglang.srt.utils.http_middleware_patch import patch_app_http_middleware
@@ -2891,21 +2937,32 @@ def add_prometheus_track_response_middleware(
 
     @app.middleware("http")
     async def track_http_status_code(request, call_next):
+        metrics = _http_request_metrics
+        if metrics is None:  # metrics are not enabled in this process
+            return await call_next(request)
+
         # With recording all requests, we have the risk of high cardinality if requests have arbitrary unhandled paths.
         # But given that SGLang engines with metrics enabled are usually behind routers this looks safe.
         path, is_handled_path = _get_fastapi_request_path(request)
         method = request.method
         routing_key = request.headers.get("x-smg-routing-key")
-
-        http_request_counter.labels(**extra_labels, endpoint=path, method=method).inc()
-        http_requests_active.labels(**extra_labels, endpoint=path, method=method).inc()
+        extra_labels = metrics.extra_labels
+
+        metrics.request_counter.labels(
+            **extra_labels, endpoint=path, method=method
+        ).inc()
+        metrics.requests_active.labels(
+            **extra_labels, endpoint=path, method=method
+        ).inc()
+        if metrics.worker_active is not None:
+            metrics.worker_active.inc()
         if routing_key:
-            routing_keys_active.inc(routing_key)
+            metrics.routing_keys_active.inc(routing_key)
 
         try:
             response = await call_next(request)
 
-            http_response_counter.labels(
+            metrics.response_counter.labels(
                 **extra_labels,
                 endpoint=path,
                 method=method,
@@ -2914,11 +2971,13 @@ async def track_http_status_code(request, call_next):
 
             return response
         finally:
-            http_requests_active.labels(
+            metrics.requests_active.labels(
                 **extra_labels, endpoint=path, method=method
             ).dec()
+            if metrics.worker_active is not None:
+                metrics.worker_active.dec()
             if routing_key:
-                routing_keys_active.dec(routing_key)
+                metrics.routing_keys_active.dec(routing_key)
 
 
 # https://github.com/blueswen/fastapi-observability/blob/132a3c576f8b09e5311c68bd553215013bc75685/fastapi_app/utils.py#L98
diff --git a/test/registered/unit/observability/test_http_request_metrics.py b/test/registered/unit/observability/test_http_request_metrics.py
new file mode 100644
index 000000000000..1725e44b860c
--- /dev/null
+++ b/test/registered/unit/observability/test_http_request_metrics.py
@@ -0,0 +1,82 @@
+"""Regression tests for the HTTP request metrics middleware.
+
+With ``--tokenizer-worker-num > 1`` the app is served by worker processes that
+uvicorn spawns and that re-import ``http_server``. The track-response
+middleware therefore has to be part of the module-level app: registering it
+later from the parent process leaves every worker without it, and
+``sglang:http_requests_total`` and its siblings disappear from ``/metrics``.
+
+Creating the metrics is deliberately a separate step
+(``init_http_request_metrics``): prometheus_client fixes its value backend when
+it is first imported, while SGLang only sets ``PROMETHEUS_MULTIPROC_DIR`` after
+this module has been imported.
+"""
+
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=25, suite="base-a-test-cpu")
+
+import subprocess
+import sys
+
+from fastapi.testclient import TestClient
+from prometheus_client import REGISTRY
+
+import sglang.srt.entrypoints.http_server as http_server
+from sglang.srt.utils import common
+from sglang.test.test_utils import CustomTestCase
+
+UNKNOWN_PATH = "/__no_such_route__"
+WORKER_ID = "7"
+
+
+def _dispatch_names(app):
+    """Names of the dispatch callables of the app's HTTP middleware entries."""
+    names = []
+    for middleware in app.user_middleware:
+        dispatch = middleware.kwargs.get("dispatch")
+        if dispatch is not None:
+            names.append(dispatch.__name__)
+    return names
+
+
+class TestHttpRequestMetrics(CustomTestCase):
+    def test_track_middleware_registered_at_import(self):
+        # It has to be on the module-level app, because that is the app the
+        # tokenizer worker processes re-import and serve.
+        self.assertIn("track_http_status_code", _dispatch_names(http_server.app))
+
+    def test_import_does_not_load_prometheus_client(self):
+        # prometheus_client picks its value backend when it is first imported:
+        # with PROMETHEUS_MULTIPROC_DIR unset it keeps values in memory, where
+        # the MultiProcessCollector behind /metrics never sees them. SGLang sets
+        # that variable only in _set_envs_and_config, which runs after this
+        # module has been imported -- so importing it must not import
+        # prometheus_client.
+        code = (
+            "import sys, sglang.srt.entrypoints.http_server; "
+            "print('prometheus_client' in sys.modules)"
+        )
+        result = subprocess.run(
+            [sys.executable, "-c", code], capture_output=True, text=True
+        )
+        self.assertEqual("False", result.stdout.strip(), result.stderr[-2000:])
+
+    def test_records_requests_and_labels_the_worker(self):
+        common.init_http_request_metrics(worker_id=WORKER_ID)
+        TestClient(http_server.app).get(UNKNOWN_PATH)
+
+        self.assertEqual(
+            1.0,
+            REGISTRY.get_sample_value(
+                "sglang:http_requests_total",
+                {"endpoint": UNKNOWN_PATH, "method": "GET"},
+            ),
+        )
+        # Present (net zero after the request) proves the per-worker gauge was
+        # incremented and decremented under this worker id.
+        self.assertIsNotNone(
+            REGISTRY.get_sample_value(
+                "sglang:http_worker_active_requests", {"worker_id": WORKER_ID}
+            )
+        )

From c3a61ce2afae04e5bbc917dda485084f0dd96fc8 Mon Sep 17 00:00:00 2001
From: hnyls2002 <lsyincs@gmail.com>
Date: Mon, 19 Oct 2026 14:00:05 -0700
Subject: [PATCH 2/4] Drop the per-worker gauge and extra_labels, init metrics
 in the diffusion server

- Remove sglang:http_worker_active_requests: out of scope, and its label was the worker PID.
- Remove the unused extra_labels parameter.
- The diffusion HTTP server registers the same middleware, so it now also calls
  init_http_request_metrics(); without it the server recorded no HTTP metrics.
- Run the test file's TestCase from __main__ so CI executes it.
---
 .../runtime/entrypoints/http_server.py        |  2 +
 python/sglang/srt/entrypoints/http_server.py  |  5 +-
 python/sglang/srt/utils/common.py             | 77 ++++---------------
 .../test_http_request_metrics.py              | 58 ++++----------
 4 files changed, 32 insertions(+), 110 deletions(-)

diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
index 6e559e0e6321..0a2525f1124e 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
@@ -50,6 +50,7 @@
 from sglang.srt.utils.common import (
     add_prometheus_middleware,
     add_prometheus_track_response_middleware,
+    init_http_request_metrics,
 )
 from sglang.srt.utils.json_response import orjson_response
 from sglang.version import __version__
@@ -436,6 +437,7 @@ def create_app(server_args: ServerArgs):
         configure_metrics()
         add_prometheus_middleware(app)
         add_prometheus_track_response_middleware(app)
+        init_http_request_metrics()
     app.add_middleware(
         CORSMiddleware,
         allow_origins=["*"],
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index e8b32464a060..44c1693531da 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -504,10 +504,7 @@ async def custom_handler(request: Request):
 
     app.add_middleware(RequestDecompressionMiddleware)
 
-# Registered while this module is imported, so that every tokenizer worker
-# process gets it. With --tokenizer-worker-num > 1 uvicorn spawns workers that
-# re-import this module; a middleware added afterwards in the parent process
-# never reaches the app those workers actually serve.
+# Registered at import so that tokenizer workers, which re-import this module, get it too.
 add_prometheus_track_response_middleware(app)
 
 # Include routers
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index adb68624b935..5f097abef3c8 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -2865,30 +2865,25 @@ def dec(self, key: str):
 
 
 class _HttpRequestMetrics:
-    """The prometheus objects behind the HTTP request metrics of one process."""
-
-    def __init__(self, extra_labels: Dict[str, str], worker_id: Optional[str]):
+    def __init__(self):
         from prometheus_client import Counter, Gauge
 
-        extra_label_names = list(extra_labels.keys())
-        self.extra_labels = extra_labels
-
         self.request_counter = Counter(
             name="sglang:http_requests_total",
             documentation="Total number of HTTP requests by endpoint and method",
-            labelnames=extra_label_names + ["endpoint", "method"],
+            labelnames=["endpoint", "method"],
         )
 
         self.response_counter = Counter(
             name="sglang:http_responses_total",
             documentation="Total number of HTTP responses by endpoint and status code",
-            labelnames=extra_label_names + ["endpoint", "status_code", "method"],
+            labelnames=["endpoint", "status_code", "method"],
         )
 
         self.requests_active = Gauge(
             name="sglang:http_requests_active",
             documentation="Number of currently active HTTP requests",
-            labelnames=extra_label_names + ["endpoint", "method"],
+            labelnames=["endpoint", "method"],
             multiprocess_mode="livesum",
         )
 
@@ -2900,51 +2895,19 @@ def __init__(self, extra_labels: Dict[str, str], worker_id: Optional[str]):
             )
         )
 
-        self.worker_active = (
-            Gauge(
-                name="sglang:http_worker_active_requests",
-                documentation=(
-                    "Number of currently active HTTP requests handled by this "
-                    "tokenizer worker"
-                ),
-                labelnames=["worker_id"],
-                multiprocess_mode="livesum",
-            ).labels(worker_id=worker_id)
-            if worker_id is not None
-            else None
-        )
-
 
 _http_request_metrics: Optional[_HttpRequestMetrics] = None
 
 
-def init_http_request_metrics(
-    *, extra_labels: Optional[Dict[str, str]] = None, worker_id: Optional[str] = None
-) -> None:
-    """Create this process's HTTP request metrics.
-
-    Called once the process has published its config rather than at import:
-    importing prometheus_client fixes its value backend from
-    ``PROMETHEUS_MULTIPROC_DIR``, which SGLang only sets in
-    ``_set_envs_and_config`` -- after this module has already been imported.
-    Creating the metrics earlier would leave them in the in-memory registry,
-    where the ``/metrics`` endpoint (which reads the multiprocess directory)
-    never sees them.
-    """
+def init_http_request_metrics():
+    # Called after PROMETHEUS_MULTIPROC_DIR is set, since importing
+    # prometheus_client earlier keeps the metrics out of /metrics.
     global _http_request_metrics
-    _http_request_metrics = _HttpRequestMetrics(extra_labels or {}, worker_id)
+    _http_request_metrics = _HttpRequestMetrics()
 
 
 def add_prometheus_track_response_middleware(app):
-    """Track per-endpoint HTTP request metrics.
-
-    Registering the middleware and creating the metrics (see
-    ``init_http_request_metrics``) are deliberately separate, because the two
-    have to happen at different times: the middleware must be in place while
-    this module is imported so that every tokenizer worker process gets it,
-    while the metrics can only be created once the process has published its
-    config.
-    """
+    """Records nothing until init_http_request_metrics() runs in this process."""
     # Fix: replace BaseHTTPMiddleware's call_next with a pure ASGI version
     # that passes `receive` through, so request.is_disconnected() keeps working.
     from sglang.srt.utils.http_middleware_patch import patch_app_http_middleware
@@ -2954,7 +2917,7 @@ def add_prometheus_track_response_middleware(app):
     @app.middleware("http")
     async def track_http_status_code(request, call_next):
         metrics = _http_request_metrics
-        if metrics is None:  # metrics are not enabled in this process
+        if metrics is None:
             return await call_next(request)
 
         # With recording all requests, we have the risk of high cardinality if requests have arbitrary unhandled paths.
@@ -2962,16 +2925,9 @@ async def track_http_status_code(request, call_next):
         path, is_handled_path = _get_fastapi_request_path(request)
         method = request.method
         routing_key = request.headers.get("x-smg-routing-key")
-        extra_labels = metrics.extra_labels
-
-        metrics.request_counter.labels(
-            **extra_labels, endpoint=path, method=method
-        ).inc()
-        metrics.requests_active.labels(
-            **extra_labels, endpoint=path, method=method
-        ).inc()
-        if metrics.worker_active is not None:
-            metrics.worker_active.inc()
+
+        metrics.request_counter.labels(endpoint=path, method=method).inc()
+        metrics.requests_active.labels(endpoint=path, method=method).inc()
         if routing_key:
             metrics.routing_keys_active.inc(routing_key)
 
@@ -2979,7 +2935,6 @@ async def track_http_status_code(request, call_next):
             response = await call_next(request)
 
             metrics.response_counter.labels(
-                **extra_labels,
                 endpoint=path,
                 method=method,
                 status_code=str(response.status_code),
@@ -2987,11 +2942,7 @@ async def track_http_status_code(request, call_next):
 
             return response
         finally:
-            metrics.requests_active.labels(
-                **extra_labels, endpoint=path, method=method
-            ).dec()
-            if metrics.worker_active is not None:
-                metrics.worker_active.dec()
+            metrics.requests_active.labels(endpoint=path, method=method).dec()
             if routing_key:
                 metrics.routing_keys_active.dec(routing_key)
 
diff --git a/test/registered/unit/observability/test_http_request_metrics.py b/test/registered/unit/observability/test_http_request_metrics.py
index 1725e44b860c..582503d60307 100644
--- a/test/registered/unit/observability/test_http_request_metrics.py
+++ b/test/registered/unit/observability/test_http_request_metrics.py
@@ -1,23 +1,10 @@
-"""Regression tests for the HTTP request metrics middleware.
-
-With ``--tokenizer-worker-num > 1`` the app is served by worker processes that
-uvicorn spawns and that re-import ``http_server``. The track-response
-middleware therefore has to be part of the module-level app: registering it
-later from the parent process leaves every worker without it, and
-``sglang:http_requests_total`` and its siblings disappear from ``/metrics``.
-
-Creating the metrics is deliberately a separate step
-(``init_http_request_metrics``): prometheus_client fixes its value backend when
-it is first imported, while SGLang only sets ``PROMETHEUS_MULTIPROC_DIR`` after
-this module has been imported.
-"""
-
 from sglang.test.ci.ci_register import register_cpu_ci
 
 register_cpu_ci(est_time=25, suite="base-a-test-cpu")
 
 import subprocess
 import sys
+import unittest
 
 from fastapi.testclient import TestClient
 from prometheus_client import REGISTRY
@@ -27,32 +14,20 @@
 from sglang.test.test_utils import CustomTestCase
 
 UNKNOWN_PATH = "/__no_such_route__"
-WORKER_ID = "7"
-
-
-def _dispatch_names(app):
-    """Names of the dispatch callables of the app's HTTP middleware entries."""
-    names = []
-    for middleware in app.user_middleware:
-        dispatch = middleware.kwargs.get("dispatch")
-        if dispatch is not None:
-            names.append(dispatch.__name__)
-    return names
 
 
 class TestHttpRequestMetrics(CustomTestCase):
     def test_track_middleware_registered_at_import(self):
-        # It has to be on the module-level app, because that is the app the
-        # tokenizer worker processes re-import and serve.
-        self.assertIn("track_http_status_code", _dispatch_names(http_server.app))
+        # Tokenizer workers re-import http_server and serve its module-level app.
+        dispatch_names = [
+            m.kwargs["dispatch"].__name__
+            for m in http_server.app.user_middleware
+            if "dispatch" in m.kwargs
+        ]
+        self.assertIn("track_http_status_code", dispatch_names)
 
     def test_import_does_not_load_prometheus_client(self):
-        # prometheus_client picks its value backend when it is first imported:
-        # with PROMETHEUS_MULTIPROC_DIR unset it keeps values in memory, where
-        # the MultiProcessCollector behind /metrics never sees them. SGLang sets
-        # that variable only in _set_envs_and_config, which runs after this
-        # module has been imported -- so importing it must not import
-        # prometheus_client.
+        # PROMETHEUS_MULTIPROC_DIR is set after http_server is imported.
         code = (
             "import sys, sglang.srt.entrypoints.http_server; "
             "print('prometheus_client' in sys.modules)"
@@ -62,8 +37,8 @@ def test_import_does_not_load_prometheus_client(self):
         )
         self.assertEqual("False", result.stdout.strip(), result.stderr[-2000:])
 
-    def test_records_requests_and_labels_the_worker(self):
-        common.init_http_request_metrics(worker_id=WORKER_ID)
+    def test_records_requests(self):
+        common.init_http_request_metrics()
         TestClient(http_server.app).get(UNKNOWN_PATH)
 
         self.assertEqual(
@@ -73,10 +48,7 @@ def test_records_requests_and_labels_the_worker(self):
                 {"endpoint": UNKNOWN_PATH, "method": "GET"},
             ),
         )
-        # Present (net zero after the request) proves the per-worker gauge was
-        # incremented and decremented under this worker id.
-        self.assertIsNotNone(
-            REGISTRY.get_sample_value(
-                "sglang:http_worker_active_requests", {"worker_id": WORKER_ID}
-            )
-        )
+
+
+if __name__ == "__main__":
+    unittest.main()

From b53bb9befb9eae8e6c4d0cdc8abd29e3da9b7561 Mon Sep 17 00:00:00 2001
From: hnyls2002 <lsyincs@gmail.com>
Date: Mon, 19 Oct 2026 14:07:36 -0700
Subject: [PATCH 3/4] Move the HTTP request metrics middleware to
 observability/http_request_metrics.py

Pure move of RefCountedGauge, the metrics holder, init_http_request_metrics,
add_prometheus_track_response_middleware and _get_fastapi_request_path out of
utils/common.py; only imports change.
---
 .../runtime/entrypoints/http_server.py        |   6 +-
 python/sglang/srt/entrypoints/http_server.py  |   6 +-
 .../srt/observability/http_request_metrics.py | 116 ++++++++++++++++++
 python/sglang/srt/utils/common.py             | 115 -----------------
 .../test_http_request_metrics.py              |   4 +-
 5 files changed, 125 insertions(+), 122 deletions(-)
 create mode 100644 python/sglang/srt/observability/http_request_metrics.py

diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
index 0a2525f1124e..95b2fb4f3858 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
@@ -46,12 +46,12 @@
     globally_suppress_loggers,
     init_logger,
 )
-from sglang.srt.observability.trace import fastapi_telemetry_disabled_kwargs
-from sglang.srt.utils.common import (
-    add_prometheus_middleware,
+from sglang.srt.observability.http_request_metrics import (
     add_prometheus_track_response_middleware,
     init_http_request_metrics,
 )
+from sglang.srt.observability.trace import fastapi_telemetry_disabled_kwargs
+from sglang.srt.utils.common import add_prometheus_middleware
 from sglang.srt.utils.json_response import orjson_response
 from sglang.version import __version__
 
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index 44c1693531da..50a9bacf8df9 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -169,6 +169,10 @@
 )
 from sglang.srt.managers.tokenizer_manager import ServerStatus, TokenizerManager
 from sglang.srt.observability.func_timer import enable_func_timer
+from sglang.srt.observability.http_request_metrics import (
+    add_prometheus_track_response_middleware,
+    init_http_request_metrics,
+)
 from sglang.srt.observability.trace import (
     fastapi_telemetry_disabled_kwargs,
     process_tracing_init,
@@ -181,12 +185,10 @@
 from sglang.srt.server_args import PortArgs, ServerArgs
 from sglang.srt.utils import (
     add_prometheus_middleware,
-    add_prometheus_track_response_middleware,
     build_server_info,
     delete_directory,
     freeze_gc,
     get_bool_env_var,
-    init_http_request_metrics,
     is_mps,
     kill_process_tree,
     set_uvicorn_logging_configs,
diff --git a/python/sglang/srt/observability/http_request_metrics.py b/python/sglang/srt/observability/http_request_metrics.py
new file mode 100644
index 000000000000..4418399d1a43
--- /dev/null
+++ b/python/sglang/srt/observability/http_request_metrics.py
@@ -0,0 +1,116 @@
+from typing import Dict, Optional, Tuple
+
+
+class RefCountedGauge:
+    def __init__(self, gauge):
+        self._gauge = gauge
+        self._refcount: Dict[str, int] = {}
+
+    def inc(self, key: str):
+        if key in self._refcount:
+            self._refcount[key] += 1
+        else:
+            self._refcount[key] = 1
+            self._gauge.inc()
+
+    def dec(self, key: str):
+        if key in self._refcount:
+            self._refcount[key] -= 1
+            if self._refcount[key] == 0:
+                del self._refcount[key]
+                self._gauge.dec()
+
+
+class _HttpRequestMetrics:
+    def __init__(self):
+        from prometheus_client import Counter, Gauge
+
+        self.request_counter = Counter(
+            name="sglang:http_requests_total",
+            documentation="Total number of HTTP requests by endpoint and method",
+            labelnames=["endpoint", "method"],
+        )
+
+        self.response_counter = Counter(
+            name="sglang:http_responses_total",
+            documentation="Total number of HTTP responses by endpoint and status code",
+            labelnames=["endpoint", "status_code", "method"],
+        )
+
+        self.requests_active = Gauge(
+            name="sglang:http_requests_active",
+            documentation="Number of currently active HTTP requests",
+            labelnames=["endpoint", "method"],
+            multiprocess_mode="livesum",
+        )
+
+        self.routing_keys_active = RefCountedGauge(
+            Gauge(
+                name="sglang:routing_keys_active",
+                documentation="Number of unique routing keys with active requests",
+                multiprocess_mode="livesum",
+            )
+        )
+
+
+_http_request_metrics: Optional[_HttpRequestMetrics] = None
+
+
+def init_http_request_metrics():
+    # Called after PROMETHEUS_MULTIPROC_DIR is set, since importing
+    # prometheus_client earlier keeps the metrics out of /metrics.
+    global _http_request_metrics
+    _http_request_metrics = _HttpRequestMetrics()
+
+
+def add_prometheus_track_response_middleware(app):
+    """Records nothing until init_http_request_metrics() runs in this process."""
+    # Fix: replace BaseHTTPMiddleware's call_next with a pure ASGI version
+    # that passes `receive` through, so request.is_disconnected() keeps working.
+    from sglang.srt.utils.http_middleware_patch import patch_app_http_middleware
+
+    patch_app_http_middleware(app)
+
+    @app.middleware("http")
+    async def track_http_status_code(request, call_next):
+        metrics = _http_request_metrics
+        if metrics is None:
+            return await call_next(request)
+
+        # With recording all requests, we have the risk of high cardinality if requests have arbitrary unhandled paths.
+        # But given that SGLang engines with metrics enabled are usually behind routers this looks safe.
+        path, is_handled_path = _get_fastapi_request_path(request)
+        method = request.method
+        routing_key = request.headers.get("x-smg-routing-key")
+
+        metrics.request_counter.labels(endpoint=path, method=method).inc()
+        metrics.requests_active.labels(endpoint=path, method=method).inc()
+        if routing_key:
+            metrics.routing_keys_active.inc(routing_key)
+
+        try:
+            response = await call_next(request)
+
+            metrics.response_counter.labels(
+                endpoint=path,
+                method=method,
+                status_code=str(response.status_code),
+            ).inc()
+
+            return response
+        finally:
+            metrics.requests_active.labels(endpoint=path, method=method).dec()
+            if routing_key:
+                metrics.routing_keys_active.dec(routing_key)
+
+
+# https://github.com/blueswen/fastapi-observability/blob/132a3c576f8b09e5311c68bd553215013bc75685/fastapi_app/utils.py#L98
+def _get_fastapi_request_path(request) -> Tuple[str, bool]:
+    from starlette.routing import Match
+
+    for route in request.app.routes:
+        match, child_scope = route.matches(request.scope)
+        if match == Match.FULL:
+            return getattr(route, "path", request.url.path), True
+
+    return request.url.path, False
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index 5f097abef3c8..c84312cceb71 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -2844,121 +2844,6 @@ def add_prometheus_middleware(app):
     app.routes.append(metrics_route)
 
 
-class RefCountedGauge:
-    def __init__(self, gauge):
-        self._gauge = gauge
-        self._refcount: Dict[str, int] = {}
-
-    def inc(self, key: str):
-        if key in self._refcount:
-            self._refcount[key] += 1
-        else:
-            self._refcount[key] = 1
-            self._gauge.inc()
-
-    def dec(self, key: str):
-        if key in self._refcount:
-            self._refcount[key] -= 1
-            if self._refcount[key] == 0:
-                del self._refcount[key]
-                self._gauge.dec()
-
-
-class _HttpRequestMetrics:
-    def __init__(self):
-        from prometheus_client import Counter, Gauge
-
-        self.request_counter = Counter(
-            name="sglang:http_requests_total",
-            documentation="Total number of HTTP requests by endpoint and method",
-            labelnames=["endpoint", "method"],
-        )
-
-        self.response_counter = Counter(
-            name="sglang:http_responses_total",
-            documentation="Total number of HTTP responses by endpoint and status code",
-            labelnames=["endpoint", "status_code", "method"],
-        )
-
-        self.requests_active = Gauge(
-            name="sglang:http_requests_active",
-            documentation="Number of currently active HTTP requests",
-            labelnames=["endpoint", "method"],
-            multiprocess_mode="livesum",
-        )
-
-        self.routing_keys_active = RefCountedGauge(
-            Gauge(
-                name="sglang:routing_keys_active",
-                documentation="Number of unique routing keys with active requests",
-                multiprocess_mode="livesum",
-            )
-        )
-
-
-_http_request_metrics: Optional[_HttpRequestMetrics] = None
-
-
-def init_http_request_metrics():
-    # Called after PROMETHEUS_MULTIPROC_DIR is set, since importing
-    # prometheus_client earlier keeps the metrics out of /metrics.
-    global _http_request_metrics
-    _http_request_metrics = _HttpRequestMetrics()
-
-
-def add_prometheus_track_response_middleware(app):
-    """Records nothing until init_http_request_metrics() runs in this process."""
-    # Fix: replace BaseHTTPMiddleware's call_next with a pure ASGI version
-    # that passes `receive` through, so request.is_disconnected() keeps working.
-    from sglang.srt.utils.http_middleware_patch import patch_app_http_middleware
-
-    patch_app_http_middleware(app)
-
-    @app.middleware("http")
-    async def track_http_status_code(request, call_next):
-        metrics = _http_request_metrics
-        if metrics is None:
-            return await call_next(request)
-
-        # With recording all requests, we have the risk of high cardinality if requests have arbitrary unhandled paths.
-        # But given that SGLang engines with metrics enabled are usually behind routers this looks safe.
-        path, is_handled_path = _get_fastapi_request_path(request)
-        method = request.method
-        routing_key = request.headers.get("x-smg-routing-key")
-
-        metrics.request_counter.labels(endpoint=path, method=method).inc()
-        metrics.requests_active.labels(endpoint=path, method=method).inc()
-        if routing_key:
-            metrics.routing_keys_active.inc(routing_key)
-
-        try:
-            response = await call_next(request)
-
-            metrics.response_counter.labels(
-                endpoint=path,
-                method=method,
-                status_code=str(response.status_code),
-            ).inc()
-
-            return response
-        finally:
-            metrics.requests_active.labels(endpoint=path, method=method).dec()
-            if routing_key:
-                metrics.routing_keys_active.dec(routing_key)
-
-
-# https://github.com/blueswen/fastapi-observability/blob/132a3c576f8b09e5311c68bd553215013bc75685/fastapi_app/utils.py#L98
-def _get_fastapi_request_path(request) -> Tuple[str, bool]:
-    from starlette.routing import Match
-
-    for route in request.app.routes:
-        match, child_scope = route.matches(request.scope)
-        if match == Match.FULL:
-            return getattr(route, "path", request.url.path), True
-
-    return request.url.path, False
-
-
 # Copy from pytorch and OpenRLHF to allow creating multiple main groups.
 # https://github.com/pytorch/pytorch/blob/main/torch/distributed/distributed_c10d.py
 # https://github.com/OpenRLHF/OpenRLHF/blob/main/openrlhf/utils/distributed_util.py
diff --git a/test/registered/unit/observability/test_http_request_metrics.py b/test/registered/unit/observability/test_http_request_metrics.py
index 582503d60307..c9919a4cde35 100644
--- a/test/registered/unit/observability/test_http_request_metrics.py
+++ b/test/registered/unit/observability/test_http_request_metrics.py
@@ -10,7 +10,7 @@
 from prometheus_client import REGISTRY
 
 import sglang.srt.entrypoints.http_server as http_server
-from sglang.srt.utils import common
+from sglang.srt.observability import http_request_metrics
 from sglang.test.test_utils import CustomTestCase
 
 UNKNOWN_PATH = "/__no_such_route__"
@@ -38,7 +38,7 @@ def test_import_does_not_load_prometheus_client(self):
         self.assertEqual("False", result.stdout.strip(), result.stderr[-2000:])
 
     def test_records_requests(self):
-        common.init_http_request_metrics()
+        http_request_metrics.init_http_request_metrics()
         TestClient(http_server.app).get(UNKNOWN_PATH)
 
         self.assertEqual(

From 02864e9645010b8cc73b7e02624ad5cd91503a49 Mon Sep 17 00:00:00 2001
From: hnyls2002 <lsyincs@gmail.com>
Date: Mon, 19 Oct 2026 14:08:17 -0700
Subject: [PATCH 4/4] Pair the HTTP request metrics names and fold the tests
 into one

- Rename add_prometheus_track_response_middleware to
  add_http_request_metrics_middleware, the pair of init_http_request_metrics.
- Name the metric attributes after their metrics (requests_total, responses_total).
- Make init_http_request_metrics idempotent.
- Merge the three tests into one: a request on the module-level app is counted
  only after init. Drop the subprocess import check, which any unrelated
  top-level prometheus_client import would break.
---
 .../runtime/entrypoints/http_server.py        |  4 +--
 python/sglang/srt/entrypoints/http_server.py  |  4 +--
 .../srt/observability/http_request_metrics.py | 17 ++++++-----
 .../test_http_request_metrics.py              | 29 ++++---------------
 4 files changed, 19 insertions(+), 35 deletions(-)

diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
index 95b2fb4f3858..0a95f581ad23 100644
--- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
+++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py
@@ -47,7 +47,7 @@
     init_logger,
 )
 from sglang.srt.observability.http_request_metrics import (
-    add_prometheus_track_response_middleware,
+    add_http_request_metrics_middleware,
     init_http_request_metrics,
 )
 from sglang.srt.observability.trace import fastapi_telemetry_disabled_kwargs
@@ -436,7 +436,7 @@ def create_app(server_args: ServerArgs):
     if server_args.enable_metrics:
         configure_metrics()
         add_prometheus_middleware(app)
-        add_prometheus_track_response_middleware(app)
+        add_http_request_metrics_middleware(app)
         init_http_request_metrics()
     app.add_middleware(
         CORSMiddleware,
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index 50a9bacf8df9..430d86b9b67b 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -170,7 +170,7 @@
 from sglang.srt.managers.tokenizer_manager import ServerStatus, TokenizerManager
 from sglang.srt.observability.func_timer import enable_func_timer
 from sglang.srt.observability.http_request_metrics import (
-    add_prometheus_track_response_middleware,
+    add_http_request_metrics_middleware,
     init_http_request_metrics,
 )
 from sglang.srt.observability.trace import (
@@ -507,7 +507,7 @@ async def custom_handler(request: Request):
     app.add_middleware(RequestDecompressionMiddleware)
 
 # Registered at import so that tokenizer workers, which re-import this module, get it too.
-add_prometheus_track_response_middleware(app)
+add_http_request_metrics_middleware(app)
 
 # Include routers
 from sglang.srt.entrypoints.v1_loads import router as v1_loads_router
diff --git a/python/sglang/srt/observability/http_request_metrics.py b/python/sglang/srt/observability/http_request_metrics.py
index 4418399d1a43..a8298a926726 100644
--- a/python/sglang/srt/observability/http_request_metrics.py
+++ b/python/sglang/srt/observability/http_request_metrics.py
@@ -25,13 +25,13 @@ class _HttpRequestMetrics:
     def __init__(self):
         from prometheus_client import Counter, Gauge
 
-        self.request_counter = Counter(
+        self.requests_total = Counter(
             name="sglang:http_requests_total",
             documentation="Total number of HTTP requests by endpoint and method",
             labelnames=["endpoint", "method"],
         )
 
-        self.response_counter = Counter(
+        self.responses_total = Counter(
             name="sglang:http_responses_total",
             documentation="Total number of HTTP responses by endpoint and status code",
             labelnames=["endpoint", "status_code", "method"],
@@ -57,13 +57,14 @@ def __init__(self):
 
 
 def init_http_request_metrics():
-    # Called after PROMETHEUS_MULTIPROC_DIR is set, since importing
-    # prometheus_client earlier keeps the metrics out of /metrics.
+    # Call after PROMETHEUS_MULTIPROC_DIR is set: prometheus_client picks its
+    # value backend on import.
     global _http_request_metrics
-    _http_request_metrics = _HttpRequestMetrics()
+    if _http_request_metrics is None:
+        _http_request_metrics = _HttpRequestMetrics()
 
 
-def add_prometheus_track_response_middleware(app):
+def add_http_request_metrics_middleware(app):
     """Records nothing until init_http_request_metrics() runs in this process."""
     # Fix: replace BaseHTTPMiddleware's call_next with a pure ASGI version
     # that passes `receive` through, so request.is_disconnected() keeps working.
@@ -83,7 +84,7 @@ async def track_http_status_code(request, call_next):
         method = request.method
         routing_key = request.headers.get("x-smg-routing-key")
 
-        metrics.request_counter.labels(endpoint=path, method=method).inc()
+        metrics.requests_total.labels(endpoint=path, method=method).inc()
         metrics.requests_active.labels(endpoint=path, method=method).inc()
         if routing_key:
             metrics.routing_keys_active.inc(routing_key)
@@ -91,7 +92,7 @@ async def track_http_status_code(request, call_next):
         try:
             response = await call_next(request)
 
-            metrics.response_counter.labels(
+            metrics.responses_total.labels(
                 endpoint=path,
                 method=method,
                 status_code=str(response.status_code),
diff --git a/test/registered/unit/observability/test_http_request_metrics.py b/test/registered/unit/observability/test_http_request_metrics.py
index c9919a4cde35..96834d31e53d 100644
--- a/test/registered/unit/observability/test_http_request_metrics.py
+++ b/test/registered/unit/observability/test_http_request_metrics.py
@@ -1,9 +1,7 @@
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=25, suite="base-a-test-cpu")
+register_cpu_ci(est_time=15, suite="base-a-test-cpu")
 
-import subprocess
-import sys
 import unittest
 
 from fastapi.testclient import TestClient
@@ -17,27 +15,12 @@
 
 
 class TestHttpRequestMetrics(CustomTestCase):
-    def test_track_middleware_registered_at_import(self):
-        # Tokenizer workers re-import http_server and serve its module-level app.
-        dispatch_names = [
-            m.kwargs["dispatch"].__name__
-            for m in http_server.app.user_middleware
-            if "dispatch" in m.kwargs
-        ]
-        self.assertIn("track_http_status_code", dispatch_names)
-
-    def test_import_does_not_load_prometheus_client(self):
-        # PROMETHEUS_MULTIPROC_DIR is set after http_server is imported.
-        code = (
-            "import sys, sglang.srt.entrypoints.http_server; "
-            "print('prometheus_client' in sys.modules)"
-        )
-        result = subprocess.run(
-            [sys.executable, "-c", code], capture_output=True, text=True
-        )
-        self.assertEqual("False", result.stdout.strip(), result.stderr[-2000:])
+    def test_module_level_app_records_requests(self):
+        # Tokenizer workers re-import http_server and serve its module-level app,
+        # so the middleware must be on it, while the metrics must wait for
+        # init_http_request_metrics().
+        self.assertIsNone(http_request_metrics._http_request_metrics)
 
-    def test_records_requests(self):
         http_request_metrics.init_http_request_metrics()
         TestClient(http_server.app).get(UNKNOWN_PATH)