[TRTLLM-17262][refactor] Move the disaggregated service types and latency logger out of responses_utils

.pre-commit-config.yaml

@@ -417,6 +417,9 @@ common-files: &common_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/latency_logger.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/service_types.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
@@ -1144,6 +1147,9 @@ legacy-files: &legacy_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/latency_logger.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/service_types.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
@@ -1620,6 +1626,7 @@ repos:
         # Run mypy with no explicit paths so it type-checks the set declared in
         # pyproject.toml [tool.mypy] `files` (single source of truth). This
         # `files:` pattern only decides when the hook triggers — any change under
-        # the sampler source/test dirs.
+        # the sampler source/test dirs or the serving frontends. A PR that adds
+        # a typed module elsewhere extends this pattern in the same PR.
         pass_filenames: false
-        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/)
+        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/|tensorrt_llm/serve/|tensorrt_llm/commands/serve_cli/)

jenkins/L0_MergeRequest.groovy

@@ -1948,6 +1948,10 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
         "tensorrt_llm/models/llama/",
         "tensorrt_llm/parameter.py",
         "tensorrt_llm/serve/__init__.py",
+        // _MsgspecRoute decodes the disaggregated frontend's msgpack bodies.
+        "tensorrt_llm/serve/_base/",
+        // /health is the disaggregated readiness probe.
+        "tensorrt_llm/serve/api/ops/",
         "tensorrt_llm/serve/cluster_storage.py",
         "tensorrt_llm/serve/disagg_auto_scaling.py",
         "tensorrt_llm/serve/extensions/",

legacy-files.txt

@@ -409,6 +409,9 @@ tensorrt_llm/scaffolding/task_collection.py
 tensorrt_llm/scaffolding/worker.py
 tensorrt_llm/scheduling_params.py
 tensorrt_llm/serve/__init__.py
+tensorrt_llm/serve/_base/latency_logger.py
+tensorrt_llm/serve/_base/msgspec_route.py
+tensorrt_llm/serve/_base/service_types.py
 tensorrt_llm/serve/chat_utils.py
 tensorrt_llm/serve/cluster_storage.py
 tensorrt_llm/serve/disagg_auto_scaling.py

pyproject.toml

@@ -474,6 +474,9 @@ exclude = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/latency_logger.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/service_types.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
@@ -862,6 +865,9 @@ files = [
     "tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py",
+    "tensorrt_llm/serve/_base/routing.py",
+    "tensorrt_llm/serve/api/ops/system/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/serving.py",
     "tests/unittest/_torch/sampler/test_torch_sampler.py",
     "tests/unittest/_torch/sampler/test_token_ban.py",
     "tests/unittest/_torch/sampler/test_beam_search.py",

ruff-legacy.toml

@@ -426,6 +426,9 @@ include = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/latency_logger.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/service_types.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",

tensorrt_llm/commands/AGENTS.md

@@ -0,0 +1,6 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+Rules for the trtllm-serve CLI (`commands/serve.py`) are in `tensorrt_llm/serve/AGENTS.md`.

tensorrt_llm/commands/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/serve/AGENTS.md

@@ -0,0 +1,97 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# AGENTS.md - tensorrt_llm/serve
+
+Scope: `tensorrt_llm/serve/` and everything under it. Supplements the repo-root `AGENTS.md`
+and `tests/AGENTS.md`; `tensorrt_llm/commands/AGENTS.md` points here for the serve CLI.
+Where a check enforces a rule, the rule names it; a failing check says what to change.
+Name files and symbols here, never line numbers.
+
+## Where code goes
+
+- New endpoint: a router and a serving class under `serve/api/<dialect>/<endpoint>/`
+  (`api_router.py` + `serving.py`). Create the router with `make_router()`
+  (`serve/_base/routing.py`) and add a `RouterSpec` to `ROUTERS`. Never add a method or a
+  route to `OpenAIServer`. Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Attach a router with `app.include_router(router)` and no `prefix=`; give the prefix to
+  `make_router(prefix=...)` instead. The composition root sets each serving object on
+  `app.state.<bucket>_serving_<endpoint>` before attaching; routers read it from
+  `request.app.state`, never from a closure.
+- Model-specific behavior: a hook on `ServingExtension` (`serve/serving_extensions.py`,
+  `serve/extensions/`). Do not branch on a model name in a handler.
+- Prompt rendering: `serve/render/` only. Handlers do not call `apply_chat_template`.
+- Request and response models: a new field needs a row in
+  `tests/unittest/api_stability/references/trtllm_serve_api.yaml`.
+  Check: `tests/unittest/api_stability/test_serve_api.py`.
+- Routes: every server configuration's routes (path, method, route class, and the order of
+  routes that can match the same request) are pinned in
+  `tests/unittest/api_stability/references/trtllm_serve_routes.yaml`. A route change
+  regenerates it and needs the API review committee.
+  Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Middleware order matters: `add_middleware` inserts at the front, so the last one added is
+  outermost. Do not reorder. Check: `tests/unittest/serve/test_middleware_order.py`.
+- Import directions between layers are tests; a PR that creates a package adds its contract.
+  Check: `tests/unittest/serve/test_layering.py`.
+- VisualGen serve tests live under a path that contains `visual_gen/`
+  (`tests/unittest/serve/api/visual_gen/`), or CI selection misses them.
+
+## Moving code
+
+- A move PR only moves code: no reformatting, no logic change.
+- A move PR is redone on the latest `main` right before merge instead of being rebased,
+  and its PR body lists every moved definition with its old and new location, and every
+  definition the PR changes on purpose.
+- In the same PR update the multi-GPU file list (`getMultiGpuFileChanged` in
+  `jenkins/L0_MergeRequest.groovy`), `.github/CODEOWNERS` and `legacy-files.txt`. A new file
+  takes the formatting group of the file its code comes from.
+- Old import paths: a move deletes the old module (or the moved names) without a shim and
+  rewrites every import of it in the repository, tests included; module paths written as
+  strings (patch targets, subprocess command lines, `_LAZY_ATTRS` in `serve/__init__.py`) are
+  fixed by hand. The PR body lists "old path → new path" for the release notes. Only five old
+  paths keep a shim, because code outside the repository imports them:
+  `tensorrt_llm/llmapi/reasoning_parser.py`, `serve/tool_parser/`,
+  `serve/openai_disagg_server.py`, `serve/openai_server.py` and `commands/serve.py`. A shim
+  re-exports public names only, lists them in `__all__` (or `_LAZY_NAMES` in
+  `commands/serve.py`), warns once with `FutureWarning`, and has its rows in
+  `tests/unittest/serve/test_shim_identity.py`.
+- `mock.patch("old.module.name")` does not follow moved code. Grep the tests for every moved
+  name, including `monkeypatch.setattr(module, "name", ...)` and dotted paths split across lines.
+
+## Tests
+
+- Register every new test file in a CI list in the same PR (`tests/AGENTS.md`). Serve tests
+  that need no GPU go in `tests/integration/test_lists/test-db/l0_cpu.yml` and must contain
+  `pytest.mark.cpu_only`, or the CPU stage collects nothing from them.
+- New serve tests mirror the package: `tests/unittest/serve/` for `tensorrt_llm/serve/`.
+- Build servers with the factory and fake generators in `tests/test_common/serve/`
+  (`isolated_serve_env`), never `object.__new__(OpenAIServer)` or
+  `object.__new__(OpenAIDisaggServer)`. Check: `tests/unittest/serve/test_no_object_new.py`.
+- List an app's routes with `effective_routes()` from `tests/test_common/serve/routes.py`, not
+  `app.routes`: FastAPI 0.139 keeps included routers as objects without `.path`.
+- The harmony tests guard whole files with try-imports. After renaming anything they import,
+  run them with `pytest -rs` and check that nothing is skipped as "not importable": a green run
+  can mean every test was skipped.
+
+## Checklists
+
+- Endpoint: package under `serve/api/`; `RouterSpec` in `ROUTERS`; regenerate the route
+  snapshot (API review); multi-GPU list entry if it runs on the disaggregated path; CPU test
+  registered in `tests/integration/test_lists/test-db/l0_cpu.yml`.
+
+## Commands
+
+- All serve CPU tests (the runner exits when `LLM_MODELS_ROOT` is unset):
+  `LLM_MODELS_ROOT=/path/to/models PYTHONPATH=tests python3 -m test_common.serve.cpu_tests`
+- Route snapshot: `pytest -m cpu_only tests/unittest/api_stability/test_serve_routes.py tests/unittest/serve/api/visual_gen/test_routes_visual_gen.py`
+- Regenerate the route snapshot: `PYTHONPATH=tests python3 -m test_common.serve.routes --write`
+- API gates: `pytest tests/unittest/api_stability/test_serve_api.py tests/unittest/api_stability/test_serve_cli.py`
+- Extensions and parsers: `pytest tests/unittest/llmapi/apps/test_serving_extensions.py tests/unittest/llmapi/apps/test_tool_parsers.py`
+- Disaggregated router and coordinator (CPU): `pytest tests/unittest/disaggregated -m cpu_only -q`
+- Which CI stage runs a test: `python scripts/test_to_stage_mapping.py --tests "<test>"`. It
+  matches the text of list entries and does not expand directories: a file listed through its
+  directory maps only when you pass the directory entry, e.g. `--tests "unittest/serve"` for
+  anything under `tests/unittest/serve/`.
+- Tests that load model files need `LLM_MODELS_ROOT` (the harmony tests' autouse fixture does).

tensorrt_llm/serve/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/serve/_base/__init__.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared serving infrastructure with no dependency on endpoint packages."""

tensorrt_llm/serve/_base/latency_logger.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Periodic latency percentiles for the disaggregated coordinator and its clients."""
+
+from typing import List
+
+from tensorrt_llm.logger import logger
+
+
+class PeriodicLatencyLogger:
+    """Periodically log latency percentiles for a named coordinator API.
+
+    This lock-free, self-resetting logger runs on one asyncio loop and profiles
+    the in-process owner and HTTP client without per-call log spam.
+    """
+
+    def __init__(self, name: str, window: int = 500):
+        self._name = name
+        self._window = window
+        self._samples: List[float] = []
+        self._n = 0
+
+    def record(self, dt_s: float) -> None:
+        self._samples.append(dt_s * 1000.0)  # ms
+        self._n += 1
+        if self._n % self._window == 0:
+            s = sorted(self._samples)
+            m = len(s)
+
+            def percentile(q):
+                return s[min(int(q * m), m - 1)]
+
+            logger.info(f"[coord_api] {self._name} n={self._n} ms: "
+                        f"mean={sum(s)/m:.2f} p50={percentile(0.5):.2f} "
+                        f"p90={percentile(0.9):.2f} "
+                        f"p99={percentile(0.99):.2f} max={s[-1]:.2f}")
+            self._samples = []

tensorrt_llm/serve/_base/msgspec_route.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Request and route classes that decode msgpack request bodies from the
+disaggregated frontend."""
+
+import json
+
+import msgspec
+
+from tensorrt_llm.serve.kv_hints import KvHintsRequest, KvHintsRoute
+
+_msgpack_decoder = msgspec.msgpack.Decoder()
+
+
+class _MsgspecRequest(KvHintsRequest):
+    """Request that decodes X-TRTLLM-Msgpack bodies with msgspec."""
+
+    async def json(self):
+        if not hasattr(self, "_json_body"):
+            body = await self.body()
+            if not body:
+                self._json_body = {}
+            elif self.headers.get("x-trtllm-msgpack") == "1":
+                try:
+                    self._json_body = _msgpack_decoder.decode(body)
+                except msgspec.DecodeError as e:
+                    raise json.JSONDecodeError(f"msgpack: {e}", "", 0) from e
+            else:
+                self._json_body = json.loads(body)
+        return self.filter_kv_hints(self._json_body)
+
+
+class _MsgspecRoute(KvHintsRoute):
+    """Parse msgpack/JSON and apply hint trust policy before model validation."""
+
+    request_type = _MsgspecRequest

tensorrt_llm/serve/_base/routing.py

@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Router creation and role-based attachment for trtllm-serve."""
+
+from __future__ import annotations
+
+import enum
+import importlib
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+from fastapi import APIRouter, FastAPI
+
+from tensorrt_llm.llmapi.disagg_utils import ServerRole
+from tensorrt_llm.serve._base.msgspec_route import _MsgspecRoute
+
+
+class RoleGroup(enum.Enum):
+    LLM = "llm"
+    MM_ENCODER = "mm_encoder"
+    EMBEDDING = "embedding"
+    VISUAL_GEN = "visual_gen"
+
+
+ALL_ROLE_GROUPS: frozenset[RoleGroup] = frozenset(RoleGroup)
+
+
+def role_group_of(role: Optional[ServerRole]) -> RoleGroup:
+    """None, CONTEXT and GENERATION serve the LLM routes, as the dispatch in OpenAIServer does."""
+    if role is ServerRole.MM_ENCODER:
+        return RoleGroup.MM_ENCODER
+    if role is ServerRole.EMBEDDING:
+        return RoleGroup.EMBEDDING
+    if role is ServerRole.VISUAL_GEN:
+        return RoleGroup.VISUAL_GEN
+    return RoleGroup.LLM
+
+
+@dataclass(frozen=True)
+class ServerCapabilities:
+    """What the composition root decides once at startup; routers are attached from it."""
+
+    role: Optional[ServerRole]
+    role_group: RoleGroup
+
+    @classmethod
+    def for_role(cls, role: Optional[ServerRole]) -> ServerCapabilities:
+        return cls(role=role, role_group=role_group_of(role))
+
+
+def _always(caps: ServerCapabilities) -> bool:
+    return True
+
+
+@dataclass(frozen=True)
+class RouterSpec:
+    """One router module and the roles and capabilities that mount it."""
+
+    module: str  # full dotted path of the api_router module
+    roles: frozenset[RoleGroup]
+    when: Callable[[ServerCapabilities], bool] = _always
+
+
+# Attach order is registration order. Keep the route snapshot's conflict-relevant
+# order: a router that registers GET /metrics must come before the Prometheus mount.
+ROUTERS: tuple[RouterSpec, ...] = (
+    RouterSpec("tensorrt_llm.serve.api.ops.system.api_router", roles=ALL_ROLE_GROUPS),
+)
+
+
+def make_router(**kwargs: Any) -> APIRouter:
+    """The only way to create a trtllm-serve router: every route decodes msgpack bodies.
+
+    The disaggregated frontend sends worker requests as msgpack; a route of a plain
+    APIRouter keeps APIRoute and rejects them.
+    """
+    if "route_class" in kwargs:
+        raise TypeError(
+            "make_router() always uses _MsgspecRoute; do not pass route_class. "
+            "A plain APIRoute breaks msgpack request bodies from the disaggregated frontend."
+        )
+    return APIRouter(route_class=_MsgspecRoute, **kwargs)
+
+
+def attach_routers(
+    app: FastAPI, caps: ServerCapabilities, routers: tuple[RouterSpec, ...] = ROUTERS
+) -> None:
+    """Import and attach only the routers this role needs."""
+    for spec in routers:
+        if caps.role_group in spec.roles and spec.when(caps):
+            importlib.import_module(spec.module).attach_router(app)