[TRTLLM-17267][refactor] Move the embeddings, mm_encoder and tokenize handlers into serving classes

.github/CODEOWNERS

@@ -371,6 +371,7 @@
 /cpp/tensorrt_llm/thop/*lysses* @NVIDIA/trt-llm-torch-visual-gen-devs
 /cpp/tensorrt_llm/kernels/ulysses* @NVIDIA/trt-llm-torch-visual-gen-devs
 /tensorrt_llm/media @NVIDIA/trt-llm-torch-visual-gen-devs
+/tensorrt_llm/serve/api/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs
 /tensorrt_llm/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs @NVIDIA/trt-llm-runtime-devs
 /tests/integration/defs/examples/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs
 /tests/integration/defs/visual_gen @NVIDIA/trt-llm-torch-visual-gen-devs

.pre-commit-config.yaml

@@ -417,13 +417,30 @@ common-files: &common_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/serving.py |
+        tensorrt_llm/serve/api/openai/embeddings/serving.py |
+        tensorrt_llm/serve/api/openai/mm_encoder/serving.py |
+        tensorrt_llm/serve/api/openai/tokenize/serving.py |
+        tensorrt_llm/serve/api/visual_gen/protocol.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
         tensorrt_llm/serve/harmony_adapter.py |
         tensorrt_llm/serve/metadata_server.py |
         tensorrt_llm/serve/openai_disagg_server.py |
-        tensorrt_llm/serve/openai_protocol.py |
+        tensorrt_llm/serve/openai_protocol/chat.py |
+        tensorrt_llm/serve/openai_protocol/common.py |
+        tensorrt_llm/serve/openai_protocol/completions.py |
+        tensorrt_llm/serve/openai_protocol/control.py |
+        tensorrt_llm/serve/openai_protocol/disagg.py |
+        tensorrt_llm/serve/openai_protocol/embeddings.py |
+        tensorrt_llm/serve/openai_protocol/kv_cache.py |
+        tensorrt_llm/serve/openai_protocol/models.py |
+        tensorrt_llm/serve/openai_protocol/responses.py |
+        tensorrt_llm/serve/openai_protocol/sampling.py |
+        tensorrt_llm/serve/openai_protocol/tokenize.py |
+        tensorrt_llm/serve/openai_protocol/tools.py |
         tensorrt_llm/serve/openai_server.py |
         tensorrt_llm/serve/postprocess_handlers.py |
         tensorrt_llm/serve/responses_utils.py |
@@ -1144,13 +1161,30 @@ legacy-files: &legacy_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/serving.py |
+        tensorrt_llm/serve/api/openai/embeddings/serving.py |
+        tensorrt_llm/serve/api/openai/mm_encoder/serving.py |
+        tensorrt_llm/serve/api/openai/tokenize/serving.py |
+        tensorrt_llm/serve/api/visual_gen/protocol.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
         tensorrt_llm/serve/harmony_adapter.py |
         tensorrt_llm/serve/metadata_server.py |
         tensorrt_llm/serve/openai_disagg_server.py |
-        tensorrt_llm/serve/openai_protocol.py |
+        tensorrt_llm/serve/openai_protocol/chat.py |
+        tensorrt_llm/serve/openai_protocol/common.py |
+        tensorrt_llm/serve/openai_protocol/completions.py |
+        tensorrt_llm/serve/openai_protocol/control.py |
+        tensorrt_llm/serve/openai_protocol/disagg.py |
+        tensorrt_llm/serve/openai_protocol/embeddings.py |
+        tensorrt_llm/serve/openai_protocol/kv_cache.py |
+        tensorrt_llm/serve/openai_protocol/models.py |
+        tensorrt_llm/serve/openai_protocol/responses.py |
+        tensorrt_llm/serve/openai_protocol/sampling.py |
+        tensorrt_llm/serve/openai_protocol/tokenize.py |
+        tensorrt_llm/serve/openai_protocol/tools.py |
         tensorrt_llm/serve/openai_server.py |
         tensorrt_llm/serve/postprocess_handlers.py |
         tensorrt_llm/serve/responses_utils.py |
@@ -1620,6 +1654,7 @@ repos:
         # Run mypy with no explicit paths so it type-checks the set declared in
         # pyproject.toml [tool.mypy] `files` (single source of truth). This
         # `files:` pattern only decides when the hook triggers — any change under
-        # the sampler source/test dirs.
+        # the sampler source/test dirs or the serving frontends. A PR that adds
+        # a typed module elsewhere extends this pattern in the same PR.
         pass_filenames: false
-        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/)
+        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/|tensorrt_llm/serve/|tensorrt_llm/commands/serve_cli/)

jenkins/L0_MergeRequest.groovy

@@ -1948,6 +1948,14 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
         "tensorrt_llm/models/llama/",
         "tensorrt_llm/parameter.py",
         "tensorrt_llm/serve/__init__.py",
+        // _MsgspecRoute decodes the disaggregated frontend's msgpack bodies.
+        "tensorrt_llm/serve/_base/",
+        // Code moved out of tensorrt_llm/serve/openai_server.py keeps its trigger.
+        "tensorrt_llm/serve/api/openai/embeddings/",
+        "tensorrt_llm/serve/api/openai/mm_encoder/",
+        "tensorrt_llm/serve/api/openai/tokenize/",
+        // /health is the disaggregated readiness probe.
+        "tensorrt_llm/serve/api/ops/",
         "tensorrt_llm/serve/cluster_storage.py",
         "tensorrt_llm/serve/disagg_auto_scaling.py",
         "tensorrt_llm/serve/extensions/",

legacy-files.txt

@@ -409,13 +409,30 @@ tensorrt_llm/scaffolding/task_collection.py
 tensorrt_llm/scaffolding/worker.py
 tensorrt_llm/scheduling_params.py
 tensorrt_llm/serve/__init__.py
+tensorrt_llm/serve/_base/msgspec_route.py
+tensorrt_llm/serve/_base/serving.py
+tensorrt_llm/serve/api/openai/embeddings/serving.py
+tensorrt_llm/serve/api/openai/mm_encoder/serving.py
+tensorrt_llm/serve/api/openai/tokenize/serving.py
+tensorrt_llm/serve/api/visual_gen/protocol.py
 tensorrt_llm/serve/chat_utils.py
 tensorrt_llm/serve/cluster_storage.py
 tensorrt_llm/serve/disagg_auto_scaling.py
 tensorrt_llm/serve/harmony_adapter.py
 tensorrt_llm/serve/metadata_server.py
 tensorrt_llm/serve/openai_disagg_server.py
-tensorrt_llm/serve/openai_protocol.py
+tensorrt_llm/serve/openai_protocol/chat.py
+tensorrt_llm/serve/openai_protocol/common.py
+tensorrt_llm/serve/openai_protocol/completions.py
+tensorrt_llm/serve/openai_protocol/control.py
+tensorrt_llm/serve/openai_protocol/disagg.py
+tensorrt_llm/serve/openai_protocol/embeddings.py
+tensorrt_llm/serve/openai_protocol/kv_cache.py
+tensorrt_llm/serve/openai_protocol/models.py
+tensorrt_llm/serve/openai_protocol/responses.py
+tensorrt_llm/serve/openai_protocol/sampling.py
+tensorrt_llm/serve/openai_protocol/tokenize.py
+tensorrt_llm/serve/openai_protocol/tools.py
 tensorrt_llm/serve/openai_server.py
 tensorrt_llm/serve/postprocess_handlers.py
 tensorrt_llm/serve/responses_utils.py

pyproject.toml

@@ -474,13 +474,30 @@ exclude = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/serving.py",
+    "tensorrt_llm/serve/api/openai/embeddings/serving.py",
+    "tensorrt_llm/serve/api/openai/mm_encoder/serving.py",
+    "tensorrt_llm/serve/api/openai/tokenize/serving.py",
+    "tensorrt_llm/serve/api/visual_gen/protocol.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
     "tensorrt_llm/serve/harmony_adapter.py",
     "tensorrt_llm/serve/metadata_server.py",
     "tensorrt_llm/serve/openai_disagg_server.py",
-    "tensorrt_llm/serve/openai_protocol.py",
+    "tensorrt_llm/serve/openai_protocol/chat.py",
+    "tensorrt_llm/serve/openai_protocol/common.py",
+    "tensorrt_llm/serve/openai_protocol/completions.py",
+    "tensorrt_llm/serve/openai_protocol/control.py",
+    "tensorrt_llm/serve/openai_protocol/disagg.py",
+    "tensorrt_llm/serve/openai_protocol/embeddings.py",
+    "tensorrt_llm/serve/openai_protocol/kv_cache.py",
+    "tensorrt_llm/serve/openai_protocol/models.py",
+    "tensorrt_llm/serve/openai_protocol/responses.py",
+    "tensorrt_llm/serve/openai_protocol/sampling.py",
+    "tensorrt_llm/serve/openai_protocol/tokenize.py",
+    "tensorrt_llm/serve/openai_protocol/tools.py",
     "tensorrt_llm/serve/openai_server.py",
     "tensorrt_llm/serve/postprocess_handlers.py",
     "tensorrt_llm/serve/responses_utils.py",
@@ -862,6 +879,13 @@ files = [
     "tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py",
+    "tensorrt_llm/llmapi/engine_client.py",
+    "tensorrt_llm/serve/_base/routing.py",
+    "tensorrt_llm/serve/api/openai/embeddings/api_router.py",
+    "tensorrt_llm/serve/api/openai/mm_encoder/api_router.py",
+    "tensorrt_llm/serve/api/openai/tokenize/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/serving.py",
     "tests/unittest/_torch/sampler/test_torch_sampler.py",
     "tests/unittest/_torch/sampler/test_token_ban.py",
     "tests/unittest/_torch/sampler/test_beam_search.py",

ruff-legacy.toml

@@ -426,13 +426,30 @@ include = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/serving.py",
+    "tensorrt_llm/serve/api/openai/embeddings/serving.py",
+    "tensorrt_llm/serve/api/openai/mm_encoder/serving.py",
+    "tensorrt_llm/serve/api/openai/tokenize/serving.py",
+    "tensorrt_llm/serve/api/visual_gen/protocol.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
     "tensorrt_llm/serve/harmony_adapter.py",
     "tensorrt_llm/serve/metadata_server.py",
     "tensorrt_llm/serve/openai_disagg_server.py",
-    "tensorrt_llm/serve/openai_protocol.py",
+    "tensorrt_llm/serve/openai_protocol/chat.py",
+    "tensorrt_llm/serve/openai_protocol/common.py",
+    "tensorrt_llm/serve/openai_protocol/completions.py",
+    "tensorrt_llm/serve/openai_protocol/control.py",
+    "tensorrt_llm/serve/openai_protocol/disagg.py",
+    "tensorrt_llm/serve/openai_protocol/embeddings.py",
+    "tensorrt_llm/serve/openai_protocol/kv_cache.py",
+    "tensorrt_llm/serve/openai_protocol/models.py",
+    "tensorrt_llm/serve/openai_protocol/responses.py",
+    "tensorrt_llm/serve/openai_protocol/sampling.py",
+    "tensorrt_llm/serve/openai_protocol/tokenize.py",
+    "tensorrt_llm/serve/openai_protocol/tools.py",
     "tensorrt_llm/serve/openai_server.py",
     "tensorrt_llm/serve/postprocess_handlers.py",
     "tensorrt_llm/serve/responses_utils.py",

tensorrt_llm/_torch/pyexecutor/py_executor.py

@@ -14,8 +14,8 @@
 from contextlib import ExitStack, contextmanager, nullcontext
 from enum import IntEnum
 from queue import Queue
-from typing import (TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List,
-                    Literal, NoReturn, Optional, Tuple, Union)
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator,
+                    List, Literal, NoReturn, Optional, Tuple, Union)
 
 import torch
 from strenum import StrEnum
@@ -2749,6 +2749,19 @@ def get_openengine_discovery(self) -> dict:
         """Return immutable-at-startup OpenEngine node and publisher discovery."""
         return self._openengine_discovery
 
+    def set_openengine_discovery(self, discovery: dict) -> None:
+        """Record OpenEngine node and KV-event publisher discovery metadata.
+
+        Called once at startup by the OpenEngine node-discovery hook
+        (tensorrt_llm/grpc/openengine/node.py); get_openengine_discovery()
+        returns it.
+        """
+        self._openengine_discovery = discovery
+
+    def set_openengine_node_server(self, server: Any) -> None:
+        """Adopt the OpenEngine node metadata server; shutdown closes it."""
+        self._openengine_node_server = server
+
     def get_kv_cache_load(self) -> dict:
         """Return the scheduler-owned KV load snapshot.
 

tensorrt_llm/commands/AGENTS.md

@@ -0,0 +1,6 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+Rules for the trtllm-serve CLI (`commands/serve.py`) are in `tensorrt_llm/serve/AGENTS.md`.

tensorrt_llm/commands/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/commands/serve.py

@@ -553,8 +553,8 @@ def _bind_launcher_uds(llm) -> Tuple[socket.socket, str]:
     It lives in the multi-frontend ipc dir, which is private to the user and
     removed, socket file included, when the executor proxy shuts down.
     """
-    # Only the classic IPC proxy has the attribute.
-    ipc_dir = getattr(llm._executor, "_multi_frontend_ipc_dir", None)
+    # Only the classic IPC proxy in multi-frontend mode has one.
+    ipc_dir = llm.multi_frontend_ipc_dir
     if not ipc_dir:
         raise ValueError(
             "num_serve_frontends > 1 requires the classic IPC executor "
@@ -2659,7 +2659,7 @@ def _run_fleet_worker_impl():
     """Build and run one fleet worker inside its telemetry boundary."""
     _init_fleet_worker_process()
     server = _build_disagg_server_from_env()
-    host, port = server._config.hostname, server._config.port
+    host, port = server.address
 
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

tensorrt_llm/executor/executor.py

@@ -340,6 +340,20 @@ def is_shutdown(self) -> bool:
         """Return True if the executor is shutting down or fatally errored."""
         return self.doing_shutdown or self._fatal_error is not None
 
+    @property
+    def fatal_error(self) -> Optional[BaseException]:
+        """The unrecoverable engine error recorded by ``_set_fatal_error``, or None."""
+        return self._fatal_error
+
+    def record_fatal_error(self, error: BaseException) -> None:
+        """Record an unrecoverable engine error; the first recorded error wins (see ``fatal_error``)."""
+        self._set_fatal_error(error)
+
+    @property
+    def engine_dead(self) -> bool:
+        """Whether the engine is known dead (an unrecoverable error was recorded)."""
+        return self._fatal_error is not None
+
     def check_health(self) -> bool:
         """Check whether the executor is healthy and able to process requests.
 
@@ -408,6 +422,11 @@ def resource_governor_queue(self):
         """Return the resource governor queue if this executor supports it."""
         return None
 
+    @property
+    def multi_frontend_ipc_dir(self) -> Optional[str]:
+        """The multi-frontend IPC directory if this executor runs one, else None."""
+        return None
+
     @property
     def enable_postprocess_parallel(self) -> bool:
         return self.postproc_config.enabled

tensorrt_llm/executor/proxy.py

@@ -480,6 +480,15 @@ def check_health(self) -> bool:
 
         return True
 
+    @property
+    def engine_dead(self) -> bool:
+        """Whether the engine is known dead (a worker died or a fatal error was recorded).
+
+        ``_mark_engine_dead`` can set the sticky flag before the fatal error
+        is recorded, so check both.
+        """
+        return self._engine_dead or self._fatal_error is not None
+
     def _set_fatal_error(self, error: BaseException) -> None:
         """Record the fatal error, then unblock every pending request.
 
@@ -671,6 +680,19 @@ def _setup_queues(self) -> WorkerCommIpcAddrs:
             frontend_result_queue_addrs=frontend_result_addrs,
         )
 
+    @property
+    def multi_frontend_ipc_dir(self) -> Optional[str]:
+        """The launcher-owned directory of the multi-frontend IPC endpoints.
+
+        None unless multi-frontend mode is active. The launcher places its
+        own Unix sockets (the attached frontends' forwarding endpoint) in it;
+        the directory is private to the user and removed, socket files
+        included, when this proxy shuts down.
+        """
+        # GenerationExecutorFrontendProxy skips this class's __init__, so the
+        # attribute can be missing there; an attached frontend owns no ipc dir.
+        return getattr(self, "_multi_frontend_ipc_dir", None)
+
     def multi_frontend_attach_info(self) -> Optional[dict]:
         """The attach payload consumed by attached serving frontends.
 
@@ -1684,8 +1706,8 @@ def __init__(
         self.garbage_collection_gen0_threshold = None
         self.workers_started = False
         self.dispatch_result_thread: Optional[ManagedThread] = None
-        # Must be None: OpenAIServer reads the resource_governor_queue
-        # property at init; the governor lives with the launcher only.
+        # Must be None: LLM.resource_governor_queue reads this property at
+        # server init; the governor lives with the launcher only.
         self._resource_governor_queue = None
 
         hmac_key = bytes.fromhex(attach_info["hmac_key"])