From 68abdb3c3865da93456b705590258f0d1e4623a0 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:18:52 -0700 Subject: [PATCH 01/24] [None][feat] serve: multi-process HTTP frontends on the classic IPC executor path At high concurrency a trtllm-serve worker is host-bound on its single serving process: one asyncio event loop (one GIL) performs every request json.loads+pydantic validate and every SSE chunk write, and queuing on that loop dominates first-token latency while the GPUs idle (measured on DSv4 disagg GEN: gen_preprocessing p50 54ms@c512 -> 1520ms@c2048). vLLM/SGLang address the same limit with multiple API-server processes. TLLM_SERVE_NUM_FRONTENDS=K (env-gated, default off) runs K HTTP frontend processes against ONE executor on the default (classic IPC) orchestrator: - Launcher/attach split: frontend 0 builds the LLM and launches workers as usual, then spawns K-1 children that re-exec the command line with TLLM_EXECUTOR_ATTACH_INFO set; their executor attaches via the new GenerationExecutorFrontendProxy (no MPI session, no worker launch, and shutdown never emits the worker's None engine-shutdown sentinel -- the launcher alone owns the engine lifecycle). - Deterministic ipc endpoints, pre-generated by the launcher (TLLM_MULTI_FRONTEND_IPC_DIR/_HMAC): the rank0 worker BINDS the request ingress (PULL) so every frontend PUSH-connects; each frontend binds its own result lane (PULL) that the worker (non-postproc) or every postprocess worker (one PUSH pipe per frontend) sends to. - client-id namespacing: the top 16 bits of the uint64 client id carry the frontend id; responses are routed by client_id>>48. Responses without a usable client id (e.g. attention-DP dummy requests carry client_id=None) route to the launcher, preserving today's silent discard semantics. Frontend 0 keeps ids bit-identical to today. - All frontends bind the public port with SO_REUSEPORT, so the kernel load-balances accepted connections across their independent processes (and GILs); clients and the disagg orchestrator still see one URL. Known limitations (documented): /metrics and /perf_metrics are served per-frontend (SO_REUSEPORT samples one frontend per request); enable_resource_governor is rejected in multi-frontend mode. Validation: 12 CPU-only unit tests (id namespacing, response-lane bucketing incl. the ADP-dummy None guard, attached-proxy submit/cancel/ never-sends-sentinel over real ipc sockets). E2E A/B on DSv4-Pro disagg (11xCTX-DEP4 + 1xGEN-DEP16, c3120, back-to-back on identical nodes): gen_preprocessing p50 1075ms -> 37.6ms (-96.5%), p95 3145ms -> 75.7ms; per-request SSE speed p50 +30%; 16 frontends, 75 min, zero tracebacks; engine-side phases (gen_queue, kv_transfer) byte-identical to baseline. Ported to main from the feat/deepseek_v4-based branch (PR #16413), with review cleanups folded in: attached-frontend child env now uses split_mpi_env() (strips SLURM_/UCX_/PMI_ etc., not just 3 prefixes); TLLM_SERVE_NUM_FRONTENDS parsing/validation unified in get_num_serve_frontends(); result-lane selection unified in frontend_lane_index(); stale rpc_common docstring reference fixed. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 100 ++++++- tensorrt_llm/executor/base_worker.py | 47 +++- tensorrt_llm/executor/executor.py | 24 ++ tensorrt_llm/executor/postproc_worker.py | 45 ++- tensorrt_llm/executor/proxy.py | 198 ++++++++++++- tensorrt_llm/executor/utils.py | 98 +++++++ tensorrt_llm/executor/worker.py | 35 ++- tensorrt_llm/llmapi/llm.py | 8 +- .../executor/test_multi_frontend_routing.py | 261 ++++++++++++++++++ 9 files changed, 774 insertions(+), 42 deletions(-) create mode 100644 tests/unittest/executor/test_multi_frontend_routing.py diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 2f39353c521c..abd4718a1d2c 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -9,6 +9,7 @@ import socket import subprocess # nosec B404 import sys +import tempfile import uuid from pathlib import Path from typing import Any, Dict, Optional, Sequence, Set @@ -26,7 +27,8 @@ from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import LlmLauncherEnvs +from tensorrt_llm.executor.utils import (LlmLauncherEnvs, + get_num_serve_frontends) from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, @@ -37,7 +39,8 @@ validate_config_bool) from tensorrt_llm.llmapi.llm_args import MultimodalConfig, TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict -from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr +from tensorrt_llm.llmapi.mpi_session import (find_free_ipc_addr, + split_mpi_env) from tensorrt_llm.llmapi.reasoning_parser import (ReasoningParserFactory, resolve_auto_reasoning_parser) from tensorrt_llm.logger import logger, severity_map @@ -358,6 +361,61 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) +def _spawn_attached_frontends(llm, num_frontends: int) -> list: + """Spawn num_frontends - 1 attached serving frontend processes. + + Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1): each child + re-execs this trtllm-serve command line with env vars pointing at the + launcher executor's attach endpoints — the multi-frontend request + ingress plus per-frontend result lanes (see + GenerationExecutorProxy._setup_queues); the child's executor attaches to + the already-running worker instead of launching a new one (see + executor.py GenerationExecutor.create). All frontends bind the serving + port with SO_REUSEPORT, so the kernel load-balances accepted + connections across their independent processes (and GILs). + """ + from tensorrt_llm.executor.proxy import GenerationExecutorProxy + + executor = getattr(llm, "_executor", None) + if not isinstance(executor, GenerationExecutorProxy) or ( + attach_info := executor.multi_frontend_attach_info()) is None: + raise ValueError( + "TLLM_SERVE_NUM_FRONTENDS > 1 requires the classic IPC executor " + f"proxy in multi-frontend mode, got {type(executor).__name__}") + # mkstemp creates the file 0600: it carries the executor HMAC keys. + fd, attach_info_path = tempfile.mkstemp(prefix="trtllm_frontend_", + suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(attach_info, f) + + children = [] + for frontend_id in range(1, num_frontends): + # Attached frontends are plain RPC clients: keep them out of the + # launcher's MPI job (an inherited PMI/SLURM rank identity would + # make mpi4py try to (re-)join it at import time). + env, _ = split_mpi_env() + env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_info_path + env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) + env["TLLM_DISABLE_MPI"] = "1" + child = subprocess.Popen([sys.executable] + sys.argv, + env=env) # nosec B603 + children.append(child) + logger.info( + f"Launched attached serving frontend {frontend_id} (pid {child.pid})" + ) + return children + + +def _terminate_attached_frontends(children: list) -> None: + for child in children: + child.terminate() + for child in children: + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + + def launch_server( host: str, port: int, @@ -376,6 +434,30 @@ def launch_server( backend = llm_args["backend"] model = served_model_name or llm_args["model"] + + # Multi-frontend serving (prototype): TLLM_SERVE_NUM_FRONTENDS=K runs K + # HTTP frontend processes against ONE executor. The launcher (this + # process, frontend 0) launches the worker as usual and spawns K-1 + # attached frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) + # skip the spawning. Supported only on the classic IPC executor path + # (the default orchestrator). + num_frontends = get_num_serve_frontends() + is_attached_frontend = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None + multi_frontend = num_frontends > 1 or is_attached_frontend + if multi_frontend and not is_attached_frontend: + if llm_args.get("orchestrator_type") is not None: + raise ValueError( + "TLLM_SERVE_NUM_FRONTENDS > 1 currently supports only the " + "default (classic IPC) executor path, not orchestrator_type=" + f"{llm_args.get('orchestrator_type')!r}") + # Opt the launcher executor into multi-frontend mode BEFORE it is + # created: pre-generate the shared ipc directory and HMAC key so the + # launcher proxy, the rank0 worker and the attached frontends agree + # on deterministic endpoints (GenerationExecutorProxy._setup_queues). + os.environ["TLLM_MULTI_FRONTEND_IPC_DIR"] = tempfile.mkdtemp( + prefix="trtllm_frontends_") + os.environ["TLLM_MULTI_FRONTEND_HMAC"] = os.urandom(32).hex() + addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) address_family = socket.AF_INET6 if all( @@ -383,6 +465,10 @@ def launch_server( with socket.socket(address_family, socket.SOCK_STREAM) as s: # If disagg cluster config is provided and port is not specified, try to find a free port, otherwise try to bind to the specified port assert port > 0 or disagg_cluster_config is not None, "Port must be specified if disagg cluster config is not provided" + if multi_frontend: + # Every frontend process binds its own listening socket on the + # same port; the kernel load-balances accepts across them. + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) try: s.bind((host, port)) if port == 0: @@ -409,6 +495,10 @@ def launch_server( f"{backend} is not a known backend, check help for available options.", param_hint="backend") + frontend_children = [] + if multi_frontend and not is_attached_frontend: + frontend_children = _spawn_attached_frontends(llm, num_frontends) + server = OpenAIServer( generator=llm, model=model, @@ -427,7 +517,11 @@ def launch_server( if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": gc.disable() - uvloop.run(server(host, port, sockets=[s])) + try: + uvloop.run(server(host, port, sockets=[s])) + finally: + if frontend_children: + _terminate_attached_frontends(frontend_children) def launch_grpc_server(host: str, diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 670d6f30508f..c86355307866 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -49,6 +49,7 @@ from .result import (GenerationResult, LogProbsResult, ResponseWrapper, compute_logprobs, get_metrics_dict) from .utils import (ErrorResponse, IntraProcessQueue, RequestError, + bucket_responses_by_frontend, frontend_lane_index, is_llm_response) if TYPE_CHECKING: @@ -118,6 +119,9 @@ def __init__( self.engine = None self.result_queue: Optional[IpcQueue] = None self.postproc_queues: Optional[List[IpcQueue]] = None + # Multi-frontend serving: one result lane per frontend process, + # selected by the frontend id in client_id's top bits. + self.frontend_result_queues: Optional[List[IpcQueue]] = None self.rank = mpi_rank() self.global_rank = global_mpi_rank() # mapping: client_id -> GenerationResult @@ -272,13 +276,24 @@ def fetch_kv_cache_events(self) -> list: def set_result_queue(self, queue): """In multi-gpu mode, result_queue will be set here to communicate between the proxy and the worker 0 process.""" assert self.postproc_queues is None + assert self.frontend_result_queues is None self.result_queue = queue def set_postproc_queues(self, queues: List["IpcQueue"]): """ Set the IPC queues for feeding post-processing processes. """ assert self.result_queue is None + assert self.frontend_result_queues is None self.postproc_queues = queues + def set_frontend_result_queues(self, queues: List["IpcQueue"]): + """Multi-frontend serving: one result lane per frontend process. + + The lane is selected by the frontend id in client_id's top bits. + """ + assert self.result_queue is None + assert self.postproc_queues is None + self.frontend_result_queues = queues + def _set_iteration_result_queue(self, it_result_queue: IterationResultQueue, queue: Union[Queue, FusedIpcQueue, IntraProcessQueue]): @@ -1087,20 +1102,20 @@ def responses_handler(self, responses: List[tllm.Response]): HandlerKind = AwaitResponseHelper.HandlerKind if self.handler_kind is HandlerKind.unknown: - if not (self.worker.result_queue is not None - or self.worker.postproc_queues is not None): + has_ipc_queues = (self.worker.result_queue is not None + or self.worker.postproc_queues is not None + or self.worker.frontend_result_queues is not None) + if not has_ipc_queues: logger_debug(f"creating await_response helper for Worker\n", color="yellow") # When ExecutorBindingWorker is used in the main process # aka the single process mode self.handler_kind = HandlerKind.single_process_worker - elif self.worker.result_queue is not None or self.worker.postproc_queues is not None: + else: # The ExecutorBindingProxy is used logger_debug(f"creating await_response helper for IPC\n", color="yellow") self.handler_kind = HandlerKind.ipc_batched - else: - raise NotImplementedError match self.handler_kind: case HandlerKind.single_process_worker: @@ -1279,7 +1294,15 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: self.worker.postproc_queues[wid].put(batch) if rsp_batch: - self.worker.result_queue.put(rsp_batch) + if (lanes := self.worker.frontend_result_queues) is not None: + # Multi-frontend serving: route each response to its origin + # frontend's lane by the id in client_id's top bits. + for frontend_id, sub_batch in enumerate( + bucket_responses_by_frontend(rsp_batch, len(lanes))): + if sub_batch: + lanes[frontend_id].put(sub_batch) + else: + self.worker.result_queue.put(rsp_batch) def _get_params_for_first_rsp( @@ -1393,7 +1416,17 @@ def _send_rsp( rsp_batch: Optional[List[tllm.Response]] = None): # if postproc_batches is set, append to batch instead of putting to IpcQueue - if worker.result_queue is not None: + if worker.frontend_result_queues is not None: + # Multi-frontend serving: route to the origin frontend's result lane + # (client_id may be None for e.g. ADP dummy requests -- those go to + # lane 0, the launcher, which silently discards them like today). + if rsp_batch is not None: + rsp_batch.append(response) + else: + lanes = worker.frontend_result_queues + lanes[frontend_lane_index(getattr(response, "client_id", None), + len(lanes))].put(response) + elif worker.result_queue is not None: if rsp_batch is not None: rsp_batch.append(response) else: diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index ba5b843ff347..6ee50e245917 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -1,6 +1,8 @@ import atexit import faulthandler +import json import multiprocessing +import os import platform import signal import traceback @@ -560,6 +562,28 @@ def create( f"Using {postproc_worker_config.num_postprocess_workers} postprocess parallel processes.\n", "green") + # Multi-frontend serving: attach to an already-running executor + # instead of launching one. Set by trtllm-serve for the attached + # frontend processes (see commands/serve.py); the frontend id is + # picked up from TLLM_EXECUTOR_FRONTEND_ID. + attach_info_path = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") + if attach_info_path: + with open(attach_info_path) as f: + attach_info = json.load(f) + if attach_info.get("mode") != "classic": + raise ValueError( + "TLLM_EXECUTOR_ATTACH_INFO only supports the classic IPC " + f"executor path, got mode={attach_info.get('mode')!r}") + from .proxy import GenerationExecutorFrontendProxy + frontend_id = int(os.environ["TLLM_EXECUTOR_FRONTEND_ID"]) + logger.info(f"Attaching executor frontend {frontend_id} to the " + f"running classic IPC worker via {attach_info_path}") + return GenerationExecutorFrontendProxy( + attach_info, + frontend_id=frontend_id, + postproc_worker_config=postproc_worker_config, + is_llm_executor=is_llm_executor) + worker_kwargs = { "engine": engine, "executor_config": executor_config, diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py index ece60b2374b1..5af13a36bc75 100644 --- a/tensorrt_llm/executor/postproc_worker.py +++ b/tensorrt_llm/executor/postproc_worker.py @@ -15,7 +15,7 @@ from ..sampling_params import SamplingParams from .ipc import ZeroMqQueue from .postprocessor_hook import load_post_processor_hook -from .utils import ErrorResponse, is_llm_response +from .utils import ErrorResponse, bucket_responses_by_frontend, is_llm_response if TYPE_CHECKING: from ..disaggregated_params import DisaggregatedParams @@ -86,7 +86,8 @@ class Output(NamedTuple): def __init__( self, pull_pipe_addr: tuple[str, Optional[bytes]], - push_pipe_addr: tuple[str, Optional[bytes]], + push_pipe_addr: Union[tuple[str, Optional[bytes]], + List[tuple[str, Optional[bytes]]]], tokenizer_dir: str, record_creator: Callable[ ["PostprocWorker.Input", TransformersTokenizer], Any], @@ -95,7 +96,10 @@ def __init__( ''' Args: pull_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the input IPC. - push_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the output IPC. + push_pipe_addr: The address and HMAC key of the output IPC, or a + list of them with multi-frontend serving -- one result lane + per frontend, selected by the frontend id in the output's + client_id top bits. tokenizer_dir (str): The directory to load tokenizer. record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request. result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result. @@ -108,11 +112,16 @@ def __init__( is_async=True, is_server=False, name="postprocess_pull_pipe") - self._push_pipe = ZeroMqQueue(address=push_pipe_addr, - is_async=True, - is_server=False, - socket_type=zmq.PUSH, - name="postprocess_push_pipe") + push_pipe_addrs = (push_pipe_addr if isinstance(push_pipe_addr, list) + else [push_pipe_addr]) + self._push_pipes = [ + ZeroMqQueue(address=addr, + is_async=True, + is_server=False, + socket_type=zmq.PUSH, + name=f"postprocess_push_pipe_{i}") + for i, addr in enumerate(push_pipe_addrs) + ] self._to_stop = asyncio.Event() self._q = deque() @@ -192,11 +201,21 @@ async def _batched_put(self): ''' Batched IPC send. ''' async for batch in self._mainloop(): if batch is None: - # notify dispatch_result corountine to quit - await self._push_pipe.put_async(None) + # notify the dispatch_result coroutine in every frontend to + # quit + for pipe in self._push_pipes: + await pipe.put_async(None) break assert isinstance(batch, list) - await self._push_pipe.put_async(batch) + if len(self._push_pipes) == 1: + await self._push_pipes[0].put_async(batch) + continue + # Multi-frontend serving: route each output to its origin + # frontend's result lane by the id in client_id's top bits. + for frontend_id, sub_batch in enumerate( + bucket_responses_by_frontend(batch, len(self._push_pipes))): + if sub_batch: + await self._push_pipes[frontend_id].put_async(sub_batch) async def _mainloop(self): ''' The loop for handle_response and keep producing outputs. ''' @@ -289,7 +308,9 @@ async def main(): @print_traceback_on_error def postproc_worker_main(feedin_ipc_addr: tuple[str, Optional[bytes]], - feedout_ipc_addr: tuple[str, Optional[bytes]], + feedout_ipc_addr: Union[tuple[str, Optional[bytes]], + List[tuple[str, + Optional[bytes]]]], tokenizer_dir: str, record_creator: Callable, post_processor_hook: Optional[str] = None): diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 4850bfcf2f5d..5e43d9f8bf0a 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -43,13 +43,16 @@ from .rpc.rpc_common import RPCError, get_unique_ipc_addr from .utils import (EngineDeadError, ErrorResponse, RequestError, WorkerCommIpcAddrs, create_mpi_comm_session, + get_multi_frontend_ipc_info, get_num_serve_frontends, get_spawn_proxy_process_env, is_llm_response, - print_alive_threads) + multi_frontend_request_addr, multi_frontend_result_addr, + namespace_client_id, print_alive_threads) from .worker import GenerationExecutorWorker, worker_main from .worker_process_monitor import WorkerProcessIdentity, WorkerProcessMonitor __all__ = [ "GenerationExecutorProxy", + "GenerationExecutorFrontendProxy", ] # Methods that are explicitly implemented for multi-rank MPI/IPC executor @@ -161,6 +164,21 @@ def __init__( self._enable_resource_governor = bool( getattr(_llm_args, "enable_resource_governor", False)) + # Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1) on the classic + # IPC path: trtllm-serve (the launcher, frontend 0) pre-generates a + # shared ipc directory + HMAC key so this proxy, the rank0 worker and + # the attached frontends agree on deterministic endpoints (see + # _setup_queues). Attached frontends never reach this class -- they + # construct GenerationExecutorFrontendProxy instead. + self._multi_frontend_info = get_multi_frontend_ipc_info() + self._num_frontends = get_num_serve_frontends( + ) if self._multi_frontend_info is not None else 1 + if self._num_frontends > 1 and self._enable_resource_governor: + raise ValueError( + "Multi-frontend serving does not support " + "enable_resource_governor: the resource-governor signal only " + "reaches the launcher frontend.") + # Generate RPC address and key for stats RPC self.rpc_addr = get_unique_ipc_addr() self.hmac_key = os.urandom(32) @@ -395,34 +413,88 @@ def _error_monitor_loop(self) -> None: self._shutdown_event.wait(timeout=5.0) def _setup_queues(self) -> WorkerCommIpcAddrs: - - self.request_queue = IpcQueue(is_server=True, - name="proxy_request_queue") + frontend_result_addrs = None + if self._num_frontends > 1: + # Multi-frontend serving: deterministic endpoints shared with the + # attached frontends. The rank0 worker BINDS the request ingress + # (PULL) so every frontend can PUSH-connect; each frontend + # (including this launcher, frontend 0) binds its own result lane + # (PULL) that the worker / postproc processes PUSH-connect to, + # selected by the frontend id in client_id's top bits. + ipc_dir, hmac_key = self._multi_frontend_info + request_addr = (multi_frontend_request_addr(ipc_dir), hmac_key) + frontend_result_addrs = [(multi_frontend_result_addr(ipc_dir, + i), hmac_key) + for i in range(self._num_frontends)] + self.request_queue = IpcQueue(request_addr, + is_server=False, + socket_type=zmq.PUSH, + name="proxy_request_queue") + self.result_queue = FusedIpcQueue(frontend_result_addrs[0], + is_server=True, + fuse_message=False, + socket_type=zmq.PULL, + name="proxy_result_queue") + else: + request_addr = None + self.request_queue = IpcQueue(is_server=True, + name="proxy_request_queue") + # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue + # Use PULL mode when enable_postprocess_parallel as there are + # multiple senders from multiple processes. + self.result_queue = FusedIpcQueue( + is_server=True, + fuse_message=False, + socket_type=zmq.PULL + if self.enable_postprocess_parallel else zmq.PAIR, + name="proxy_result_queue") self.worker_init_status_queue = IpcQueue( is_server=True, socket_type=zmq.ROUTER, name="worker_init_status_queue") - # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue - # Use PULL mode when enable_postprocess_parallel as there are - # multiple senders from multiple processes. - self.result_queue = FusedIpcQueue( - is_server=True, - fuse_message=False, - socket_type=zmq.PULL - if self.enable_postprocess_parallel else zmq.PAIR, - name="proxy_result_queue") self._resource_governor_queue = IpcQueue( is_server=True, name="proxy_resource_governor_queue" ) if self._enable_resource_governor else None # Stats and KV events are now fetched via RPC, not IPC queues. return WorkerCommIpcAddrs( - request_queue_addr=self.request_queue.address, + # A connect-mode queue has no bound .address; use the preset one. + request_queue_addr=request_addr + if request_addr is not None else self.request_queue.address, worker_init_status_queue_addr=self.worker_init_status_queue.address, result_queue_addr=self.result_queue.address, resource_governor_queue_addr=self._resource_governor_queue.address if self._resource_governor_queue is not None else None, + frontend_result_queue_addrs=frontend_result_addrs, ) + def multi_frontend_attach_info(self) -> Optional[dict]: + """The attach payload consumed by attached serving frontends. + + See GenerationExecutorFrontendProxy and commands/serve.py. Returns + None unless multi-frontend mode is active. + """ + if self._num_frontends <= 1: + return None + ipc_dir, hmac_key = self._multi_frontend_info + return { + "mode": + "classic", + "request_addr": + multi_frontend_request_addr(ipc_dir), + "result_addrs": [ + multi_frontend_result_addr(ipc_dir, i) + for i in range(self._num_frontends) + ], + "hmac_key": + hmac_key.hex(), + # Stats / KV events / disagg params RPC endpoint on the rank0 + # worker (ROUTER socket, natively multi-client). + "rpc_addr": + self.rpc_addr, + "rpc_hmac_key": + self.hmac_key.hex(), + } + @property def resource_governor_queue(self): return self._resource_governor_queue @@ -911,3 +983,101 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.shutdown() return False # propagate the exception + + +class GenerationExecutorFrontendProxy(GenerationExecutorProxy): + """An attached serving frontend for the classic IPC executor path. + + Used for multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1). + PUSH-connects to the request ingress bound by the rank0 worker and binds + its own per-frontend result lane (PULL); the worker routes responses to + this lane by the frontend id embedded in the top bits of client_id (see + utils.namespace_client_id). It never owns the engine: no MPI + session, no worker launch, and shutdown never emits the worker's None + shutdown sentinel -- that right is the launcher frontend's alone. + """ + + def __init__( + self, + attach_info: dict, + *, + frontend_id: int, + postproc_worker_config: Optional[PostprocWorkerConfig] = None, + is_llm_executor: Optional[bool] = None, + ) -> None: + if not 0 < frontend_id < (1 << 16): + raise ValueError(f"frontend_id out of range: {frontend_id}") + if frontend_id >= len(attach_info["result_addrs"]): + raise ValueError( + f"frontend_id {frontend_id} has no result lane: only " + f"{len(attach_info['result_addrs'])} lanes were provisioned") + postproc_worker_config = postproc_worker_config or PostprocWorkerConfig( + ) + # Deliberately skip GenerationExecutorProxy.__init__: it creates an + # MPI session, launches workers, and registers the pre_shutdown + # atexit hook that emits the engine shutdown sentinel. + GenerationExecutor.__init__( + self, + num_postprocess_workers=postproc_worker_config. + num_postprocess_workers, + postprocess_tokenizer_dir=postproc_worker_config. + postprocess_tokenizer_dir, + is_llm_executor=is_llm_executor) + + self._frontend_id = frontend_id + self._results: Dict[int, GenerationResult] = {} + self.garbage_collection_gen0_threshold = None + self.workers_started = False + self.dispatch_result_thread: Optional[ManagedThread] = None + # The resource governor lives with the launcher frontend only; the + # inherited resource_governor_queue property must return None here so + # OpenAIServer takes its governor-disabled path (openai_server.py). + self._resource_governor_queue = None + + hmac_key = bytes.fromhex(attach_info["hmac_key"]) + self.request_queue = IpcQueue( + (attach_info["request_addr"], hmac_key), + is_server=False, + socket_type=zmq.PUSH, + name=f"frontend_{frontend_id}_request_queue") + self.result_queue = FusedIpcQueue( + (attach_info["result_addrs"][frontend_id], hmac_key), + is_server=True, + fuse_message=False, + socket_type=zmq.PULL, + name=f"frontend_{frontend_id}_result_queue") + + # Stats / KV events / disagg params share the rank0 worker's stats + # RPC server with the launcher (ROUTER socket, natively + # multi-client; per-frontend sampling). + self.rpc_client: Optional[RPCClient] = None + if attach_info.get("rpc_addr"): + self.rpc_client = RPCClient(attach_info["rpc_addr"], + hmac_key=bytes.fromhex( + attach_info["rpc_hmac_key"])) + + def _get_next_client_id(self) -> int: + # Embed the frontend id in the top bits so the worker routes the + # responses back to this frontend's result lane. + return namespace_client_id(self._frontend_id, + super()._get_next_client_id()) + + def pre_shutdown(self): + if self.doing_shutdown: + return + self.doing_shutdown = True + # Abort this frontend's in-flight requests so the engine frees their + # slots. Never send the None engine-shutdown sentinel: the launcher + # frontend owns the engine lifecycle (see the class docstring). + self._abort_all_requests() + + def shutdown(self): + self.pre_shutdown() + if self.rpc_client is not None: + self.rpc_client.close() + self.rpc_client = None + # The dispatch thread blocks on result_queue.get(); it is a daemon + # ManagedThread that exits with the process or on the worker's + # per-lane None sentinel at engine teardown. Closing its socket from + # another thread is not ZMQ-safe, so leave the queues to process + # teardown. diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 3bd0106241f8..bf93b81a6ae3 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -229,6 +229,104 @@ class WorkerCommIpcAddrs(NamedTuple): worker_init_status_queue_addr: tuple[str, Optional[bytes]] result_queue_addr: tuple[str, Optional[bytes]] resource_governor_queue_addr: Optional[tuple[str, Optional[bytes]]] = None + # Multi-frontend serving (classic IPC path): one result lane per frontend + # process, selected by the frontend id in client_id's top bits. When set, + # the rank0 worker BINDS the request queue (PULL) so every frontend can + # PUSH-connect, and routes responses to these lanes instead of + # result_queue_addr (which then aliases lane 0, the launcher's). + frontend_result_queue_addrs: Optional[list[tuple[str, + Optional[bytes]]]] = None + + +# Multi-frontend client_id namespacing: the top bits of the uint64 client id +# carry the frontend id, the low FRONTEND_ID_SHIFT bits carry the per-frontend +# request counter. Frontend id 0 keeps client ids bit-identical to the legacy +# single-frontend scheme. +FRONTEND_ID_SHIFT = 48 +FRONTEND_COUNTER_MASK = (1 << FRONTEND_ID_SHIFT) - 1 + + +def get_frontend_id(client_id: Optional[int]) -> int: + """Extract the originating frontend id from a namespaced client id.""" + if not isinstance(client_id, int): + return 0 + return client_id >> FRONTEND_ID_SHIFT + + +def namespace_client_id(frontend_id: int, client_id: int) -> int: + """Embed frontend_id in the top bits of a per-frontend client id.""" + return (frontend_id << FRONTEND_ID_SHIFT) | (client_id + & FRONTEND_COUNTER_MASK) + + +def frontend_lane_index(client_id: Optional[int], num_lanes: int) -> int: + """The result-lane index for a response's originating frontend. + + Responses without a usable client_id (e.g. ADP dummy requests carry + client_id=None) and ids with an out-of-range frontend go to lane 0 + (the launcher), matching legacy single-client visibility where such + responses are silently discarded by the launcher's dispatcher. + """ + frontend_id = get_frontend_id(client_id) + return frontend_id if frontend_id < num_lanes else 0 + + +def bucket_responses_by_frontend(responses: list, + num_frontends: int) -> list[list]: + """Bucket responses by their originating frontend id (client_id top bits). + + Lane selection (including the route-to-launcher fallback) follows + frontend_lane_index. + """ + buckets = [[] for _ in range(num_frontends)] + for rsp in responses: + buckets[frontend_lane_index(getattr(rsp, "client_id", None), + num_frontends)].append(rsp) + return buckets + + +def get_num_serve_frontends() -> int: + """The number of serving frontend processes (TLLM_SERVE_NUM_FRONTENDS). + + Single source for parsing and range-validating the env knob; used by + trtllm-serve (the launcher) and GenerationExecutorProxy so they always + agree on the frontend count. 1 (single frontend) when unset. + """ + num_frontends = int(os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") or "1") + if not 0 < num_frontends <= (1 << 16): + raise ValueError( + f"TLLM_SERVE_NUM_FRONTENDS out of range: {num_frontends}") + return num_frontends + + +def get_multi_frontend_ipc_info() -> Optional[tuple[str, bytes]]: + """The shared ipc dir and HMAC key for multi-frontend serving. + + Pre-generated by trtllm-serve (the launcher) on the classic IPC executor + path before the executor is created. None outside that mode. + """ + ipc_dir = os.getenv("TLLM_MULTI_FRONTEND_IPC_DIR") + hmac_hex = os.getenv("TLLM_MULTI_FRONTEND_HMAC") + if ipc_dir and hmac_hex: + return ipc_dir, bytes.fromhex(hmac_hex) + return None + + +def multi_frontend_request_addr(ipc_dir: str) -> str: + """The request ingress endpoint bound by the rank0 worker (PULL). + + Every frontend PUSH-connects to it. + """ + return f"ipc://{os.path.join(ipc_dir, 'request.sock')}" + + +def multi_frontend_result_addr(ipc_dir: str, frontend_id: int) -> str: + """The result lane endpoint bound by frontend ``frontend_id`` (PULL). + + The worker/postproc processes PUSH-connect to it. Deterministic so a + respawned process can rebind the same lane. + """ + return f"ipc://{os.path.join(ipc_dir, f'result_{frontend_id}.sock')}" def is_llm_response(instance): diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index 5958d296e1c4..1d876ee29e9a 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -221,6 +221,10 @@ def _print_stacks(): postproc_worker_config = postproc_worker_config or PostprocWorkerConfig() is_leader: bool = mpi_rank() == 0 + # Multi-frontend serving (classic IPC path): per-frontend result lanes; + # the request queue is bound here (PULL) so every frontend PUSH-connects. + multi_frontend_addrs = worker_queues.frontend_result_queue_addrs + frontend_result_queues: Optional[List[FusedIpcQueue]] = None if tracer_init_kwargs is not None and is_leader: tracer = VizTracer(**tracer_init_kwargs) tracer.register_exit() @@ -238,7 +242,9 @@ def _print_stacks(): # inherit the log level from "TLLM_LOG_LEVEL" environment variable logger.set_level(log_level) request_queue = IpcQueue(worker_queues.request_queue_addr, - is_server=False, + is_server=multi_frontend_addrs is not None, + socket_type=zmq.PULL if multi_frontend_addrs + is not None else zmq.PAIR, name="worker_request_queue") worker_init_status_queue = IpcQueue( worker_queues.worker_init_status_queue_addr, @@ -260,6 +266,18 @@ def _print_stacks(): name=f"postprocess_{i}_feedin_queue") for i in range(postproc_worker_config.num_postprocess_workers) ] + elif multi_frontend_addrs is not None: + # Multi-frontend serving: one PUSH lane per frontend, selected by + # the frontend id in client_id's top bits (see base_worker + # _send_rsp). + frontend_result_queues = [ + FusedIpcQueue(addr, + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name=f"worker_result_queue_{i}") + for i, addr in enumerate(multi_frontend_addrs) + ] else: # IPC queue for sending results back to the proxy, and let the # Proxy process to handle the postprocess @@ -269,9 +287,12 @@ def _print_stacks(): name="worker_result_queue") def notify_proxy_threads_to_quit(): - # Signal the dispatcher thread in the proxy to quit + # Signal the dispatcher thread in every frontend proxy to quit if result_queue is not None: result_queue.put(None) + elif frontend_result_queues is not None: + for q in frontend_result_queues: + q.put(None) else: assert result_queues is not None for q in result_queues: @@ -281,13 +302,15 @@ def notify_proxy_threads_to_quit(): if is_leader and postproc_worker_config.enabled: logger_debug(f"initiate postprocess workers...", "yellow") - proxy_result_queue: tuple[ - str, Optional[bytes]] = worker_queues.result_queue_addr + # With multi-frontend serving each postproc worker gets every + # frontend's result lane and routes outputs by client_id's top bits. + proxy_result_queue = (multi_frontend_addrs if multi_frontend_addrs + is not None else worker_queues.result_queue_addr) assert result_queues is not None postproc_worker_pool = ProcessPoolExecutor( max_workers=postproc_worker_config.num_postprocess_workers) - assert isinstance(proxy_result_queue, tuple) + assert isinstance(proxy_result_queue, (tuple, list)) for i in range(postproc_worker_config.num_postprocess_workers): fut = postproc_worker_pool.submit( postproc_worker_main, @@ -349,6 +372,8 @@ def notify_proxy_threads_to_quit(): if is_leader: if postproc_worker_config.enabled: worker.set_postproc_queues(result_queues) + elif frontend_result_queues is not None: + worker.set_frontend_result_queues(frontend_result_queues) else: worker.set_result_queue(result_queue) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 5d19670b0991..f1bdf1628128 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -313,7 +313,13 @@ def __init__(self, load_post_processor_hook(_post_processor_path) if _post_processor_path else None) - if self.args.parallel_config.is_multi_gpu: + # Attached serving frontends (TLLM_EXECUTOR_ATTACH_INFO) connect to + # an already-running executor worker: they need no MPI session of + # their own and must not spawn one (see executor.py + # GenerationExecutor.create). + is_attached_frontend = os.getenv( + "TLLM_EXECUTOR_ATTACH_INFO") is not None + if self.args.parallel_config.is_multi_gpu and not is_attached_frontend: if os.getenv("RAY_LOCAL_WORLD_SIZE") is None and get_device_count( ) < self.args.parallel_config.world_size_per_node: raise RuntimeError( diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py new file mode 100644 index 000000000000..2258a7b98f47 --- /dev/null +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU-only tests for classic-path multi-frontend serving. + +Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS) on the classic IPC +executor path. Covers client-id namespacing, worker-side response-lane routing, and the +attached-frontend proxy lifecycle over real ipc:// sockets. +""" + +import os +import tempfile +import time +from types import SimpleNamespace + +from tensorrt_llm.executor.utils import ( + FRONTEND_COUNTER_MASK, + bucket_responses_by_frontend, + frontend_lane_index, + get_frontend_id, + get_num_serve_frontends, + namespace_client_id, +) + + +class TestClientIdNamespacing: + def test_frontend_zero_keeps_legacy_ids(self): + for client_id in (1, 42, FRONTEND_COUNTER_MASK): + assert namespace_client_id(0, client_id) == client_id + + def test_roundtrip(self): + for frontend_id in (0, 1, 7, (1 << 16) - 1): + for counter in (1, 12345, FRONTEND_COUNTER_MASK): + client_id = namespace_client_id(frontend_id, counter) + assert get_frontend_id(client_id) == frontend_id + assert client_id & FRONTEND_COUNTER_MASK == counter + assert client_id < (1 << 64) + + def test_counter_wraparound_stays_in_namespace(self): + # A counter larger than 48 bits must not leak into the frontend bits. + client_id = namespace_client_id(3, FRONTEND_COUNTER_MASK + 5) + assert get_frontend_id(client_id) == 3 + assert client_id & FRONTEND_COUNTER_MASK == 4 + + def test_non_int_client_id_routes_to_launcher(self): + assert get_frontend_id(None) == 0 + + def test_lane_index_clamps_to_launcher(self): + assert frontend_lane_index(namespace_client_id(2, 7), 3) == 2 + assert frontend_lane_index(namespace_client_id(7, 1), 2) == 0 + assert frontend_lane_index(None, 4) == 0 + + +class TestNumServeFrontendsEnv: + def test_defaults_to_one(self, monkeypatch): + monkeypatch.delenv("TLLM_SERVE_NUM_FRONTENDS", raising=False) + assert get_num_serve_frontends() == 1 + monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", "") + assert get_num_serve_frontends() == 1 + + def test_parses_value(self, monkeypatch): + monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", "16") + assert get_num_serve_frontends() == 16 + + def test_rejects_out_of_range(self, monkeypatch): + import pytest + for bad in ("0", "-1", str((1 << 16) + 1)): + monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", bad) + with pytest.raises(ValueError): + get_num_serve_frontends() + + +def _response(client_id): + return SimpleNamespace(client_id=client_id) + + +class TestClassicResponseBucketing: + """The classic IPC path routes batches with bucket_responses_by_frontend.""" + + def test_buckets_by_namespace(self): + responses = [ + _response(namespace_client_id(0, 1)), + _response(namespace_client_id(1, 1)), + _response(namespace_client_id(1, 2)), + _response(namespace_client_id(2, 1)), + ] + buckets = bucket_responses_by_frontend(responses, 3) + assert [len(b) for b in buckets] == [1, 2, 1] + assert all(get_frontend_id(r.client_id) == 1 for r in buckets[1]) + + def test_none_client_id_routes_to_launcher(self): + # ADP dummy responses carry client_id=None: they must land in the + # launcher's lane (which silently discards them), not raise. + buckets = bucket_responses_by_frontend([_response(None)], 4) + assert len(buckets[0]) == 1 + assert all(not b for b in buckets[1:]) + + def test_out_of_range_frontend_routes_to_launcher(self): + buckets = bucket_responses_by_frontend([_response(namespace_client_id(7, 1))], 2) + assert len(buckets[0]) == 1 and not buckets[1] + + +class _LaneStub: + def __init__(self): + self.items = [] + + def put(self, obj): + self.items.append(obj) + + +class TestClassicSendRspLaneRouting: + """_send_rsp selects the origin frontend's lane on the non-postproc path.""" + + @staticmethod + def _fake_worker(num_lanes): + pops = [] + return SimpleNamespace( + result_queue=None, + postproc_queues=None, + frontend_result_queues=[_LaneStub() for _ in range(num_lanes)], + _pop_result=pops.append, + ), pops + + def test_error_response_routes_to_origin_lane(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, pops = self._fake_worker(3) + client_id = namespace_client_id(2, 7) + _send_rsp(worker, ErrorResponse(client_id, "boom", 1)) + assert [len(q.items) for q in worker.frontend_result_queues] == [0, 0, 1] + assert pops == [client_id] + + def test_none_client_id_routes_to_launcher_lane(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, _ = self._fake_worker(2) + _send_rsp(worker, ErrorResponse(None, "adp dummy", 1)) + assert len(worker.frontend_result_queues[0].items) == 1 + assert not worker.frontend_result_queues[1].items + + def test_rsp_batch_defers_lane_selection(self): + from tensorrt_llm.executor.base_worker import _send_rsp + from tensorrt_llm.executor.utils import ErrorResponse + + worker, _ = self._fake_worker(2) + rsp_batch = [] + _send_rsp(worker, ErrorResponse(namespace_client_id(1, 3), "x", 1), rsp_batch=rsp_batch) + assert len(rsp_batch) == 1 + assert all(not q.items for q in worker.frontend_result_queues) + + +class TestClassicFrontendProxyEndToEnd: + """GenerationExecutorFrontendProxy against a fake rank0 worker. + + Real ipc:// sockets: namespaced submit, cancel-on-shutdown, and -- + critically -- that an attached frontend NEVER emits the None + engine-shutdown sentinel. + """ + + @staticmethod + def _make_proxy_and_fake_worker(tmpdir, frontend_id=1, num_frontends=2): + import zmq + + from tensorrt_llm.executor.ipc import IpcQueue + from tensorrt_llm.executor.proxy import GenerationExecutorFrontendProxy + + hmac_key = os.urandom(32) + request_addr = f"ipc://{os.path.join(tmpdir, 'request.sock')}" + result_addrs = [ + f"ipc://{os.path.join(tmpdir, f'result_{i}.sock')}" for i in range(num_frontends) + ] + # The fake rank0 worker binds the request ingress (PULL), exactly + # like worker_main does in multi-frontend mode. + worker_ingress = IpcQueue( + (request_addr, hmac_key), + is_server=True, + socket_type=zmq.PULL, + name="fake_worker_request_queue", + ) + proxy = GenerationExecutorFrontendProxy( + { + "mode": "classic", + "request_addr": request_addr, + "result_addrs": result_addrs, + "hmac_key": hmac_key.hex(), + }, + frontend_id=frontend_id, + ) + return proxy, worker_ingress, hmac_key, result_addrs + + def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): + from tensorrt_llm.executor.request import CancellingRequest, GenerationRequest + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, worker_ingress, _, _ = self._make_proxy_and_fake_worker(tmpdir) + + # Attributes read by OpenAIServer at init must exist (a missing + # _resource_governor_queue crashed all siblings in the first e2e). + assert proxy.resource_governor_queue is None + + result = proxy.submit(GenerationRequest([1, 2, 3], SamplingParams())) + assert get_frontend_id(result.request_id) == 1 + assert worker_ingress.poll(5) + received = worker_ingress.get() + assert isinstance(received, GenerationRequest) + assert received.id == result.request_id + + # Frontend shutdown aborts its in-flight requests (cancel) but + # must NOT emit the None engine-shutdown sentinel. + proxy.shutdown() + assert worker_ingress.poll(5) + cancel = worker_ingress.get() + assert isinstance(cancel, CancellingRequest) + assert cancel.id == result.request_id + assert not worker_ingress.poll(1), ( + "an attached frontend must never send the engine-shutdown sentinel" + ) + + def test_dispatch_routes_own_lane_responses(self): + import zmq + + from tensorrt_llm.executor.ipc import FusedIpcQueue + from tensorrt_llm.executor.request import GenerationRequest + from tensorrt_llm.executor.utils import ErrorResponse + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, _, hmac_key, result_addrs = self._make_proxy_and_fake_worker(tmpdir) + result = proxy.submit(GenerationRequest([1, 2, 3], SamplingParams())) + client_id = result.request_id + assert client_id in proxy._results + + # The fake worker pushes an ErrorResponse down this frontend's + # result lane; the dispatcher must deliver it and retire the + # request. + worker_lane = FusedIpcQueue( + (result_addrs[1], hmac_key), + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name="fake_worker_result_lane", + ) + worker_lane.put(ErrorResponse(client_id, "boom", 1)) + deadline = time.time() + 5 + while client_id in proxy._results and time.time() < deadline: + time.sleep(0.01) + assert client_id not in proxy._results From 06dbbd3b5f94da5d5676107324749fb29fb21f74 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:56:12 -0700 Subject: [PATCH 02/24] [None][fix] serve multi-frontend: clean up ipc artifacts and scope the env gate Review follow-ups on the multi-frontend prototype: - The attach-info JSON (mkstemp, carries the executor HMAC keys in hex) and the shared ipc directory (mkdtemp, holds the request/result .sock files) were never removed: every multi-frontend run leaked a secret-bearing file plus a directory in /tmp. launch_server's finally now unlinks the attach file and rmtree's the ipc dir after the attached frontends have been terminated (established zmq connections are not disturbed by unlinking ipc paths; the engine can still drain its lanes at teardown). - launch_server read TLLM_SERVE_NUM_FRONTENDS from the global env unconditionally, but disaggregated ctx/gen MPI workers reach launch_server too (_launch_disaggregated_server) and inherit the submitter's env under Slurm export-all: an exported knob would mis-switch them into multi-frontend mode. launch_server gains multi_frontend_enabled (default True); the disagg worker path passes False and an ignored knob logs a warning. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 60 +++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index abd4718a1d2c..7a00f1c60057 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -5,6 +5,7 @@ import json import os import secrets +import shutil import signal import socket import subprocess # nosec B404 @@ -361,7 +362,7 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) -def _spawn_attached_frontends(llm, num_frontends: int) -> list: +def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: """Spawn num_frontends - 1 attached serving frontend processes. Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1): each child @@ -373,6 +374,9 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> list: executor.py GenerationExecutor.create). All frontends bind the serving port with SO_REUSEPORT, so the kernel load-balances accepted connections across their independent processes (and GILs). + + Returns (children, attach_info_path); the caller owns terminating the + children and removing the secret-bearing attach-info file. """ from tensorrt_llm.executor.proxy import GenerationExecutorProxy @@ -403,7 +407,7 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> list: logger.info( f"Launched attached serving frontend {frontend_id} (pid {child.pid})" ) - return children + return children, attach_info_path def _terminate_attached_frontends(children: list) -> None: @@ -416,6 +420,24 @@ def _terminate_attached_frontends(children: list) -> None: child.kill() +def _cleanup_multi_frontend_artifacts(attach_info_path: Optional[str]) -> None: + """Remove the attach-info file (it carries HMAC keys) and the ipc dir. + + Launcher-only, after the attached frontends have been terminated. + Unlinking ipc socket paths does not disturb established zmq connections + (the engine may still be draining its result lanes at teardown); it only + prevents new connects. + """ + if attach_info_path is not None: + try: + os.unlink(attach_info_path) + except OSError: + pass + ipc_dir = os.environ.get("TLLM_MULTI_FRONTEND_IPC_DIR") + if ipc_dir: + shutil.rmtree(ipc_dir, ignore_errors=True) + + def launch_server( host: str, port: int, @@ -430,7 +452,8 @@ def launch_server( served_model_name: Optional[str] = None, allow_request_chat_template: bool = False, num_input_processor_workers: int = 8, - num_media_load_workers: int = 8): + num_media_load_workers: int = 8, + multi_frontend_enabled: bool = True): backend = llm_args["backend"] model = served_model_name or llm_args["model"] @@ -440,9 +463,22 @@ def launch_server( # process, frontend 0) launches the worker as usual and spawns K-1 # attached frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) # skip the spawning. Supported only on the classic IPC executor path - # (the default orchestrator). - num_frontends = get_num_serve_frontends() - is_attached_frontend = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None + # (the default orchestrator). Entry points that reach launch_server but + # must not honor the env knob (e.g. disaggregated MPI workers, which + # inherit the caller's env under Slurm export-all) pass + # multi_frontend_enabled=False. + if multi_frontend_enabled: + num_frontends = get_num_serve_frontends() + is_attached_frontend = os.getenv( + "TLLM_EXECUTOR_ATTACH_INFO") is not None + else: + if os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") not in ("", "1"): + logger.warning( + "TLLM_SERVE_NUM_FRONTENDS is ignored on this entry point; " + "multi-frontend serving is only supported on plain " + "trtllm-serve.") + num_frontends = 1 + is_attached_frontend = False multi_frontend = num_frontends > 1 or is_attached_frontend if multi_frontend and not is_attached_frontend: if llm_args.get("orchestrator_type") is not None: @@ -496,8 +532,10 @@ def launch_server( param_hint="backend") frontend_children = [] + attach_info_path = None if multi_frontend and not is_attached_frontend: - frontend_children = _spawn_attached_frontends(llm, num_frontends) + frontend_children, attach_info_path = _spawn_attached_frontends( + llm, num_frontends) server = OpenAIServer( generator=llm, @@ -522,6 +560,8 @@ def launch_server( finally: if frontend_children: _terminate_attached_frontends(frontend_children) + if multi_frontend and not is_attached_frontend: + _cleanup_multi_frontend_artifacts(attach_info_path) def launch_grpc_server(host: str, @@ -1892,7 +1932,11 @@ def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): host=server_cfg.hostname, port=server_cfg.port, llm_args=llm_args, - allow_request_chat_template=disagg_config.allow_request_chat_template) + allow_request_chat_template=disagg_config.allow_request_chat_template, + # Disagg ctx/gen workers inherit the submitter's env (e.g. Slurm + # export-all); an exported TLLM_SERVE_NUM_FRONTENDS must not switch + # them into multi-frontend mode. + multi_frontend_enabled=False) def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, From 21663cfb8c47366517460eb5a23bb6cc81a55ea9 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:03:17 -0700 Subject: [PATCH 03/24] [None][chore] serve: fold multi-frontend mode resolution into a helper Extract the env-gate block at the top of launch_server into _init_multi_frontend_mode(), returning a MultiFrontendMode NamedTuple whose .active / .is_launcher properties replace the repeated 'multi_frontend and not is_attached_frontend' expressions at the SO_REUSEPORT, spawn, and cleanup sites. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 106 ++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 7a00f1c60057..f16d6eb8b6b1 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -13,7 +13,7 @@ import tempfile import uuid from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Set +from typing import Any, Dict, NamedTuple, Optional, Sequence, Set import click import torch @@ -362,6 +362,65 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) +class MultiFrontendMode(NamedTuple): + """Resolved multi-frontend serving mode for a launch_server invocation.""" + num_frontends: int + is_attached_frontend: bool + + @property + def active(self) -> bool: + """Any multi-frontend role: the launcher or an attached frontend.""" + return self.num_frontends > 1 or self.is_attached_frontend + + @property + def is_launcher(self) -> bool: + """The frontend that owns the engine and spawns/cleans the others.""" + return self.active and not self.is_attached_frontend + + +def _init_multi_frontend_mode(llm_args: dict, + enabled: bool) -> MultiFrontendMode: + """Resolve the multi-frontend serving mode (prototype). + + TLLM_SERVE_NUM_FRONTENDS=K runs K HTTP frontend processes against ONE + executor. The launcher (frontend 0) launches the worker as usual and + spawns K-1 attached frontends; attached frontends + (TLLM_EXECUTOR_ATTACH_INFO set) skip the spawning. Supported only on + the classic IPC executor path (the default orchestrator). + + On the launcher this opts the executor into multi-frontend mode BEFORE + it is created: it pre-generates the shared ipc directory and HMAC key + (env) so the launcher proxy, the rank0 worker and the attached + frontends agree on deterministic endpoints + (GenerationExecutorProxy._setup_queues). + + Entry points that reach launch_server but must not honor the env knob + (e.g. disaggregated MPI workers, which inherit the caller's env under + Slurm export-all) pass enabled=False. + """ + if not enabled: + if os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") not in ("", "1"): + logger.warning( + "TLLM_SERVE_NUM_FRONTENDS is ignored on this entry point; " + "multi-frontend serving is only supported on plain " + "trtllm-serve.") + return MultiFrontendMode(1, False) + + mode = MultiFrontendMode( + get_num_serve_frontends(), + os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) + if mode.is_launcher: + if llm_args.get("orchestrator_type") is not None: + raise ValueError( + "TLLM_SERVE_NUM_FRONTENDS > 1 currently supports only the " + "default (classic IPC) executor path, not orchestrator_type=" + f"{llm_args.get('orchestrator_type')!r}") + os.environ["TLLM_MULTI_FRONTEND_IPC_DIR"] = tempfile.mkdtemp( + prefix="trtllm_frontends_") + os.environ["TLLM_MULTI_FRONTEND_HMAC"] = os.urandom(32).hex() + return mode + + def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: """Spawn num_frontends - 1 attached serving frontend processes. @@ -458,41 +517,8 @@ def launch_server( backend = llm_args["backend"] model = served_model_name or llm_args["model"] - # Multi-frontend serving (prototype): TLLM_SERVE_NUM_FRONTENDS=K runs K - # HTTP frontend processes against ONE executor. The launcher (this - # process, frontend 0) launches the worker as usual and spawns K-1 - # attached frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) - # skip the spawning. Supported only on the classic IPC executor path - # (the default orchestrator). Entry points that reach launch_server but - # must not honor the env knob (e.g. disaggregated MPI workers, which - # inherit the caller's env under Slurm export-all) pass - # multi_frontend_enabled=False. - if multi_frontend_enabled: - num_frontends = get_num_serve_frontends() - is_attached_frontend = os.getenv( - "TLLM_EXECUTOR_ATTACH_INFO") is not None - else: - if os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") not in ("", "1"): - logger.warning( - "TLLM_SERVE_NUM_FRONTENDS is ignored on this entry point; " - "multi-frontend serving is only supported on plain " - "trtllm-serve.") - num_frontends = 1 - is_attached_frontend = False - multi_frontend = num_frontends > 1 or is_attached_frontend - if multi_frontend and not is_attached_frontend: - if llm_args.get("orchestrator_type") is not None: - raise ValueError( - "TLLM_SERVE_NUM_FRONTENDS > 1 currently supports only the " - "default (classic IPC) executor path, not orchestrator_type=" - f"{llm_args.get('orchestrator_type')!r}") - # Opt the launcher executor into multi-frontend mode BEFORE it is - # created: pre-generate the shared ipc directory and HMAC key so the - # launcher proxy, the rank0 worker and the attached frontends agree - # on deterministic endpoints (GenerationExecutorProxy._setup_queues). - os.environ["TLLM_MULTI_FRONTEND_IPC_DIR"] = tempfile.mkdtemp( - prefix="trtllm_frontends_") - os.environ["TLLM_MULTI_FRONTEND_HMAC"] = os.urandom(32).hex() + multi_frontend = _init_multi_frontend_mode(llm_args, + multi_frontend_enabled) addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) @@ -501,7 +527,7 @@ def launch_server( with socket.socket(address_family, socket.SOCK_STREAM) as s: # If disagg cluster config is provided and port is not specified, try to find a free port, otherwise try to bind to the specified port assert port > 0 or disagg_cluster_config is not None, "Port must be specified if disagg cluster config is not provided" - if multi_frontend: + if multi_frontend.active: # Every frontend process binds its own listening socket on the # same port; the kernel load-balances accepts across them. s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) @@ -533,9 +559,9 @@ def launch_server( frontend_children = [] attach_info_path = None - if multi_frontend and not is_attached_frontend: + if multi_frontend.is_launcher: frontend_children, attach_info_path = _spawn_attached_frontends( - llm, num_frontends) + llm, multi_frontend.num_frontends) server = OpenAIServer( generator=llm, @@ -560,7 +586,7 @@ def launch_server( finally: if frontend_children: _terminate_attached_frontends(frontend_children) - if multi_frontend and not is_attached_frontend: + if multi_frontend.is_launcher: _cleanup_multi_frontend_artifacts(attach_info_path) From 8840733ecb596fe5ae52208cd12f8c83be0ec18f Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:07:04 -0700 Subject: [PATCH 04/24] [None][chore] serve multi-frontend: direct client_id access on the response routing hot path Every response type that reaches _send_rsp and bucket_responses_by_frontend (tllm.Response, ErrorResponse, ResponseWrapper via __getattr__ delegation, PostprocWorker.Output) defines client_id, and the None-VALUE case (ADP dummy requests) is already handled by frontend_lane_index. The getattr(..., None) default was never exercised for a missing attribute; worse, it would silently route a hypothetical malformed response to lane 0 (where the launcher discards it, hanging the client) instead of failing loudly. Use direct attribute access: cheaper on the per-response loop and loud on real bugs. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/executor/base_worker.py | 2 +- tensorrt_llm/executor/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index c86355307866..c50a966f5e5a 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -1424,7 +1424,7 @@ def _send_rsp( rsp_batch.append(response) else: lanes = worker.frontend_result_queues - lanes[frontend_lane_index(getattr(response, "client_id", None), + lanes[frontend_lane_index(response.client_id, len(lanes))].put(response) elif worker.result_queue is not None: if rsp_batch is not None: diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index bf93b81a6ae3..8ef100ccaef3 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -280,7 +280,7 @@ def bucket_responses_by_frontend(responses: list, """ buckets = [[] for _ in range(num_frontends)] for rsp in responses: - buckets[frontend_lane_index(getattr(rsp, "client_id", None), + buckets[frontend_lane_index(rsp.client_id, num_frontends)].append(rsp) return buckets From 257d7378b3159a7a0eccf73d138350d25d0bf927 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:39:14 -0700 Subject: [PATCH 05/24] [None][feat] serve multi-frontend: replace the env gate with the num_serve_frontends knob Promote the multi-frontend switch from TLLM_SERVE_NUM_FRONTENDS to a typed config knob, following the vLLM --api-server-count shape: - llm_args gains num_serve_frontends (int, default 1, ge=1 le=65536, status=prototype) next to the other serving knobs; exposed on trtllm-serve as --num_serve_frontends and via the config yaml. api_stability reference updated. - GenerationExecutorProxy now reads the count from llm_args and OWNS the shared ipc directory + HMAC key (generated in __init__, removed in shutdown): the TLLM_MULTI_FRONTEND_IPC_DIR / TLLM_MULTI_FRONTEND_HMAC env side-channel between serve.py and the proxy is deleted, and the ipc-dir cleanup moves from serve.py to the proxy that created it. - serve.py _init_multi_frontend_mode normalizes the knob into llm_args; TLLM_SERVE_NUM_FRONTENDS is kept as a fallback for when the knob is unset (existing deployments), translated to the knob at the CLI entry. The disabled entry points (disagg MPI workers) now also scrub the knob from llm_args so the executor cannot enter multi-frontend mode there. - executor.py validates the TLLM_EXECUTOR_FRONTEND_ID / TLLM_EXECUTOR_ATTACH_INFO pairing with a clear error instead of a raw KeyError. The attach-info env+file bootstrap remains: it crosses the child re-exec boundary, which a config knob cannot. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 77 +++++++++++-------- tensorrt_llm/executor/executor.py | 8 +- tensorrt_llm/executor/proxy.py | 53 +++++++++---- tensorrt_llm/executor/utils.py | 21 +---- tensorrt_llm/llmapi/llm_args.py | 11 +++ .../api_stability/references/llm.yaml | 4 + 6 files changed, 106 insertions(+), 68 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index f16d6eb8b6b1..c4a210b5320f 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -5,7 +5,6 @@ import json import os import secrets -import shutil import signal import socket import subprocess # nosec B404 @@ -201,6 +200,7 @@ def get_llm_args( free_gpu_memory_fraction: float = 0.9, kv_cache_dtype: str = "auto", num_postprocess_workers: int = 0, + num_serve_frontends: int = 1, trust_remote_code: bool = False, revision: Optional[str] = None, reasoning_parser: Optional[str] = None, @@ -274,6 +274,8 @@ def get_llm_args( max_seq_len, "num_postprocess_workers": num_postprocess_workers, + "num_serve_frontends": + num_serve_frontends, "enable_chunked_prefill": enable_chunked_prefill, "enable_attention_dp": @@ -382,49 +384,53 @@ def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: """Resolve the multi-frontend serving mode (prototype). - TLLM_SERVE_NUM_FRONTENDS=K runs K HTTP frontend processes against ONE - executor. The launcher (frontend 0) launches the worker as usual and - spawns K-1 attached frontends; attached frontends - (TLLM_EXECUTOR_ATTACH_INFO set) skip the spawning. Supported only on - the classic IPC executor path (the default orchestrator). + num_serve_frontends=K (--num_serve_frontends / config yaml; the + TLLM_SERVE_NUM_FRONTENDS env is honored as a fallback when the knob is + unset) runs K HTTP frontend processes against ONE executor. The + launcher (frontend 0) launches the worker as usual and spawns K-1 + attached frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) + skip the spawning. Supported only on the classic IPC executor path + (the default orchestrator). - On the launcher this opts the executor into multi-frontend mode BEFORE - it is created: it pre-generates the shared ipc directory and HMAC key - (env) so the launcher proxy, the rank0 worker and the attached - frontends agree on deterministic endpoints - (GenerationExecutorProxy._setup_queues). + The executor proxy reads num_serve_frontends from llm_args and + provisions the shared ipc endpoints itself + (GenerationExecutorProxy._setup_queues); this function only normalizes + the knob into llm_args before the LLM is created. - Entry points that reach launch_server but must not honor the env knob + Entry points that reach launch_server but must not honor the knob (e.g. disaggregated MPI workers, which inherit the caller's env under Slurm export-all) pass enabled=False. """ + num_frontends = llm_args.get( + "num_serve_frontends") or get_num_serve_frontends() if not enabled: - if os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") not in ("", "1"): + if num_frontends > 1: logger.warning( - "TLLM_SERVE_NUM_FRONTENDS is ignored on this entry point; " + "num_serve_frontends is ignored on this entry point; " "multi-frontend serving is only supported on plain " "trtllm-serve.") + llm_args.pop("num_serve_frontends", None) return MultiFrontendMode(1, False) mode = MultiFrontendMode( - get_num_serve_frontends(), + num_frontends, os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) if mode.is_launcher: if llm_args.get("orchestrator_type") is not None: raise ValueError( - "TLLM_SERVE_NUM_FRONTENDS > 1 currently supports only the " + "num_serve_frontends > 1 currently supports only the " "default (classic IPC) executor path, not orchestrator_type=" f"{llm_args.get('orchestrator_type')!r}") - os.environ["TLLM_MULTI_FRONTEND_IPC_DIR"] = tempfile.mkdtemp( - prefix="trtllm_frontends_") - os.environ["TLLM_MULTI_FRONTEND_HMAC"] = os.urandom(32).hex() + # Make the resolved count visible to the executor proxy even when it + # came from the env fallback. + llm_args["num_serve_frontends"] = num_frontends return mode def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: """Spawn num_frontends - 1 attached serving frontend processes. - Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1): each child + Multi-frontend serving (num_serve_frontends > 1): each child re-execs this trtllm-serve command line with env vars pointing at the launcher executor's attach endpoints — the multi-frontend request ingress plus per-frontend result lanes (see @@ -443,7 +449,7 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: if not isinstance(executor, GenerationExecutorProxy) or ( attach_info := executor.multi_frontend_attach_info()) is None: raise ValueError( - "TLLM_SERVE_NUM_FRONTENDS > 1 requires the classic IPC executor " + "num_serve_frontends > 1 requires the classic IPC executor " f"proxy in multi-frontend mode, got {type(executor).__name__}") # mkstemp creates the file 0600: it carries the executor HMAC keys. fd, attach_info_path = tempfile.mkstemp(prefix="trtllm_frontend_", @@ -480,21 +486,17 @@ def _terminate_attached_frontends(children: list) -> None: def _cleanup_multi_frontend_artifacts(attach_info_path: Optional[str]) -> None: - """Remove the attach-info file (it carries HMAC keys) and the ipc dir. + """Remove the attach-info file (it carries HMAC keys). - Launcher-only, after the attached frontends have been terminated. - Unlinking ipc socket paths does not disturb established zmq connections - (the engine may still be draining its result lanes at teardown); it only - prevents new connects. + Launcher-only, after the attached frontends have been terminated. The + shared ipc directory is owned and removed by the launcher's executor + proxy (GenerationExecutorProxy.shutdown). """ if attach_info_path is not None: try: os.unlink(attach_info_path) except OSError: pass - ipc_dir = os.environ.get("TLLM_MULTI_FRONTEND_IPC_DIR") - if ipc_dir: - shutil.rmtree(ipc_dir, ignore_errors=True) def launch_server( @@ -1046,6 +1048,13 @@ def launch_visual_gen_server( help="Number of workers to postprocess raw responses " "to comply with OpenAI protocol.", status="prototype") +@stability_option("--num_serve_frontends", + type=click.IntRange(min=1, max=1 << 16), + default=1, + help="Number of HTTP frontend processes serving one " + "executor; values > 1 share the serving port via " + "SO_REUSEPORT (classic IPC executor path only).", + status="prototype") @stability_option("--num_input_processor_workers", type=click.IntRange(min=1), default=8, @@ -1236,7 +1245,8 @@ def serve( moe_expert_parallel_size: Optional[int], moe_cluster_parallel_size: Optional[int], gpus_per_node: Optional[int], free_gpu_memory_fraction: float, kv_cache_dtype: str, - num_postprocess_workers: int, num_input_processor_workers: int, + num_postprocess_workers: int, num_serve_frontends: int, + num_input_processor_workers: int, num_media_load_workers: int, trust_remote_code: bool, revision: Optional[str], extra_llm_api_options: Optional[str], reasoning_parser: Optional[str], tool_parser: Optional[str], @@ -1331,6 +1341,7 @@ def _serve_llm(): free_gpu_memory_fraction=free_gpu_memory_fraction, kv_cache_dtype=kv_cache_dtype, num_postprocess_workers=num_postprocess_workers, + num_serve_frontends=num_serve_frontends, trust_remote_code=trust_remote_code, revision=revision, reasoning_parser=reasoning_parser, @@ -1959,9 +1970,9 @@ def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): port=server_cfg.port, llm_args=llm_args, allow_request_chat_template=disagg_config.allow_request_chat_template, - # Disagg ctx/gen workers inherit the submitter's env (e.g. Slurm - # export-all); an exported TLLM_SERVE_NUM_FRONTENDS must not switch - # them into multi-frontend mode. + # Disagg ctx/gen workers must not enter multi-frontend mode (e.g. + # via a num_serve_frontends knob or an env fallback inherited under + # Slurm export-all). multi_frontend_enabled=False) diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 6ee50e245917..960f54625c98 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -575,7 +575,13 @@ def create( "TLLM_EXECUTOR_ATTACH_INFO only supports the classic IPC " f"executor path, got mode={attach_info.get('mode')!r}") from .proxy import GenerationExecutorFrontendProxy - frontend_id = int(os.environ["TLLM_EXECUTOR_FRONTEND_ID"]) + frontend_id_env = os.getenv("TLLM_EXECUTOR_FRONTEND_ID") + if frontend_id_env is None: + raise ValueError( + "TLLM_EXECUTOR_ATTACH_INFO is set but " + "TLLM_EXECUTOR_FRONTEND_ID is not; both are set together " + "by trtllm-serve when spawning attached frontends.") + frontend_id = int(frontend_id_env) logger.info(f"Attaching executor frontend {frontend_id} to the " f"running classic IPC worker via {attach_info_path}") return GenerationExecutorFrontendProxy( diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 5e43d9f8bf0a..28c5b4ebc75f 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -16,6 +16,8 @@ import concurrent.futures import json import os +import shutil +import tempfile import threading import weakref from queue import Empty @@ -43,7 +45,6 @@ from .rpc.rpc_common import RPCError, get_unique_ipc_addr from .utils import (EngineDeadError, ErrorResponse, RequestError, WorkerCommIpcAddrs, create_mpi_comm_session, - get_multi_frontend_ipc_info, get_num_serve_frontends, get_spawn_proxy_process_env, is_llm_response, multi_frontend_request_addr, multi_frontend_result_addr, namespace_client_id, print_alive_threads) @@ -164,20 +165,24 @@ def __init__( self._enable_resource_governor = bool( getattr(_llm_args, "enable_resource_governor", False)) - # Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1) on the classic - # IPC path: trtllm-serve (the launcher, frontend 0) pre-generates a - # shared ipc directory + HMAC key so this proxy, the rank0 worker and - # the attached frontends agree on deterministic endpoints (see - # _setup_queues). Attached frontends never reach this class -- they - # construct GenerationExecutorFrontendProxy instead. - self._multi_frontend_info = get_multi_frontend_ipc_info() - self._num_frontends = get_num_serve_frontends( - ) if self._multi_frontend_info is not None else 1 - if self._num_frontends > 1 and self._enable_resource_governor: - raise ValueError( - "Multi-frontend serving does not support " - "enable_resource_governor: the resource-governor signal only " - "reaches the launcher frontend.") + # Multi-frontend serving (llm_args.num_serve_frontends > 1) on the + # classic IPC path: this launcher proxy owns the shared ipc directory + # + HMAC key for the per-frontend endpoints (see _setup_queues); + # trtllm-serve hands them to the attached frontends via + # multi_frontend_attach_info(). Attached frontends never reach this + # class -- they construct GenerationExecutorFrontendProxy instead. + self._num_frontends = getattr(_llm_args, "num_serve_frontends", 1) or 1 + self._multi_frontend_ipc_dir: Optional[str] = None + self._multi_frontend_hmac: Optional[bytes] = None + if self._num_frontends > 1: + if self._enable_resource_governor: + raise ValueError( + "Multi-frontend serving does not support " + "enable_resource_governor: the resource-governor signal " + "only reaches the launcher frontend.") + self._multi_frontend_ipc_dir = tempfile.mkdtemp( + prefix="trtllm_frontends_") + self._multi_frontend_hmac = os.urandom(32) # Generate RPC address and key for stats RPC self.rpc_addr = get_unique_ipc_addr() @@ -421,7 +426,8 @@ def _setup_queues(self) -> WorkerCommIpcAddrs: # (including this launcher, frontend 0) binds its own result lane # (PULL) that the worker / postproc processes PUSH-connect to, # selected by the frontend id in client_id's top bits. - ipc_dir, hmac_key = self._multi_frontend_info + ipc_dir = self._multi_frontend_ipc_dir + hmac_key = self._multi_frontend_hmac request_addr = (multi_frontend_request_addr(ipc_dir), hmac_key) frontend_result_addrs = [(multi_frontend_result_addr(ipc_dir, i), hmac_key) @@ -475,7 +481,8 @@ def multi_frontend_attach_info(self) -> Optional[dict]: """ if self._num_frontends <= 1: return None - ipc_dir, hmac_key = self._multi_frontend_info + ipc_dir = self._multi_frontend_ipc_dir + hmac_key = self._multi_frontend_hmac return { "mode": "classic", @@ -702,8 +709,19 @@ def pre_shutdown(self): if not self.mpi_futures or any(not f.done() for f in self.mpi_futures): self.request_queue.put_noblock(None, retry=4) + def _cleanup_multi_frontend_ipc_dir(self): + """Remove the launcher-owned multi-frontend ipc directory. + + Unlinking ipc socket paths does not disturb established zmq + connections; it only prevents new connects. + """ + if self._multi_frontend_ipc_dir is not None: + shutil.rmtree(self._multi_frontend_ipc_dir, ignore_errors=True) + self._multi_frontend_ipc_dir = None + def shutdown(self): if not self.workers_started: + self._cleanup_multi_frontend_ipc_dir() return if not self.doing_shutdown: @@ -743,6 +761,7 @@ def shutdown(self): self.result_queue.close() if self._resource_governor_queue is not None: self._resource_governor_queue.close() + self._cleanup_multi_frontend_ipc_dir() self.workers_started = False if self._owns_mpi_session: diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 8ef100ccaef3..9043fff41ff4 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -286,11 +286,11 @@ def bucket_responses_by_frontend(responses: list, def get_num_serve_frontends() -> int: - """The number of serving frontend processes (TLLM_SERVE_NUM_FRONTENDS). + """The number of serving frontend processes from TLLM_SERVE_NUM_FRONTENDS. - Single source for parsing and range-validating the env knob; used by - trtllm-serve (the launcher) and GenerationExecutorProxy so they always - agree on the frontend count. 1 (single frontend) when unset. + Env fallback for the llm_args.num_serve_frontends knob, honored by + trtllm-serve when the knob is unset (see commands/serve.py + _init_multi_frontend_mode). 1 (single frontend) when unset. """ num_frontends = int(os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") or "1") if not 0 < num_frontends <= (1 << 16): @@ -299,19 +299,6 @@ def get_num_serve_frontends() -> int: return num_frontends -def get_multi_frontend_ipc_info() -> Optional[tuple[str, bytes]]: - """The shared ipc dir and HMAC key for multi-frontend serving. - - Pre-generated by trtllm-serve (the launcher) on the classic IPC executor - path before the executor is created. None outside that mode. - """ - ipc_dir = os.getenv("TLLM_MULTI_FRONTEND_IPC_DIR") - hmac_hex = os.getenv("TLLM_MULTI_FRONTEND_HMAC") - if ipc_dir and hmac_hex: - return ipc_dir, bytes.fromhex(hmac_hex) - return None - - def multi_frontend_request_addr(ipc_dir: str) -> str: """The request ingress endpoint bound by the rank0 worker (PULL). diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 57c3de25dcfe..df61c7e1aa19 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -4121,6 +4121,17 @@ class BaseLlmArgs(StrictBaseModel): description="The path to the tokenizer directory for postprocessing.", status="prototype") + num_serve_frontends: int = Field( + default=1, + ge=1, + le=1 << 16, + description= + "The number of HTTP frontend processes serving one executor. Used by " + "trtllm-serve: values > 1 run additional attached frontend processes " + "that share the serving port via SO_REUSEPORT (classic IPC executor " + "path only).", + status="prototype") + reasoning_parser: Optional[str] = Field( default=None, description="The parser to separate reasoning content from output.", diff --git a/tests/unittest/api_stability/references/llm.yaml b/tests/unittest/api_stability/references/llm.yaml index 7ff208c41ce3..959f249183e8 100644 --- a/tests/unittest/api_stability/references/llm.yaml +++ b/tests/unittest/api_stability/references/llm.yaml @@ -60,6 +60,10 @@ methods: annotation: Optional[str] default: null status: prototype + num_serve_frontends: + annotation: int + default: 1 + status: prototype custom_tokenizer: annotation: Optional[str] default: null From c55c1ae6af2e9c20a70ac936640d6a1126afa489 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:48:11 -0700 Subject: [PATCH 06/24] [None][chore] serve multi-frontend: cap num_serve_frontends at 64, drop the env fallback 64 frontends already exceeds what one node's cores can usefully serve; the previous 1<<16 bound was the client-id encoding capacity, not a sane policy limit (the 16-bit wire format is unchanged). TLLM_SERVE_NUM_FRONTENDS is no longer honored: the knob is the only switch. A set env now logs a pointer to --num_serve_frontends instead of being silently ignored. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 46 +++++++++---------- tensorrt_llm/executor/proxy.py | 2 +- tensorrt_llm/executor/utils.py | 14 ------ tensorrt_llm/llmapi/llm_args.py | 2 +- .../executor/test_multi_frontend_routing.py | 22 +-------- 5 files changed, 24 insertions(+), 62 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index c4a210b5320f..fc6128862e71 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -27,8 +27,7 @@ from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import (LlmLauncherEnvs, - get_num_serve_frontends) +from tensorrt_llm.executor.utils import LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, @@ -384,25 +383,26 @@ def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: """Resolve the multi-frontend serving mode (prototype). - num_serve_frontends=K (--num_serve_frontends / config yaml; the - TLLM_SERVE_NUM_FRONTENDS env is honored as a fallback when the knob is - unset) runs K HTTP frontend processes against ONE executor. The - launcher (frontend 0) launches the worker as usual and spawns K-1 - attached frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) - skip the spawning. Supported only on the classic IPC executor path - (the default orchestrator). + num_serve_frontends=K (--num_serve_frontends / config yaml) runs K + HTTP frontend processes against ONE executor. The launcher + (frontend 0) launches the worker as usual and spawns K-1 attached + frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) skip + the spawning. Supported only on the classic IPC executor path (the + default orchestrator). The executor proxy reads num_serve_frontends from llm_args and provisions the shared ipc endpoints itself - (GenerationExecutorProxy._setup_queues); this function only normalizes - the knob into llm_args before the LLM is created. + (GenerationExecutorProxy._setup_queues). Entry points that reach launch_server but must not honor the knob - (e.g. disaggregated MPI workers, which inherit the caller's env under - Slurm export-all) pass enabled=False. + (e.g. disaggregated MPI workers) pass enabled=False. """ - num_frontends = llm_args.get( - "num_serve_frontends") or get_num_serve_frontends() + if os.getenv("TLLM_SERVE_NUM_FRONTENDS") is not None: + logger.warning( + "TLLM_SERVE_NUM_FRONTENDS is not supported; use the " + "num_serve_frontends llm-args knob (--num_serve_frontends or " + "the config yaml) instead.") + num_frontends = llm_args.get("num_serve_frontends", 1) if not enabled: if num_frontends > 1: logger.warning( @@ -415,15 +415,11 @@ def _init_multi_frontend_mode(llm_args: dict, mode = MultiFrontendMode( num_frontends, os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) - if mode.is_launcher: - if llm_args.get("orchestrator_type") is not None: - raise ValueError( - "num_serve_frontends > 1 currently supports only the " - "default (classic IPC) executor path, not orchestrator_type=" - f"{llm_args.get('orchestrator_type')!r}") - # Make the resolved count visible to the executor proxy even when it - # came from the env fallback. - llm_args["num_serve_frontends"] = num_frontends + if mode.is_launcher and llm_args.get("orchestrator_type") is not None: + raise ValueError( + "num_serve_frontends > 1 currently supports only the " + "default (classic IPC) executor path, not orchestrator_type=" + f"{llm_args.get('orchestrator_type')!r}") return mode @@ -1049,7 +1045,7 @@ def launch_visual_gen_server( "to comply with OpenAI protocol.", status="prototype") @stability_option("--num_serve_frontends", - type=click.IntRange(min=1, max=1 << 16), + type=click.IntRange(min=1, max=64), default=1, help="Number of HTTP frontend processes serving one " "executor; values > 1 share the serving port via " diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 28c5b4ebc75f..e5c6685b04e1 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -1007,7 +1007,7 @@ def __exit__(self, exc_type, exc_value, traceback): class GenerationExecutorFrontendProxy(GenerationExecutorProxy): """An attached serving frontend for the classic IPC executor path. - Used for multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS > 1). + Used for multi-frontend serving (num_serve_frontends > 1). PUSH-connects to the request ingress bound by the rank0 worker and binds its own per-frontend result lane (PULL); the worker routes responses to this lane by the frontend id embedded in the top bits of client_id (see diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 9043fff41ff4..3fd4627d3b18 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -285,20 +285,6 @@ def bucket_responses_by_frontend(responses: list, return buckets -def get_num_serve_frontends() -> int: - """The number of serving frontend processes from TLLM_SERVE_NUM_FRONTENDS. - - Env fallback for the llm_args.num_serve_frontends knob, honored by - trtllm-serve when the knob is unset (see commands/serve.py - _init_multi_frontend_mode). 1 (single frontend) when unset. - """ - num_frontends = int(os.getenv("TLLM_SERVE_NUM_FRONTENDS", "1") or "1") - if not 0 < num_frontends <= (1 << 16): - raise ValueError( - f"TLLM_SERVE_NUM_FRONTENDS out of range: {num_frontends}") - return num_frontends - - def multi_frontend_request_addr(ipc_dir: str) -> str: """The request ingress endpoint bound by the rank0 worker (PULL). diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index df61c7e1aa19..13fac08b933e 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -4124,7 +4124,7 @@ class BaseLlmArgs(StrictBaseModel): num_serve_frontends: int = Field( default=1, ge=1, - le=1 << 16, + le=64, description= "The number of HTTP frontend processes serving one executor. Used by " "trtllm-serve: values > 1 run additional attached frontend processes " diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index 2258a7b98f47..4876623260e8 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -14,7 +14,7 @@ # limitations under the License. """CPU-only tests for classic-path multi-frontend serving. -Multi-frontend serving (TLLM_SERVE_NUM_FRONTENDS) on the classic IPC +Multi-frontend serving (num_serve_frontends) on the classic IPC executor path. Covers client-id namespacing, worker-side response-lane routing, and the attached-frontend proxy lifecycle over real ipc:// sockets. """ @@ -29,7 +29,6 @@ bucket_responses_by_frontend, frontend_lane_index, get_frontend_id, - get_num_serve_frontends, namespace_client_id, ) @@ -62,25 +61,6 @@ def test_lane_index_clamps_to_launcher(self): assert frontend_lane_index(None, 4) == 0 -class TestNumServeFrontendsEnv: - def test_defaults_to_one(self, monkeypatch): - monkeypatch.delenv("TLLM_SERVE_NUM_FRONTENDS", raising=False) - assert get_num_serve_frontends() == 1 - monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", "") - assert get_num_serve_frontends() == 1 - - def test_parses_value(self, monkeypatch): - monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", "16") - assert get_num_serve_frontends() == 16 - - def test_rejects_out_of_range(self, monkeypatch): - import pytest - for bad in ("0", "-1", str((1 << 16) + 1)): - monkeypatch.setenv("TLLM_SERVE_NUM_FRONTENDS", bad) - with pytest.raises(ValueError): - get_num_serve_frontends() - - def _response(client_id): return SimpleNamespace(client_id=client_id) From f1e72aed9a63ef7ce6f6d35b7c771fbbb9e53674 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:53:58 -0700 Subject: [PATCH 07/24] [None][chore] serve multi-frontend: drop the leftover env warning, tighten comments TLLM_SERVE_NUM_FRONTENDS never shipped in a release, so there is nothing to deprecate -- remove the warning shim entirely; the knob is the only switch. Also condense the multi-frontend comments/docstrings: the architecture is described once (MultiFrontendMode / GenerationExecutorFrontendProxy); other sites keep only the local constraint they enforce. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 50 +++++++----------------- tensorrt_llm/executor/base_worker.py | 12 ++---- tensorrt_llm/executor/executor.py | 5 +-- tensorrt_llm/executor/postproc_worker.py | 9 ++--- tensorrt_llm/executor/proxy.py | 25 +++++------- tensorrt_llm/executor/utils.py | 31 ++++----------- tensorrt_llm/executor/worker.py | 8 ++-- tensorrt_llm/llmapi/llm.py | 6 +-- 8 files changed, 44 insertions(+), 102 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index fc6128862e71..9c53b1b37038 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -383,25 +383,12 @@ def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: """Resolve the multi-frontend serving mode (prototype). - num_serve_frontends=K (--num_serve_frontends / config yaml) runs K - HTTP frontend processes against ONE executor. The launcher - (frontend 0) launches the worker as usual and spawns K-1 attached - frontends; attached frontends (TLLM_EXECUTOR_ATTACH_INFO set) skip - the spawning. Supported only on the classic IPC executor path (the - default orchestrator). - - The executor proxy reads num_serve_frontends from llm_args and - provisions the shared ipc endpoints itself - (GenerationExecutorProxy._setup_queues). - - Entry points that reach launch_server but must not honor the knob - (e.g. disaggregated MPI workers) pass enabled=False. + num_serve_frontends=K runs K HTTP frontend processes against ONE + executor: the launcher (frontend 0) owns the engine and spawns K-1 + attached frontends. Classic IPC executor path only. Entry points that + must not honor the knob (e.g. disaggregated MPI workers) pass + enabled=False. """ - if os.getenv("TLLM_SERVE_NUM_FRONTENDS") is not None: - logger.warning( - "TLLM_SERVE_NUM_FRONTENDS is not supported; use the " - "num_serve_frontends llm-args knob (--num_serve_frontends or " - "the config yaml) instead.") num_frontends = llm_args.get("num_serve_frontends", 1) if not enabled: if num_frontends > 1: @@ -426,15 +413,10 @@ def _init_multi_frontend_mode(llm_args: dict, def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: """Spawn num_frontends - 1 attached serving frontend processes. - Multi-frontend serving (num_serve_frontends > 1): each child - re-execs this trtllm-serve command line with env vars pointing at the - launcher executor's attach endpoints — the multi-frontend request - ingress plus per-frontend result lanes (see - GenerationExecutorProxy._setup_queues); the child's executor attaches to - the already-running worker instead of launching a new one (see - executor.py GenerationExecutor.create). All frontends bind the serving - port with SO_REUSEPORT, so the kernel load-balances accepted - connections across their independent processes (and GILs). + Each child re-execs this trtllm-serve command line with env vars + pointing at the launcher executor's attach endpoints; its executor + attaches to the already-running worker instead of launching one (see + GenerationExecutor.create / GenerationExecutorFrontendProxy). Returns (children, attach_info_path); the caller owns terminating the children and removing the secret-bearing attach-info file. @@ -455,9 +437,8 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: children = [] for frontend_id in range(1, num_frontends): - # Attached frontends are plain RPC clients: keep them out of the - # launcher's MPI job (an inherited PMI/SLURM rank identity would - # make mpi4py try to (re-)join it at import time). + # Strip MPI/SLURM identity vars: an inherited rank identity would + # make the child's mpi4py try to (re-)join the launcher's job. env, _ = split_mpi_env() env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_info_path env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) @@ -484,9 +465,8 @@ def _terminate_attached_frontends(children: list) -> None: def _cleanup_multi_frontend_artifacts(attach_info_path: Optional[str]) -> None: """Remove the attach-info file (it carries HMAC keys). - Launcher-only, after the attached frontends have been terminated. The - shared ipc directory is owned and removed by the launcher's executor - proxy (GenerationExecutorProxy.shutdown). + The shared ipc directory is owned and removed by the launcher's + executor proxy instead. """ if attach_info_path is not None: try: @@ -1966,9 +1946,7 @@ def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): port=server_cfg.port, llm_args=llm_args, allow_request_chat_template=disagg_config.allow_request_chat_template, - # Disagg ctx/gen workers must not enter multi-frontend mode (e.g. - # via a num_serve_frontends knob or an env fallback inherited under - # Slurm export-all). + # Disagg ctx/gen MPI workers must not enter multi-frontend mode. multi_frontend_enabled=False) diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index c50a966f5e5a..ec0403666c42 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -286,10 +286,7 @@ def set_postproc_queues(self, queues: List["IpcQueue"]): self.postproc_queues = queues def set_frontend_result_queues(self, queues: List["IpcQueue"]): - """Multi-frontend serving: one result lane per frontend process. - - The lane is selected by the frontend id in client_id's top bits. - """ + """Multi-frontend serving: one result lane per frontend process.""" assert self.result_queue is None assert self.postproc_queues is None self.frontend_result_queues = queues @@ -1295,8 +1292,6 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: if rsp_batch: if (lanes := self.worker.frontend_result_queues) is not None: - # Multi-frontend serving: route each response to its origin - # frontend's lane by the id in client_id's top bits. for frontend_id, sub_batch in enumerate( bucket_responses_by_frontend(rsp_batch, len(lanes))): if sub_batch: @@ -1417,9 +1412,8 @@ def _send_rsp( # if postproc_batches is set, append to batch instead of putting to IpcQueue if worker.frontend_result_queues is not None: - # Multi-frontend serving: route to the origin frontend's result lane - # (client_id may be None for e.g. ADP dummy requests -- those go to - # lane 0, the launcher, which silently discards them like today). + # Route to the origin frontend's result lane; None/out-of-range ids + # fall back to lane 0 (see frontend_lane_index). if rsp_batch is not None: rsp_batch.append(response) else: diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 960f54625c98..726102b68431 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -563,9 +563,8 @@ def create( "green") # Multi-frontend serving: attach to an already-running executor - # instead of launching one. Set by trtllm-serve for the attached - # frontend processes (see commands/serve.py); the frontend id is - # picked up from TLLM_EXECUTOR_FRONTEND_ID. + # instead of launching one (set by trtllm-serve for attached + # frontend processes). attach_info_path = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") if attach_info_path: with open(attach_info_path) as f: diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py index 5af13a36bc75..182b160db471 100644 --- a/tensorrt_llm/executor/postproc_worker.py +++ b/tensorrt_llm/executor/postproc_worker.py @@ -96,10 +96,9 @@ def __init__( ''' Args: pull_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the input IPC. - push_pipe_addr: The address and HMAC key of the output IPC, or a - list of them with multi-frontend serving -- one result lane - per frontend, selected by the frontend id in the output's - client_id top bits. + push_pipe_addr: The address and HMAC key of the output IPC, or + a list of them (one result lane per frontend) with + multi-frontend serving. tokenizer_dir (str): The directory to load tokenizer. record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request. result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result. @@ -210,8 +209,6 @@ async def _batched_put(self): if len(self._push_pipes) == 1: await self._push_pipes[0].put_async(batch) continue - # Multi-frontend serving: route each output to its origin - # frontend's result lane by the id in client_id's top bits. for frontend_id, sub_batch in enumerate( bucket_responses_by_frontend(batch, len(self._push_pipes))): if sub_batch: diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index e5c6685b04e1..504b26b1b6bb 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -165,12 +165,10 @@ def __init__( self._enable_resource_governor = bool( getattr(_llm_args, "enable_resource_governor", False)) - # Multi-frontend serving (llm_args.num_serve_frontends > 1) on the - # classic IPC path: this launcher proxy owns the shared ipc directory - # + HMAC key for the per-frontend endpoints (see _setup_queues); + # Multi-frontend serving: this launcher proxy owns the shared ipc + # dir + HMAC key for the per-frontend endpoints (_setup_queues); # trtllm-serve hands them to the attached frontends via - # multi_frontend_attach_info(). Attached frontends never reach this - # class -- they construct GenerationExecutorFrontendProxy instead. + # multi_frontend_attach_info(). self._num_frontends = getattr(_llm_args, "num_serve_frontends", 1) or 1 self._multi_frontend_ipc_dir: Optional[str] = None self._multi_frontend_hmac: Optional[bytes] = None @@ -420,12 +418,9 @@ def _error_monitor_loop(self) -> None: def _setup_queues(self) -> WorkerCommIpcAddrs: frontend_result_addrs = None if self._num_frontends > 1: - # Multi-frontend serving: deterministic endpoints shared with the - # attached frontends. The rank0 worker BINDS the request ingress - # (PULL) so every frontend can PUSH-connect; each frontend - # (including this launcher, frontend 0) binds its own result lane - # (PULL) that the worker / postproc processes PUSH-connect to, - # selected by the frontend id in client_id's top bits. + # The rank0 worker BINDS the request ingress (PULL) so every + # frontend can PUSH-connect; each frontend (incl. this launcher, + # frontend 0) binds its own result lane (PULL). ipc_dir = self._multi_frontend_ipc_dir hmac_key = self._multi_frontend_hmac request_addr = (multi_frontend_request_addr(ipc_dir), hmac_key) @@ -1048,9 +1043,8 @@ def __init__( self.garbage_collection_gen0_threshold = None self.workers_started = False self.dispatch_result_thread: Optional[ManagedThread] = None - # The resource governor lives with the launcher frontend only; the - # inherited resource_governor_queue property must return None here so - # OpenAIServer takes its governor-disabled path (openai_server.py). + # Must be None: OpenAIServer reads the resource_governor_queue + # property at init; the governor lives with the launcher only. self._resource_governor_queue = None hmac_key = bytes.fromhex(attach_info["hmac_key"]) @@ -1067,8 +1061,7 @@ def __init__( name=f"frontend_{frontend_id}_result_queue") # Stats / KV events / disagg params share the rank0 worker's stats - # RPC server with the launcher (ROUTER socket, natively - # multi-client; per-frontend sampling). + # RPC server with the launcher (ROUTER socket, multi-client). self.rpc_client: Optional[RPCClient] = None if attach_info.get("rpc_addr"): self.rpc_client = RPCClient(attach_info["rpc_addr"], diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 3fd4627d3b18..469067133a04 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -229,11 +229,9 @@ class WorkerCommIpcAddrs(NamedTuple): worker_init_status_queue_addr: tuple[str, Optional[bytes]] result_queue_addr: tuple[str, Optional[bytes]] resource_governor_queue_addr: Optional[tuple[str, Optional[bytes]]] = None - # Multi-frontend serving (classic IPC path): one result lane per frontend - # process, selected by the frontend id in client_id's top bits. When set, - # the rank0 worker BINDS the request queue (PULL) so every frontend can - # PUSH-connect, and routes responses to these lanes instead of - # result_queue_addr (which then aliases lane 0, the launcher's). + # Multi-frontend serving: one result lane per frontend. When set, the + # rank0 worker BINDS the request queue (PULL) and routes responses to + # these lanes; result_queue_addr then aliases lane 0 (the launcher). frontend_result_queue_addrs: Optional[list[tuple[str, Optional[bytes]]]] = None @@ -262,10 +260,8 @@ def namespace_client_id(frontend_id: int, client_id: int) -> int: def frontend_lane_index(client_id: Optional[int], num_lanes: int) -> int: """The result-lane index for a response's originating frontend. - Responses without a usable client_id (e.g. ADP dummy requests carry - client_id=None) and ids with an out-of-range frontend go to lane 0 - (the launcher), matching legacy single-client visibility where such - responses are silently discarded by the launcher's dispatcher. + None (e.g. ADP dummy requests) and out-of-range frontend ids go to + lane 0, the launcher, which silently discards them like today. """ frontend_id = get_frontend_id(client_id) return frontend_id if frontend_id < num_lanes else 0 @@ -273,11 +269,7 @@ def frontend_lane_index(client_id: Optional[int], num_lanes: int) -> int: def bucket_responses_by_frontend(responses: list, num_frontends: int) -> list[list]: - """Bucket responses by their originating frontend id (client_id top bits). - - Lane selection (including the route-to-launcher fallback) follows - frontend_lane_index. - """ + """Bucket responses by frontend_lane_index of their client_id.""" buckets = [[] for _ in range(num_frontends)] for rsp in responses: buckets[frontend_lane_index(rsp.client_id, @@ -286,19 +278,12 @@ def bucket_responses_by_frontend(responses: list, def multi_frontend_request_addr(ipc_dir: str) -> str: - """The request ingress endpoint bound by the rank0 worker (PULL). - - Every frontend PUSH-connects to it. - """ + """The request ingress bound by the rank0 worker; frontends PUSH-connect.""" return f"ipc://{os.path.join(ipc_dir, 'request.sock')}" def multi_frontend_result_addr(ipc_dir: str, frontend_id: int) -> str: - """The result lane endpoint bound by frontend ``frontend_id`` (PULL). - - The worker/postproc processes PUSH-connect to it. Deterministic so a - respawned process can rebind the same lane. - """ + """The result lane bound by a frontend; worker/postproc PUSH-connect.""" return f"ipc://{os.path.join(ipc_dir, f'result_{frontend_id}.sock')}" diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index 1d876ee29e9a..d083b8c975b7 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -221,8 +221,8 @@ def _print_stacks(): postproc_worker_config = postproc_worker_config or PostprocWorkerConfig() is_leader: bool = mpi_rank() == 0 - # Multi-frontend serving (classic IPC path): per-frontend result lanes; - # the request queue is bound here (PULL) so every frontend PUSH-connects. + # Multi-frontend serving: the worker binds the request ingress (PULL) + # and pushes responses to per-frontend result lanes. multi_frontend_addrs = worker_queues.frontend_result_queue_addrs frontend_result_queues: Optional[List[FusedIpcQueue]] = None if tracer_init_kwargs is not None and is_leader: @@ -267,9 +267,7 @@ def _print_stacks(): for i in range(postproc_worker_config.num_postprocess_workers) ] elif multi_frontend_addrs is not None: - # Multi-frontend serving: one PUSH lane per frontend, selected by - # the frontend id in client_id's top bits (see base_worker - # _send_rsp). + # One PUSH lane per frontend (see base_worker._send_rsp). frontend_result_queues = [ FusedIpcQueue(addr, is_server=False, diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index f1bdf1628128..b02c5eab23b0 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -313,10 +313,8 @@ def __init__(self, load_post_processor_hook(_post_processor_path) if _post_processor_path else None) - # Attached serving frontends (TLLM_EXECUTOR_ATTACH_INFO) connect to - # an already-running executor worker: they need no MPI session of - # their own and must not spawn one (see executor.py - # GenerationExecutor.create). + # Attached serving frontends connect to an already-running worker: + # they must not spawn an MPI session (see GenerationExecutor.create). is_attached_frontend = os.getenv( "TLLM_EXECUTOR_ATTACH_INFO") is not None if self.args.parallel_config.is_multi_gpu and not is_attached_frontend: From e089a40a0cf4d546d5de58d5f2c1c10c465c870a Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:20:48 -0700 Subject: [PATCH 08/24] [None][chore] serve multi-frontend: explicit None-check instead of getattr for the frontend count llm_args can legitimately be None (legacy TensorRT path), but any non-None llm_args has the num_serve_frontends field (BaseLlmArgs), so handle the None case explicitly like the adjacent lines and let a renamed field fail loud. The 'or 1' was dead: pydantic enforces ge=1. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/executor/proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 504b26b1b6bb..93e9070780b3 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -169,7 +169,8 @@ def __init__( # dir + HMAC key for the per-frontend endpoints (_setup_queues); # trtllm-serve hands them to the attached frontends via # multi_frontend_attach_info(). - self._num_frontends = getattr(_llm_args, "num_serve_frontends", 1) or 1 + self._num_frontends = (_llm_args.num_serve_frontends + if _llm_args is not None else 1) self._multi_frontend_ipc_dir: Optional[str] = None self._multi_frontend_hmac: Optional[bytes] = None if self._num_frontends > 1: From a4f54aa6a178f0175f2819637ba266493d6394fe Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:28:33 -0700 Subject: [PATCH 09/24] [None][chore] serve multi-frontend: merge FrontendProxy id checks The 1<<16 upper bound (wire-format capacity) was dead code after the num_serve_frontends<=64 cap: the lane-count check already rejects anything >= len(result_addrs). Fold both into one range check with a message that names the valid ids. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/executor/proxy.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 93e9070780b3..31bf197b6381 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -1020,12 +1020,11 @@ def __init__( postproc_worker_config: Optional[PostprocWorkerConfig] = None, is_llm_executor: Optional[bool] = None, ) -> None: - if not 0 < frontend_id < (1 << 16): - raise ValueError(f"frontend_id out of range: {frontend_id}") - if frontend_id >= len(attach_info["result_addrs"]): + num_lanes = len(attach_info["result_addrs"]) + if not 0 < frontend_id < num_lanes: raise ValueError( - f"frontend_id {frontend_id} has no result lane: only " - f"{len(attach_info['result_addrs'])} lanes were provisioned") + f"frontend_id {frontend_id} out of range: attached frontends " + f"use ids 1..{num_lanes - 1} (id 0 is the launcher)") postproc_worker_config = postproc_worker_config or PostprocWorkerConfig( ) # Deliberately skip GenerationExecutorProxy.__init__: it creates an From 757660a11ac67e24ac56fb43b3168f237b83ef02 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:25:56 -0700 Subject: [PATCH 10/24] [None][chore] apply pre-commit yapf formatting Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 59 ++++++++++++++++------------------ tensorrt_llm/executor/utils.py | 3 +- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 9c53b1b37038..69e4c93bb498 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -38,8 +38,7 @@ validate_config_bool) from tensorrt_llm.llmapi.llm_args import MultimodalConfig, TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict -from tensorrt_llm.llmapi.mpi_session import (find_free_ipc_addr, - split_mpi_env) +from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr, split_mpi_env from tensorrt_llm.llmapi.reasoning_parser import (ReasoningParserFactory, resolve_auto_reasoning_parser) from tensorrt_llm.logger import logger, severity_map @@ -399,9 +398,8 @@ def _init_multi_frontend_mode(llm_args: dict, llm_args.pop("num_serve_frontends", None) return MultiFrontendMode(1, False) - mode = MultiFrontendMode( - num_frontends, - os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) + mode = MultiFrontendMode(num_frontends, + os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) if mode.is_launcher and llm_args.get("orchestrator_type") is not None: raise ValueError( "num_serve_frontends > 1 currently supports only the " @@ -495,8 +493,7 @@ def launch_server( backend = llm_args["backend"] model = served_model_name or llm_args["model"] - multi_frontend = _init_multi_frontend_mode(llm_args, - multi_frontend_enabled) + multi_frontend = _init_multi_frontend_mode(llm_args, multi_frontend_enabled) addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) @@ -1212,30 +1209,30 @@ def launch_visual_gen_server( help= "Types of agents to schedule. Now Only Support Open Deep Research agent.", status="prototype") -def serve( - model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], - post_processor_hook: Optional[str], host: str, port: int, - log_level: str, backend: str, max_beam_width: int, max_batch_size: int, - max_num_tokens: int, max_seq_len: int, tensor_parallel_size: int, - pipeline_parallel_size: int, context_parallel_size: int, - moe_expert_parallel_size: Optional[int], - moe_cluster_parallel_size: Optional[int], gpus_per_node: Optional[int], - free_gpu_memory_fraction: float, kv_cache_dtype: str, - num_postprocess_workers: int, num_serve_frontends: int, - num_input_processor_workers: int, - num_media_load_workers: int, trust_remote_code: bool, - revision: Optional[str], extra_llm_api_options: Optional[str], - reasoning_parser: Optional[str], tool_parser: Optional[str], - metadata_server_config_file: Optional[str], server_role: Optional[str], - fail_fast_on_attention_window_too_large: bool, - otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, - enable_attention_dp: bool, disagg_cluster_uri: Optional[str], - media_io_kwargs: Optional[str], agent_percentage: float, - agent_types: Optional[str], video_pruning_rate: Optional[float], - telemetry: bool, custom_module_dirs: list[Path], - chat_template: Optional[str], allow_request_chat_template: bool, - middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, - served_model_name: Optional[str], visual_gen_args: Optional[str]): +def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], + post_processor_hook: Optional[str], host: str, port: int, + log_level: str, backend: str, max_beam_width: int, + max_batch_size: int, max_num_tokens: int, max_seq_len: int, + tensor_parallel_size: int, pipeline_parallel_size: int, + context_parallel_size: int, moe_expert_parallel_size: Optional[int], + moe_cluster_parallel_size: Optional[int], + gpus_per_node: Optional[int], free_gpu_memory_fraction: float, + kv_cache_dtype: str, num_postprocess_workers: int, + num_serve_frontends: int, num_input_processor_workers: int, + num_media_load_workers: int, trust_remote_code: bool, + revision: Optional[str], extra_llm_api_options: Optional[str], + reasoning_parser: Optional[str], tool_parser: Optional[str], + metadata_server_config_file: Optional[str], + server_role: Optional[str], + fail_fast_on_attention_window_too_large: bool, + otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, + enable_attention_dp: bool, disagg_cluster_uri: Optional[str], + media_io_kwargs: Optional[str], agent_percentage: float, + agent_types: Optional[str], video_pruning_rate: Optional[float], + telemetry: bool, custom_module_dirs: list[Path], + chat_template: Optional[str], allow_request_chat_template: bool, + middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, + served_model_name: Optional[str], visual_gen_args: Optional[str]): """Running an OpenAI API compatible server MODEL: model name | HF checkpoint path | TensorRT engine path diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 469067133a04..6628906377f3 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -272,8 +272,7 @@ def bucket_responses_by_frontend(responses: list, """Bucket responses by frontend_lane_index of their client_id.""" buckets = [[] for _ in range(num_frontends)] for rsp in responses: - buckets[frontend_lane_index(rsp.client_id, - num_frontends)].append(rsp) + buckets[frontend_lane_index(rsp.client_id, num_frontends)].append(rsp) return buckets From 9a086ffde9ea4d539f2aca04a1ccbdc5f6de6774 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:03:08 -0700 Subject: [PATCH 11/24] [None][chore] serve multi-frontend: pass attach info via env value, not a temp file Review feedback (reasonsolo): serialize the attach payload directly into TLLM_EXECUTOR_ATTACH_INFO instead of writing a 0600 temp file and pointing the env at it. This deletes the file lifecycle entirely (no mkstemp, no unlink, no key left on disk after a hard kill), and the child pops the env once consumed so the HMAC keys cannot leak into descendant processes. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 38 +++++++------------------------ tensorrt_llm/executor/executor.py | 12 ++++++---- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 69e4c93bb498..d5afd3b6b717 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -9,7 +9,6 @@ import socket import subprocess # nosec B404 import sys -import tempfile import uuid from pathlib import Path from typing import Any, Dict, NamedTuple, Optional, Sequence, Set @@ -408,16 +407,13 @@ def _init_multi_frontend_mode(llm_args: dict, return mode -def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: +def _spawn_attached_frontends(llm, num_frontends: int) -> list: """Spawn num_frontends - 1 attached serving frontend processes. Each child re-execs this trtllm-serve command line with env vars - pointing at the launcher executor's attach endpoints; its executor + carrying the launcher executor's attach endpoints; its executor attaches to the already-running worker instead of launching one (see GenerationExecutor.create / GenerationExecutorFrontendProxy). - - Returns (children, attach_info_path); the caller owns terminating the - children and removing the secret-bearing attach-info file. """ from tensorrt_llm.executor.proxy import GenerationExecutorProxy @@ -427,18 +423,16 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: raise ValueError( "num_serve_frontends > 1 requires the classic IPC executor " f"proxy in multi-frontend mode, got {type(executor).__name__}") - # mkstemp creates the file 0600: it carries the executor HMAC keys. - fd, attach_info_path = tempfile.mkstemp(prefix="trtllm_frontend_", - suffix=".json") - with os.fdopen(fd, "w") as f: - json.dump(attach_info, f) + # Carries the executor HMAC keys; the child deletes it from its env + # once consumed (GenerationExecutor.create). + attach_env = json.dumps(attach_info) children = [] for frontend_id in range(1, num_frontends): # Strip MPI/SLURM identity vars: an inherited rank identity would # make the child's mpi4py try to (re-)join the launcher's job. env, _ = split_mpi_env() - env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_info_path + env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_env env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) env["TLLM_DISABLE_MPI"] = "1" child = subprocess.Popen([sys.executable] + sys.argv, @@ -447,7 +441,7 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> tuple[list, str]: logger.info( f"Launched attached serving frontend {frontend_id} (pid {child.pid})" ) - return children, attach_info_path + return children def _terminate_attached_frontends(children: list) -> None: @@ -460,19 +454,6 @@ def _terminate_attached_frontends(children: list) -> None: child.kill() -def _cleanup_multi_frontend_artifacts(attach_info_path: Optional[str]) -> None: - """Remove the attach-info file (it carries HMAC keys). - - The shared ipc directory is owned and removed by the launcher's - executor proxy instead. - """ - if attach_info_path is not None: - try: - os.unlink(attach_info_path) - except OSError: - pass - - def launch_server( host: str, port: int, @@ -533,9 +514,8 @@ def launch_server( param_hint="backend") frontend_children = [] - attach_info_path = None if multi_frontend.is_launcher: - frontend_children, attach_info_path = _spawn_attached_frontends( + frontend_children = _spawn_attached_frontends( llm, multi_frontend.num_frontends) server = OpenAIServer( @@ -561,8 +541,6 @@ def launch_server( finally: if frontend_children: _terminate_attached_frontends(frontend_children) - if multi_frontend.is_launcher: - _cleanup_multi_frontend_artifacts(attach_info_path) def launch_grpc_server(host: str, diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 726102b68431..43adcbba93e6 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -565,10 +565,12 @@ def create( # Multi-frontend serving: attach to an already-running executor # instead of launching one (set by trtllm-serve for attached # frontend processes). - attach_info_path = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") - if attach_info_path: - with open(attach_info_path) as f: - attach_info = json.load(f) + attach_env = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") + if attach_env: + attach_info = json.loads(attach_env) + # Consumed: it carries the HMAC keys and must not leak into + # descendant processes. + os.environ.pop("TLLM_EXECUTOR_ATTACH_INFO", None) if attach_info.get("mode") != "classic": raise ValueError( "TLLM_EXECUTOR_ATTACH_INFO only supports the classic IPC " @@ -582,7 +584,7 @@ def create( "by trtllm-serve when spawning attached frontends.") frontend_id = int(frontend_id_env) logger.info(f"Attaching executor frontend {frontend_id} to the " - f"running classic IPC worker via {attach_info_path}") + "running classic IPC worker") return GenerationExecutorFrontendProxy( attach_info, frontend_id=frontend_id, From a8d7b23c4b28278f8bca6c4e87e6e8d15517c9e0 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:03:57 -0700 Subject: [PATCH 12/24] [None][chore] serve multi-frontend: slim MultiFrontendMode Review feedback (reasonsolo): drop the single-use 'active' property (its one call site reads clearer as launcher-or-attached), fold its definition into is_launcher, and shorten the docstrings. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index d5afd3b6b717..dce118db7066 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -362,30 +362,22 @@ def _diagnose_port_in_use(port: int) -> str: class MultiFrontendMode(NamedTuple): - """Resolved multi-frontend serving mode for a launch_server invocation.""" + """This process's role under multi-frontend serving (prototype).""" num_frontends: int is_attached_frontend: bool - @property - def active(self) -> bool: - """Any multi-frontend role: the launcher or an attached frontend.""" - return self.num_frontends > 1 or self.is_attached_frontend - @property def is_launcher(self) -> bool: - """The frontend that owns the engine and spawns/cleans the others.""" - return self.active and not self.is_attached_frontend + """Owns the engine and spawns/cleans the attached frontends.""" + return self.num_frontends > 1 and not self.is_attached_frontend def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: - """Resolve the multi-frontend serving mode (prototype). - - num_serve_frontends=K runs K HTTP frontend processes against ONE + """num_serve_frontends=K runs K HTTP frontend processes against ONE executor: the launcher (frontend 0) owns the engine and spawns K-1 - attached frontends. Classic IPC executor path only. Entry points that - must not honor the knob (e.g. disaggregated MPI workers) pass - enabled=False. + attached frontends (classic IPC executor path only). Entry points + that must not honor the knob pass enabled=False. """ num_frontends = llm_args.get("num_serve_frontends", 1) if not enabled: @@ -483,7 +475,7 @@ def launch_server( with socket.socket(address_family, socket.SOCK_STREAM) as s: # If disagg cluster config is provided and port is not specified, try to find a free port, otherwise try to bind to the specified port assert port > 0 or disagg_cluster_config is not None, "Port must be specified if disagg cluster config is not provided" - if multi_frontend.active: + if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: # Every frontend process binds its own listening socket on the # same port; the kernel load-balances accepts across them. s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) From bd1fab24a2b15dddd1ffe40feec73dd5516d6b6d Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:05:15 -0700 Subject: [PATCH 13/24] [None][chore] postproc: take the result-lane address list unconditionally Review feedback (reasonsolo): no backward compatibility needed for the in-tree call chain -- worker_main always passes a list (a single lane in single-frontend mode), so PostprocWorker/postproc_worker_main drop the Union[tuple, List[tuple]] signatures and the isinstance normalization. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/executor/postproc_worker.py | 17 ++++++----------- tensorrt_llm/executor/worker.py | 12 ++++++------ tests/unittest/llmapi/test_executor.py | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py index 182b160db471..552c5c8f2bd3 100644 --- a/tensorrt_llm/executor/postproc_worker.py +++ b/tensorrt_llm/executor/postproc_worker.py @@ -86,8 +86,7 @@ class Output(NamedTuple): def __init__( self, pull_pipe_addr: tuple[str, Optional[bytes]], - push_pipe_addr: Union[tuple[str, Optional[bytes]], - List[tuple[str, Optional[bytes]]]], + push_pipe_addrs: List[tuple[str, Optional[bytes]]], tokenizer_dir: str, record_creator: Callable[ ["PostprocWorker.Input", TransformersTokenizer], Any], @@ -96,9 +95,9 @@ def __init__( ''' Args: pull_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the input IPC. - push_pipe_addr: The address and HMAC key of the output IPC, or - a list of them (one result lane per frontend) with - multi-frontend serving. + push_pipe_addrs: The addresses and HMAC keys of the output IPC + lanes, one per frontend (a single-element list in + single-frontend mode). tokenizer_dir (str): The directory to load tokenizer. record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request. result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result. @@ -111,8 +110,6 @@ def __init__( is_async=True, is_server=False, name="postprocess_pull_pipe") - push_pipe_addrs = (push_pipe_addr if isinstance(push_pipe_addr, list) - else [push_pipe_addr]) self._push_pipes = [ ZeroMqQueue(address=addr, is_async=True, @@ -305,15 +302,13 @@ async def main(): @print_traceback_on_error def postproc_worker_main(feedin_ipc_addr: tuple[str, Optional[bytes]], - feedout_ipc_addr: Union[tuple[str, Optional[bytes]], - List[tuple[str, - Optional[bytes]]]], + feedout_ipc_addrs: List[tuple[str, Optional[bytes]]], tokenizer_dir: str, record_creator: Callable, post_processor_hook: Optional[str] = None): # Pass the hook import path; PostprocWorker builds it once. worker = PostprocWorker(feedin_ipc_addr, - feedout_ipc_addr, + feedout_ipc_addrs, tokenizer_dir=tokenizer_dir, record_creator=record_creator, post_processor_hook=post_processor_hook) diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index d083b8c975b7..233e394e7c77 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -300,20 +300,20 @@ def notify_proxy_threads_to_quit(): if is_leader and postproc_worker_config.enabled: logger_debug(f"initiate postprocess workers...", "yellow") - # With multi-frontend serving each postproc worker gets every - # frontend's result lane and routes outputs by client_id's top bits. - proxy_result_queue = (multi_frontend_addrs if multi_frontend_addrs - is not None else worker_queues.result_queue_addr) + # Each postproc worker pushes to every frontend result lane (a + # single lane in single-frontend mode). + proxy_result_addrs = (multi_frontend_addrs + if multi_frontend_addrs is not None else + [worker_queues.result_queue_addr]) assert result_queues is not None postproc_worker_pool = ProcessPoolExecutor( max_workers=postproc_worker_config.num_postprocess_workers) - assert isinstance(proxy_result_queue, (tuple, list)) for i in range(postproc_worker_config.num_postprocess_workers): fut = postproc_worker_pool.submit( postproc_worker_main, result_queues[i].address, - proxy_result_queue, + proxy_result_addrs, postproc_worker_config.postprocess_tokenizer_dir, PostprocWorker.default_record_creator, postproc_worker_config.post_processor_hook, diff --git a/tests/unittest/llmapi/test_executor.py b/tests/unittest/llmapi/test_executor.py index 1b11587f7e15..50eee4f768a4 100644 --- a/tests/unittest/llmapi/test_executor.py +++ b/tests/unittest/llmapi/test_executor.py @@ -318,7 +318,7 @@ def ResponsePostprocessWorker_worker_task(pull_pipe_addr, push_pipe_addr, tokenizer_dir): worker = PostprocWorker( pull_pipe_addr=pull_pipe_addr, - push_pipe_addr=push_pipe_addr, + push_pipe_addrs=[push_pipe_addr], tokenizer_dir=tokenizer_dir, record_creator=ResponsePostprocessWorker_record_creator) worker.start() From cbbf9ab6f1c24b08d0470322fc95b706c1ef604a Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:10:17 -0700 Subject: [PATCH 14/24] [None][chore] serve multi-frontend: tighten _init_multi_frontend_mode Review feedback (reasonsolo): fold the disabled-path get/warn/pop into one pop-with-default, drop the duplicated knob read, and shorten the docstring and error text. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index dce118db7066..ef477fe99d8f 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -376,25 +376,21 @@ def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: """num_serve_frontends=K runs K HTTP frontend processes against ONE executor: the launcher (frontend 0) owns the engine and spawns K-1 - attached frontends (classic IPC executor path only). Entry points - that must not honor the knob pass enabled=False. + attached frontends (classic IPC executor path only). enabled=False + entry points (e.g. disaggregated MPI workers) never honor the knob. """ - num_frontends = llm_args.get("num_serve_frontends", 1) if not enabled: - if num_frontends > 1: - logger.warning( - "num_serve_frontends is ignored on this entry point; " - "multi-frontend serving is only supported on plain " - "trtllm-serve.") - llm_args.pop("num_serve_frontends", None) + if llm_args.pop("num_serve_frontends", 1) > 1: + logger.warning("num_serve_frontends is only supported on plain " + "trtllm-serve; ignored on this entry point.") return MultiFrontendMode(1, False) - mode = MultiFrontendMode(num_frontends, + mode = MultiFrontendMode(llm_args.get("num_serve_frontends", 1), os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) if mode.is_launcher and llm_args.get("orchestrator_type") is not None: raise ValueError( - "num_serve_frontends > 1 currently supports only the " - "default (classic IPC) executor path, not orchestrator_type=" + "num_serve_frontends > 1 requires the default (classic IPC) " + "executor path, not orchestrator_type=" f"{llm_args.get('orchestrator_type')!r}") return mode From 6f0d39d09baa83f38a559266c076a42d2b751ce1 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:31:44 -0700 Subject: [PATCH 15/24] [None][fix] serve multi-frontend: harden the client_id namespace Review feedback (reasonsolo) on the id encoding: - The launcher (lane 0) now applies the same namespace rule as attached frontends whenever multi-frontend is active, so a long-lived request counter can never bleed into the frontend-id bits and misroute responses (it was an unguarded invariant before; the re-encode is a numeric no-op until the counter wraps). Single-frontend mode keeps raw ids. - The frontend-id field width is now the single source of truth: FRONTEND_ID_BITS = 6 derives MAX_NUM_FRONTENDS (64), the shift and the counter mask. The CLI cap imports the constant; the llm_args bound cannot (import cycle) and is pinned by a unit test instead. - The field sits just below the sign bit (shift 57): bit 63 stays clear, so ids remain positive in signed-int64 contexts. Field order stays frontend-id-high: the launcher's ids keep their numeric values, and a stray un-namespaced id degrades to lane 0 (the launcher) instead of spraying across lanes. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 4 ++-- tensorrt_llm/executor/proxy.py | 10 ++++++++++ tensorrt_llm/executor/utils.py | 15 ++++++++++----- tensorrt_llm/llmapi/llm_args.py | 2 ++ .../executor/test_multi_frontend_routing.py | 14 +++++++++++--- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index ef477fe99d8f..ee124201918a 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -26,7 +26,7 @@ from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import LlmLauncherEnvs +from tensorrt_llm.executor.utils import (LlmLauncherEnvs, MAX_NUM_FRONTENDS) from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, @@ -988,7 +988,7 @@ def launch_visual_gen_server( "to comply with OpenAI protocol.", status="prototype") @stability_option("--num_serve_frontends", - type=click.IntRange(min=1, max=64), + type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS), default=1, help="Number of HTTP frontend processes serving one " "executor; values > 1 share the serving port via " diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 31bf197b6381..d0329d3c9ead 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -705,6 +705,15 @@ def pre_shutdown(self): if not self.mpi_futures or any(not f.done() for f in self.mpi_futures): self.request_queue.put_noblock(None, retry=4) + def _get_next_client_id(self) -> int: + client_id = super()._get_next_client_id() + if self._num_frontends > 1: + # Lane 0 follows the same namespace rule as attached frontends + # so a long-lived counter can never bleed into the frontend-id + # bits (a no-op re-encode until the counter wraps). + client_id = namespace_client_id(0, client_id) + return client_id + def _cleanup_multi_frontend_ipc_dir(self): """Remove the launcher-owned multi-frontend ipc directory. @@ -1039,6 +1048,7 @@ def __init__( is_llm_executor=is_llm_executor) self._frontend_id = frontend_id + self._num_frontends = num_lanes self._results: Dict[int, GenerationResult] = {} self.garbage_collection_gen0_threshold = None self.workers_started = False diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 6628906377f3..e4a9f3333f4b 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -236,11 +236,16 @@ class WorkerCommIpcAddrs(NamedTuple): Optional[bytes]]]] = None -# Multi-frontend client_id namespacing: the top bits of the uint64 client id -# carry the frontend id, the low FRONTEND_ID_SHIFT bits carry the per-frontend -# request counter. Frontend id 0 keeps client ids bit-identical to the legacy -# single-frontend scheme. -FRONTEND_ID_SHIFT = 48 +# Multi-frontend client_id namespacing: a FRONTEND_ID_BITS-wide frontend id +# sits just below the sign bit -- bit 63 stays clear so ids remain positive +# in signed-int64 contexts -- and the low bits carry the per-frontend +# request counter. A stray un-namespaced (small) id reads as frontend 0, +# the launcher. +FRONTEND_ID_BITS = 6 +# Keep llm_args.num_serve_frontends le= in sync (it cannot import this +# module; test_multi_frontend_routing pins the two together). +MAX_NUM_FRONTENDS = 1 << FRONTEND_ID_BITS +FRONTEND_ID_SHIFT = 63 - FRONTEND_ID_BITS FRONTEND_COUNTER_MASK = (1 << FRONTEND_ID_SHIFT) - 1 diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 13fac08b933e..f0baa32afaf1 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -4124,6 +4124,8 @@ class BaseLlmArgs(StrictBaseModel): num_serve_frontends: int = Field( default=1, ge=1, + # = executor.utils.MAX_NUM_FRONTENDS (cannot be imported here); + # test_multi_frontend_routing pins the two together. le=64, description= "The number of HTTP frontend processes serving one executor. Used by " diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index 4876623260e8..f1631ec8abb8 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -26,6 +26,7 @@ from tensorrt_llm.executor.utils import ( FRONTEND_COUNTER_MASK, + MAX_NUM_FRONTENDS, bucket_responses_by_frontend, frontend_lane_index, get_frontend_id, @@ -39,15 +40,22 @@ def test_frontend_zero_keeps_legacy_ids(self): assert namespace_client_id(0, client_id) == client_id def test_roundtrip(self): - for frontend_id in (0, 1, 7, (1 << 16) - 1): + for frontend_id in (0, 1, 7, MAX_NUM_FRONTENDS - 1): for counter in (1, 12345, FRONTEND_COUNTER_MASK): client_id = namespace_client_id(frontend_id, counter) assert get_frontend_id(client_id) == frontend_id assert client_id & FRONTEND_COUNTER_MASK == counter - assert client_id < (1 << 64) + # Bit 63 stays clear: ids remain positive as signed int64. + assert 0 < client_id < (1 << 63) + + def test_knob_cap_matches_encoding(self): + from tensorrt_llm.llmapi.llm_args import BaseLlmArgs + meta = BaseLlmArgs.model_fields["num_serve_frontends"].metadata + assert any( + getattr(m, "le", None) == MAX_NUM_FRONTENDS for m in meta) def test_counter_wraparound_stays_in_namespace(self): - # A counter larger than 48 bits must not leak into the frontend bits. + # A counter overflowing its field must not leak into the frontend bits. client_id = namespace_client_id(3, FRONTEND_COUNTER_MASK + 5) assert get_frontend_id(client_id) == 3 assert client_id & FRONTEND_COUNTER_MASK == 4 From d6a2eb60ce7d46f8654528c75abc019363e9dbd1 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:39:52 -0700 Subject: [PATCH 16/24] [None][chore] apply pre-commit isort/yapf formatting Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 2 +- tests/unittest/executor/test_multi_frontend_routing.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index ee124201918a..d483850c28e8 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -26,7 +26,7 @@ from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import (LlmLauncherEnvs, MAX_NUM_FRONTENDS) +from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS, LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index f1631ec8abb8..9dca95db998a 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -50,9 +50,9 @@ def test_roundtrip(self): def test_knob_cap_matches_encoding(self): from tensorrt_llm.llmapi.llm_args import BaseLlmArgs + meta = BaseLlmArgs.model_fields["num_serve_frontends"].metadata - assert any( - getattr(m, "le", None) == MAX_NUM_FRONTENDS for m in meta) + assert any(getattr(m, "le", None) == MAX_NUM_FRONTENDS for m in meta) def test_counter_wraparound_stays_in_namespace(self): # A counter overflowing its field must not leak into the frontend bits. From eb3e42570f4926523c1fd83b074666d128f26db9 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:45:20 -0700 Subject: [PATCH 17/24] [None][chore] fix D205 docstring layout in _init_multi_frontend_mode Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index d483850c28e8..056668b65c98 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -374,7 +374,9 @@ def is_launcher(self) -> bool: def _init_multi_frontend_mode(llm_args: dict, enabled: bool) -> MultiFrontendMode: - """num_serve_frontends=K runs K HTTP frontend processes against ONE + """Resolve this process's multi-frontend serving role. + + num_serve_frontends=K runs K HTTP frontend processes against ONE executor: the launcher (frontend 0) owns the engine and spawns K-1 attached frontends (classic IPC executor path only). enabled=False entry points (e.g. disaggregated MPI workers) never honor the knob. From e87306ec32b347f88da052e761c2bfa588532664 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:56:15 -0700 Subject: [PATCH 18/24] [None][chore] regenerate the LLM args telemetry golden manifest num_serve_frontends was added to the llm args; the telemetry capture manifest gains the corresponding row (plain int value field, mirroring num_postprocess_workers). Requires telemetry/privacy CODEOWNER review per the manifest policy. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/usage/llm_args_golden_manifest.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index ca6e762490cf..4d0261bf0131 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -1016,6 +1016,13 @@ "kind": "value", "path": "num_postprocess_workers" }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "num_serve_frontends" + }, { "allowed_values": [ "cutlass", From d17d1ccd979f73c9833af37108f984481e778eba Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:59:07 -0700 Subject: [PATCH 19/24] [None][chore] add --num_serve_frontends to the serve CLI stability reference status: prototype, matching the stability_option declaration. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- .../api_stability/references/trtllm_serve_cli.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 5d681cdeb9d8..c3430712d780 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -303,6 +303,15 @@ commands: is_flag: false flags: - "--num_postprocess_workers" + num_serve_frontends: + type: int + default: 1 + status: prototype + required: false + multiple: false + is_flag: false + flags: + - "--num_serve_frontends" num_input_processor_workers: type: int default: 8 From 909faaeeda02c967d8c432b1bc3b8f207f93f0a9 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Mon, 20 Jul 2026 02:54:47 -0700 Subject: [PATCH 20/24] [None][fix] serve: initialize attached-frontend proxy state and cover it in CI GenerationExecutorFrontendProxy deliberately skips GenerationExecutorProxy.__init__ but still inherits submit(), check_health() and collective_rpc(), which read _engine_dead, _worker_process_monitor and model_world_size: the first submit() on an attached frontend raised AttributeError. Initialize that state explicitly, carry model_world_size in the attach payload so the collective_rpc guard matches the launcher, and give the attached frontend an explicit health contract (it owns no workers to poll). unittest/executor tests are listed per-file in the CI test db and test_multi_frontend_routing.py was in no list, so its own submit test never ran. Add it to l0_cpu_x86 and add a health/submit regression test. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/executor/proxy.py | 31 +++++++++++++++++++ .../test_lists/test-db/l0_cpu_x86.yml | 1 + .../executor/test_multi_frontend_routing.py | 22 +++++++++++++ 3 files changed, 54 insertions(+) diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index d0329d3c9ead..aa661ab02789 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -490,6 +490,10 @@ def multi_frontend_attach_info(self) -> Optional[dict]: ], "hmac_key": hmac_key.hex(), + # Attached frontends must apply the same collective_rpc guard + # as the launcher (see _check_collective_rpc_guard). + "model_world_size": + self.model_world_size, # Stats / KV events / disagg params RPC endpoint on the rank0 # worker (ROUTER socket, natively multi-client). "rpc_addr": @@ -1047,6 +1051,15 @@ def __init__( postprocess_tokenizer_dir, is_llm_executor=is_llm_executor) + # State consumed by methods inherited from GenerationExecutorProxy + # (submit / check_health / collective_rpc). The engine lives with + # the launcher: there are no local MPI workers, so the monitor + # stays empty and worker death reaches this frontend through its + # result lane / error queue instead. + self._engine_dead = False + self.model_world_size = attach_info.get("model_world_size", 1) + self._worker_process_monitor = WorkerProcessMonitor() + self._frontend_id = frontend_id self._num_frontends = num_lanes self._results: Dict[int, GenerationResult] = {} @@ -1084,6 +1097,23 @@ def _get_next_client_id(self) -> int: return namespace_client_id(self._frontend_id, super()._get_next_client_id()) + def check_health(self) -> bool: + """Health contract of an attached frontend. + + An attached frontend owns no workers, so there is no process or + MPI-future liveness to poll: it is healthy while no fatal error + has been recorded and shutdown has not begun. Engine death + reaches it through the per-lane result socket / dispatch-thread + error path, which records the fatal error checked here. + """ + if self.doing_shutdown or self._fatal_error is not None: + return False + + if self._drain_error_queue(): + return self._fatal_error is None and not self.doing_shutdown + + return True + def pre_shutdown(self): if self.doing_shutdown: return @@ -1098,6 +1128,7 @@ def shutdown(self): if self.rpc_client is not None: self.rpc_client.close() self.rpc_client = None + self._worker_process_monitor.close() # The dispatch thread blocks on result_queue.get(); it is a daemon # ManagedThread that exits with the process or on the worker's # per-lane None sentinel at engine teardown. Closing its socket from diff --git a/tests/integration/test_lists/test-db/l0_cpu_x86.yml b/tests/integration/test_lists/test-db/l0_cpu_x86.yml index 9a39993347b8..a23f8b9a8f46 100644 --- a/tests/integration/test_lists/test-db/l0_cpu_x86.yml +++ b/tests/integration/test_lists/test-db/l0_cpu_x86.yml @@ -14,3 +14,4 @@ l0_cpu_x86: orchestrator: mpi tests: - unittest/executor/test_rpc.py + - unittest/executor/test_multi_frontend_routing.py diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index 9dca95db998a..22c6d7d76997 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -218,6 +218,28 @@ def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): "an attached frontend must never send the engine-shutdown sentinel" ) + def test_check_health_and_submit_reflect_fatal_error(self): + import pytest + + from tensorrt_llm.executor.request import GenerationRequest + from tensorrt_llm.executor.utils import EngineDeadError + from tensorrt_llm.sampling_params import SamplingParams + + with tempfile.TemporaryDirectory() as tmpdir: + proxy, _, _, _ = self._make_proxy_and_fake_worker(tmpdir) + + # Regression: this state lives in GenerationExecutorProxy.__init__, + # which an attached frontend deliberately skips, so it must be + # initialized explicitly (a missing _engine_dead broke the first + # submit(); a missing _worker_process_monitor broke /health). + assert proxy.check_health() + assert proxy.model_world_size == 1 + + proxy._set_fatal_error(RuntimeError("rank0 worker died")) + assert not proxy.check_health() + with pytest.raises(EngineDeadError): + proxy.submit(GenerationRequest([1], SamplingParams())) + def test_dispatch_routes_own_lane_responses(self): import zmq From 0b63e891c550cebcc7b1034d73e5a6bf687e3f97 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Mon, 20 Jul 2026 03:18:04 -0700 Subject: [PATCH 21/24] [None][fix] serve: READY handshake and exception-safe cleanup for attached frontends A successful Popen only proves the child process exists: an attached frontend can still fail during executor attach or server setup, and num_serve_frontends=K would silently keep serving with fewer frontends, skewing benchmarks. Each child now inherits a pipe (TLLM_FRONTEND_READY_FD) and writes a READY byte once everything fallible in its startup has succeeded; the launcher blocks on every byte (TLLM_FRONTEND_READY_TIMEOUT, default 300s) and fails the whole group if any child exits or misses the deadline. Also move the cleanup boundary so children cannot leak: spawning cleans up after itself on partial failure, and launch_server's try/finally now covers spawn, OpenAIServer construction, middleware registration and runtime instead of runtime alone. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 144 +++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 35 deletions(-) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 056668b65c98..9eed3c593b87 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -5,10 +5,12 @@ import json import os import secrets +import select import signal import socket import subprocess # nosec B404 import sys +import time import uuid from pathlib import Path from typing import Any, Dict, NamedTuple, Optional, Sequence, Set @@ -404,6 +406,13 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> list: carrying the launcher executor's attach endpoints; its executor attaches to the already-running worker instead of launching one (see GenerationExecutor.create / GenerationExecutorFrontendProxy). + + Blocks until every child signals READY over its inherited pipe: a + successful Popen only proves the process exists, while the frontend + can still fail during executor attach or server setup. Any child + failure (or a missed deadline) fails the whole group, terminating + the children already started, so num_serve_frontends=K never + silently degrades to fewer frontends. """ from tensorrt_llm.executor.proxy import GenerationExecutorProxy @@ -417,23 +426,83 @@ def _spawn_attached_frontends(llm, num_frontends: int) -> list: # once consumed (GenerationExecutor.create). attach_env = json.dumps(attach_info) - children = [] - for frontend_id in range(1, num_frontends): - # Strip MPI/SLURM identity vars: an inherited rank identity would - # make the child's mpi4py try to (re-)join the launcher's job. - env, _ = split_mpi_env() - env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_env - env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) - env["TLLM_DISABLE_MPI"] = "1" - child = subprocess.Popen([sys.executable] + sys.argv, - env=env) # nosec B603 - children.append(child) - logger.info( - f"Launched attached serving frontend {frontend_id} (pid {child.pid})" - ) + children, ready_fds = [], [] + try: + for frontend_id in range(1, num_frontends): + # Strip MPI/SLURM identity vars: an inherited rank identity would + # make the child's mpi4py try to (re-)join the launcher's job. + env, _ = split_mpi_env() + env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_env + env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) + env["TLLM_DISABLE_MPI"] = "1" + read_fd, write_fd = os.pipe() + ready_fds.append(read_fd) + env["TLLM_FRONTEND_READY_FD"] = str(write_fd) + try: + child = subprocess.Popen([sys.executable] + sys.argv, + env=env, + pass_fds=(write_fd, )) # nosec B603 + finally: + # The child now holds the only write end; its exit before + # READY surfaces as EOF on read_fd. + os.close(write_fd) + children.append(child) + logger.info( + f"Launched attached serving frontend {frontend_id} (pid {child.pid})" + ) + _wait_attached_frontends_ready(children, ready_fds) + except BaseException: + _terminate_attached_frontends(children) + raise + finally: + for fd in ready_fds: + os.close(fd) return children +def _wait_attached_frontends_ready(children: list, ready_fds: list) -> None: + """Block until every attached frontend writes its READY byte.""" + timeout = float(os.getenv("TLLM_FRONTEND_READY_TIMEOUT", "300")) + deadline = time.monotonic() + timeout + pending = dict(zip(ready_fds, children)) + while pending: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError( + f"{len(pending)} attached frontend(s) not ready within " + f"{timeout:.0f}s (TLLM_FRONTEND_READY_TIMEOUT)") + readable, _, _ = select.select(list(pending), [], [], + min(remaining, 1.0)) + for fd in readable: + child = pending.pop(fd) + if os.read(fd, 1) != b"R": # EOF: pipe closed without READY + raise RuntimeError( + f"Attached frontend (pid {child.pid}) exited before " + "signaling READY") + logger.info(f"Attached frontend (pid {child.pid}) is ready") + for fd, child in list(pending.items()): + if child.poll() is not None: + raise RuntimeError( + f"Attached frontend (pid {child.pid}) exited with code " + f"{child.returncode} before signaling READY") + + +def _signal_frontend_ready(multi_frontend: MultiFrontendMode) -> None: + """Report READY to the launcher over the inherited pipe. + + Called once everything fallible in an attached frontend's startup + (port bind, executor attach, LLM and OpenAIServer construction, + middleware registration) has succeeded; the launcher blocks group + startup on this byte (see _wait_attached_frontends_ready). + """ + ready_fd = os.environ.pop("TLLM_FRONTEND_READY_FD", None) + if not (multi_frontend.is_attached_frontend and ready_fd): + return + fd = int(ready_fd) + os.write(fd, b"R") + os.close(fd) + + def _terminate_attached_frontends(children: list) -> None: for child in children: child.terminate() @@ -503,30 +572,35 @@ def launch_server( f"{backend} is not a known backend, check help for available options.", param_hint="backend") + # The finally below is the cleanup boundary for the attached + # frontends: it must cover everything from their spawn through + # server construction, middleware registration, and runtime, or a + # failure in between leaks the child processes. frontend_children = [] - if multi_frontend.is_launcher: - frontend_children = _spawn_attached_frontends( - llm, multi_frontend.num_frontends) - - server = OpenAIServer( - generator=llm, - model=model, - tool_parser=tool_parser, - server_role=server_role, - metadata_server_cfg=metadata_server_cfg, - disagg_cluster_config=disagg_cluster_config, - multimodal_server_config=multimodal_server_config, - chat_template=chat_template, - allow_request_chat_template=allow_request_chat_template, - input_processor_workers=num_input_processor_workers, - media_load_workers=num_media_load_workers) - _apply_fastapi_middlewares(server.app, middleware) + try: + if multi_frontend.is_launcher: + frontend_children = _spawn_attached_frontends( + llm, multi_frontend.num_frontends) + + server = OpenAIServer( + generator=llm, + model=model, + tool_parser=tool_parser, + server_role=server_role, + metadata_server_cfg=metadata_server_cfg, + disagg_cluster_config=disagg_cluster_config, + multimodal_server_config=multimodal_server_config, + chat_template=chat_template, + allow_request_chat_template=allow_request_chat_template, + input_processor_workers=num_input_processor_workers, + media_load_workers=num_media_load_workers) + _apply_fastapi_middlewares(server.app, middleware) - # Optionally disable GC (default: not disabled) - if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": - gc.disable() + # Optionally disable GC (default: not disabled) + if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": + gc.disable() - try: + _signal_frontend_ready(multi_frontend) uvloop.run(server(host, port, sockets=[s])) finally: if frontend_children: From 25a21de0c357d06b78937f3297a3d3b3da5161ff Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Mon, 20 Jul 2026 03:21:11 -0700 Subject: [PATCH 22/24] [None][fix] serve: disable stateful Responses API storage in multi-frontend mode The Responses API store is per-process in-memory and ResponsesRequest.store defaults to true: with several frontends behind one SO_REUSEPORT port, a follow-up request can land on a sibling that has no record of the previous response, so retrieval, deletion and previous_response_id resolution silently break. Force TRTLLM_RESPONSES_API_DISABLE_STORE=1 for every frontend in multi-frontend mode (reusing the existing enable_store gate), and reject previous_response_id with an explicit 400 when storage is disabled instead of silently treating the request as stateless. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tensorrt_llm/commands/serve.py | 11 +++++++++++ tensorrt_llm/serve/openai_server.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 9eed3c593b87..c40d7e43be16 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -534,6 +534,17 @@ def launch_server( model = served_model_name or llm_args["model"] multi_frontend = _init_multi_frontend_mode(llm_args, multi_frontend_enabled) + if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: + # The Responses API store is per-process in-memory: with several + # frontends behind one SO_REUSEPORT port, a follow-up request may + # land on a sibling that has no record of the previous response. + # Disable storage group-wide (OpenAIServer.enable_store). + if not os.getenv("TRTLLM_RESPONSES_API_DISABLE_STORE"): + logger.warning( + "num_serve_frontends > 1: stateful Responses API storage " + "(store/previous_response_id) is disabled; the per-frontend " + "in-memory store cannot be shared across frontends.") + os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1" addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 53ab14af4234..1d7c471f550c 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -2180,6 +2180,17 @@ async def create_streaming_generator(promise: RequestOutput, "Request.background is not supported yet, will fallback to foreground processing." ) + # Reject rather than silently ignore: with storage disabled + # (TRTLLM_RESPONSES_API_DISABLE_STORE, postproc workers, or + # multi-frontend serving) the previous response can never be + # resolved. + if request.previous_response_id is not None and not self.enable_store: + return self.create_error_response( + err_type="InvalidRequestError", + message=("'previous_response_id' requires response " + "storage, which is disabled on this server."), + ) + # Get prev response prev_response = None if self.enable_store: From 4e95c68424694abdabccd9df247cbf6af37e1db4 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:54:28 -0700 Subject: [PATCH 23/24] [None][test] mark multi-frontend routing tests cpu_only The CI CPU stages collect unittests with -m "cpu_only and not disabled" and skip files that don't mention pytest.mark.cpu_only, so the file was collected as 15 items / 15 deselected and pytest exited with code 5 (no tests ran), which the test wrapper reports as a failure. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- tests/unittest/executor/test_multi_frontend_routing.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index 22c6d7d76997..d37ec200da36 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -24,6 +24,8 @@ import time from types import SimpleNamespace +import pytest + from tensorrt_llm.executor.utils import ( FRONTEND_COUNTER_MASK, MAX_NUM_FRONTENDS, @@ -33,6 +35,10 @@ namespace_client_id, ) +# The CI CPU stages collect with -m "cpu_only and not disabled" and skip +# files that don't mention pytest.mark.cpu_only (see unittest/conftest.py). +pytestmark = pytest.mark.cpu_only + class TestClientIdNamespacing: def test_frontend_zero_keeps_legacy_ids(self): @@ -219,8 +225,6 @@ def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): ) def test_check_health_and_submit_reflect_fatal_error(self): - import pytest - from tensorrt_llm.executor.request import GenerationRequest from tensorrt_llm.executor.utils import EngineDeadError from tensorrt_llm.sampling_params import SamplingParams From f01226309e0510bd3bac49ade3ee928f70c3f852 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:07:55 -0700 Subject: [PATCH 24/24] [None][test] stop the dispatch thread in the frontend-proxy lifecycle tests Frontend shutdown deliberately leaves the dispatch thread to process teardown (closing its ZMQ socket from another thread is not safe), so the two proxy end-to-end tests left proxy_dispatch_result_thread running and pytest-threadleak failed them in the CI CPU stage. End each test the way the worker ends the thread at engine teardown: push the per-lane None sentinel from the fake worker and join the thread, which also covers the sentinel exit path. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- .../executor/test_multi_frontend_routing.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/unittest/executor/test_multi_frontend_routing.py b/tests/unittest/executor/test_multi_frontend_routing.py index d37ec200da36..d34aa62d8dde 100644 --- a/tests/unittest/executor/test_multi_frontend_routing.py +++ b/tests/unittest/executor/test_multi_frontend_routing.py @@ -195,12 +195,36 @@ def _make_proxy_and_fake_worker(tmpdir, frontend_id=1, num_frontends=2): ) return proxy, worker_ingress, hmac_key, result_addrs + @staticmethod + def _stop_dispatch_thread(proxy, result_addrs, hmac_key): + """End the dispatch thread the way the worker does at engine teardown. + + The worker fans a per-lane None sentinel to every frontend (see + notify_proxy_threads_to_quit). Frontend shutdown deliberately leaves + the thread to process teardown, which the threadleak checker would + report as a leak. + """ + import zmq + + from tensorrt_llm.executor.ipc import FusedIpcQueue + + worker_lane = FusedIpcQueue( + (result_addrs[proxy._frontend_id], hmac_key), + is_server=False, + fuse_message=False, + socket_type=zmq.PUSH, + name="fake_worker_shutdown_lane", + ) + worker_lane.put(None) + proxy.dispatch_result_thread.join(timeout=10) + assert not proxy.dispatch_result_thread.is_alive() + def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): from tensorrt_llm.executor.request import CancellingRequest, GenerationRequest from tensorrt_llm.sampling_params import SamplingParams with tempfile.TemporaryDirectory() as tmpdir: - proxy, worker_ingress, _, _ = self._make_proxy_and_fake_worker(tmpdir) + proxy, worker_ingress, hmac_key, result_addrs = self._make_proxy_and_fake_worker(tmpdir) # Attributes read by OpenAIServer at init must exist (a missing # _resource_governor_queue crashed all siblings in the first e2e). @@ -223,6 +247,7 @@ def test_submit_namespaces_and_shutdown_never_sends_sentinel(self): assert not worker_ingress.poll(1), ( "an attached frontend must never send the engine-shutdown sentinel" ) + self._stop_dispatch_thread(proxy, result_addrs, hmac_key) def test_check_health_and_submit_reflect_fatal_error(self): from tensorrt_llm.executor.request import GenerationRequest @@ -273,3 +298,5 @@ def test_dispatch_routes_own_lane_responses(self): while client_id in proxy._results and time.time() < deadline: time.sleep(0.01) assert client_id not in proxy._results + + self._stop_dispatch_thread(proxy, result_addrs, hmac_key)