diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f768882cd7e1..6334f8551c7e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -501,6 +501,7 @@ /pyproject.toml @NVIDIA/trt-llm-oss-compliance /requirements-dev.txt @NVIDIA/trt-llm-oss-compliance /requirements-grpc-smg.txt @NVIDIA/trt-llm-oss-compliance +/requirements-openengine.txt @NVIDIA/trt-llm-oss-compliance /requirements.txt @NVIDIA/trt-llm-oss-compliance /setup.py @NVIDIA/trt-llm-oss-compliance /tests/unittest/api_stability/ @NVIDIA/trt-llm-noncommitted-api-review-committee diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index 9a3c17a83f11..d1fb578115eb 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -130,7 +130,7 @@ COPY scripts scripts COPY tensorrt_llm tensorrt_llm COPY triton_kernels triton_kernels COPY 3rdparty 3rdparty -COPY setup.py requirements.txt requirements-dev.txt requirements-grpc-smg.txt constraints.txt LICENSE README.md ./ +COPY setup.py requirements.txt requirements-dev.txt requirements-grpc-smg.txt requirements-openengine.txt constraints.txt LICENSE README.md ./ ENV CCACHE_DIR=/root/.cache/ccache # Build the TRT-LLM wheel diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index afc883f1468a..b883aec89d79 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608191543-17891 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608191543-17891 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608191543-17891 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608191543-17891 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608191543-17891 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202608271702-17084 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202608271702-17084 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202608271702-17084 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202608271702-17084 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202608271702-17084 diff --git a/requirements-openengine.txt b/requirements-openengine.txt new file mode 100644 index 000000000000..3ac09eceb170 --- /dev/null +++ b/requirements-openengine.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Source: https://github.com/ai-dynamo/openengine/releases/tag/v0.1.0 +# BSR module: https://buf.build/openengine/openengine +# Immutable BSR commit: 768a93c7b44e40f28c692ad0b471a8f2 +--extra-index-url https://buf.build/gen/python +openengine-openengine-grpc-python==1.67.1.2.20260730172104+768a93c7b44e +openengine-openengine-protocolbuffers-python==31.1.0.2.20260730172104+768a93c7b44e +openengine-openengine-protocolbuffers-pyi==31.1.0.2.20260730172104+768a93c7b44e +grpcio>=1.67.1,<2 +protobuf>=6.31.1,<7 diff --git a/setup.py b/setup.py index d1d4dc643d98..205735a4daae 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,7 @@ def has_ext_modules(self): devel_deps, _ = parse_requirements( Path("requirements-dev-windows.txt" if on_windows else "requirements-dev.txt")) +openengine_deps, _ = parse_requirements(Path("requirements-openengine.txt")) mx_deps = ["modelexpress>=0.4.1,<0.6.0"] # Gateway protocol adapters are opt-in extras: the default installation must # not carry any gateway protobuf package. Each gateway owns a dedicated @@ -563,6 +564,7 @@ def get_build_state_options(): scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'], extras_require={ "devel": devel_deps + grpc_smg_deps, + "openengine": openengine_deps, "mx": mx_deps, "grpc-smg": grpc_smg_deps, }, diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 7becb894ce86..cc03245a7c91 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -1205,10 +1205,13 @@ def launch_visual_gen_server( "--grpc", is_flag=True, default=False, - help="Run gRPC server instead of OpenAI HTTP server. " - "gRPC server accepts pre-tokenized requests and returns raw token IDs. " - "Requires the tensorrt_llm[grpc-smg] extra.", + help="Run the selected gRPC protocol instead of the OpenAI HTTP server.", status="prototype") +@stability_option("--grpc-protocol", + type=click.Choice(["smg", "openengine"]), + default="smg", + help="Protocol used when --grpc is enabled.", + status="prototype") @stability_option( "--served_model_name", type=str, @@ -1276,9 +1279,9 @@ def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], agent_types: Optional[str], video_pruning_rate: Optional[float], telemetry: bool, custom_module_dirs: list[Path], chat_template: Optional[str], allow_request_chat_template: bool, - middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, - served_model_name: Optional[str], visual_gen_args: Optional[str], - report_addr: Optional[str]): + middleware: tuple[str, ...], grpc: bool, grpc_protocol: str, + enable_visual_gen: bool, served_model_name: Optional[str], + visual_gen_args: Optional[str], report_addr: Optional[str]) -> None: """Running an OpenAI API compatible server MODEL: model name | HF checkpoint path | TensorRT engine path @@ -1292,6 +1295,9 @@ def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], "to the 'pytorch' backend. See " "https://github.com/NVIDIA/TensorRT-LLM/issues/15638 for details.") + if not grpc and grpc_protocol != "smg": + raise click.UsageError("--grpc-protocol requires --grpc") + if moe_cluster_parallel_size is not None: logger.warning( "--moe_cluster_parallel_size / --cluster_size is deprecated and " @@ -1439,6 +1445,10 @@ def _serve_llm(): media_io_kwargs=parsed_media_io_kwargs) if grpc: + if num_serve_frontends != 1: + raise click.UsageError( + "--num_serve_frontends must be 1 when --grpc is enabled.") + # gRPC mode: launch gRPC server instead of OpenAI HTTP server # Check for unsupported arguments that are silently ignored in gRPC mode unsupported_args = { @@ -1466,18 +1476,32 @@ def _serve_llm(): f"Argument '{name}' is not supported when running in gRPC mode. " f"The gRPC server is designed for use with external routers that handle " f"these features (e.g., tool parsing, chat templates).") - if find_spec("smg_grpc_proto") is None: - raise ValueError( - "gRPC serving with the SMG protocol requires the optional " - "'smg-grpc-proto' package. Install it with: " - 'pip install "tensorrt_llm[grpc-smg]"') - - from tensorrt_llm.grpc.smg.server import launch_smg_server - - launch_smg_server(host, - port, - llm_args, - served_model_name=served_model_name) + if grpc_protocol == "smg": + if find_spec("smg_grpc_proto") is None: + raise ValueError( + "gRPC serving with the SMG protocol requires the optional " + "'smg-grpc-proto' package. Install it with: " + 'pip install "tensorrt_llm[grpc-smg]"') + + from tensorrt_llm.grpc.smg.server import launch_smg_server + + launch_smg_server(host, + port, + llm_args, + served_model_name=served_model_name) + else: + try: + from tensorrt_llm.grpc.openengine.server import \ + launch_server as launch_grpc_server + except ImportError as error: + raise click.ClickException( + f"Failed to import OpenEngine support: {error}. " + "Install the optional Python bindings with `python -m " + "pip install --extra-index-url " + "https://buf.build/gen/python " + "\"tensorrt_llm[openengine]\"`.") from error + + launch_grpc_server(host, port) else: # Default: launch OpenAI HTTP server launch_server( @@ -1519,6 +1543,9 @@ def _serve_visual_gen(): "--report_addr is only supported for the OpenAI HTTP server, not " f"the {'gRPC' if grpc else 'VisualGen'} server.") if is_visual_gen: + if grpc: + raise click.UsageError( + "--grpc is not supported by the VisualGen server") _serve_visual_gen() else: _serve_llm() diff --git a/tensorrt_llm/grpc/openengine/README.md b/tensorrt_llm/grpc/openengine/README.md new file mode 100644 index 000000000000..da8d1f915bd5 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/README.md @@ -0,0 +1,46 @@ + + + +# TensorRT-LLM OpenEngine stub server + +`trtllm-serve` can expose an experimental OpenEngine gRPC server instead of its normal OpenAI HTTP server. SMG remains the default gRPC protocol. + +Install the optional Python bindings from the Buf Schema Registry: + +```bash +python -m pip install \ + --extra-index-url https://buf.build/gen/python \ + "tensorrt_llm[openengine]" +``` + +Then select OpenEngine when starting the gRPC server: + +```bash +trtllm-serve \ + --grpc \ + --grpc-protocol openengine \ + --host 0.0.0.0 \ + --port 50051 +``` + +Existing `--grpc` invocations continue to select SMG. OpenEngine and VisualGen cannot be enabled together. + +This initial integration is a protocol stub. Every OpenEngine RPC returns gRPC status `UNIMPLEMENTED`; no request reaches the TensorRT-LLM engine. OpenEngine and SMG are independent protocol integrations. This integration does not make a replacement or convergence decision between them. + +## Dependency provenance + +The schema source is the Apache-2.0-licensed [`ai-dynamo/openengine`](https://github.com/ai-dynamo/openengine) repository at signed Git tag [`v0.1.0`](https://github.com/ai-dynamo/openengine/releases/tag/v0.1.0). That release maps to the public [`buf.build/openengine/openengine`](https://buf.build/openengine/openengine) module at immutable BSR commit `768a93c7b44e40f28c692ad0b471a8f2`. + +The BSR generated the pinned wheels from that module commit: + +| Package | Generator | Version | SHA-256 | +| --- | --- | --- | --- | +| `openengine-openengine-grpc-python` | [`grpc/python`](https://buf.build/grpc/python) | `1.67.1.2.20260730172104+768a93c7b44e` | `1485aed9799c4eb9367d1a261ca5cc5319f1e9b8d950ac98a26f3cb3641b8cf6` | +| `openengine-openengine-protocolbuffers-python` | [`protocolbuffers/python`](https://buf.build/protocolbuffers/python) | `31.1.0.2.20260730172104+768a93c7b44e` | `6eae12c3d8d06147fccf608da9772d6391139031fabdafdb7cf4c71a19c1f25e` | +| `openengine-openengine-protocolbuffers-pyi` | [`protocolbuffers/pyi`](https://buf.build/protocolbuffers/pyi) | `31.1.0.2.20260730172104+768a93c7b44e` | `8b0a054dbdaaa67459b3fa4786f13d8f6f4d30cf30be325f5416dbd97aba46a6` | + +Buf documents the package naming and version format in its [Python-generated SDK guide](https://buf.build/docs/bsr/generated-sdks/python/). The final version segment is the BSR commit prefix. The exact requirements are pinned in `requirements-openengine.txt`. + +## Maintenance boundary + +The OpenEngine contributor community owns this adapter, its tests, protocol version updates, and integration bugs. TensorRT-LLM internal APIs do not provide compatibility guarantees to protocol adapters. Adapter updates must follow core runtime changes and must not block normal TensorRT-LLM development or releases. diff --git a/tensorrt_llm/grpc/openengine/__init__.py b/tensorrt_llm/grpc/openengine/__init__.py new file mode 100644 index 000000000000..d1049d02ea13 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""OpenEngine gRPC integration for TensorRT-LLM.""" + +from .server import OpenEngineServer, launch_server + +__all__ = ["OpenEngineServer", "launch_server"] diff --git a/tensorrt_llm/grpc/openengine/server.py b/tensorrt_llm/grpc/openengine/server.py new file mode 100644 index 000000000000..641ff7a90eb0 --- /dev/null +++ b/tensorrt_llm/grpc/openengine/server.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""OpenEngine gRPC server lifecycle for TensorRT-LLM.""" + +import asyncio +import signal + +import grpc +import uvloop +from openengine.v1 import openengine_pb2_grpc + +from tensorrt_llm.logger import logger + +__all__ = ["OpenEngineServer", "launch_server"] + + +def _format_bind_address(host: str, port: int) -> str: + """Format a host and port as a gRPC bind address.""" + if ":" in host and not (host.startswith("[") and host.endswith("]")): + host = f"[{host}]" + return f"{host}:{port}" + + +class OpenEngineServer: + """OpenEngine gRPC server with intentionally unimplemented RPCs. + + Args: + host: Interface on which the server listens. + port: Port on which the server listens. Use zero to select a free port. + """ + + def __init__(self, host: str, port: int) -> None: + self.host = host + self.port = port + self._server = grpc.aio.server() + openengine_pb2_grpc.add_InferenceServicer_to_server( + openengine_pb2_grpc.InferenceServicer(), self._server + ) + openengine_pb2_grpc.add_ControlServicer_to_server( + openengine_pb2_grpc.ControlServicer(), self._server + ) + self._bind_address = _format_bind_address(host, port) + bound_port = self._server.add_insecure_port(self._bind_address) + if bound_port == 0: + raise RuntimeError(f"Failed to bind OpenEngine server to {self._bind_address}") + if port == 0: + self.port = bound_port + + async def start(self) -> None: + """Start accepting OpenEngine requests.""" + await self._server.start() + address = _format_bind_address(self.host, self.port) + logger.info(f"OpenEngine stub server started on {address}") + + async def stop(self, grace: float = 5.0) -> None: + """Stop accepting OpenEngine requests. + + Args: + grace: Maximum time in seconds to allow active RPCs to finish. + """ + await self._server.stop(grace=grace) + logger.info("OpenEngine stub server stopped") + + async def wait_for_termination(self) -> None: + """Wait until the OpenEngine server terminates.""" + await self._server.wait_for_termination() + + +def launch_server(host: str, port: int) -> None: + """Launch the dedicated OpenEngine gRPC server. + + Args: + host: Interface on which the server listens. + port: Port on which the server listens. + """ + + async def serve() -> None: + server = OpenEngineServer(host=host, port=port) + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + + def signal_handler() -> None: + logger.info("Received shutdown signal") + stop_event.set() + + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, signal_handler) + + try: + logger.warning( + "OpenEngine protocol support is a stub: no model is loaded and " + "all RPCs return UNIMPLEMENTED." + ) + await server.start() + await stop_event.wait() + finally: + await server.stop() + + uvloop.run(serve()) diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 4d4cabd2777f..5970af6dbb94 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -193,6 +193,15 @@ commands: is_flag: true flags: - "--grpc" + grpc_protocol: + type: Choice(['openengine', 'smg']) + default: smg + status: prototype + required: false + multiple: false + is_flag: false + flags: + - "--grpc-protocol" host: type: str default: localhost