[None][refactor] Organize SMG gRPC adapter by protocol

.github/CODEOWNERS

@@ -478,6 +478,7 @@
 /jenkins/license_cpp.json @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance
 /pyproject.toml @NVIDIA/trt-llm-oss-compliance
 /requirements-dev.txt @NVIDIA/trt-llm-oss-compliance
+/requirements-grpc-smg.txt @NVIDIA/trt-llm-oss-compliance
 /requirements.txt @NVIDIA/trt-llm-oss-compliance
 /setup.py @NVIDIA/trt-llm-oss-compliance
 /tests/unittest/api_stability/ @NVIDIA/trt-llm-noncommitted-api-review-committee

docker/Dockerfile.multi

@@ -110,7 +110,7 @@ COPY scripts scripts
 COPY tensorrt_llm tensorrt_llm
 COPY triton_kernels triton_kernels
 COPY 3rdparty 3rdparty
-COPY .gitmodules setup.py requirements.txt requirements-dev.txt constraints.txt README.md ./
+COPY .gitmodules setup.py requirements.txt requirements-dev.txt requirements-grpc-smg.txt constraints.txt README.md ./
 
 ENV CCACHE_DIR=/root/.cache/ccache
 # Build the TRT-LLM wheel

jenkins/L0_Test.groovy

@@ -4227,6 +4227,20 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                 sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt"
             }
             trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt")
+            // Gateway adapters are opt-in extras excluded from requirements.txt;
+            // each gateway declares its pins in a dedicated
+            // requirements-<gateway>.txt, and a test stage installs exactly
+            // zero or one gateway file so every adapter is tested under the
+            // dependency set its real opt-in users receive. A gateway whose
+            // pins co-resolve with the default environment (SMG today) is
+            // installed in the shared stages so its unit tests run from the
+            // regular shard pool instead of being skipped at collection; a
+            // gateway whose pins conflict with the default environment (for
+            // example a protobuf major-version floor or a custom package
+            // index) must instead install its file behind a dedicated stage
+            // guard and skip this one (see the Ray install below for the
+            // stage-scoped pattern).
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-grpc-smg.txt")
             trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install opencv-python-headless")
             if (stageName.contains("-Ray-")) {
                 trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install ray[default]==2.55.1")
@@ -4764,6 +4778,7 @@ def runLLMBuild(
     }
 
     trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && pip3 install -r requirements-dev.txt")
+    trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && pip3 install -r requirements-grpc-smg.txt")
     if (env.alternativeTRT) {
         trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver)
     }

jenkins/scripts/slurm_install.sh

@@ -31,6 +31,7 @@ slurm_install_setup() {
         fi
         retry_command --timeout 2700 bash -c "pip3 install --retries 10 opencv-python-headless"
         retry_command --timeout 2700 bash -c "cd $llmSrcNode && pip3 install --retries 10 -r requirements-dev.txt"
+        retry_command --timeout 2700 bash -c "cd $llmSrcNode && pip3 install --retries 10 -r requirements-grpc-smg.txt"
         retry_command --timeout 2700 bash -c "cd $resourcePathNode && pip3 install --retries 10 --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl"
         gpuUuids=$(nvidia-smi -q | grep "GPU UUID" | awk '{print $4}' | tr '\n' ',' || true)
         hostNodeName="${HOST_NODE_NAME:-$(hostname -f || hostname)}"

requirements-grpc-smg.txt

@@ -0,0 +1 @@
+smg-grpc-proto>=0.4.2

requirements.txt

@@ -92,8 +92,9 @@ llist
 cuda-tile>=1.0.1
 nvidia-cuda-tileiras>=13.1,<13.2
 etcd-sdk-python==0.0.7
+# etcd-sdk-python imports google.protobuf but omits it from its package metadata.
+protobuf>=5.27.2
 python-multipart
-smg-grpc-proto>=0.4.2
 cache-dit>=1.3.5
 librosa
 msgpack

setup.py

@@ -141,6 +141,13 @@ def has_ext_modules(self):
     Path("requirements-dev-windows.txt"
          if on_windows else "requirements-dev.txt"))
 mx_deps = ["modelexpress==0.4.1"]
+# Gateway protocol adapters are opt-in extras: the default installation must
+# not carry any gateway protobuf package. Each gateway owns a dedicated
+# requirements-<gateway>.txt as the single source of truth for its pins; CI
+# stages that exercise a gateway install that file explicitly, and the file
+# may carry gateway-specific options (such as an --extra-index-url) without
+# affecting the default dependency graph.
+grpc_smg_deps, _ = parse_requirements(Path("requirements-grpc-smg.txt"))
 constraints_file = Path("constraints.txt")
 if constraints_file.exists():
     constraints, _ = parse_requirements(constraints_file)
@@ -484,8 +491,9 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str],
     },
     scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'],
     extras_require={
-        "devel": devel_deps,
+        "devel": devel_deps + grpc_smg_deps,
         "mx": mx_deps,
+        "grpc-smg": grpc_smg_deps,
     },
     zip_safe=True,
     install_requires=required_deps,

tensorrt_llm/commands/serve.py

@@ -13,6 +13,7 @@
 import sys
 import time
 import uuid
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Sequence, Set
 
@@ -60,9 +61,6 @@
 # Global variable to store the Popen object of the child process
 _child_p_global: Optional[subprocess.Popen] = None
 
-# Bound gRPC messages while leaving room for multimodal image payloads.
-_GRPC_MAX_MESSAGE_LENGTH_BYTES = 32 * 1024 * 1024
-
 
 def _pop_bool_config_option(config: dict[str, Any], key: str) -> bool:
     return validate_config_bool(config.pop(key, False), key)
@@ -637,129 +635,6 @@ def launch_server(
                 _terminate_attached_frontends(frontend_children)
 
 
-def launch_grpc_server(host: str,
-                       port: int,
-                       llm_args: dict,
-                       served_model_name: Optional[str] = None):
-    """
-    Launch a gRPC server for TensorRT-LLM.
-
-    This provides a high-performance gRPC interface designed for external routers
-    (e.g., sgl-router) using pre-tokenized input and raw token ID output.
-
-    Args:
-        host: Host to bind to
-        port: Port to bind to
-        llm_args: Arguments for LLM initialization (from get_llm_args)
-        served_model_name: Custom model name for API responses (defaults to model path)
-    """
-    import grpc
-
-    try:
-        from grpc_reflection.v1alpha import reflection
-        REFLECTION_AVAILABLE = True
-    except ImportError:
-        REFLECTION_AVAILABLE = False
-
-    from tensorrt_llm.grpc import trtllm_service_pb2, trtllm_service_pb2_grpc
-    from tensorrt_llm.grpc.grpc_request_manager import GrpcRequestManager
-    from tensorrt_llm.grpc.grpc_servicer import TrtllmServiceServicer
-
-    async def serve_grpc_async():
-        logger.info("Initializing TensorRT-LLM gRPC server...")
-
-        backend = llm_args.get("backend")
-        model_path = served_model_name or llm_args.get("model", "")
-
-        if backend == "pytorch":
-            llm_args.pop("build_config", None)
-            llm = PyTorchLLM(**llm_args)
-        elif backend == "_autodeploy":
-            from tensorrt_llm._torch.auto_deploy import LLM as AutoDeployLLM
-            llm_args.pop("build_config", None)
-            llm = AutoDeployLLM(**llm_args)
-        else:
-            raise click.BadParameter(
-                f"{backend} is not a known backend, check help for available options.",
-                param_hint="backend")
-
-        logger.info("Model loaded successfully")
-
-        # Create request manager
-        request_manager = GrpcRequestManager(llm)
-
-        # Create servicer
-        servicer = TrtllmServiceServicer(request_manager, model_path=model_path)
-
-        # Create gRPC server
-        server = grpc.aio.server(
-            options=[
-                ("grpc.max_send_message_length",
-                 _GRPC_MAX_MESSAGE_LENGTH_BYTES),
-                ("grpc.max_receive_message_length",
-                 _GRPC_MAX_MESSAGE_LENGTH_BYTES),
-                ("grpc.keepalive_time_ms", 30000),  # 30s keepalive
-                ("grpc.keepalive_timeout_ms", 10000),  # 10s timeout
-                ("grpc.keepalive_permit_without_calls", True),
-                ("grpc.http2.min_recv_ping_interval_without_data_ms", 10000),
-            ], )
-
-        # Add servicer to server
-        trtllm_service_pb2_grpc.add_TrtllmServiceServicer_to_server(
-            servicer, server)
-
-        # Enable reflection for grpcurl and other tools
-        if REFLECTION_AVAILABLE:
-            service_names = (
-                trtllm_service_pb2.DESCRIPTOR.services_by_name["TrtllmService"].
-                full_name,
-                reflection.SERVICE_NAME,
-            )
-            reflection.enable_server_reflection(service_names, server)
-            logger.info("gRPC reflection enabled")
-
-        # Bind to address
-        address = f"{host}:{port}"
-        server.add_insecure_port(address)
-
-        # Start server
-        await server.start()
-        logger.info(f"TensorRT-LLM gRPC server started on {address}")
-        logger.info("Server is ready to accept requests")
-
-        # Handle shutdown signals
-        loop = asyncio.get_running_loop()
-        stop_event = asyncio.Event()
-
-        def signal_handler():
-            logger.info("Received shutdown signal")
-            stop_event.set()
-
-        for sig in (signal.SIGTERM, signal.SIGINT):
-            loop.add_signal_handler(sig, signal_handler)
-
-        # Serve until shutdown signal
-        try:
-            await stop_event.wait()
-        except KeyboardInterrupt:
-            logger.info("Interrupted by user")
-        finally:
-            logger.info("Shutting down TensorRT-LLM gRPC server...")
-
-            # Stop gRPC server
-            await server.stop(grace=5.0)
-            logger.info("gRPC server stopped")
-
-            # Shutdown LLM
-            if hasattr(llm, "shutdown"):
-                llm.shutdown()
-            logger.info("LLM engine stopped")
-
-            logger.info("Shutdown complete")
-
-    uvloop.run(serve_grpc_async())
-
-
 def launch_mm_encoder_server(
     host: str,
     port: int,
@@ -1252,7 +1127,8 @@ def launch_visual_gen_server(
     is_flag=True,
     default=False,
     help="Run gRPC server instead of OpenAI HTTP server. "
-    "gRPC server accepts pre-tokenized requests and returns raw token IDs.",
+    "gRPC server accepts pre-tokenized requests and returns raw token IDs. "
+    "Requires the tensorrt_llm[grpc-smg] extra.",
     status="prototype")
 @stability_option(
     "--served_model_name",
@@ -1494,10 +1370,18 @@ def _serve_llm():
                         f"Argument '{name}' is not supported when running in gRPC mode. "
                         f"The gRPC server is designed for use with external routers that handle "
                         f"these features (e.g., tool parsing, chat templates).")
-            launch_grpc_server(host,
-                               port,
-                               llm_args,
-                               served_model_name=served_model_name)
+            if find_spec("smg_grpc_proto") is None:
+                raise ValueError(
+                    "gRPC serving with the SMG protocol requires the optional "
+                    "'smg-grpc-proto' package. Install it with: "
+                    'pip install "tensorrt_llm[grpc-smg]"')
+
+            from tensorrt_llm.grpc.smg.server import launch_smg_server
+
+            launch_smg_server(host,
+                              port,
+                              llm_args,
+                              served_model_name=served_model_name)
         else:
             # Default: launch OpenAI HTTP server
             launch_server(

tensorrt_llm/grpc/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,75 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-r"""TensorRT-LLM gRPC module for high-performance communication with external routers.
-
-This module provides a gRPC server interface that accepts pre-tokenized requests
-and returns raw token IDs, enabling efficient binary communication with Rust-based
-routers like sgl-router.
-
-Key Features:
-- Pre-tokenized input (no Python tokenization overhead)
-- Raw token ID output (no Python detokenization overhead)
-- Streaming support with delta tokens
-- Full sampling parameter support
-- Guided decoding (JSON schema, regex, grammar)
-- LoRA and prompt tuning support
-- Disaggregated inference support
-
-Proto definitions are provided by the smg-grpc-proto package (pip install smg-grpc-proto).
-
-Usage:
-    python -m tensorrt_llm.commands.serve /path/to/model \
-        --grpc \
-        --host 0.0.0.0 \
-        --port 50051
-"""
-
-# Try to import generated protobuf modules from smg-grpc-proto package
-try:
-    from smg_grpc_proto.generated import trtllm_service_pb2, trtllm_service_pb2_grpc
-
-    PROTOS_AVAILABLE = True
-except ImportError:
-    PROTOS_AVAILABLE = False
-    trtllm_service_pb2 = None
-    trtllm_service_pb2_grpc = None
-
-# Try to import request manager
-try:
-    from .grpc_request_manager import (
-        GrpcRequestManager,
-        create_disaggregated_params_from_proto,
-        create_lora_request_from_proto,
-        create_sampling_params_from_proto,
-    )
-
-    REQUEST_MANAGER_AVAILABLE = True
-except ImportError:
-    REQUEST_MANAGER_AVAILABLE = False
-    GrpcRequestManager = None
-    create_sampling_params_from_proto = None
-    create_lora_request_from_proto = None
-    create_disaggregated_params_from_proto = None
-
-# Try to import servicer
-try:
-    from .grpc_servicer import TrtllmServiceServicer
-
-    SERVICER_AVAILABLE = True
-except ImportError:
-    SERVICER_AVAILABLE = False
-    TrtllmServiceServicer = None
-
-__all__ = [
-    "PROTOS_AVAILABLE",
-    "REQUEST_MANAGER_AVAILABLE",
-    "SERVICER_AVAILABLE",
-    "trtllm_service_pb2",
-    "trtllm_service_pb2_grpc",
-    "GrpcRequestManager",
-    "TrtllmServiceServicer",
-    "create_sampling_params_from_proto",
-    "create_lora_request_from_proto",
-    "create_disaggregated_params_from_proto",
-]
+"""gRPC protocol integrations for TensorRT-LLM."""

tensorrt_llm/grpc/smg/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""SMG gRPC protocol integration for TensorRT-LLM."""