diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 9a1aa72c560d..51c85e63023f 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -265,6 +265,7 @@ async def init_multi_tokenizer() -> ServerArgs: @asynccontextmanager async def lifespan(fast_api_app: FastAPI): grpc_handle = None + sidecar = None warmup_thread = None if getattr(fast_api_app, "is_single_tokenizer_mode", False): server_args = fast_api_app.server_args @@ -397,6 +398,10 @@ async def lifespan(fast_api_app: FastAPI): template_manager=_global_state.template_manager, scheduler_info=_global_state.scheduler_info, ) + if server_args.sidecar is not None: + from sglang.srt.entrypoints.sidecar import start_sidecar + + sidecar = start_sidecar(server_args) # Execute the general warmup warmup_thread = threading.Thread( @@ -408,6 +413,11 @@ async def lifespan(fast_api_app: FastAPI): # Start the HTTP server yield finally: + if sidecar is not None: + try: + sidecar.stop() + except Exception: + logger.exception("Failed to stop sidecar") _shutdown_native_grpc_server(grpc_handle) if tool_server is not None and hasattr(tool_server, "aclose"): await tool_server.aclose() diff --git a/python/sglang/srt/entrypoints/sidecar.py b/python/sglang/srt/entrypoints/sidecar.py new file mode 100644 index 000000000000..ddf58b81e33a --- /dev/null +++ b/python/sglang/srt/entrypoints/sidecar.py @@ -0,0 +1,130 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Lifecycle management for an optional local native gRPC sidecar.""" + +import argparse +import importlib +import logging +import multiprocessing as mp +import os + +from sglang.srt.utils.common import kill_itself_when_parent_died, kill_process_tree +from sglang.srt.utils.network import NetworkAddress +from sglang.srt.utils.watchdog import SubprocessWatchdog + +logger = logging.getLogger(__name__) + +SGLANG_GRPC_ENDPOINT_ENV = "SGLANG_GRPC_ENDPOINT" +_DEFAULT_SIDECAR_SHUTDOWN_TIMEOUT = 45.0 + + +def _loopback_host(host: str) -> str: + if not host or host == "0.0.0.0": + return "127.0.0.1" + if host in ("::", "[::]"): + return "::1" + return host + + +def build_sidecar_endpoint(server_args) -> str: + return NetworkAddress( + _loopback_host(server_args.host), server_args.grpc_port + ).to_url() + + +def _parse_sidecar_args(args: list[str] | None) -> tuple[list[str], float]: + parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + parser.add_argument( + "--sidecar-shutdown-timeout", + type=float, + default=_DEFAULT_SIDECAR_SHUTDOWN_TIMEOUT, + ) + parsed, provider_args = parser.parse_known_args(args or []) + if parsed.sidecar_shutdown_timeout <= 0: + raise ValueError("--sidecar-shutdown-timeout must be greater than 0.") + return provider_args, parsed.sidecar_shutdown_timeout + + +def _run_sidecar(module_name: str, args: list[str], endpoint: str) -> None: + kill_itself_when_parent_died() + os.environ[SGLANG_GRPC_ENDPOINT_ENV] = endpoint + try: + main = getattr(importlib.import_module(module_name), "main") + except (AttributeError, ImportError) as e: + raise RuntimeError( + f"--sidecar requires importable module {module_name!r} " + "with a main(argv) function." + ) from e + + if not callable(main): + raise RuntimeError( + f"--sidecar requires module {module_name!r} to expose " + "a callable main(argv)." + ) + + main(args) + + +class Sidecar: + def __init__( + self, + proc, + module_name: str, + shutdown_timeout: float, + ): + self.proc = proc + self.module_name = module_name + self.shutdown_timeout = shutdown_timeout + self._watchdog = SubprocessWatchdog( + processes=[proc], process_names=[module_name] + ) + + def start(self) -> None: + self.proc.start() + self._watchdog.start() + logger.info( + "Sidecar module %s started pid=%s", + self.module_name, + self.proc.pid, + ) + + def stop(self) -> None: + self._watchdog.stop() + if self.proc.is_alive(): + self.proc.terminate() + self.proc.join(timeout=self.shutdown_timeout) + else: + self.proc.join(timeout=0) + + if self.proc.is_alive(): + logger.warning("Sidecar module did not terminate; killing process tree") + kill_process_tree(self.proc.pid, wait_timeout=self.shutdown_timeout) + + +def start_sidecar(server_args) -> Sidecar: + module_name = server_args.sidecar + assert module_name is not None + sidecar_args, shutdown_timeout = _parse_sidecar_args(server_args.sidecar_args) + endpoint = build_sidecar_endpoint(server_args) + proc = mp.get_context("spawn").Process( + name=f"sglang_sidecar_{module_name}", + target=_run_sidecar, + args=(module_name, sidecar_args, endpoint), + ) + sidecar = Sidecar( + proc, + module_name, + shutdown_timeout=shutdown_timeout, + ) + sidecar.start() + return sidecar diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index be7028baf28f..838b925a34db 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1070,6 +1070,22 @@ class ServerArgs: "default. In legacy --smg-grpc-mode this is the SMG server port and " "defaults to --port + 10000.", ] = None + sidecar: A[ + Optional[str], + "Start a locally managed sidecar against the native gRPC server. " + "The selected module must expose main(argv) and read the resolved " + "native gRPC endpoint from SGLANG_GRPC_ENDPOINT. Requires --grpc-port " + "or SGLANG_GRPC_PORT.", + ] = None + sidecar_args: A[ + Optional[List[str]], + Arg( + help="JSON array passed to the selected sidecar module's " + "main(argv) function. --sidecar-shutdown-timeout SECONDS is " + "consumed by SGLang.", + type_parser=json_list_type, + ), + ] = None skip_server_warmup: A[bool, "If set, skip warmup."] = False warmups: A[ Optional[str], @@ -3325,6 +3341,23 @@ def _handle_deprecated_args(self): # Native gRPC is incompatible with launch paths it doesn't wire into. # Legacy takes precedence over grpc_port, keeping re-runs idempotent. native_grpc = self.grpc_port is not None and not legacy_grpc + if self.sidecar_args is not None: + if self.sidecar is None: + raise ValueError("--sidecar-args requires --sidecar.") + if not isinstance(self.sidecar_args, list) or not all( + isinstance(arg, str) for arg in self.sidecar_args + ): + raise ValueError("--sidecar-args must be a JSON array of strings.") + if self.sidecar is not None: + if not self.sidecar.strip(): + raise ValueError("--sidecar must not be empty.") + if legacy_grpc: + raise ValueError( + "--sidecar requires SGLang's native gRPC server; " + "it cannot be combined with --smg-grpc-mode/--grpc-mode." + ) + if self.grpc_port is None: + raise ValueError("--sidecar requires --grpc-port or SGLANG_GRPC_PORT.") if native_grpc: if self.use_ray: raise ValueError( diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 3887cbb7c03c..677e13b80511 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -9,6 +9,13 @@ import sglang.srt.server_args as server_args_module from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding +from sglang.srt.entrypoints.sidecar import ( + SGLANG_GRPC_ENDPOINT_ENV, + Sidecar, + _run_sidecar, + build_sidecar_endpoint, + start_sidecar, +) from sglang.srt.environ import envs from sglang.srt.layers.cp.base import is_cp_enabled, is_interleave from sglang.srt.model_executor.cuda_graph_config import ( @@ -1536,6 +1543,133 @@ def test_env_grpc_port_enables_native(self): sa._handle_deprecated_args() self.assertEqual(sa.grpc_port, 45000) + @staticmethod + def _sidecar_parser(): + parser = server_args_module.argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + return parser + + def test_sidecar_builds_loopback_grpc_endpoints(self): + self.assertEqual( + build_sidecar_endpoint(SimpleNamespace(host="0.0.0.0", grpc_port=50051)), + "http://127.0.0.1:50051", + ) + self.assertEqual( + build_sidecar_endpoint(SimpleNamespace(host="::", grpc_port=50051)), + "http://[::1]:50051", + ) + self.assertEqual( + build_sidecar_endpoint(SimpleNamespace(host="[::]", grpc_port=50051)), + "http://[::1]:50051", + ) + + def test_sidecar_args_parse_as_exact_json_argv(self): + argv = ["--flag", "value"] + parsed = self._sidecar_parser().parse_args( + ["--model-path", "dummy", "--sidecar-args", json.dumps(argv)] + ) + self.assertEqual(parsed.sidecar_args, argv) + + def test_start_sidecar_passes_endpoint_and_provider_argv_separately(self): + server_args = SimpleNamespace( + sidecar="example.sidecar", + sidecar_args=[ + "--sidecar-shutdown-timeout", + "42", + "--grpc-connections", + "2", + ], + host="127.0.0.1", + grpc_port=50051, + ) + with ( + patch("sglang.srt.entrypoints.sidecar.mp.get_context") as get_context, + patch("sglang.srt.entrypoints.sidecar.Sidecar") as sidecar_class, + ): + start_sidecar(server_args) + + process_kwargs = get_context.return_value.Process.call_args.kwargs + self.assertEqual(process_kwargs["name"], "sglang_sidecar_example.sidecar") + self.assertEqual(process_kwargs["target"], _run_sidecar) + self.assertEqual( + process_kwargs["args"], + ( + "example.sidecar", + ["--grpc-connections", "2"], + "http://127.0.0.1:50051", + ), + ) + sidecar_class.assert_called_once_with( + get_context.return_value.Process.return_value, + "example.sidecar", + shutdown_timeout=42.0, + ) + + def test_sidecar_requires_native_grpc(self): + sa = self._args(sidecar="example.sidecar") + with self.assertRaisesRegex(ValueError, "requires --grpc-port"): + sa._handle_deprecated_args() + + def test_sidecar_rejects_legacy_grpc(self): + sa = self._args(sidecar="example.sidecar", smg_grpc_mode=True) + with self.assertRaisesRegex(ValueError, "native gRPC server"): + sa._handle_deprecated_args() + + def test_sidecar_rejects_empty_value(self): + sa = self._args(sidecar="", grpc_port=50051) + with self.assertRaisesRegex(ValueError, "must not be empty"): + sa._handle_deprecated_args() + + def test_sidecar_sets_endpoint_env_before_import_and_calls_main(self): + main = MagicMock() + + def import_module(module_name): + self.assertEqual(module_name, "example.sidecar") + self.assertEqual( + os.environ[SGLANG_GRPC_ENDPOINT_ENV], + "http://127.0.0.1:50051", + ) + self.assertEqual(os.environ["DYN_NAMESPACE"], "pluh") + return SimpleNamespace(main=main) + + with ( + patch.dict( + os.environ, + { + SGLANG_GRPC_ENDPOINT_ENV: "http://stale.example:1", + "DYN_NAMESPACE": "pluh", + }, + ), + patch("sglang.srt.entrypoints.sidecar.kill_itself_when_parent_died"), + patch( + "sglang.srt.entrypoints.sidecar.importlib.import_module", + side_effect=import_module, + ), + ): + _run_sidecar( + "example.sidecar", + ["--provider-flag", "value"], + "http://127.0.0.1:50051", + ) + + main.assert_called_once_with(["--provider-flag", "value"]) + + def test_sidecar_stop_uses_configured_shutdown_timeout(self): + proc = MagicMock(pid=1234) + proc.is_alive.side_effect = [True, True] + sidecar = Sidecar( + proc, + "example.sidecar", + shutdown_timeout=42.0, + ) + + with patch("sglang.srt.entrypoints.sidecar.kill_process_tree") as kill_tree: + sidecar.stop() + + proc.terminate.assert_called_once_with() + proc.join.assert_called_once_with(timeout=42.0) + kill_tree.assert_called_once_with(1234, wait_timeout=42.0) + def test_legacy_smg_derives_grpc_port_from_http_port(self): sa = self._args(port=30000, smg_grpc_mode=True) sa._handle_deprecated_args()