From 47e3adf3ba70c174d836fd94f9e4194e4316d842 Mon Sep 17 00:00:00 2001 From: "L.B.R." Date: Wed, 13 May 2026 12:58:20 +0100 Subject: [PATCH 1/2] [gRPC] Expose multimodal flag + tokenizer metadata via get_init_info GetModelInfo over gRPC always reports `supports_vision: false` (and falls back to default vocab_size / eos_token_ids / pad / bos / is_generation) for every model. The smg-grpc-servicer SGLang bridge reads these fields out of `scheduler_info` via `.get(..., default)` (see smg_grpc_servicer/sglang/server.py and servicer.py), but `Scheduler.get_init_info()` only ever returned `{status, max_total_num_tokens, max_req_input_len}`, so every other key silently falls back to its hardcoded default. For multimodal models like Nemotron-3-Nano-Omni this makes the worker invisible to image/audio routing through the gateway. This patch populates `is_generation`, `supports_vision`, `vocab_size`, `eos_token_ids`, `pad_token_id`, and `bos_token_id` from `self.model_config` and `self.is_generation`, mirroring what the vLLM servicer reads from `model_config.is_multimodal_model` at smg_grpc_servicer/vllm/servicer.py:273. A second hunk fixes `DataParallelController`, which rebuilds the upstream handshake dict from scratch at line 651 and would otherwise strip the new fields when `dp_size > 1`; the controller now caches the first child scheduler's full init dict and spreads it through the pipe send. Verified on a live `--grpc-mode` Spark worker serving nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8: GetModelInfo now correctly returns supports_vision=True, vocab_size=131072, and eos_token_ids=[2, 11] where they previously came back as False / 128256 / []. HTTP-mode consumers are unaffected. --- .../srt/managers/data_parallel_controller.py | 12 +++++++++--- python/sglang/srt/managers/scheduler.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 57632e146d94..ff7c15b805cd 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -559,6 +559,10 @@ def launch_tensor_parallel_group( self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"] self.max_req_input_len = scheduler_info[0]["max_req_input_len"] + # Retain the full first child init dict so callers that build + # the upstream handshake (run_data_parallel_controller_process, + # below) can forward every field — not just the two limits. + self.scheduler_init_info = scheduler_info[0] def maybe_external_dp_rank_routing(self, req: Req): if req.routed_dp_rank is not None: @@ -649,10 +653,12 @@ def run_data_parallel_controller_process( proc.pid for proc in controller.scheduler_procs if proc is not None ] pipe_writer.send( + # Spread the first child scheduler's full init dict (status + + # both max_* limits + the multimodal/tokenizer metadata added + # by the get_init_info patch above) so dp_size > 1 doesn't + # strip them before they reach the gRPC servicer. { - "status": "ready", - "max_total_num_tokens": controller.max_total_num_tokens, - "max_req_input_len": controller.max_req_input_len, + **controller.scheduler_init_info, SCHEDULER_PIDS_ARG: scheduler_pids, } ) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 689cf9837bcc..116d02d648b8 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1494,10 +1494,28 @@ def get_init_info(self) -> Dict[str, Any]: This method provides the initialization info needed by the tokenizer manager and other components to verify the scheduler is ready. """ + # Fields below `max_req_input_len` are read by smg-grpc-servicer's + # GetModelInfo bridge (sglang/server.py:81-97, servicer.py:325-329) + # out of scheduler_info via .get(..., default). Without them the + # gateway sees defaults — supports_vision in particular is always + # False, which makes multimodal workers invisible to image/audio + # routing. They are no-ops for HTTP-mode consumers. + hf_cfg = self.model_config.hf_config + pad_id = getattr(hf_cfg, "pad_token_id", None) + bos_id = getattr(hf_cfg, "bos_token_id", None) result_dict = { "status": "ready", "max_total_num_tokens": self.max_total_num_tokens, "max_req_input_len": self.max_req_input_len, + "is_generation": self.is_generation, + "supports_vision": self.model_config.is_multimodal, + "vocab_size": self.model_config.vocab_size, + "eos_token_ids": sorted(self.model_config.hf_eos_token_id or []), + # int32 proto field; coerce missing IDs to the smg-grpc-servicer + # int defaults (0 for pad, 1 for bos) so encoding doesn't crash + # on `None`. + "pad_token_id": pad_id if pad_id is not None else 0, + "bos_token_id": bos_id if bos_id is not None else 1, } return result_dict From 83c2f893036d5881c6db08f7f8bbf1cc7463101c Mon Sep 17 00:00:00 2001 From: "L.B.R." Date: Wed, 13 May 2026 15:49:41 +0100 Subject: [PATCH 2/2] [gRPC] Harden eos_token_ids coercion against int/None inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sorted(hf_eos_token_id or []) silently relied on ModelConfig._get_hf_eos_token_id normalizing int|list|None into a Set[int] before this site runs. That contract is invisible from this hunk — the producer's annotation is Optional[Set[int]] — so a future refactor could legitimately break the handshake. Coerce inline (wrap a bare int, treat None as empty) so the serialization is robust without depending on a cross-file invariant that the type system doesn't enforce. --- python/sglang/srt/managers/scheduler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 116d02d648b8..6b9fdcc4cea6 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1503,6 +1503,10 @@ def get_init_info(self) -> Dict[str, Any]: hf_cfg = self.model_config.hf_config pad_id = getattr(hf_cfg, "pad_token_id", None) bos_id = getattr(hf_cfg, "bos_token_id", None) + # `hf_eos_token_id` is annotated Optional[Set[int]] but the + # producer accepts int/list/None inputs; coerce defensively. + eos_ids = self.model_config.hf_eos_token_id + eos_token_ids = sorted([eos_ids] if isinstance(eos_ids, int) else eos_ids or []) result_dict = { "status": "ready", "max_total_num_tokens": self.max_total_num_tokens, @@ -1510,7 +1514,7 @@ def get_init_info(self) -> Dict[str, Any]: "is_generation": self.is_generation, "supports_vision": self.model_config.is_multimodal, "vocab_size": self.model_config.vocab_size, - "eos_token_ids": sorted(self.model_config.hf_eos_token_id or []), + "eos_token_ids": eos_token_ids, # int32 proto field; coerce missing IDs to the smg-grpc-servicer # int defaults (0 for pad, 1 for bos) so encoding doesn't crash # on `None`.