diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 11f75bae2447..25cac48f6923 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -130,7 +130,7 @@ For agentic MXFP4 serving on MI355X, select **Low Latency** with **L1 + L2**. Th ### Multimodal memory -All strategies enable multimodal serving. Image and video input need a Transformers build that ships `Glm5NextProcessor` (the current `transformers==5.17.0` dependency includes it and requires `tokenizers>=0.23.1`). With older `transformers==5.12.1`, the server loads only the tokenizer and silently answers image requests from the text alone. On MI355X, text, image, and an image-bearing follow-up were qualified with Transformers commit `e4052f55`; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. +All strategies enable multimodal serving. Image and video input use `Glm5NextProcessor`, which the pinned Transformers release ships (the MI355X checks predate that release and used commit `e4052f55`). On MI355X, text, image, and an image-bearing follow-up were qualified with that build; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. The default multimodal feature transport is automatic, and on a single CUDA node auto resolves to CPU transport. The MI355X selection explicitly emits CPU transport, matching its image-serving check. CUDA IPC is opt-in: pass `--mm-feature-transport cuda_ipc` when lower transfer latency matters more than the GPU memory the IPC pool reserves. CUDA VMM transport applies only to multi-node GB200/GB300 systems on the MNNVL fabric, where auto selects it. diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py index a02c746cfb52..9dc312f39204 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py @@ -33,7 +33,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): return np.concatenate([emb_sin, emb_cos], axis=1) -# transformers 5.19 renamed Dinov2's attention projections and split its SwiGLU input +# transformers 5.18 renamed Dinov2's attention projections and split its SwiGLU input # projection into gate/up. from_pretrained remaps legacy checkpoints on its own, but # Hunyuan3D builds Dinov2Model from a config and loads its own state dict, where a # non-strict load would silently leave these weights randomly initialized. @@ -48,7 +48,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): def _remap_legacy_dinov2_keys(module, state_dict, prefix, *args): if not hasattr(module.encoder.layer[0].attention, "q_proj"): - return # transformers < 5.19 still uses the checkpoint's layout + return # transformers < 5.18 still uses the checkpoint's layout layer_prefix = prefix + "encoder.layer." for key in [k for k in state_dict if k.startswith(layer_prefix)]: for old, new in _DINOV2_LEGACY_RENAMES: diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 1a6ceef1bfc3..05957e990296 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -97,10 +97,7 @@ def _probe_wo_a_weight_dtype(model_config: Any, download_dir: str | None) -> str from transformers.utils.hub import cached_file model_path = model_config.model_path - revision = ( - getattr(model_config.hf_config, "_commit_hash", None) - or model_config.revision - ) + revision = model_config.revision index_path = cached_file( model_path, "model.safetensors.index.json", diff --git a/python/sglang/srt/configs/granitemoehybrid.py b/python/sglang/srt/configs/granitemoehybrid.py index e34dec2a10ea..690ab998f2ec 100644 --- a/python/sglang/srt/configs/granitemoehybrid.py +++ b/python/sglang/srt/configs/granitemoehybrid.py @@ -272,8 +272,7 @@ def __init__( **kwargs, ) - # HF's `PreTrainedConfig.validate_layer_type` rewrites layer_types to - # transformers >= 5.15 schema, so map them back afterwards. + # `PreTrainedConfig` rewrites legacy layer_types on init, so map them back. self.layer_types = _to_sglang_layer_types(self.layer_types) @property diff --git a/python/sglang/srt/configs/hy_v4.py b/python/sglang/srt/configs/hy_v4.py index 4d9c0c547dd8..11aa95d1ee67 100644 --- a/python/sglang/srt/configs/hy_v4.py +++ b/python/sglang/srt/configs/hy_v4.py @@ -133,8 +133,13 @@ def _validate_hy_v4(self): ) if set(self.mlp_layer_types) - {"dense", "sparse"}: raise ValueError("mlp_layer_types only supports dense and sparse") - if set(self.layer_types) != {"deepseek_sparse_attention"}: - raise ValueError("HYV4 only supports deepseek_sparse_attention") + if not set(self.layer_types) <= { + "deepseek_sparse_attention", + "indexed_attention", + }: + raise ValueError( + "HYV4 only supports deepseek_sparse_attention (indexed_attention)" + ) if set(self.indexer_types) - {"full", "shared"}: raise ValueError("indexer_types only supports full and shared") if self.indexer_types[0] != "full": diff --git a/python/sglang/srt/configs/qwen4_exp.py b/python/sglang/srt/configs/qwen4_exp.py index 01e18c713aea..fc956fc86fe3 100644 --- a/python/sglang/srt/configs/qwen4_exp.py +++ b/python/sglang/srt/configs/qwen4_exp.py @@ -85,7 +85,8 @@ def layers_block_type(self): return [ ( "attention" - if layer_type in ("full_attention", "qwen_sparse_attention") + if layer_type + in ("full_attention", "qwen_sparse_attention", "indexed_attention") else layer_type ) for layer_type in self.layer_types diff --git a/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py b/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py index 68459980ac48..1c9e186e0997 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py +++ b/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py @@ -181,7 +181,6 @@ def npu_wrapper_minimax_m3_video_preprocess(func): def _preprocess( self, videos: List[torch.Tensor], - do_convert_rgb: bool, do_resize: bool, size: SizeDict, resample: PILImageResampling | InterpolationMode | int | None, diff --git a/python/sglang/srt/model_loader/expert_pack_runtime.py b/python/sglang/srt/model_loader/expert_pack_runtime.py index 5f5ea5b81047..2fd81ab6bb38 100644 --- a/python/sglang/srt/model_loader/expert_pack_runtime.py +++ b/python/sglang/srt/model_loader/expert_pack_runtime.py @@ -335,8 +335,16 @@ def value(name: str) -> object: def _write_deepseek_tokenizer_from_gguf( reader: object, output_dir: Path, config: dict[str, Any] ) -> None: - from tokenizers import AddedToken, Regex, normalizers, pre_tokenizers - from transformers.integrations.ggml import convert_gguf_tokenizer + from tokenizers import ( + AddedToken, + Regex, + Tokenizer, + decoders, + models, + normalizers, + pre_tokenizers, + processors, + ) tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.model") pre_tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.pre") @@ -347,18 +355,21 @@ def _write_deepseek_tokenizer_from_gguf( ) tokens = list(_deepseek_gguf_value(reader, "tokenizer.ggml.tokens")) token_types = list(_deepseek_gguf_value(reader, "tokenizer.ggml.token_type")) - tokenizer_data = { - "tokenizer_type": tokenizer_type, - "tokens": tokens, - "token_type": token_types, - "merges": list(_deepseek_gguf_value(reader, "tokenizer.ggml.merges")), - "bos_token_id": config["bos_token_id"], - "eos_token_id": config["eos_token_id"], - "pad_token_id": int( - _deepseek_gguf_value(reader, "tokenizer.ggml.padding_token_id") - ), - } - tokenizer, _ = convert_gguf_tokenizer("gpt2", tokenizer_data) + tokenizer = Tokenizer( + models.BPE( + vocab={token: index for index, token in enumerate(tokens)}, + merges=[ + tuple(merge.split(" ")) + for merge in _deepseek_gguf_value(reader, "tokenizer.ggml.merges") + ], + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) tokenizer.add_special_tokens( [ AddedToken(token, normalized=False, special=True) diff --git a/python/sglang/srt/utils/hf_transformers_patches.py b/python/sglang/srt/utils/hf_transformers_patches.py index 3e945992d37b..ffbf1e110d77 100644 --- a/python/sglang/srt/utils/hf_transformers_patches.py +++ b/python/sglang/srt/utils/hf_transformers_patches.py @@ -62,7 +62,6 @@ def apply_all(): _patch_flash_attn_availability() _patch_rope_parameters_validation() _patch_layer_types_validation() - _patch_nested_rope_validation() _patch_removed_symbols() _patch_image_processor_kwargs() _patch_image_process_cuda_tensor() @@ -237,36 +236,6 @@ def validate_layer_type(self, _orig=validator): break -def _patch_nested_rope_validation(): - from transformers import PretrainedConfig - - validators = PretrainedConfig.__class_validators__ - for index, validator in enumerate(validators): - if validator.__name__ != "validate_rope": - continue - - def validate_rope(self, _orig=validator): - try: - return _orig(self) - except AttributeError: - # Backport of huggingface/transformers#48798; drop once transformers >= 5.18. - rope_parameters = getattr(self, "rope_parameters", None) - layer_types = getattr(self, "layer_types", None) or () - if not isinstance(rope_parameters, dict): - raise - nested = {k: v for k, v in rope_parameters.items() if k in layer_types} - if not nested or len(nested) == len(rope_parameters): - raise - self.rope_parameters = nested - try: - return _orig(self) - finally: - self.rope_parameters = rope_parameters - - validators[index] = validate_rope - break - - def _patch_removed_symbols(): """Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.