[vllm + v5 fix] handle TokenizersBackend fallback properly for v5

src/transformers/convert_slow_tokenizer.py

@@ -635,6 +635,54 @@ class SpmConverter(Converter):
     SpmExtractor = SentencePieceExtractor
     special_tokens = {}
 
+    @staticmethod
+    def build_tokenizer_from_spm_proto(proto, vocab, merges=None):
+        """
+        Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
+        """
+        byte_fallback = proto.trainer_spec.byte_fallback
+        unk_piece = proto.trainer_spec.unk_piece
+        precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
+
+        # model
+        if isinstance(vocab, dict):
+            tokenizer = Tokenizer(
+                BPE(
+                    vocab=vocab,
+                    merges=merges or [],
+                    unk_token=unk_piece,
+                    fuse_unk=True,
+                    byte_fallback=byte_fallback,
+                    dropout=None,
+                )
+            )
+        elif isinstance(vocab, list) and vocab and isinstance(vocab[0], (tuple, list)):
+            tokenizer = Tokenizer(
+                Unigram(
+                    vocab=vocab,
+                    unk_id=proto.trainer_spec.unk_id,
+                    byte_fallback=byte_fallback,
+                )
+            )
+        else:
+            return None
+
+        # normalizer
+        _normalizers = [normalizers.Replace(" ", "▁")]
+        if precompiled_charsmap:
+            _normalizers.insert(0, normalizers.Precompiled(precompiled_charsmap))
+        tokenizer.normalizer = normalizers.Sequence(_normalizers)
+
+        # decoder
+        if byte_fallback:
+            tokenizer.decoder = decoders.Sequence(
+                [decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()]
+            )
+        else:
+            tokenizer.decoder = decoders.Sequence([decoders.Replace("▁", " ")])
+
+        return tokenizer
+
     @classmethod
     def convert_from_spm(cls, vocab=None, **kwargs):
         """

src/transformers/models/auto/tokenization_auto.py

@@ -171,7 +171,6 @@
         ("lighton_ocr", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
         ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None),
         ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None),
-        ("longt5", "T5Tokenizer" if is_tokenizers_available() else None),
         ("luke", "LukeTokenizer"),
         ("lxmert", "LxmertTokenizer" if is_tokenizers_available() else None),
         ("m2m_100", "M2M100Tokenizer" if is_sentencepiece_available() else None),
@@ -342,13 +341,16 @@
 MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS: set[str] = {
     "arctic",
     "deepseek_vl",
+    "deepseek_vl_v2",
     "deepseek_vl_hybrid",
     "fuyu",
     "hyperclovax_vlm",
+    "internlm2",
     "janus",
     "jamba",
     "llava",
     "llava_next",
+    "longt5",
     "opencua",
     "phi3",
     "step3p5",
@@ -706,6 +708,12 @@ def from_pretrained(
                 or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None
             )
         )
+
+        # V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class
+        if has_remote_code and config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
+            has_remote_code = False
+            tokenizer_auto_map = None
+
         if has_remote_code:
             # V5: Always prefer fast tokenizer (index 1), fallback to slow (index 0)
             if tokenizer_auto_map[1] is not None:

src/transformers/models/blenderbot/tokenization_blenderbot.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Tokenization class for Blenderbot."""
 
-from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers import Tokenizer, decoders, pre_tokenizers
 from tokenizers.models import BPE
 
 from ...tokenization_utils_base import AddedToken
@@ -170,12 +170,6 @@ def __init__(
             add_prefix_space=add_prefix_space,
             **kwargs,
         )
-        self._tokenizer.post_processor = processors.RobertaProcessing(
-            sep=(str(eos_token), self.eos_token_id),
-            cls=(str(bos_token), self.bos_token_id),
-            add_prefix_space=add_prefix_space,
-            trim_offsets=True,
-        )
 
 
 __all__ = ["BlenderbotTokenizer"]

src/transformers/models/gemma/tokenization_gemma.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from tokenizers import Tokenizer, decoders, normalizers
+from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers
 from tokenizers.models import BPE
 
 from ...tokenization_utils_tokenizers import TokenizersBackend
@@ -88,6 +88,9 @@ def __init__(
                 byte_fallback=True,
             )
         )
+        self._tokenizer.pre_tokenizer = pre_tokenizers.Split(
+            pattern=" ", behavior="merged_with_previous", invert=False
+        )
 
         self._tokenizer.decoder = decoders.Sequence(
             [decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()]
@@ -102,9 +105,5 @@ def __init__(
             **kwargs,
         )
 
-    def _unk_id(self) -> int:
-        # Align with historical Gemma convention: pad, eos, bos, unk
-        return 3
-
 
 __all__ = ["GemmaTokenizer"]

src/transformers/models/gpt_neox/tokenization_gpt_neox.py

@@ -127,7 +127,7 @@ def __init__(
         self._tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
             add_prefix_space=add_prefix_space, trim_offsets=trim_offsets
         )
-        self._tokenizer.decoder = decoders.ByteLevel(add_prefix_space=False, trim_offsets=True)
+        self._tokenizer.decoder = decoders.ByteLevel()
 
         super().__init__(
             errors=errors,

src/transformers/models/lasr/modular_lasr.py

@@ -16,7 +16,7 @@
 from collections.abc import Callable
 
 import torch
-from tokenizers import Tokenizer
+from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers
 from tokenizers.models import Unigram
 from torch import nn
 
@@ -46,6 +46,7 @@ def __init__(
         eos_token="</s>",
         unk_token="<unk>",
         pad_token="<pad>",
+        _spm_precompiled_charsmap=None,
         extra_ids=100,
         additional_special_tokens=None,
         vocab=None,
@@ -56,6 +57,7 @@ def __init__(
             eos_token=eos_token,
             unk_token=unk_token,
             pad_token=pad_token,
+            _spm_precompiled_charsmap=_spm_precompiled_charsmap,
             extra_ids=extra_ids,
             additional_special_tokens=additional_special_tokens,
             vocab=vocab,
@@ -70,6 +72,17 @@ def __init__(
             )
         )
 
+        if _spm_precompiled_charsmap is not None:
+            self._tokenizer.normalizer = normalizers.Precompiled(_spm_precompiled_charsmap)
+
+        self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+            [
+                pre_tokenizers.WhitespaceSplit(),
+                pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True),
+            ]
+        )
+        self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+
     def _decode(
         self,
         token_ids: int | list[int],

src/transformers/models/lasr/tokenization_lasr.py

@@ -21,8 +21,9 @@
 import itertools
 import re
 
-from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
+from tokenizers.normalizers import Replace
 
 from ...tokenization_utils_tokenizers import TokenizersBackend
 
@@ -76,6 +77,7 @@ def __init__(
         eos_token="</s>",
         unk_token="<unk>",
         pad_token="<pad>",
+        _spm_precompiled_charsmap=None,
         extra_ids=100,
         additional_special_tokens=None,
         vocab=None,
@@ -119,15 +121,17 @@ def __init__(
             )
         )
 
-        self._tokenizer.normalizer = None
+        if _spm_precompiled_charsmap is not None:
+            self._tokenizer.normalizer = normalizers.Sequence(
+                [normalizers.Precompiled(_spm_precompiled_charsmap), Replace(pattern=Regex(" {2,}"), content=" ")]
+            )
 
         self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
             [
                 pre_tokenizers.WhitespaceSplit(),
                 pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True),
             ]
         )
-
         self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
 
         super().__init__(
@@ -147,6 +151,9 @@ def __init__(
             ],
         )
 
+        if _spm_precompiled_charsmap is not None:
+            self._tokenizer.normalizer = normalizers.Precompiled(_spm_precompiled_charsmap)
+
     def get_sentinel_tokens(self):
         """Get the list of sentinel tokens (extra_id tokens) from additional_special_tokens."""
         return list(

src/transformers/models/openai/tokenization_openai.py

@@ -80,13 +80,7 @@ def __init__(
 
         # Set normalizer and pre-tokenizer to mimic OpenAI GPT behavior
         # OpenAI GPT uses BERT BasicTokenizer with lower_case=True
-        self._tokenizer.normalizer = normalizers.Sequence(
-            [
-                normalizers.NFD(),
-                normalizers.Lowercase(),
-                normalizers.StripAccents(),
-            ]
-        )
+        self._tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)
 
         self._tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
         self._tokenizer.decoder = decoders.BPEDecoder(suffix="</w>")

src/transformers/models/pegasus/tokenization_pegasus.py

@@ -85,6 +85,7 @@ def __init__(
         unk_token="<unk>",
         mask_token="<mask_2>",
         mask_token_sent="<mask_1>",
+        _spm_precompiled_charsmap=None,
         additional_special_tokens=None,
         offset=103,
         **kwargs,
@@ -100,9 +101,14 @@ def __init__(
 
         self._vocab = vocab
         self._tokenizer = Tokenizer(Unigram(vocab=vocab, unk_id=self._vocab.index((str(unk_token), 0.0), 1)))
-        self._tokenizer.normalizer = normalizers.Sequence(
-            [normalizers.Replace(Regex(r"\n"), " "), normalizers.Replace(Regex(r" {2,}"), " ")]
-        )
+        if _spm_precompiled_charsmap is not None:
+            self._tokenizer.normalizer = normalizers.Sequence(
+                [normalizers.Precompiled(_spm_precompiled_charsmap), normalizers.Replace(Regex(r" {2,}"), " ")]
+            )
+        else:
+            self._tokenizer.normalizer = normalizers.Sequence(
+                [normalizers.Replace(Regex(r"\n"), " "), normalizers.Replace(Regex(r" {2,}"), " ")]
+            )
 
         self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
         self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)

src/transformers/models/siglip2/modular_siglip2.py

@@ -82,9 +82,6 @@ def __init__(
         if backend is not None and backend.normalizer is not None:
             backend.normalizer = normalizers.Sequence([normalizers.Lowercase(), backend.normalizer])
 
-    def _unk_id(self) -> int:
-        raise AttributeError("_unk_id is not needed for SigLIP2.")
-
 
 class Siglip2TextConfig(SiglipTextConfig):
     pass

src/transformers/models/siglip2/tokenization_siglip2.py

@@ -18,7 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from tokenizers import Tokenizer, decoders, normalizers
+from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers
 from tokenizers.models import BPE
 
 from ...tokenization_utils_tokenizers import TokenizersBackend
@@ -69,6 +69,9 @@ def __init__(
                 byte_fallback=True,
             )
         )
+        self._tokenizer.pre_tokenizer = pre_tokenizers.Split(
+            pattern=" ", behavior="merged_with_previous", invert=False
+        )
 
         self._tokenizer.decoder = decoders.Sequence(
             [decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()]

src/transformers/models/t5/tokenization_t5.py

@@ -15,8 +15,9 @@
 
 import re
 
-from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
+from tokenizers.normalizers import Replace
 
 from ...tokenization_utils_tokenizers import TokenizersBackend
 from ...utils import logging
@@ -74,6 +75,7 @@ def __init__(
         eos_token="</s>",
         unk_token="<unk>",
         pad_token="<pad>",
+        _spm_precompiled_charsmap=None,
         extra_ids=100,
         additional_special_tokens=None,
         **kwargs,
@@ -116,15 +118,17 @@ def __init__(
             )
         )
 
-        self._tokenizer.normalizer = None
+        if _spm_precompiled_charsmap is not None:
+            self._tokenizer.normalizer = normalizers.Sequence(
+                [normalizers.Precompiled(_spm_precompiled_charsmap), Replace(pattern=Regex(" {2,}"), content=" ")]
+            )
 
         self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
             [
                 pre_tokenizers.WhitespaceSplit(),
                 pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True),
             ]
         )
-
         self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
 
         super().__init__(

src/transformers/tokenization_utils_base.py

@@ -1669,7 +1669,7 @@ def from_pretrained(
         if "tokenizer_file" in vocab_files and not re.search(vocab_files["tokenizer_file"], "".join(remote_files)):
             # mistral tokenizer names are different, but we can still convert them if
             # mistral common is not there
-            other_pattern = r"tekken\.json|tokenizer\.model\.*"
+            other_pattern = r"tekken\.json|tokenizer\.model\.*|tiktoken\.model|spiece\.model"
             if match := re.search(other_pattern, "\n".join(remote_files)):
                 vocab_files["vocab_file"] = match.group()