Make tokenization_mistral_common importable without mistral_common installed

src/transformers/tokenization_mistral_common.py

@@ -48,6 +48,13 @@
         get_one_valid_tokenizer_file,
     )
 
+    _MAP_SPECIAL_TOKENS: dict[str, str] = {
+        "bos_token": SpecialTokens.bos.value,
+        "eos_token": SpecialTokens.eos.value,
+        "pad_token": SpecialTokens.pad.value,
+        "unk_token": SpecialTokens.unk.value,
+    }
+
 
 if is_torch_available():
     import torch
@@ -172,13 +179,6 @@ def _maybe_remove_lang(text: str | list[str], skip_special_tokens: bool) -> str
     return [re.sub(r"^lang:[a-z]{2}", "", string) for string in text]
 
 
-_MAP_SPECIAL_TOKENS = {
-    "bos_token": SpecialTokens.bos.value,
-    "eos_token": SpecialTokens.eos.value,
-    "pad_token": SpecialTokens.pad.value,
-    "unk_token": SpecialTokens.unk.value,
-}
-
 _VALID_INIT_KWARGS = {"_from_auto", "backend", "files_loaded"}
 
 
@@ -222,7 +222,7 @@ class MistralCommonBackend(PreTrainedTokenizerBase):
     def __init__(
         self,
         tokenizer_path: str | os.PathLike | Path,
-        mode: ValidationMode = ValidationMode.test,
+        mode: "str | ValidationMode" = "test",
         model_max_length: int = VERY_LARGE_INTEGER,
         padding_side: str = "left",
         truncation_side: str = "right",
@@ -304,7 +304,7 @@ def __init__(
         )
 
     @property
-    def mode(self) -> ValidationMode:
+    def mode(self) -> "ValidationMode":
         """
         `ValidationMode`: The mode used by the tokenizer. Possible values are:
             - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
@@ -1034,7 +1034,7 @@ def apply_chat_template(  # type: ignore[override]
         max_length: int | None = None,
         return_tensors: str | TensorType | None = None,
         return_dict: bool = True,
-        reasoning_effort: ReasoningEffort | None = None,
+        reasoning_effort: "ReasoningEffort | None" = None,
         **kwargs,
     ) -> str | list[int] | list[str] | list[list[int]] | BatchEncoding:
         """
@@ -1422,7 +1422,7 @@ def from_pretrained(
         cls,
         pretrained_model_name_or_path: str | os.PathLike,
         *init_inputs,
-        mode: str | ValidationMode = ValidationMode.test,
+        mode: "str | ValidationMode" = "test",
         cache_dir: str | os.PathLike | None = None,
         force_download: bool = False,
         local_files_only: bool = False,
@@ -1587,7 +1587,7 @@ def save_pretrained(  # type: ignore[override]
         return (str(save_directory / self._tokenizer_path.name),)
 
     @staticmethod
-    def _get_validation_mode(mode: str | ValidationMode) -> ValidationMode:
+    def _get_validation_mode(mode: "str | ValidationMode") -> "ValidationMode":
         """Get the validation mode from a string or a ValidationMode."""
         _invalid_mode_msg = f"Invalid `mistral-common` tokenizer mode: {mode}. Possible values are {', '.join([vm.value for vm in list(ValidationMode)])}."
         if isinstance(mode, str):

tests/test_tokenization_mistral_common.py

@@ -14,7 +14,9 @@
 
 import base64
 import gc
+import importlib.util
 import io
+import sys
 import tempfile
 import unittest
 from unittest.mock import patch
@@ -2265,3 +2267,27 @@ def test_prepare_for_model(self):
         # unsupported kwargs should raise ValueError
         with self.assertRaises(ValueError):
             self.tokenizer.prepare_for_model(token_ids, add_special_tokens=False, unsupported_arg="")
+
+
+class TestMistralCommonImport(unittest.TestCase):
+    def test_import_does_not_raise_regardless_of_mistral_common_availability(self) -> None:
+        # Regression test: importing `transformers.tokenization_mistral_common` must not raise, whether or not
+        # `mistral_common` is installed.
+        # The module body is executed under a throwaway module name via `importlib.util` so that the real
+        # `sys.modules["transformers.tokenization_mistral_common"]` entry is never swapped out.
+        module_origin = importlib.util.find_spec("transformers.tokenization_mistral_common").origin
+
+        for available in (False, True):
+            with self.subTest(mistral_common_available=available):
+                with patch("transformers.utils.import_utils.is_mistral_common_available", return_value=available):
+                    throwaway_name = f"_regression_tokenization_mistral_common_available_{available}"
+                    spec = importlib.util.spec_from_file_location(throwaway_name, module_origin)
+                    module = importlib.util.module_from_spec(spec)
+                    sys.modules[throwaway_name] = module
+                    try:
+                        # Executing the module body must not raise (this is what the regression guards against).
+                        spec.loader.exec_module(module)
+                        # `_MAP_SPECIAL_TOKENS` is only bound when `mistral_common` is available.
+                        self.assertEqual(hasattr(module, "_MAP_SPECIAL_TOKENS"), available)
+                    finally:
+                        sys.modules.pop(throwaway_name, None)