diff --git a/src/transformers/tokenization_mistral_common.py b/src/transformers/tokenization_mistral_common.py index a0dab55e0722..43e4139c5e54 100644 --- a/src/transformers/tokenization_mistral_common.py +++ b/src/transformers/tokenization_mistral_common.py @@ -48,6 +48,13 @@ get_one_valid_tokenizer_file, ) + _MAP_SPECIAL_TOKENS: dict[str, str] = { + "bos_token": SpecialTokens.bos.value, + "eos_token": SpecialTokens.eos.value, + "pad_token": SpecialTokens.pad.value, + "unk_token": SpecialTokens.unk.value, + } + if is_torch_available(): import torch @@ -172,13 +179,6 @@ def _maybe_remove_lang(text: str | list[str], skip_special_tokens: bool) -> str return [re.sub(r"^lang:[a-z]{2}", "", string) for string in text] -_MAP_SPECIAL_TOKENS = { - "bos_token": SpecialTokens.bos.value, - "eos_token": SpecialTokens.eos.value, - "pad_token": SpecialTokens.pad.value, - "unk_token": SpecialTokens.unk.value, -} - _VALID_INIT_KWARGS = {"_from_auto", "backend", "files_loaded"} @@ -222,7 +222,7 @@ class MistralCommonBackend(PreTrainedTokenizerBase): def __init__( self, tokenizer_path: str | os.PathLike | Path, - mode: ValidationMode = ValidationMode.test, + mode: "str | ValidationMode" = "test", model_max_length: int = VERY_LARGE_INTEGER, padding_side: str = "left", truncation_side: str = "right", @@ -304,7 +304,7 @@ def __init__( ) @property - def mode(self) -> ValidationMode: + def mode(self) -> "ValidationMode": """ `ValidationMode`: The mode used by the tokenizer. Possible values are: - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode. @@ -1034,7 +1034,7 @@ def apply_chat_template( # type: ignore[override] max_length: int | None = None, return_tensors: str | TensorType | None = None, return_dict: bool = True, - reasoning_effort: ReasoningEffort | None = None, + reasoning_effort: "ReasoningEffort | None" = None, **kwargs, ) -> str | list[int] | list[str] | list[list[int]] | BatchEncoding: """ @@ -1422,7 +1422,7 @@ def from_pretrained( cls, pretrained_model_name_or_path: str | os.PathLike, *init_inputs, - mode: str | ValidationMode = ValidationMode.test, + mode: "str | ValidationMode" = "test", cache_dir: str | os.PathLike | None = None, force_download: bool = False, local_files_only: bool = False, @@ -1587,7 +1587,7 @@ def save_pretrained( # type: ignore[override] return (str(save_directory / self._tokenizer_path.name),) @staticmethod - def _get_validation_mode(mode: str | ValidationMode) -> ValidationMode: + def _get_validation_mode(mode: "str | ValidationMode") -> "ValidationMode": """Get the validation mode from a string or a ValidationMode.""" _invalid_mode_msg = f"Invalid `mistral-common` tokenizer mode: {mode}. Possible values are {', '.join([vm.value for vm in list(ValidationMode)])}." if isinstance(mode, str): diff --git a/tests/test_tokenization_mistral_common.py b/tests/test_tokenization_mistral_common.py index a6b195239641..f497a06a64a6 100644 --- a/tests/test_tokenization_mistral_common.py +++ b/tests/test_tokenization_mistral_common.py @@ -14,7 +14,9 @@ import base64 import gc +import importlib.util import io +import sys import tempfile import unittest from unittest.mock import patch @@ -2265,3 +2267,27 @@ def test_prepare_for_model(self): # unsupported kwargs should raise ValueError with self.assertRaises(ValueError): self.tokenizer.prepare_for_model(token_ids, add_special_tokens=False, unsupported_arg="") + + +class TestMistralCommonImport(unittest.TestCase): + def test_import_does_not_raise_regardless_of_mistral_common_availability(self) -> None: + # Regression test: importing `transformers.tokenization_mistral_common` must not raise, whether or not + # `mistral_common` is installed. + # The module body is executed under a throwaway module name via `importlib.util` so that the real + # `sys.modules["transformers.tokenization_mistral_common"]` entry is never swapped out. + module_origin = importlib.util.find_spec("transformers.tokenization_mistral_common").origin + + for available in (False, True): + with self.subTest(mistral_common_available=available): + with patch("transformers.utils.import_utils.is_mistral_common_available", return_value=available): + throwaway_name = f"_regression_tokenization_mistral_common_available_{available}" + spec = importlib.util.spec_from_file_location(throwaway_name, module_origin) + module = importlib.util.module_from_spec(spec) + sys.modules[throwaway_name] = module + try: + # Executing the module body must not raise (this is what the regression guards against). + spec.loader.exec_module(module) + # `_MAP_SPECIAL_TOKENS` is only bound when `mistral_common` is available. + self.assertEqual(hasattr(module, "_MAP_SPECIAL_TOKENS"), available) + finally: + sys.modules.pop(throwaway_name, None)