From a400464418dd8b28bb097f4edd497d5347be323f Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Fri, 27 Jun 2025 16:14:50 +0700 Subject: [PATCH 1/8] fix: multiprocessing pickle error with tokenizer --- .../tokens/tokenizers/mistral.py | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 4a5cd9e5..70ab16dc 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -1,6 +1,6 @@ import warnings from pathlib import Path -from typing import Callable, Dict, Generic, List, Optional, Union +from typing import Callable, Dict, Generic, List, Optional, Tuple, Union, Any from mistral_common.exceptions import ( TokenizerException, @@ -95,6 +95,7 @@ def __init__( request_normalizer: InstructRequestNormalizer[ UserMessageType, AssistantMessageType, ToolMessageType, SystemMessageType, InstructRequestType ], + _reconstruction_args: Optional[Dict[str, Any]] = None, ): r"""Initializes a `MistralTokenizer`. @@ -102,13 +103,41 @@ def __init__( instruct_tokenizer: The instruct tokenizer to use. validator: The request validator to use. request_normalizer: The request normalizer to use. + _reconstruction_args: The reconstruction arguments to use. """ self._chat_completion_request_validator = validator self._instruct_request_normalizer = request_normalizer + self._reconstruction_args = _reconstruction_args self.instruct_tokenizer: InstructTokenizer[InstructRequest, FIMRequest, TokenizedType, AssistantMessageType] = ( instruct_tokenizer ) + def __reduce__(self) -> Tuple[Callable, Tuple[Any, ...]]: + """ + Provides a recipe for pickling (serializing) this object, which is + necessary for use with multiprocessing. Instead of pickling the complex + internal state, we provide a factory function and the arguments to + reconstruct the object from its source file. + + Returns: + A tuple of the factory function and the arguments to reconstruct the object from its source file. + """ + if self._reconstruction_args is None: + raise TokenizerException( + "This MistralTokenizer instance cannot be pickled because it was not created " + "using a supported factory method (e.g., from_file, from_hf_hub)." + ) + + # Get the name of the factory method (e.g., 'from_file') + method_name = self._reconstruction_args["method"] + # Get the actual classmethod from the class object + factory = getattr(self.__class__, method_name) + # Get the arguments for that factory + args = self._reconstruction_args["args"] + + # Return the recipe: (callable, arguments_tuple) + return (factory, args) + @classmethod def _data_path(cls) -> Path: return Path(__file__).parents[2] / "data" @@ -250,6 +279,11 @@ def from_file( Returns: The loaded tokenizer. """ + reconstruction_args: Dict[str, Any] = { + "method": "from_file", + "args": (tokenizer_filename, mode), + } + tokenizer: Union[SentencePieceTokenizer, Tekkenizer] if is_tekken(tokenizer_filename): @@ -265,37 +299,42 @@ def from_file( request_normalizer = normalizer_for_tokenizer_version(tokenizer.version) + tokenizer_kwargs: Dict[str, Any] = { + "_reconstruction_args": reconstruction_args, + "request_normalizer": request_normalizer, + } + if tokenizer.version == TokenizerVersion.v1: assert mm_encoder is None, "Tokenizer version needs to be >= v3" return MistralTokenizer( InstructTokenizerV1(tokenizer), validator=MistralRequestValidator(mode=mode), - request_normalizer=request_normalizer, + **tokenizer_kwargs, ) elif tokenizer.version == TokenizerVersion.v2: assert mm_encoder is None, "Tokenizer version needs to be >= v3" return MistralTokenizer( InstructTokenizerV2(tokenizer), validator=MistralRequestValidator(mode=mode), - request_normalizer=request_normalizer, + **tokenizer_kwargs, ) elif tokenizer.version == TokenizerVersion.v3: return MistralTokenizer( InstructTokenizerV3(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV3(mode=mode), - request_normalizer=request_normalizer, + **tokenizer_kwargs, ) elif tokenizer.version == TokenizerVersion.v7: return MistralTokenizer( InstructTokenizerV7(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV5(mode=mode), - request_normalizer=request_normalizer, + **tokenizer_kwargs, ) elif tokenizer.version == TokenizerVersion.v11: return MistralTokenizer( InstructTokenizerV11(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV5(mode=mode), - request_normalizer=request_normalizer, + **tokenizer_kwargs, ) raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}") From 753d51ad989b5fd05ccecc6b08f91e2a68cacb2a Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 27 Jun 2025 11:54:31 +0200 Subject: [PATCH 2/8] wip: reconstruct based on file_path --- src/mistral_common/tokens/tokenizers/base.py | 7 +++ .../tokens/tokenizers/mistral.py | 49 ++++--------------- .../tokens/tokenizers/sentencepiece.py | 7 +++ .../tokens/tokenizers/tekken.py | 11 ++++- 4 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/mistral_common/tokens/tokenizers/base.py b/src/mistral_common/tokens/tokenizers/base.py index 1ea1cb10..a31c1eed 100644 --- a/src/mistral_common/tokens/tokenizers/base.py +++ b/src/mistral_common/tokens/tokenizers/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum +from pathlib import Path from typing import Generic, List, Optional, Protocol, Tuple, TypeVar, Union import numpy as np @@ -236,6 +237,12 @@ def to_string(self, tokens: List[int]) -> str: @abstractmethod def _to_string(self, tokens: List[int]) -> str: ... + @property + @abstractmethod + def file_path(self) -> Path: + r"""The file path of the tokenizer.""" + ... + InstructRequestType = TypeVar("InstructRequestType", bound=InstructRequest) FIMRequestType = TypeVar("FIMRequestType", bound=FIMRequest) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 70ab16dc..36461e08 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -1,6 +1,6 @@ import warnings from pathlib import Path -from typing import Callable, Dict, Generic, List, Optional, Tuple, Union, Any +from typing import Any, Callable, Dict, Generic, List, Optional, Tuple, Union from mistral_common.exceptions import ( TokenizerException, @@ -95,7 +95,6 @@ def __init__( request_normalizer: InstructRequestNormalizer[ UserMessageType, AssistantMessageType, ToolMessageType, SystemMessageType, InstructRequestType ], - _reconstruction_args: Optional[Dict[str, Any]] = None, ): r"""Initializes a `MistralTokenizer`. @@ -103,40 +102,19 @@ def __init__( instruct_tokenizer: The instruct tokenizer to use. validator: The request validator to use. request_normalizer: The request normalizer to use. - _reconstruction_args: The reconstruction arguments to use. """ self._chat_completion_request_validator = validator self._instruct_request_normalizer = request_normalizer - self._reconstruction_args = _reconstruction_args - self.instruct_tokenizer: InstructTokenizer[InstructRequest, FIMRequest, TokenizedType, AssistantMessageType] = ( - instruct_tokenizer - ) + self.instruct_tokenizer = instruct_tokenizer def __reduce__(self) -> Tuple[Callable, Tuple[Any, ...]]: """ - Provides a recipe for pickling (serializing) this object, which is - necessary for use with multiprocessing. Instead of pickling the complex - internal state, we provide a factory function and the arguments to - reconstruct the object from its source file. + Provides a recipe for pickling (serializing) this object, which is necessary for use with multiprocessing. Returns: A tuple of the factory function and the arguments to reconstruct the object from its source file. """ - if self._reconstruction_args is None: - raise TokenizerException( - "This MistralTokenizer instance cannot be pickled because it was not created " - "using a supported factory method (e.g., from_file, from_hf_hub)." - ) - - # Get the name of the factory method (e.g., 'from_file') - method_name = self._reconstruction_args["method"] - # Get the actual classmethod from the class object - factory = getattr(self.__class__, method_name) - # Get the arguments for that factory - args = self._reconstruction_args["args"] - - # Return the recipe: (callable, arguments_tuple) - return (factory, args) + return self.from_file, (self.instruct_tokenizer.tokenizer.file_path,) @classmethod def _data_path(cls) -> Path: @@ -279,10 +257,6 @@ def from_file( Returns: The loaded tokenizer. """ - reconstruction_args: Dict[str, Any] = { - "method": "from_file", - "args": (tokenizer_filename, mode), - } tokenizer: Union[SentencePieceTokenizer, Tekkenizer] @@ -299,42 +273,37 @@ def from_file( request_normalizer = normalizer_for_tokenizer_version(tokenizer.version) - tokenizer_kwargs: Dict[str, Any] = { - "_reconstruction_args": reconstruction_args, - "request_normalizer": request_normalizer, - } - if tokenizer.version == TokenizerVersion.v1: assert mm_encoder is None, "Tokenizer version needs to be >= v3" return MistralTokenizer( InstructTokenizerV1(tokenizer), validator=MistralRequestValidator(mode=mode), - **tokenizer_kwargs, + request_normalizer=request_normalizer, ) elif tokenizer.version == TokenizerVersion.v2: assert mm_encoder is None, "Tokenizer version needs to be >= v3" return MistralTokenizer( InstructTokenizerV2(tokenizer), validator=MistralRequestValidator(mode=mode), - **tokenizer_kwargs, + request_normalizer=request_normalizer, ) elif tokenizer.version == TokenizerVersion.v3: return MistralTokenizer( InstructTokenizerV3(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV3(mode=mode), - **tokenizer_kwargs, + request_normalizer=request_normalizer, ) elif tokenizer.version == TokenizerVersion.v7: return MistralTokenizer( InstructTokenizerV7(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV5(mode=mode), - **tokenizer_kwargs, + request_normalizer=request_normalizer, ) elif tokenizer.version == TokenizerVersion.v11: return MistralTokenizer( InstructTokenizerV11(tokenizer, mm_encoder=mm_encoder), validator=MistralRequestValidatorV5(mode=mode), - **tokenizer_kwargs, + request_normalizer=request_normalizer, ) raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}") diff --git a/src/mistral_common/tokens/tokenizers/sentencepiece.py b/src/mistral_common/tokens/tokenizers/sentencepiece.py index 6ed65bbf..2db62bd6 100644 --- a/src/mistral_common/tokens/tokenizers/sentencepiece.py +++ b/src/mistral_common/tokens/tokenizers/sentencepiece.py @@ -81,8 +81,15 @@ def __init__(self, model_path: str, tokenizer_version: Optional[TokenizerVersion self._version: TokenizerVersion = tokenizer_version or get_spm_version(model_path, raise_deprecated=False) + self._file_path = Path(model_path) + super().__init__() + @property + def file_path(self) -> Path: + r"""The path to the tokenizer model.""" + return self._file_path + @property def version(self) -> TokenizerVersion: r"""The version of the tokenizer.""" diff --git a/src/mistral_common/tokens/tokenizers/tekken.py b/src/mistral_common/tokens/tokenizers/tekken.py index e7f2dec8..cac3a6a6 100644 --- a/src/mistral_common/tokens/tokenizers/tekken.py +++ b/src/mistral_common/tokens/tokenizers/tekken.py @@ -137,7 +137,7 @@ def __init__( version: TokenizerVersion, *, name: str = "tekkenizer", - _path: Optional[str] = None, + _path: Optional[Union[str, Path]] = None, mm_config: Optional[MultimodalConfig] = None, ): r"""Initialize the tekken tokenizer. @@ -199,6 +199,14 @@ def __init__( self._special_tokens_reverse_vocab = {t["token_str"]: t["rank"] for t in special_tokens} self._vocab = [self.id_to_piece(i) for i in range(vocab_size)] self._special_token_policy = SpecialTokenPolicy.IGNORE + self._file_path = Path(_path) if _path is not None else None + + @property + def file_path(self) -> Path: + r"""The path to the tokenizer file.""" + if self._file_path is None: + raise ValueError("The tokenizer was not loaded from a file.") + return self._file_path @classmethod def from_file(cls: Type["Tekkenizer"], path: Union[str, Path]) -> "Tekkenizer": @@ -255,6 +263,7 @@ def from_file(cls: Type["Tekkenizer"], path: Union[str, Path]) -> "Tekkenizer": version=version, name=path.name.replace(".json", ""), mm_config=model_data.get("multimodal"), + _path=path, ) @property From 318aed009dd084c952d5fce451cab1be7206e3df Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 27 Jun 2025 12:00:16 +0200 Subject: [PATCH 3/8] revert erased typing --- src/mistral_common/tokens/tokenizers/mistral.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 36461e08..fe7c4a2c 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -105,7 +105,9 @@ def __init__( """ self._chat_completion_request_validator = validator self._instruct_request_normalizer = request_normalizer - self.instruct_tokenizer = instruct_tokenizer + self.instruct_tokenizer: InstructTokenizer[InstructRequest, FIMRequest, TokenizedType, AssistantMessageType] = ( + instruct_tokenizer + ) def __reduce__(self) -> Tuple[Callable, Tuple[Any, ...]]: """ From 38977d1c20468a1b4be7648814fd3c61e03ddff3 Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Fri, 27 Jun 2025 17:40:45 +0700 Subject: [PATCH 4/8] feat: add test for multiprocessing --- tests/test_multiprocessing.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_multiprocessing.py diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py new file mode 100644 index 00000000..7ebe1f52 --- /dev/null +++ b/tests/test_multiprocessing.py @@ -0,0 +1,33 @@ +import unittest +import multiprocessing + +from mistral_common.tokens.tokenizers.mistral import MistralTokenizer + + +def _worker_decode_function(tokenizer_instance: MistralTokenizer) -> str: + """ + A simple worker that receives a tokenizer, performs a decode operation, + and returns the result. + + Example tokens were taken from `tests/test_tokenize_v1.py.test_system_multiturn`. + """ + tokens_to_decode = [1, 733, 16289, 28793, 17121, 22526, 13, 13, 28708, 733, 28748, 16289, 28793] + return tokenizer_instance.decode(tokens_to_decode) + + +class TestTokenizerPickling(unittest.TestCase): + """ + Test suite to ensure MistralTokenizer can be pickled and used across + different processes, which is essential for multiprocessing. + """ + + def test_tokenizer_is_pickleable_with_multiprocessing(self) -> None: + tokenizer_path = str(MistralTokenizer._data_path() / "tokenizer.model.v1") + tokenizer = MistralTokenizer.from_file(tokenizer_path) + + with multiprocessing.Pool(processes=2) as pool: + results = pool.map(_worker_decode_function, [tokenizer]) + + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], str) + self.assertEqual(results[0], "[INST] SYSTEM\n\na [/INST]") From 84429d9f5109bfff39035afdf592719d2c28ceb9 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 27 Jun 2025 13:31:40 +0200 Subject: [PATCH 5/8] fix --- .../tokens/tokenizers/mistral.py | 5 ++- .../tokens/tokenizers/sentencepiece.py | 15 +++++--- tests/test_mistral_tokenizer.py | 34 ++++++++++++++++++- tests/test_multiprocessing.py | 33 ------------------ 4 files changed, 45 insertions(+), 42 deletions(-) delete mode 100644 tests/test_multiprocessing.py diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index fe7c4a2c..234c2768 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -116,7 +116,7 @@ def __reduce__(self) -> Tuple[Callable, Tuple[Any, ...]]: Returns: A tuple of the factory function and the arguments to reconstruct the object from its source file. """ - return self.from_file, (self.instruct_tokenizer.tokenizer.file_path,) + return MistralTokenizer.from_file, (self.instruct_tokenizer.tokenizer.file_path,) @classmethod def _data_path(cls) -> Path: @@ -247,7 +247,7 @@ def from_hf_hub( @classmethod def from_file( cls, - tokenizer_filename: str, + tokenizer_filename: Union[str, Path], mode: ValidationMode = ValidationMode.test, ) -> "MistralTokenizer": r"""Loads a tokenizer from a file. @@ -259,7 +259,6 @@ def from_file( Returns: The loaded tokenizer. """ - tokenizer: Union[SentencePieceTokenizer, Tekkenizer] if is_tekken(tokenizer_filename): diff --git a/src/mistral_common/tokens/tokenizers/sentencepiece.py b/src/mistral_common/tokens/tokenizers/sentencepiece.py index 2db62bd6..c75dfffe 100644 --- a/src/mistral_common/tokens/tokenizers/sentencepiece.py +++ b/src/mistral_common/tokens/tokenizers/sentencepiece.py @@ -28,8 +28,11 @@ def is_sentencepiece(path: Union[str, Path]) -> bool: return path.is_file() and any(path.name.endswith(suffix) for suffix in suffixes) -def get_spm_version(tokenizer_filename: str, raise_deprecated: bool = False) -> TokenizerVersion: +def get_spm_version(tokenizer_filename: Union[str, Path], raise_deprecated: bool = False) -> TokenizerVersion: r"""Get the version of the tokenizer from the filename.""" + if isinstance(tokenizer_filename, Path): + tokenizer_filename = str(tokenizer_filename) + _version_str = tokenizer_filename.split(".")[-1] if _version_str != "model": # filter tokenizer_filename == "/path/to/tokenizer.model" case _version_str = _version_str.split("m")[0] @@ -47,8 +50,11 @@ def get_spm_version(tokenizer_filename: str, raise_deprecated: bool = False) -> return TokenizerVersion(_version_str) -def get_mm_config(tokenizer_filename: str) -> Optional[MultimodalConfig]: +def get_mm_config(tokenizer_filename: Union[str, Path]) -> Optional[MultimodalConfig]: r"""Get the multimodal config from the tokenizer filename.""" + if isinstance(tokenizer_filename, Path): + tokenizer_filename = str(tokenizer_filename) + _version_str = tokenizer_filename.split(".")[-1] if _version_str == "model" or "m" not in _version_str: return None @@ -64,7 +70,7 @@ def get_mm_config(tokenizer_filename: str) -> Optional[MultimodalConfig]: class SentencePieceTokenizer(Tokenizer): r"""[SentencePiece](https://github.com/google/sentencepiece) tokenizer.""" - def __init__(self, model_path: str, tokenizer_version: Optional[TokenizerVersion] = None) -> None: + def __init__(self, model_path: Union[str, Path], tokenizer_version: Optional[TokenizerVersion] = None) -> None: r"""Initialize the `SentencePieceTokenizer`. Args: @@ -74,7 +80,7 @@ def __init__(self, model_path: str, tokenizer_version: Optional[TokenizerVersion self._logger = logging.getLogger(self.__class__.__name__) # reload tokenizer assert os.path.isfile(model_path), model_path - self._model = SentencePieceProcessor(model_file=model_path) + self._model = SentencePieceProcessor(model_file=model_path if isinstance(model_path, str) else model_path.as_posix()) assert self._model.vocab_size() == self._model.get_piece_size() self._vocab = [self._model.id_to_piece(i) for i in range(self.n_words)] @@ -82,7 +88,6 @@ def __init__(self, model_path: str, tokenizer_version: Optional[TokenizerVersion self._version: TokenizerVersion = tokenizer_version or get_spm_version(model_path, raise_deprecated=False) self._file_path = Path(model_path) - super().__init__() @property diff --git a/tests/test_mistral_tokenizer.py b/tests/test_mistral_tokenizer.py index 5d53ea6c..9c1a8ab6 100644 --- a/tests/test_mistral_tokenizer.py +++ b/tests/test_mistral_tokenizer.py @@ -1,4 +1,5 @@ -from typing import Optional, Union +import multiprocessing +from typing import List, Optional, Tuple, Union from unittest.mock import patch import pytest @@ -103,3 +104,34 @@ def _mocked_hf_download( tokenizer = MistralTokenizer.from_hf_hub("mistralai/Pixtral-Large-Instruct-2411") assert isinstance(tokenizer.instruct_tokenizer, InstructTokenizerV3) + + +def _worker_decode_function(tokenizer_instance_and_token_ids: Tuple[MistralTokenizer, List[int]]) -> str: + tokenizer_instance, token_ids = tokenizer_instance_and_token_ids + return tokenizer_instance.decode(token_ids) + + +@pytest.mark.parametrize( + ["tokenizer_file", "token_ids", "expected"], + [ + ( + "tokenizer.model.v1", + [1, 733, 16289, 28793, 17121, 22526, 13, 13, 28708, 733, 28748, 16289, 28793], + "[INST] SYSTEM\n\na [/INST]", + ), + ( + "tekken_240911.json", + [1091, 3174, 3074, 1093, 126205, 1267, 1097, 1766, 1047, 3174, 3074, 1093], + "[INST] SYSTEM\n\na [/INST]", + ), + ], +) +def test_tokenizer_is_pickleable_with_multiprocessing(tokenizer_file: str, token_ids: List[int], expected: str) -> None: + tokenizer_path = str(MistralTokenizer._data_path() / tokenizer_file) + tokenizer = MistralTokenizer.from_file(tokenizer_path) + + with multiprocessing.Pool(processes=1) as pool: + results = pool.map(_worker_decode_function, [(tokenizer, token_ids)]) + + assert len(results) == 1 + assert results[0] == expected diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py deleted file mode 100644 index 7ebe1f52..00000000 --- a/tests/test_multiprocessing.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest -import multiprocessing - -from mistral_common.tokens.tokenizers.mistral import MistralTokenizer - - -def _worker_decode_function(tokenizer_instance: MistralTokenizer) -> str: - """ - A simple worker that receives a tokenizer, performs a decode operation, - and returns the result. - - Example tokens were taken from `tests/test_tokenize_v1.py.test_system_multiturn`. - """ - tokens_to_decode = [1, 733, 16289, 28793, 17121, 22526, 13, 13, 28708, 733, 28748, 16289, 28793] - return tokenizer_instance.decode(tokens_to_decode) - - -class TestTokenizerPickling(unittest.TestCase): - """ - Test suite to ensure MistralTokenizer can be pickled and used across - different processes, which is essential for multiprocessing. - """ - - def test_tokenizer_is_pickleable_with_multiprocessing(self) -> None: - tokenizer_path = str(MistralTokenizer._data_path() / "tokenizer.model.v1") - tokenizer = MistralTokenizer.from_file(tokenizer_path) - - with multiprocessing.Pool(processes=2) as pool: - results = pool.map(_worker_decode_function, [tokenizer]) - - self.assertEqual(len(results), 1) - self.assertIsInstance(results[0], str) - self.assertEqual(results[0], "[INST] SYSTEM\n\na [/INST]") From bfe526323de85013d6e40dc85fa0fa73aa9f6d5e Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 27 Jun 2025 13:33:58 +0200 Subject: [PATCH 6/8] revert processes to 2 --- tests/test_mistral_tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mistral_tokenizer.py b/tests/test_mistral_tokenizer.py index 9c1a8ab6..43a6fe3f 100644 --- a/tests/test_mistral_tokenizer.py +++ b/tests/test_mistral_tokenizer.py @@ -130,7 +130,7 @@ def test_tokenizer_is_pickleable_with_multiprocessing(tokenizer_file: str, token tokenizer_path = str(MistralTokenizer._data_path() / tokenizer_file) tokenizer = MistralTokenizer.from_file(tokenizer_path) - with multiprocessing.Pool(processes=1) as pool: + with multiprocessing.Pool(processes=2) as pool: results = pool.map(_worker_decode_function, [(tokenizer, token_ids)]) assert len(results) == 1 From 012a44a09dd391d192aaa3647e95b2d9830970d8 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 27 Jun 2025 13:34:41 +0200 Subject: [PATCH 7/8] format --- src/mistral_common/tokens/tokenizers/sentencepiece.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mistral_common/tokens/tokenizers/sentencepiece.py b/src/mistral_common/tokens/tokenizers/sentencepiece.py index c75dfffe..9254ff85 100644 --- a/src/mistral_common/tokens/tokenizers/sentencepiece.py +++ b/src/mistral_common/tokens/tokenizers/sentencepiece.py @@ -80,7 +80,9 @@ def __init__(self, model_path: Union[str, Path], tokenizer_version: Optional[Tok self._logger = logging.getLogger(self.__class__.__name__) # reload tokenizer assert os.path.isfile(model_path), model_path - self._model = SentencePieceProcessor(model_file=model_path if isinstance(model_path, str) else model_path.as_posix()) + self._model = SentencePieceProcessor( + model_file=model_path if isinstance(model_path, str) else model_path.as_posix() + ) assert self._model.vocab_size() == self._model.get_piece_size() self._vocab = [self._model.id_to_piece(i) for i in range(self.n_words)] From be0c9088e1a88e12b2e9f2a07b1912be7e21b241 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Mon, 30 Jun 2025 09:54:40 +0200 Subject: [PATCH 8/8] wip --- .../tokens/tokenizers/mistral.py | 5 ++++- .../tokens/tokenizers/sentencepiece.py | 6 ++---- tests/test_mistral_tokenizer.py | 20 +++++++++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 234c2768..db31cf09 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -116,7 +116,10 @@ def __reduce__(self) -> Tuple[Callable, Tuple[Any, ...]]: Returns: A tuple of the factory function and the arguments to reconstruct the object from its source file. """ - return MistralTokenizer.from_file, (self.instruct_tokenizer.tokenizer.file_path,) + return MistralTokenizer.from_file, ( + self.instruct_tokenizer.tokenizer.file_path, + self._chat_completion_request_validator._mode, + ) @classmethod def _data_path(cls) -> Path: diff --git a/src/mistral_common/tokens/tokenizers/sentencepiece.py b/src/mistral_common/tokens/tokenizers/sentencepiece.py index 9254ff85..cfbf76e9 100644 --- a/src/mistral_common/tokens/tokenizers/sentencepiece.py +++ b/src/mistral_common/tokens/tokenizers/sentencepiece.py @@ -30,8 +30,7 @@ def is_sentencepiece(path: Union[str, Path]) -> bool: def get_spm_version(tokenizer_filename: Union[str, Path], raise_deprecated: bool = False) -> TokenizerVersion: r"""Get the version of the tokenizer from the filename.""" - if isinstance(tokenizer_filename, Path): - tokenizer_filename = str(tokenizer_filename) + tokenizer_filename = str(tokenizer_filename) _version_str = tokenizer_filename.split(".")[-1] if _version_str != "model": # filter tokenizer_filename == "/path/to/tokenizer.model" case @@ -52,8 +51,7 @@ def get_spm_version(tokenizer_filename: Union[str, Path], raise_deprecated: bool def get_mm_config(tokenizer_filename: Union[str, Path]) -> Optional[MultimodalConfig]: r"""Get the multimodal config from the tokenizer filename.""" - if isinstance(tokenizer_filename, Path): - tokenizer_filename = str(tokenizer_filename) + tokenizer_filename = str(tokenizer_filename) _version_str = tokenizer_filename.split(".")[-1] if _version_str == "model" or "m" not in _version_str: diff --git a/tests/test_mistral_tokenizer.py b/tests/test_mistral_tokenizer.py index 43a6fe3f..b5733f05 100644 --- a/tests/test_mistral_tokenizer.py +++ b/tests/test_mistral_tokenizer.py @@ -5,6 +5,7 @@ import pytest from mistral_common.exceptions import TokenizerException +from mistral_common.protocol.instruct.validator import ValidationMode from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy from mistral_common.tokens.tokenizers.instruct import ( InstructTokenizerV1, @@ -106,32 +107,39 @@ def _mocked_hf_download( assert isinstance(tokenizer.instruct_tokenizer, InstructTokenizerV3) -def _worker_decode_function(tokenizer_instance_and_token_ids: Tuple[MistralTokenizer, List[int]]) -> str: - tokenizer_instance, token_ids = tokenizer_instance_and_token_ids +def _worker_decode_function( + tokenizer_instance_and_token_ids_and_validation_mode: Tuple[MistralTokenizer, List[int], ValidationMode], +) -> str: + tokenizer_instance, token_ids, validation_mode = tokenizer_instance_and_token_ids_and_validation_mode + assert tokenizer_instance._chat_completion_request_validator._mode == validation_mode return tokenizer_instance.decode(token_ids) @pytest.mark.parametrize( - ["tokenizer_file", "token_ids", "expected"], + ["tokenizer_file", "validation_mode", "token_ids", "expected"], [ ( "tokenizer.model.v1", + ValidationMode.test, [1, 733, 16289, 28793, 17121, 22526, 13, 13, 28708, 733, 28748, 16289, 28793], "[INST] SYSTEM\n\na [/INST]", ), ( "tekken_240911.json", + ValidationMode.finetuning, [1091, 3174, 3074, 1093, 126205, 1267, 1097, 1766, 1047, 3174, 3074, 1093], "[INST] SYSTEM\n\na [/INST]", ), ], ) -def test_tokenizer_is_pickleable_with_multiprocessing(tokenizer_file: str, token_ids: List[int], expected: str) -> None: +def test_tokenizer_is_pickleable_with_multiprocessing( + tokenizer_file: str, validation_mode: ValidationMode, token_ids: List[int], expected: str +) -> None: tokenizer_path = str(MistralTokenizer._data_path() / tokenizer_file) - tokenizer = MistralTokenizer.from_file(tokenizer_path) + tokenizer = MistralTokenizer.from_file(tokenizer_path, validation_mode) with multiprocessing.Pool(processes=2) as pool: - results = pool.map(_worker_decode_function, [(tokenizer, token_ids)]) + results = pool.map(_worker_decode_function, [(tokenizer, token_ids, validation_mode)]) assert len(results) == 1 assert results[0] == expected