WIP

vllm-project · Aug 24, 2024 · 4e99ef4 · 4e99ef4
1 parent 25ceeb1
commit 4e99ef4
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py
@@ -13,6 +13,7 @@
 from mistral_common.tokens.tokenizers.sentencepiece import (
     SentencePieceTokenizer)
 from mistral_common.tokens.tokenizers.tekken import Tekkenizer
+from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
 
 if TYPE_CHECKING:
     from vllm.entrypoints.chat_utils import ConversationMessage
@@ -53,6 +54,10 @@ def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
                               self.tokenizer)
         self._is_tekken = isinstance(self.tokenizer, Tekkenizer)
 
+        if self._is_tekken:
+            # Make sure special tokens will not raise
+            self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
+
         # the following attributes are set to fit VLLM's design
         self.is_fast = True
         self.chat_template = True