[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers

vllm/multimodal/processing/context.py

@@ -268,28 +268,6 @@ def call_hf_processor(
         try:
             output = hf_processor(**data, **allowed_kwargs)
         except Exception as exc:
-            # See https://github.com/huggingface/tokenizers/issues/537
-            if (
-                isinstance(exc, RuntimeError)
-                and exc
-                and exc.args[0] == "Already borrowed"
-                and num_tries < max_tries
-            ):
-                logger.warning(
-                    "Failed to acquire tokenizer in current thread. "
-                    "Retrying (%d/%d)...",
-                    num_tries,
-                    max_tries,
-                )
-                time.sleep(0.5)
-                return self.call_hf_processor(
-                    hf_processor,
-                    data,
-                    kwargs,
-                    num_tries=num_tries + 1,
-                    max_tries=max_tries,
-                )
-
             msg = (
                 f"Failed to apply {type(hf_processor).__name__} "
                 f"on data={data} with kwargs={allowed_kwargs}"

vllm/renderers/base.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import asyncio
-import copy
 import time
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
@@ -108,17 +107,10 @@ def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:
         if config.model_config.is_multimodal_model:
             mm_processor_cache = mm_registry.processor_cache_from_config(config)
 
-            # Deep-copy the tokenizer so the multimodal processor gets its
-            # own Rust tokenizer backend.  Without this, concurrent access
-            # from AsyncMicrobatchTokenizer and call_hf_processor causes
-            # "RuntimeError: Already borrowed" from the Rust RefCell.
-            # See: https://github.com/huggingface/tokenizers/issues/537
-            mm_tokenizer = copy.deepcopy(tokenizer)
-
             with set_default_torch_num_threads():
                 self.mm_processor = mm_registry.create_processor(
                     config.model_config,
-                    tokenizer=mm_tokenizer,
+                    tokenizer=tokenizer,
                     cache=mm_processor_cache,
                 )
 
@@ -130,11 +122,10 @@ def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:
             # requests don't pollute the sender cache.
             ro_cache = mm_registry.processor_only_cache_from_config(config)
             if ro_cache is not None:
-                ro_tokenizer = copy.deepcopy(tokenizer)
                 with set_default_torch_num_threads():
                     self._readonly_mm_processor = mm_registry.create_processor(
                         config.model_config,
-                        tokenizer=ro_tokenizer,
+                        tokenizer=tokenizer,
                         cache=ro_cache,
                     )
 

vllm/tokenizers/deepseek_v32.py

@@ -8,7 +8,7 @@
 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
 
 from .deepseek_v32_encoding import encode_messages
-from .hf import HfTokenizer, get_cached_tokenizer
+from .hf import HfTokenizer, get_cached_tokenizer, make_backend_thread_local
 from .protocol import TokenizerLike
 
 
@@ -86,4 +86,5 @@ class DeepseekV32Tokenizer(TokenizerLike):
     @classmethod
     def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
         tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
+        tokenizer = make_backend_thread_local(tokenizer)
         return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer))

vllm/tokenizers/deepseek_v4.py

@@ -8,7 +8,7 @@
 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
 
 from .deepseek_v4_encoding import encode_messages
-from .hf import HfTokenizer, get_cached_tokenizer
+from .hf import HfTokenizer, get_cached_tokenizer, make_backend_thread_local
 from .protocol import TokenizerLike
 
 
@@ -93,4 +93,5 @@ class DeepseekV4Tokenizer(TokenizerLike):
     @classmethod
     def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
         tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
+        tokenizer = make_backend_thread_local(tokenizer)
         return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer))

vllm/tokenizers/hf.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import contextlib
 import copy
+import threading
 from pathlib import Path
 from typing import TypeAlias
 
@@ -14,6 +15,38 @@
 HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
 
 
+def make_backend_thread_local(tokenizer: HfTokenizer) -> HfTokenizer:
+    """Route operations through a per-thread deep-copied backend tokenizer."""
+    if not isinstance(tokenizer, PreTrainedTokenizerFast):
+        return tokenizer
+
+    thread_safe_tokenizer = copy.copy(tokenizer)
+
+    backend_tokenizer = thread_safe_tokenizer._tokenizer
+
+    # Concurrent dict insertion is safe here thanks to the GIL.
+    thread_local = {threading.get_ident(): copy.deepcopy(backend_tokenizer)}
+
+    class ThreadLocalTokenizer(tokenizer.__class__):  # type: ignore
+        @property
+        def _tokenizer(self):
+            current_thread_id = threading.get_ident()
+            try:
+                return thread_local[current_thread_id]
+            except KeyError:
+                backend_copy = copy.deepcopy(backend_tokenizer)
+                thread_local[current_thread_id] = backend_copy
+                return backend_copy
+
+        def __reduce__(self):
+            return make_backend_thread_local, (tokenizer,)
+
+    ThreadLocalTokenizer.__name__ = f"ThreadLocal{tokenizer.__class__.__name__}"
+
+    thread_safe_tokenizer.__class__ = ThreadLocalTokenizer
+    return thread_safe_tokenizer
+
+
 def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
     """
     By default, transformers will recompute multiple tokenizer properties
@@ -122,4 +155,4 @@ def from_pretrained(
             }
             tokenizer.add_special_tokens(special_tokens_map)
 
-        return get_cached_tokenizer(tokenizer)
+        return get_cached_tokenizer(make_backend_thread_local(tokenizer))

vllm/tokenizers/qwen_vl.py

@@ -6,7 +6,7 @@
 
 from transformers import AutoTokenizer
 
-from .hf import HfTokenizer, get_cached_tokenizer
+from .hf import HfTokenizer, get_cached_tokenizer, make_backend_thread_local
 from .protocol import TokenizerLike
 
 
@@ -68,4 +68,5 @@ class QwenVLTokenizer(TokenizerLike):
     @classmethod
     def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
         tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
+        tokenizer = make_backend_thread_local(tokenizer)
         return get_cached_tokenizer(get_qwen_vl_tokenizer(tokenizer))