[Frontend] Make tokenizer batch wait timeout configurable

tests/engine/test_arg_utils.py

@@ -232,6 +232,14 @@ def test_hf_token_cli_arg(cli_args, expected):
     assert args.hf_token == expected
 
 
+def test_tokenizer_batch_wait_timeout_cli_arg():
+    parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
+
+    args = parser.parse_args(["--tokenizer-batch-wait-timeout-s", "0"])
+
+    assert args.tokenizer_batch_wait_timeout_s == 0
+
+
 @pytest.mark.parametrize(
     ("arg", "expected"),
     [

tests/entrypoints/openai/chat_completion/test_serving_chat.py

@@ -538,6 +538,7 @@ class MockModelConfig:
     is_encoder_decoder: bool = False
     is_multimodal_model: bool = False
     renderer_num_workers: int = 1
+    tokenizer_batch_wait_timeout_s: float = 0.002
     enable_prompt_embeds: bool = False
 
     def get_diff_sampling_param(self):

tests/entrypoints/openai/completion/test_completion_error.py

@@ -55,6 +55,7 @@ class MockModelConfig:
     is_encoder_decoder: bool = False
     is_multimodal_model: bool = False
     renderer_num_workers: int = 1
+    tokenizer_batch_wait_timeout_s: float = 0.002
 
     def get_diff_sampling_param(self):
         return self.diff_sampling_param or {}

tests/entrypoints/openai/completion/test_lora_resolvers.py

@@ -55,6 +55,7 @@ class MockModelConfig:
     is_encoder_decoder: bool = False
     is_multimodal_model: bool = False
     renderer_num_workers: int = 1
+    tokenizer_batch_wait_timeout_s: float = 0.002
 
     def get_diff_sampling_param(self):
         return self.diff_sampling_param or {}

tests/renderers/test_completions.py

@@ -39,6 +39,7 @@ class MockModelConfig:
     is_encoder_decoder: bool = False
     is_multimodal_model: bool = False
     renderer_num_workers: int = 1
+    tokenizer_batch_wait_timeout_s: float = 0.002
     hidden_size: int = 768
     dtype: torch.dtype = torch.float32
 
@@ -132,6 +133,12 @@ def test_invalid_type(self):
 
 
 class TestRenderPrompt:
+    @pytest.mark.asyncio
+    async def test_async_tokenizer_uses_configured_batch_wait_timeout(self):
+        renderer = _build_renderer(MockModelConfig(tokenizer_batch_wait_timeout_s=0))
+
+        assert renderer.get_async_tokenizer().batch_wait_timeout_s == 0
+
     def test_tokens_input(self):
         renderer = _build_renderer(MockModelConfig())
 

vllm/config/model.py

@@ -339,6 +339,11 @@ class ModelConfig:
     The offline `LLM` entrypoint uses the synchronous renderer path and
     processes prompts (including multimodal preprocessing) serially, so
     this setting has no effect there."""
+    tokenizer_batch_wait_timeout_s: float = Field(default=0.002, ge=0)
+    """Maximum time in seconds for the async renderer tokenizer to wait for
+    additional encode/decode requests before running a micro-batch. Set to 0
+    to disable the waiting delay for latency-sensitive, low-concurrency
+    serving."""
 
     # Pooler config
     pooler_config: PoolerConfig | None = None

vllm/engine/arg_utils.py

@@ -574,6 +574,7 @@ class EngineArgs:
     )
     io_processor_plugin: str | None = None
     renderer_num_workers: int = 1
+    tokenizer_batch_wait_timeout_s: float = ModelConfig.tokenizer_batch_wait_timeout_s
     skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
     video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
     mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
@@ -866,6 +867,10 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
             "--renderer-num-workers",
             **model_kwargs["renderer_num_workers"],
         )
+        model_group.add_argument(
+            "--tokenizer-batch-wait-timeout-s",
+            **model_kwargs["tokenizer_batch_wait_timeout_s"],
+        )
 
         # Model loading arguments
         load_kwargs = get_kwargs(LoadConfig)
@@ -1624,6 +1629,7 @@ def create_model_config(self) -> ModelConfig:
             mm_tensor_ipc=self.mm_tensor_ipc,
             io_processor_plugin=self.io_processor_plugin,
             renderer_num_workers=self.renderer_num_workers,
+            tokenizer_batch_wait_timeout_s=self.tokenizer_batch_wait_timeout_s,
         )
 
     def validate_tensorizer_args(self):

vllm/renderers/base.py

@@ -149,7 +149,9 @@ def get_tokenizer(self) -> _T:
     def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:
         if self._async_tokenizer is None:
             self._async_tokenizer = AsyncMicrobatchTokenizer(
-                self.get_tokenizer(), executor=self._executor
+                self.get_tokenizer(),
+                batch_wait_timeout_s=self.model_config.tokenizer_batch_wait_timeout_s,
+                executor=self._executor,
             )
 
         return self._async_tokenizer