From 48528e20f3d3d5b51fda9659cb92872952aca538 Mon Sep 17 00:00:00 2001 From: Gagan Dhakrey Date: Wed, 27 May 2026 19:47:43 +0530 Subject: [PATCH] offload prompt_embeds decode in render_prompts_async to avoid blocking the event loop Signed-off-by: Gagan Dhakrey --- vllm/renderers/base.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 41d8c0075fb1..9fab3aff04e4 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -104,6 +104,9 @@ def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None: self._process_multimodal_async = make_async( self._process_multimodal, executor=self._mm_executor ) + self._safe_load_prompt_embeds_async = make_async( + safe_load_prompt_embeds, executor=self._executor + ) if mm_registry.supports_multimodal_inputs(config.model_config): mm_processor_cache = mm_registry.processor_cache_from_config(config) @@ -376,11 +379,28 @@ def render_prompts( return [self.render_prompt(prompt) for prompt in prompts] + async def _render_prompt_async( + self, + prompt: DictPrompt | bytes, + ) -> DictPrompt: + if isinstance(prompt, bytes): + embeds = await self._safe_load_prompt_embeds_async( + self.model_config, prompt + ) + return EmbedsPrompt(prompt_embeds=embeds) + + return prompt + async def render_prompts_async( self, prompts: Sequence[DictPrompt | bytes], ) -> list[DictPrompt]: - return self.render_prompts(prompts) + if len(prompts) == 0: + raise ValueError("You must pass at least one prompt") + + return await asyncio.gather( + *(self._render_prompt_async(prompt) for prompt in prompts) + ) @abstractmethod def render_messages(