Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 96 additions & 1 deletion tests/entrypoints/openai/chat_completion/test_chat_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.exceptions import (
GenerationError,
RetryableRequestError,
VLLMValidationError,
)
from vllm.multimodal.cache import MultiModalCacheMissError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM

Expand Down Expand Up @@ -196,6 +202,95 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
)


@pytest.mark.asyncio
async def test_chat_retries_cache_miss_with_uncached_prompt():
mock_engine = MagicMock(spec=AsyncLLM)
serving_chat = object.__new__(OpenAIServingChat)
serving_chat.engine_client = mock_engine
serving_chat.model_config = MockModelConfig()

request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=1,
)
initial_prompt = {"prompt_token_ids": [1, 2, 3]}
retry_prompt = {"prompt_token_ids": [1, 2, 3]}
serving_chat.render_chat_request = AsyncMock(
return_value=([{"role": "user", "content": "Test prompt"}], [retry_prompt])
)
expected = MagicMock(spec=RequestOutput)

async def generate(prompt, *args, **kwargs):
if prompt is initial_prompt:
raise MultiModalCacheMissError(["missing-hash"])
yield expected

mock_engine.generate = MagicMock(side_effect=generate)

outputs = [
output
async for output in serving_chat._generate_with_cache_retry(
request,
initial_prompt,
SamplingParams(max_tokens=1),
"request-id",
0,
[1, 2, 3],
{},
None,
{},
)
]

assert outputs == [expected]
assert mock_engine.generate.call_count == 2
assert mock_engine.generate.call_args_list[1].args[0] is retry_prompt
serving_chat.render_chat_request.assert_awaited_once_with(
request, skip_mm_cache=True
)


@pytest.mark.asyncio
async def test_chat_does_not_retry_after_output():
mock_engine = MagicMock(spec=AsyncLLM)
serving_chat = object.__new__(OpenAIServingChat)
serving_chat.engine_client = mock_engine
serving_chat.render_chat_request = AsyncMock()

request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=1,
)
expected = MagicMock(spec=RequestOutput)

async def generate(*args, **kwargs):
yield expected
raise MultiModalCacheMissError(["missing-hash"])

mock_engine.generate = MagicMock(side_effect=generate)
outputs = []

with pytest.raises(RetryableRequestError) as exc_info:
async for output in serving_chat._generate_with_cache_retry(
request,
{"prompt_token_ids": [1, 2, 3]},
SamplingParams(max_tokens=1),
"request-id",
0,
[1, 2, 3],
{},
None,
{},
):
outputs.append(output)

assert outputs == [expected]
assert isinstance(exc_info.value.__cause__, MultiModalCacheMissError)
serving_chat.render_chat_request.assert_not_awaited()


def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
Expand Down
95 changes: 94 additions & 1 deletion tests/entrypoints/openai/completion/test_completion_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,18 @@
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.exceptions import (
GenerationError,
RetryableRequestError,
VLLMValidationError,
)
from vllm.logprobs import Logprob
from vllm.multimodal.cache import MultiModalCacheMissError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
Expand Down Expand Up @@ -565,6 +571,93 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
)


@pytest.mark.asyncio
async def test_completion_retries_cache_miss_with_matching_uncached_prompt():
mock_engine = MagicMock(spec=AsyncLLM)
serving_completion = object.__new__(OpenAIServingCompletion)
serving_completion.engine_client = mock_engine

request = CompletionRequest(
model=MODEL_NAME,
prompt=["first", "second"],
max_tokens=1,
)
initial_prompt = {"prompt_token_ids": [4, 5]}
retry_prompts = [
{"prompt_token_ids": [1, 2, 3]},
{"prompt_token_ids": [4, 5]},
]
serving_completion.render_completion_request = AsyncMock(return_value=retry_prompts)
expected = MagicMock(spec=RequestOutput)

async def generate(prompt, *args, **kwargs):
if prompt is initial_prompt:
raise MultiModalCacheMissError(["missing-hash"])
yield expected

mock_engine.generate = MagicMock(side_effect=generate)

outputs = [
output
async for output in serving_completion._generate_with_cache_retry(
request,
initial_prompt,
SamplingParams(max_tokens=1),
"request-id-1",
1,
{},
)
]

assert outputs == [expected]
assert mock_engine.generate.call_count == 2
assert mock_engine.generate.call_args_list[1].args[0] is retry_prompts[1]
serving_completion.render_completion_request.assert_awaited_once_with(
request, skip_mm_cache=True
)


@pytest.mark.asyncio
async def test_completion_second_cache_miss_is_retryable_error():
mock_engine = MagicMock(spec=AsyncLLM)
serving_completion = object.__new__(OpenAIServingCompletion)
serving_completion.engine_client = mock_engine

request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=1,
)
retry_prompt = {"prompt_token_ids": [1, 2, 3]}
serving_completion.render_completion_request = AsyncMock(
return_value=[retry_prompt]
)

async def generate(*args, **kwargs):
if False:
yield
raise MultiModalCacheMissError(["missing-hash"])

mock_engine.generate = MagicMock(side_effect=generate)

with pytest.raises(RetryableRequestError) as exc_info:
async for _ in serving_completion._generate_with_cache_retry(
request,
{"prompt_token_ids": [1, 2, 3]},
SamplingParams(max_tokens=1),
"request-id-0",
0,
{},
):
pass

assert isinstance(exc_info.value.__cause__, MultiModalCacheMissError)
assert mock_engine.generate.call_count == 2
serving_completion.render_completion_request.assert_awaited_once_with(
request, skip_mm_cache=True
)


def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
Expand Down
102 changes: 102 additions & 0 deletions tests/entrypoints/pooling/embed/test_serving.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

from unittest.mock import AsyncMock, MagicMock, call

import pytest
from fastapi.responses import Response

from vllm.entrypoints.pooling.embed.protocol import (
CohereEmbedRequest,
EmbeddingCompletionRequest,
)
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding
from vllm.exceptions import RetryableRequestError
from vllm.multimodal.cache import MultiModalCacheMissError


def _mock_serving(monkeypatch, collect_side_effect):
serving = object.__new__(ServingEmbedding)
io_processor = MagicMock()
ctx = MagicMock()
expected = Response()
collect_batch = AsyncMock(side_effect=collect_side_effect)

monkeypatch.setattr(
serving, "get_io_processor", MagicMock(return_value=io_processor)
)
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=ctx))
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
monkeypatch.setattr(serving, "_prepare_generators", AsyncMock())
monkeypatch.setattr(serving, "_collect_batch", collect_batch)
monkeypatch.setattr(
serving,
"_postprocessing_async",
AsyncMock(return_value=expected),
raising=False,
)
return serving, io_processor, ctx, expected, collect_batch


@pytest.mark.asyncio
async def test_embedding_retries_cache_miss_without_mm_cache(monkeypatch):
cache_miss = MultiModalCacheMissError(["missing-hash"])
serving, io_processor, ctx, expected, collect_batch = _mock_serving(
monkeypatch, [cache_miss, None]
)
request = EmbeddingCompletionRequest(model="test", input="hello")

result = await serving(request)

assert result is expected
assert collect_batch.await_count == 2
assert serving._preprocessing.await_args_list == [
call(io_processor, ctx, skip_mm_cache=False),
call(io_processor, ctx, skip_mm_cache=True),
]


@pytest.mark.asyncio
async def test_embedding_second_cache_miss_is_retryable_error(monkeypatch):
cache_miss = MultiModalCacheMissError(["missing-hash"])
serving, _, _, _, collect_batch = _mock_serving(
monkeypatch, [cache_miss, cache_miss]
)
request = EmbeddingCompletionRequest(model="test", input="hello")

with pytest.raises(RetryableRequestError) as exc_info:
await serving(request)

assert isinstance(exc_info.value.__cause__, MultiModalCacheMissError)
assert collect_batch.await_count == 2


@pytest.mark.asyncio
async def test_cohere_embedding_does_not_retry_cache_miss(monkeypatch):
cache_miss = MultiModalCacheMissError(["missing-hash"])
serving, _, _, _, collect_batch = _mock_serving(monkeypatch, [cache_miss])
request = CohereEmbedRequest(model="test", texts=["hello"])

with pytest.raises(RetryableRequestError) as exc_info:
await serving(request)

assert isinstance(exc_info.value.__cause__, MultiModalCacheMissError)
assert collect_batch.await_count == 1


@pytest.mark.asyncio
async def test_embedding_retry_render_skips_mm_cache():
serving = object.__new__(ServingEmbedding)
render_request = {"skip_mm_cache": False}
io_processor = MagicMock()
io_processor.get_request_factory_online.return_value = [render_request]
io_processor.render_async = AsyncMock(return_value={"prompts": {}})
io_processor.maybe_pre_process_chunked = MagicMock()
serving.io_processor = io_processor
ctx = MagicMock()

await serving._preprocessing(io_processor, ctx, skip_mm_cache=True)

assert render_request["skip_mm_cache"] is True
io_processor.render_async.assert_awaited_once_with(render_request)
io_processor.maybe_pre_process_chunked.assert_called_once_with(ctx)
Loading
Loading