Skip to content

[Bugfix] Reject empty logprob_token_ids and fix falsy validation in OpenAI request protocols #57394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nandan2003 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[Bugfix] Reject empty logprob_token_ids and fix falsy validation in OpenAI request protocols #57394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions tests/entrypoints/openai/chat_completion/test_logprob_token_ids_validation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0

import pytest
from pydantic import ValidationError

from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.exceptions import VLLMValidationError

MODELS = [ChatCompletionRequest, CompletionRequest, BatchChatCompletionRequest]
BASE = {
ChatCompletionRequest: {
"model": "m",
"messages": [{"role": "user", "content": "hi"}],
},
CompletionRequest: {"model": "m", "prompt": "hi"},
BatchChatCompletionRequest: {
"messages": [[{"role": "user", "content": "hi"}]],
},
}


@pytest.mark.parametrize("cls", MODELS)
@pytest.mark.parametrize("logprobs", [None, False, True])
def test_rejects_empty_token_ids(cls, logprobs):
data = {**BASE[cls], "logprob_token_ids": []}
if logprobs is not None:
data["logprobs"] = logprobs
with pytest.raises(VLLMValidationError, match="must not be an empty list") as exc:
cls.model_validate(data)
assert exc.value.parameter == "logprob_token_ids"


@pytest.mark.parametrize("cls", MODELS)
def test_rejects_token_ids_without_logprobs(cls):
with pytest.raises(
VLLMValidationError, match="when using `logprob_token_ids`"
) as exc:
cls.model_validate({**BASE[cls], "logprob_token_ids": [100]})
assert exc.value.parameter == "logprob_token_ids"


@pytest.mark.parametrize("cls", MODELS)
@pytest.mark.parametrize("val", [123, "123"])
def test_defers_non_list_to_pydantic(cls, val):
with pytest.raises(ValidationError):
cls.model_validate({**BASE[cls], "logprob_token_ids": val})
51 changes: 34 additions & 17 deletions vllm/entrypoints/openai/chat_completion/protocol.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
prompt_logprobs: int | None = None
logprob_token_ids: list[int] | None = Field(
default=None,
min_length=1,
description=(
"Specific vocab token IDs to return logprobs for at each generated "
"position, in addition to the sampled token. More efficient than "
Expand Down Expand Up @@ -778,7 +779,7 @@ def to_sampling_params(
else None
),
prompt_logprobs=prompt_logprobs,
logprob_token_ids=self.logprob_token_ids or None,
logprob_token_ids=self.logprob_token_ids,
ignore_eos=self.ignore_eos,
max_tokens=max_tokens,
min_tokens=self.min_tokens,
Expand Down Expand Up @@ -851,17 +852,24 @@ def validate_stream_options(cls, data):
def check_logprobs(cls, data):
if not isinstance(data, dict):
return data
if data.get("logprob_token_ids") and data.get("use_beam_search"):
raise VLLMValidationError(
"`logprob_token_ids` is not supported with beam search.",
parameter="logprob_token_ids",
)

if data.get("logprob_token_ids") and not data.get("logprobs"):
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set to true.",
parameter="logprob_token_ids",
)
if data.get("logprob_token_ids") is not None:
if not isinstance(data["logprob_token_ids"], list):
return data
if not data["logprob_token_ids"]:
raise VLLMValidationError(
"`logprob_token_ids` must not be an empty list.",
parameter="logprob_token_ids",
)
if data.get("use_beam_search"):
raise VLLMValidationError(
"`logprob_token_ids` is not supported with beam search.",
parameter="logprob_token_ids",
)
if not data.get("logprobs"):
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set to true.",
parameter="logprob_token_ids",
)

# These fields are integers, but `mode="before"` runs on the raw
# request data, so a non-numeric value (e.g. a JSON string) would
Expand Down Expand Up @@ -1143,6 +1151,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
top_logprobs: TopLogprobsParam = 0
logprob_token_ids: list[int] | None = Field(
default=None,
min_length=1,
description=(
"Specific vocab token IDs to return logprobs for at each generated "
"position, in addition to the sampled token. Requires "
Expand Down Expand Up @@ -1202,11 +1211,19 @@ def check_batch_mode(cls, data: Any) -> Any:
"Please set `use_beam_search` to False.",
parameter="use_beam_search",
)
if data.get("logprob_token_ids") and not data.get("logprobs"):
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set to true.",
parameter="logprob_token_ids",
)
if data.get("logprob_token_ids") is not None:
if not isinstance(data["logprob_token_ids"], list):
return data
if not data["logprob_token_ids"]:
raise VLLMValidationError(
"`logprob_token_ids` must not be an empty list.",
parameter="logprob_token_ids",
)
if not data.get("logprobs"):
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set to true.",
parameter="logprob_token_ids",
)
kv_transfer_params = data.get("kv_transfer_params")
if (
isinstance(kv_transfer_params, dict)
Expand Down
49 changes: 26 additions & 23 deletions vllm/entrypoints/openai/completion/protocol.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ class CompletionRequest(OpenAIBaseModel):
prompt_logprobs: int | None = None
logprob_token_ids: list[int] | None = Field(
default=None,
min_length=1,
description=(
"Specific vocab token IDs to return logprobs for at each generated "
"position, in addition to the sampled token. More efficient than "
Expand Down Expand Up @@ -406,7 +407,7 @@ def to_sampling_params(
max_tokens=max_tokens if not echo_without_generation else 1,
min_tokens=self.min_tokens,
prompt_logprobs=prompt_logprobs,
logprob_token_ids=self.logprob_token_ids or None,
logprob_token_ids=self.logprob_token_ids,
skip_special_tokens=self.skip_special_tokens,
spaces_between_special_tokens=self.spaces_between_special_tokens,
include_stop_str_in_output=self.include_stop_str_in_output,
Expand Down Expand Up @@ -514,11 +515,30 @@ def check_structured_outputs_count(cls, data):
def check_logprobs(cls, data):
if not isinstance(data, dict):
return data
if data.get("logprob_token_ids") and data.get("use_beam_search"):
raise VLLMValidationError(
"`logprob_token_ids` is not supported with beam search.",
parameter="logprob_token_ids",
)
if data.get("logprob_token_ids") is not None:
if not isinstance(data["logprob_token_ids"], list):
return data
if not data["logprob_token_ids"]:
raise VLLMValidationError(
"`logprob_token_ids` must not be an empty list.",
parameter="logprob_token_ids",
)
if data.get("use_beam_search"):
raise VLLMValidationError(
"`logprob_token_ids` is not supported with beam search.",
parameter="logprob_token_ids",
)
if data.get("echo") and data.get("max_tokens") == 0:
raise VLLMValidationError(
"`logprob_token_ids` is not supported when `echo=True` and "
"`max_tokens=0` because no output tokens are generated.",
parameter="logprob_token_ids",
)
if data.get("logprobs") is None:
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set.",
parameter="logprob_token_ids",
)

if (
data.get("use_beam_search")
Expand All @@ -531,23 +551,6 @@ def check_logprobs(cls, data):
parameter="echo",
)

if (
data.get("logprob_token_ids")
and data.get("echo")
and data.get("max_tokens") == 0
):
raise VLLMValidationError(
"`logprob_token_ids` is not supported when `echo=True` and "
"`max_tokens=0` because no output tokens are generated.",
parameter="logprob_token_ids",
)

if data.get("logprob_token_ids") and data.get("logprobs") is None:
raise VLLMValidationError(
"when using `logprob_token_ids`, `logprobs` must be set.",
parameter="logprob_token_ids",
)

# These fields are integers, but `mode="before"` runs on the raw
# request data, so a non-numeric value (e.g. a JSON string) would
# reach the comparisons below and raise TypeError -> HTTP 500. Reject
Expand Down
Loading