Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -1553,3 +1553,109 @@ def test_count_tokens_validation_error_returns_bad_request(self):

assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json()["error"]["type"] == "BadRequestError"


# ======================================================================
# thinking configuration pass-through
# ======================================================================


class TestThinkingConfig:
def test_absent_thinking_leaves_reasoning_untouched(self):
"""Requests without `thinking` must convert exactly as before."""
request = _make_request([{"role": "user", "content": "Hello"}])

result = _convert(request)
assert result.reasoning_effort is None
assert result.thinking_token_budget is None
assert result.include_reasoning is True

def test_disabled_clears_reasoning_effort(self):
"""`disabled` maps to reasoning_effort="none", which is what clears
enable_thinking for templates that honor it."""
request = _make_request(
[{"role": "user", "content": "Hello"}],
thinking={"type": "disabled"},
)

result = _convert(request)
assert result.reasoning_effort == "none"

def test_disabled_overrides_output_config_effort(self):
"""`thinking` is applied after output_config so an explicit opt-out wins
over an inherited effort ceiling."""
request = _make_request(
[{"role": "user", "content": "Hello"}],
output_config={"effort": "high"},
thinking={"type": "disabled"},
)

result = _convert(request)
assert result.reasoning_effort == "none"

def test_enabled_with_budget_sets_thinking_token_budget(self):
request = _make_request(
[{"role": "user", "content": "Hello"}],
thinking={"type": "enabled", "budget_tokens": 2048},
)

result = _convert(request)
assert result.thinking_token_budget == 2048

def test_enabled_without_budget_pins_nothing(self):
request = _make_request(
[{"role": "user", "content": "Hello"}],
thinking={"type": "enabled"},
)

result = _convert(request)
assert result.thinking_token_budget is None
assert result.reasoning_effort is None

def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self):
"""`adaptive` lets the model choose depth, so only the ceiling from
output_config.effort should survive."""
request = _make_request(
[{"role": "user", "content": "Hello"}],
output_config={"effort": "low"},
thinking={"type": "adaptive"},
)

result = _convert(request)
assert result.reasoning_effort == "low"
assert result.thinking_token_budget is None

def test_display_omitted_suppresses_reasoning_in_response(self):
"""`display` controls visibility only -- it must not touch depth."""
request = _make_request(
[{"role": "user", "content": "Hello"}],
thinking={"type": "adaptive", "display": "omitted"},
)

result = _convert(request)
assert result.include_reasoning is False
assert result.reasoning_effort is None
assert result.thinking_token_budget is None

def test_display_summarized_keeps_reasoning_included(self):
request = _make_request(
[{"role": "user", "content": "Hello"}],
thinking={"type": "adaptive", "display": "summarized"},
)

result = _convert(request)
assert result.include_reasoning is True

def test_claude_code_payload(self):
"""The combination Claude Code sends on every request: an effort ceiling
plus adaptive thinking with reasoning display omitted."""
request = _make_request(
[{"role": "user", "content": "Hello"}],
output_config={"effort": "high"},
thinking={"type": "adaptive", "display": "omitted"},
)

result = _convert(request)
assert result.reasoning_effort == "high"
assert result.include_reasoning is False
assert result.thinking_token_budget is None
2 changes: 2 additions & 0 deletions tests/entrypoints/anthropic/test_protocol_exports.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
AnthropicMessagesResponse,
AnthropicOutputConfig,
AnthropicStreamEvent,
AnthropicThinkingConfig,
AnthropicUsage,
)

Expand All @@ -35,6 +36,7 @@
AnthropicMessagesResponse,
AnthropicOutputConfig,
AnthropicStreamEvent,
AnthropicThinkingConfig,
AnthropicUsage,
)

Expand Down
13 changes: 13 additions & 0 deletions vllm/entrypoints/anthropic/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,18 @@ class AnthropicOutputConfig(BaseModel):
format: AnthropicJsonOutputFormat | None = None


class AnthropicThinkingConfig(BaseModel):
"""Extended-thinking configuration.

``display`` controls visibility only: reasoning still runs and is still
billed under every setting.
"""

type: Literal["enabled", "disabled", "adaptive"] = "enabled"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason it hasn’t been introduced yet is that, if I remember correctly, these fields are going to be deprecated.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecated in Claude Code? Is it not beneficial to be able to control reasoning. It's a huge difference using different harness like OpenCode (OpenAI api) vs. Claude Code (Anthropic API) since vLLM handles reasoning well via OpenAI API but not in Anthropic.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cf. https://platform.claude.com/doc/en/build-with-claude/extended-thinking

Although I do not reckon that the whole thinking field is deprecated, thinking.budget_tokens is no longer supported since 4.7

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@chaunceyjiang I think this is a good addition since it's not going to be deprecated. The newly released DeepSeek Harness does support it as one example: https://github.com/earendil-works/pi/blob/5cd93f688aaab89dbb6dfa4aca535f21796ae185/packages/ai/src/api/anthropic-messages.ts#L1069 This is what DeepSeek Harness are using, dynamic thinking / effort for each prompt.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I’ll test it locally.

budget_tokens: int | None = None
display: Literal["summarized", "omitted"] | None = None

Comment on lines +129 to +131

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per Anthropic Messages API spec, budget_tokens is a required field when type is enabled (BetaThinkingConfigEnabled) and unsupported otherwise (BetaThinkingConfigDisabled, BetaThinkingConfigAdaptive). It might be better to add schema validation for consistency I think.

Suggested change
budget_tokens: int | None = None
display: Literal["summarized", "omitted"] | None = None
budget_tokens: int | None = None
display: Literal["summarized", "omitted"] | None = None
@model_validator(mode="after")
def validate_budget_tokens(self) -> "AnthropicThinkingConfig":
if self.type == "enabled" and self.budget_tokens is None:
raise ValueError("thinking.budget_tokens is required when thinking.type is 'enabled'.")
elif self.budget_tokens is not None:
raise ValueError(
f"thinking.budget_tokens must not be set when thinking.type is '{self.type}'."
)
return self

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the goal are just to be able to emulate Anthropics API to support Claude Code harness. vLLM will not be able to handle budget_tokens anyway? I don't think it really matters so I let the maintainer decide if vLLM need that kind of consistency


class AnthropicMessagesRequest(BaseModel):
"""Anthropic Messages API request"""

Expand All @@ -126,6 +138,7 @@ class AnthropicMessagesRequest(BaseModel):
max_tokens: int
metadata: dict[str, Any] | None = None
output_config: AnthropicOutputConfig | None = None
thinking: AnthropicThinkingConfig | None = None
stop_sequences: (
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
) = None
Expand Down
26 changes: 26 additions & 0 deletions vllm/entrypoints/anthropic/serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
AnthropicMessagesResponse,
AnthropicOutputConfig,
AnthropicStreamEvent,
AnthropicThinkingConfig,
AnthropicUsage,
)
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
Expand Down Expand Up @@ -213,6 +214,7 @@ def _convert_anthropic_to_openai_request(
req = cls._build_base_request(anthropic_request, openai_messages)
cls._handle_streaming_options(req, anthropic_request)
cls._handle_output_config(req, anthropic_request)
cls._handle_thinking(req, anthropic_request)
cls._convert_tool_choice(anthropic_request, req)
cls._convert_tools(anthropic_request, req)
return req
Expand Down Expand Up @@ -492,6 +494,30 @@ def _build_base_request(
chat_template_kwargs=anthropic_request.chat_template_kwargs,
)

@classmethod
def _handle_thinking(
cls,
req: ChatCompletionRequest,
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
) -> None:
"""Handle extended-thinking configuration"""
if isinstance(anthropic_request, AnthropicCountTokensRequest):
return
thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
if thinking is None:
return

if thinking.type == "disabled":
# "none" is what clears enable_thinking for templates that honor it.
req.reasoning_effort = "none"
elif thinking.type == "enabled" and thinking.budget_tokens is not None:
req.thinking_token_budget = thinking.budget_tokens
# "adaptive" pins nothing: the model chooses depth beneath the ceiling
# already set from output_config.effort.

if thinking.display == "omitted":
req.include_reasoning = False

@classmethod
def _handle_output_config(
cls,
Expand Down
Loading