Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions litellm/llms/openai/chat/gpt_5_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,9 +244,11 @@ def map_openai_params(
),
status_code=400,
)
elif effective_effort == "minimal":
# minimal is opt-out: unknown models pass through; only block when
# the model map explicitly sets supports_minimal_reasoning_effort=false.
elif effective_effort in ("minimal", "low"):
# minimal/low are opt-out: unknown models pass through; only block when
# the model map explicitly sets supports_{level}_reasoning_effort=false.
# Example: gpt-5.5-pro only accepts {medium, high, xhigh}, so it sets
# supports_low_reasoning_effort=false (and supports_minimal=false).
if self._is_reasoning_effort_level_explicitly_disabled(
model, effective_effort
):
Expand Down
2 changes: 2 additions & 0 deletions litellm/llms/vertex_ai/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,8 @@ def process_items(schema, depth=0):
and ("items" not in schema or schema.get("items") == {})
):
schema["items"] = {"type": "object"}
elif schema.get("type") == "array" and "items" not in schema:
schema["items"] = {"type": "object"}
for key, value in schema.items():
if isinstance(value, dict):
process_items(value, depth + 1)
Expand Down
10 changes: 6 additions & 4 deletions litellm/model_prices_and_context_window_backup.json
Original file line number Diff line number Diff line change
Expand Up @@ -19928,7 +19928,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false
},
"gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5e-07,
Expand Down Expand Up @@ -19976,7 +19976,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false
},
"gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
Expand Down Expand Up @@ -20019,7 +20019,8 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"gpt-5.5-pro-2026-04-23": {
"cache_read_input_token_cost": 3e-06,
Expand Down Expand Up @@ -20062,7 +20063,8 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
Expand Down
1 change: 1 addition & 0 deletions litellm/types/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_url_context: Optional[bool]
supports_none_reasoning_effort: Optional[bool]
supports_minimal_reasoning_effort: Optional[bool]
supports_low_reasoning_effort: Optional[bool]
supports_xhigh_reasoning_effort: Optional[bool]
supports_max_reasoning_effort: Optional[bool]

Expand Down
3 changes: 3 additions & 0 deletions litellm/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5896,6 +5896,9 @@ def _get_model_info_helper( # noqa: PLR0915
supports_minimal_reasoning_effort=_model_info.get(
"supports_minimal_reasoning_effort", None
),
supports_low_reasoning_effort=_model_info.get(
"supports_low_reasoning_effort", None
),
supports_xhigh_reasoning_effort=_model_info.get(
"supports_xhigh_reasoning_effort", None
),
Expand Down
10 changes: 6 additions & 4 deletions model_prices_and_context_window.json
Original file line number Diff line number Diff line change
Expand Up @@ -19942,7 +19942,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false
},
"gpt-5.5-2026-04-23": {
"cache_read_input_token_cost": 5e-07,
Expand Down Expand Up @@ -19990,7 +19990,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false
},
"gpt-5.5-pro": {
"cache_read_input_token_cost": 3e-06,
Expand Down Expand Up @@ -20033,7 +20033,8 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"gpt-5.5-pro-2026-04-23": {
"cache_read_input_token_cost": 3e-06,
Expand Down Expand Up @@ -20076,7 +20077,8 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
"supports_minimal_reasoning_effort": false,
"supports_low_reasoning_effort": false
},
"gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,46 @@ def test_generic_cost_per_token_gpt55_pro():
)


@pytest.mark.parametrize(
"model,expected_none,expected_xhigh,expected_minimal",
[
# Verified against OpenAI's live API on 2026-04-24:
# gpt-5.5 -> supports: none, low, medium, high, xhigh
# gpt-5.5-pro -> supports: medium, high, xhigh
# Neither supports "minimal"; gpt-5.5-pro additionally does not support "none".
# The JSON must reflect this so LiteLLM rejects unsupported values locally
# (or drops them with drop_params=True) instead of round-tripping to OpenAI
# for a 400.
("gpt-5.5", True, True, False),
("gpt-5.5-2026-04-23", True, True, False),
("gpt-5.5-pro", False, True, False),
("gpt-5.5-pro-2026-04-23", False, True, False),
],
)
def test_gpt55_reasoning_effort_flags_match_live_openai_api(
model, expected_none, expected_xhigh, expected_minimal
):
"""Pin reasoning_effort capability flags to OpenAI's actual API contract.

Observed via `POST /v1/chat/completions` with reasoning_effort=minimal:
``Unsupported value: 'reasoning_effort' does not support 'minimal' with
this model``. gpt-5.5-pro additionally rejects 'none' and 'low'.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
Comment on lines +439 to +440

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Missing env var teardown pollutes later tests

os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] is set to "True" and litellm.model_cost is overwritten, but neither is restored after the test. If a test running later in the same process expects the default (remote-fetched) cost map, it will silently get the local backup instead. The same pattern exists in test_generic_cost_per_token_gpt55_pro, so this wasn't introduced here, but adding the same unguarded pattern again compounds the issue.

A monkeypatch fixture or pytest.fixture with yield + cleanup would prevent leaking into the test suite.

Suggested change
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")

(requires adding monkeypatch as a parameter to the test function)


m = litellm.model_cost[model]
assert (
m.get("supports_none_reasoning_effort") is expected_none
), f"{model}: supports_none_reasoning_effort expected {expected_none}"
assert (
m.get("supports_xhigh_reasoning_effort") is expected_xhigh
), f"{model}: supports_xhigh_reasoning_effort expected {expected_xhigh}"
assert (
m.get("supports_minimal_reasoning_effort") is expected_minimal
), f"{model}: supports_minimal_reasoning_effort expected {expected_minimal}"


@pytest.mark.parametrize(
"base_model,dated_model",
[
Expand Down
75 changes: 75 additions & 0 deletions tests/test_litellm/llms/openai/test_gpt5_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,81 @@ def test_gpt5_unknown_model_passes_through_minimal(config: OpenAIConfig):
assert params["reasoning_effort"] == "minimal"


def test_gpt5_5_pro_rejects_reasoning_effort_low(config: OpenAIConfig):
"""gpt-5.5-pro only accepts {medium, high, xhigh} — 'low' must raise.

Verified against OpenAI's live API: /v1/chat/completions with
reasoning_effort='low' on gpt-5.5-pro returns HTTP 400.
"""
with pytest.raises(litellm.utils.UnsupportedParamsError):
config.map_openai_params(
non_default_params={"reasoning_effort": "low"},
optional_params={},
model="gpt-5.5-pro",
drop_params=False,
)


def test_gpt5_5_pro_dated_rejects_reasoning_effort_low(config: OpenAIConfig):
"""Dated snapshot must inherit the base alias's low-rejection behavior."""
with pytest.raises(litellm.utils.UnsupportedParamsError):
config.map_openai_params(
non_default_params={"reasoning_effort": "low"},
optional_params={},
model="gpt-5.5-pro-2026-04-23",
drop_params=False,
)


def test_gpt5_5_pro_drops_reasoning_effort_low_when_requested(config: OpenAIConfig):
"""drop_params=True silently strips 'low' instead of round-tripping a 400."""
params = config.map_openai_params(
non_default_params={"reasoning_effort": "low"},
optional_params={},
model="gpt-5.5-pro",
drop_params=True,
)
assert "reasoning_effort" not in params


def test_gpt5_5_chat_allows_reasoning_effort_low(config: OpenAIConfig):
"""gpt-5.5 (chat) supports 'low'; flag absent → opt-out check passes."""
params = config.map_openai_params(
non_default_params={"reasoning_effort": "low"},
optional_params={},
model="gpt-5.5",
drop_params=False,
)
assert params["reasoning_effort"] == "low"


def test_gpt5_unknown_model_passes_through_low(config: OpenAIConfig):
"""Unknown gpt-5 models pass 'low' through (opt-out, not opt-in)."""
params = config.map_openai_params(
non_default_params={"reasoning_effort": "low"},
optional_params={},
model="gpt-5.4-turbo-preview",
drop_params=False,
)
assert params["reasoning_effort"] == "low"


def test_gpt5_low_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config):
"""supports_low_reasoning_effort=false → disabled; missing/true → not disabled."""
assert gpt5_config._is_reasoning_effort_level_explicitly_disabled(
"gpt-5.5-pro", "low"
)
assert gpt5_config._is_reasoning_effort_level_explicitly_disabled(
"gpt-5.5-pro-2026-04-23", "low"
)
assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled(
"gpt-5.5", "low"
)
assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled(
"gpt-5.4", "low"
)


def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig):
"""Dict with summary/generate_summary is normalized for chat completions."""
params = config.map_openai_params(
Expand Down
12 changes: 12 additions & 0 deletions tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,18 @@ def test_process_items_basic():
process_items(schema)
assert schema["items"] == {"type": "object"}

# Test array missing items inside anyOf branch
schema = {
"type": "object",
"properties": {
"callbacks": {
"anyOf": [{"type": "array"}, {"type": "object"}],
}
},
}
process_items(schema)
assert schema["properties"]["callbacks"]["anyOf"][0]["items"] == {"type": "object"}


def test_build_vertex_schema_array_branch_missing_items_in_anyof():
"""
Expand Down
Loading