From 05eb57cc454852e57d68ee9b6201c70a98e950e1 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Thu, 24 Sep 2026 11:33:46 +0300 Subject: [PATCH] fix(cost-map): carry openai reasoning annotations onto their chatgpt twins Every chatgpt/* row in the cost map lacked the reasoning annotations its openai twin carries, so /model/info reported twelve reasoning models as non-reasoning. The generalization fallback cannot rescue them: _get_model_info_from_generalization returns None when any candidate is an exact cost-map key, and chatgpt/ is one, so the unannotated row wins over the annotated twin it proxies. The flags are ours to assert rather than a vendor claim - ChatGPTConfig subclasses OpenAIConfig and ChatGPTResponsesAPIConfig subclasses OpenAIResponsesAPIConfig, so a chatgpt row's reasoning behaviour is whatever the openai row describes. Two rows are left alone because they have no openai twin to copy from: chatgpt/gpt-5.3-codex-spark and chatgpt/gpt-5.3-instant. The new test pins the twin invariant rather than the twelve values, so the next chatgpt row added cannot repeat this. It is scoped to openai twins: the azure registry already diverges here and is not this repository's to pin. --- ...odel_prices_and_context_window_backup.json | 62 +++++++++++++++---- model_prices_and_context_window.json | 62 +++++++++++++++---- .../test_litellm/test_model_prices_schema.py | 51 +++++++++++++++ 3 files changed, 151 insertions(+), 24 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d2aec118d31b..a9d335df39ed 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30333,7 +30333,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.6-luna": { "litellm_provider": "chatgpt", @@ -30349,7 +30353,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.6-sol": { "litellm_provider": "chatgpt", @@ -30365,7 +30373,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.6-terra": { "litellm_provider": "chatgpt", @@ -30381,7 +30393,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -30396,7 +30412,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.4-pro": { "litellm_provider": "chatgpt", @@ -30410,7 +30431,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex": { "litellm_provider": "chatgpt", @@ -30424,7 +30449,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -30468,7 +30497,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.2-codex": { "litellm_provider": "chatgpt", @@ -30482,7 +30512,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.2": { "litellm_provider": "chatgpt", @@ -30497,7 +30528,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.1-codex-max": { "litellm_provider": "chatgpt", @@ -30511,7 +30547,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.1-codex-mini": { "litellm_provider": "chatgpt", @@ -30525,7 +30562,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "gigachat/GigaChat-2": { "input_cost_per_token": 0.0, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d2aec118d31b..a9d335df39ed 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30333,7 +30333,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.6-luna": { "litellm_provider": "chatgpt", @@ -30349,7 +30353,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.6-sol": { "litellm_provider": "chatgpt", @@ -30365,7 +30373,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.6-terra": { "litellm_provider": "chatgpt", @@ -30381,7 +30393,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_reasoning": true, + "supports_xhigh_reasoning_effort": true }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -30396,7 +30412,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.4-pro": { "litellm_provider": "chatgpt", @@ -30410,7 +30431,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex": { "litellm_provider": "chatgpt", @@ -30424,7 +30449,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -30468,7 +30497,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.2-codex": { "litellm_provider": "chatgpt", @@ -30482,7 +30512,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.2": { "litellm_provider": "chatgpt", @@ -30497,7 +30528,12 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": true, + "default_reasoning_effort": "none", + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.1-codex-max": { "litellm_provider": "chatgpt", @@ -30511,7 +30547,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "chatgpt/gpt-5.1-codex-mini": { "litellm_provider": "chatgpt", @@ -30525,7 +30562,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_reasoning": true }, "gigachat/GigaChat-2": { "input_cost_per_token": 0.0, diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 052278631e21..bf08a22cbae2 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -266,6 +266,57 @@ def test_openai_reasoning_family_entries_carry_supports_reasoning(prices: dict): ) +REASONING_ANNOTATION_KEYS: Final = ( + "supports_reasoning", + "supports_minimal_reasoning_effort", + "supports_none_reasoning_effort", + "supports_xhigh_reasoning_effort", + "default_reasoning_effort", +) + + +def chatgpt_openai_twins(prices: dict) -> list[tuple[str, str]]: + """`chatgpt/` rows paired with the bare `` row served by the openai provider. + + Scoped to openai twins on purpose. `ChatGPTConfig` and `ChatGPTResponsesAPIConfig` subclass + their openai counterparts, so a chatgpt row's reasoning behaviour is whatever the openai row + describes. The azure rows are a separate registry that already diverges from openai here, and + pinning them to each other would assert something this repository does not control. + """ + pairs = [] + for name, entry in prices.items(): + if not isinstance(entry, dict) or not name.startswith("chatgpt/"): + continue + bare = name.split("/", 1)[1] + twin = prices.get(bare) + if isinstance(twin, dict) and twin.get("litellm_provider") == "openai": + pairs.append((name, bare)) + return pairs + + +def test_chatgpt_rows_carry_their_openai_twin_reasoning_annotations(prices: dict): + """A chatgpt row must not silently drop the reasoning annotations of the model it proxies. + + `litellm.utils._get_model_info_from_generalization` refuses to fall back when an exact cost-map + key exists, so an unannotated `chatgpt/` row wins over its annotated twin and + `/model/info` reports the model as non-reasoning. + """ + twins = chatgpt_openai_twins(prices) + assert twins, "no chatgpt/* row has an openai twin any more; this guard has stopped guarding" + + mismatched = [] + for name, bare in twins: + for key in REASONING_ANNOTATION_KEYS: + if key in prices[bare] and prices[name].get(key) != prices[bare][key]: + mismatched.append(f"{name}.{key} is {prices[name].get(key)!r}, {bare}.{key} is {prices[bare][key]!r}") + + assert mismatched == [], ( + "chatgpt/* entries proxy their openai twin through ChatGPTConfig, so they must carry the " + "same reasoning annotations; an exact cost-map key blocks the generalization fallback, so " + "a missing flag here is reported to callers as 'not a reasoning model':\n" + "\n".join(mismatched) + ) + + def test_chat_latest_declares_the_one_effort_openai_accepts(prices: dict): """OpenAI rejects every reasoning.effort on chat-latest except medium, and a reasoning entry with no declared levels resolves to None, which lets /model_group/info and the dashboard offer