fix(openai,azure): return a length-truncated 200 when the output budget fits no token

litellm/llms/azure/azure.py

@@ -10,6 +10,7 @@
     AsyncAzureOpenAI,
     AsyncOpenAI,
     AzureOpenAI,
+    BadRequestError,
     OpenAI,
 )
 
@@ -37,6 +38,10 @@
 
 from ...types.llms.openai import HttpxBinaryResponseContent
 from ..base import BaseLLM
+from ..openai.common_utils import (
+    build_output_token_limit_response,
+    is_output_token_limit_error,
+)
 from .common_utils import (
     AzureOpenAIError,
     BaseAzureLLM,
@@ -147,6 +152,10 @@ def make_sync_azure_openai_chat_completion_request(
             headers: Final = dict(raw_response.headers)
             response: Final = raw_response.parse()
             return headers, response
+        except BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=False)
         except Exception as e:
             raise e
 
@@ -175,6 +184,10 @@ async def make_azure_openai_chat_completion_request(
             time_delta: Final = round(end_time - start_time, 2)
             e.message += f" - timeout value={timeout}, time taken={time_delta} seconds"
             raise e
+        except BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=True)
         except Exception as e:
             raise e
 

litellm/llms/openai/common_utils.py

@@ -7,16 +7,25 @@
 import json
 import os
 import ssl
+import time
+import uuid
+from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
 
 import httpx
 import openai
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from openai.types.chat.chat_completion_chunk import ChoiceDelta
+from openai.types.completion_usage import CompletionUsage
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
 
 import litellm
+from litellm.litellm_core_utils.token_counter import token_counter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
@@ -111,6 +120,79 @@ def drop_params_from_unprocessable_entity_error(
     return new_data
 
 
+_OUTPUT_TOKEN_LIMIT_ERROR_MARKER: Final[str] = (
+    "could not finish the message because max_tokens or model output limit was reached"
+)
+
+
+def is_output_token_limit_error(e: openai.BadRequestError) -> bool:
+    """
+    True when OpenAI/Azure rejected a chat request because the output budget could not fit a single visible token.
+
+    GPT-5.x turns that case into a 400 while returning a length-truncated 200 for marginally larger budgets, so the
+    match has to stay pinned to the full provider sentence to avoid swallowing genuine bad requests.
+    """
+    return _OUTPUT_TOKEN_LIMIT_ERROR_MARKER in e.message.lower()
+
+
+def _output_token_limit_completion(model: str, prompt_tokens: int) -> ChatCompletion:
+    return ChatCompletion(
+        id=f"chatcmpl-{uuid.uuid4()}",
+        choices=(
+            Choice(
+                index=0,
+                finish_reason="length",
+                message=ChatCompletionMessage(role="assistant", content=""),
+            ),
+        ),
+        created=int(time.time()),
+        model=model,
+        object="chat.completion",
+        usage=CompletionUsage(completion_tokens=0, prompt_tokens=prompt_tokens, total_tokens=prompt_tokens),
+    )
+
+
+def _output_token_limit_chunk(model: str) -> ChatCompletionChunk:
+    return ChatCompletionChunk(
+        id=f"chatcmpl-{uuid.uuid4()}",
+        choices=(
+            ChunkChoice(
+                index=0,
+                finish_reason="length",
+                delta=ChoiceDelta(role="assistant", content=""),
+            ),
+        ),
+        created=int(time.time()),
+        model=model,
+        object="chat.completion.chunk",
+    )
+
+
+def _iter_once(chunk: ChatCompletionChunk) -> Iterator[ChatCompletionChunk]:
+    yield chunk
+
+
+async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletionChunk]:
+    yield chunk
+
+
+def build_output_token_limit_response(
+    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool
+) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]:
+    """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets.
+
+    The provider billed the prompt it processed but sends no usage object with the 400, so the prompt is estimated
+    the way every other usage-less path estimates it: reporting zero would spend input tokens against no budget.
+    """
+    model: Final[str] = str(data.get("model", ""))
+    messages: Final = data.get("messages")
+    prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0
+    if not data.get("stream"):
+        return e.response.headers, _output_token_limit_completion(model, prompt_tokens)
+    chunk: Final = _output_token_limit_chunk(model)
+    return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
+
+
 class BaseOpenAILLM:
     """
     Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings

litellm/llms/openai/openai.py

@@ -46,7 +46,9 @@
 from .common_utils import (
     BaseOpenAILLM,
     OpenAIError,
+    build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
+    is_output_token_limit_error,
 )
 
 openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
@@ -436,6 +438,10 @@ async def make_openai_chat_completion_request(
             time_delta: Final = round(end_time - start_time, 2)
             e.message += f" - timeout value={timeout}, time taken={time_delta} seconds"
             raise e
+        except openai.BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=True)
         except Exception as e:
             raise e
 
@@ -469,6 +475,10 @@ def make_sync_openai_chat_completion_request(
             return headers, response
         except OpenAIError:
             raise
+        except openai.BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=False)
         except Exception as e:
             if raw_response is not None:
                 raise Exception(

tests/test_litellm/llms/openai/test_openai_common_utils.py

@@ -2,13 +2,16 @@
 import sys
 from unittest.mock import MagicMock, call, patch
 
+import httpx
+import openai
 import pytest
 
 sys.path.insert(
     0, os.path.abspath("../../..")
 )  # Adds the parent directory to the system path
 
 import litellm
+from litellm.litellm_core_utils.token_counter import token_counter
 from litellm.llms.openai.common_utils import BaseOpenAILLM
 
 # Test parameters for different API functions
@@ -247,3 +250,145 @@ def test_a_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypa
     closer.reap()
 
     assert wrapper.is_closed() is True
+
+
+OUTPUT_LIMIT_400_MESSAGE = (
+    "Could not finish the message because max_tokens or model output limit was reached. "
+    "Please try again with higher max_tokens."
+)
+GENUINE_400_MESSAGE = "Invalid value for 'max_tokens': integer above maximum value. Expected <= 128000, got 999999999."
+LONG_PROMPT = "please summarise the following notes for me: " + ("token " * 200)
+
+CALL_KWARGS_BY_PROVIDER = {
+    "openai": {"model": "gpt-5.6-sol", "api_key": "sk-not-a-real-key"},
+    "azure": {
+        "model": "azure/gpt-5.6-sol",
+        "api_key": "not-a-real-key",
+        "api_base": "https://not-a-real-resource.openai.azure.com",
+        "api_version": "2024-10-21",
+    },
+}
+
+
+def _transport(message: str) -> httpx.MockTransport:
+    def _handler(_request: httpx.Request) -> httpx.Response:
+        return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}})
+
+    return httpx.MockTransport(_handler)
+
+
+def _sync_client_raising(provider: str, message: str):
+    http_client = httpx.Client(transport=_transport(message))
+    if provider == "azure":
+        return openai.AzureOpenAI(
+            api_key="not-a-real-key",
+            azure_endpoint="https://not-a-real-resource.openai.azure.com",
+            api_version="2024-10-21",
+            http_client=http_client,
+        )
+    return openai.OpenAI(api_key="sk-not-a-real-key", http_client=http_client)
+
+
+def _async_client_raising(provider: str, message: str):
+    http_client = httpx.AsyncClient(transport=_transport(message))
+    if provider == "azure":
+        return openai.AsyncAzureOpenAI(
+            api_key="not-a-real-key",
+            azure_endpoint="https://not-a-real-resource.openai.azure.com",
+            api_version="2024-10-21",
+            http_client=http_client,
+        )
+    return openai.AsyncOpenAI(api_key="sk-not-a-real-key", http_client=http_client)
+
+
+def _completion_kwargs(provider: str, client, **overrides) -> dict:
+    return {
+        **CALL_KWARGS_BY_PROVIDER[provider],
+        "messages": [{"role": "user", "content": "hi"}],
+        "max_tokens": 1,
+        "client": client,
+        **overrides,
+    }
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_sync_output_limit_400_maps_to_length_truncated_response(provider):
+    response = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE))
+    )
+
+    assert response.choices[0].finish_reason == "length"
+    assert response.choices[0].message.content == ""
+    assert response.usage.completion_tokens == 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_mapped_response_still_bills_the_prompt_the_provider_processed(provider):
+    messages = [{"role": "user", "content": LONG_PROMPT}]
+    expected_prompt_tokens = token_counter(model="gpt-5.6-sol", messages=messages)
+    assert expected_prompt_tokens > 100, "the fixture prompt must be big enough for a zeroed count to stand out"
+
+    response = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), messages=messages)
+    )
+
+    assert response.usage.prompt_tokens == expected_prompt_tokens
+    assert response.usage.completion_tokens == 0
+    assert litellm.completion_cost(completion_response=response) > 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.asyncio
+async def test_async_output_limit_400_maps_to_length_truncated_response(provider):
+    response = await litellm.acompletion(
+        **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE))
+    )
+
+    assert response.choices[0].finish_reason == "length"
+    assert response.choices[0].message.content == ""
+    assert response.usage.completion_tokens == 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_sync_streaming_output_limit_400_maps_to_length_truncated_stream(provider):
+    stream = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True)
+    )
+    chunks = list(stream)
+
+    assert [c.choices[0].finish_reason for c in chunks].count("length") == 1
+    assert all(not c.choices[0].delta.content for c in chunks)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.asyncio
+async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream(provider):
+    stream = await litellm.acompletion(
+        **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True)
+    )
+    chunks = [chunk async for chunk in stream]
+
+    assert [c.choices[0].finish_reason for c in chunks].count("length") == 1
+    assert all(not c.choices[0].delta.content for c in chunks)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.parametrize("stream", [False, True])
+def test_sync_genuine_bad_request_still_raises(provider, stream):
+    with pytest.raises(litellm.BadRequestError):
+        result = litellm.completion(
+            **_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
+        )
+        list(result)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.parametrize("stream", [False, True])
+@pytest.mark.asyncio
+async def test_async_genuine_bad_request_still_raises(provider, stream):
+    with pytest.raises(litellm.BadRequestError):
+        result = await litellm.acompletion(
+            **_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
+        )
+        async for _ in result:
+            pass