From db687bfeca2d1a5bdf9845a9693997226e6c97df Mon Sep 17 00:00:00 2001 From: James Myatt Date: Mon, 27 Apr 2026 18:13:48 +0100 Subject: [PATCH 1/4] fix(ollama): Include provider in model list for ollama (#26135) * Include provider in model names for ollama * Fix unit tests --- litellm/llms/ollama/common_utils.py | 2 +- tests/test_litellm/llms/ollama/test_ollama_model_info.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 8aedd9b35001..e7264ed6e0bb 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -108,7 +108,7 @@ def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]: continue nm = entry.get("name") or entry.get("model") if isinstance(nm, str): - names.add(nm) + names.add(f"ollama/{nm}") except Exception as e: verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}") # If tags endpoint fails, fall back to static list diff --git a/tests/test_litellm/llms/ollama/test_ollama_model_info.py b/tests/test_litellm/llms/ollama/test_ollama_model_info.py index 95fc80b7fd65..83d120d877d6 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_model_info.py +++ b/tests/test_litellm/llms/ollama/test_ollama_model_info.py @@ -73,7 +73,7 @@ def mock_get(url, headers): info = OllamaModelInfo() models = info.get_models() # Only 'alpha' and 'zeta' should be returned, sorted alphabetically - assert models == ["alpha", "zeta"] + assert models == ["ollama/alpha", "ollama/zeta"] # Ensure correct endpoint was called assert calls and calls[0].endswith("/api/tags") assert call_headers and call_headers[0] == {} @@ -122,7 +122,7 @@ def mock_get(url, headers): monkeypatch.setattr(httpx, "get", mock_get) info = OllamaModelInfo() models = info.get_models() - assert models == ["m1", "m2"] + assert models == ["ollama/m1", "ollama/m2"] def test_get_models_fallback_on_error(self, monkeypatch): """ From 764c128120616b4b95dd32e113aec16883df3ca7 Mon Sep 17 00:00:00 2001 From: VHash <225398745+vhash0@users.noreply.github.com> Date: Tue, 28 Apr 2026 02:17:30 +0900 Subject: [PATCH 2/4] fix(ollama): process both thinking and content in same streaming chunk (#26098) --- litellm/llms/ollama/chat/transformation.py | 2 +- .../ollama/test_ollama_chat_transformation.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 48534799c970..bba2358b6a7c 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -510,7 +510,7 @@ def chunk_parser(self, chunk: dict) -> ModelResponseStream: if chunk["message"].get("thinking") is not None: reasoning_content = chunk["message"].get("thinking") self.started_reasoning_content = True - elif chunk["message"].get("content") is not None: + if chunk["message"].get("content") is not None: if ( self.started_reasoning_content and not self.finished_reasoning_content diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 05b96b88228b..2ba2e1db9bea 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -694,6 +694,30 @@ def test_thinking_to_content_transition(self): # reasoning_content is not set when there's no thinking in the chunk assert getattr(result2.choices[0].delta, "reasoning_content", None) is None + def test_thinking_and_content_in_same_chunk(self): + """ + Test that a chunk containing both thinking and content preserves both fields. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "model": "deepseek-r1", + "message": { + "role": "assistant", + "thinking": "Let me reason first.", + "content": "Final answer.", + }, + "done": False, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.reasoning_content == "Let me reason first." + assert result.choices[0].delta.content == "Final answer." + def test_think_tags_in_content(self): """ Test that tags embedded in content are properly parsed. From d1dbd574f101395704777df96d3f88ebbcb43f98 Mon Sep 17 00:00:00 2001 From: hayden Date: Tue, 28 Apr 2026 08:28:56 +0900 Subject: [PATCH 3/4] fix(health_check): skip max_tokens for image_generation mode (#26417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(health_check): skip max_tokens for image_generation mode `_update_litellm_params_for_health_check` injected `max_tokens` for every deployment. OpenAI `/v1/images/generations` strictly rejects unknown fields, so health checks for dall-e-* and gpt-image-1 always failed with `400 "Unknown parameter: 'max_tokens'"` even though the actual image endpoint calls succeed. Skip the `max_tokens` injection when `model_info.mode == "image_generation"`. `messages` still gets injected (downstream `_filter_model_params` already strips it for non-chat handlers). * Switch to allow-list with per-deployment override Per @krrishdholakia review: deny-listing image_generation only re-introduces the same bug for every other non-chat mode (embedding, audio_*, rerank, video_generation, ocr, search, moderation, ...). Replace the single image_generation skip with `_MAX_TOKEN_SUPPORT_MODES = {chat, completion, responses}`. Missing `mode` is treated as chat for backward compatibility. New modes are safe by default. Add `model_info.health_check_supports_max_tokens` as an operator escape hatch — True forces injection on a non-listed deployment (operator wants to bound probe tokens), False suppresses it on a chat-style deployment behind a strict-schema provider. Tests: parametrize over 3 chat-style + 10 non-chat modes, plus override on/off and the no-mode legacy path. --- litellm/proxy/health_check.py | 38 +++++- .../proxy/test_health_check_max_tokens.py | 126 ++++++++++++++++++ 2 files changed, 161 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 7d67750c78fa..c59c61a9aa4d 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -32,6 +32,30 @@ MINIMAL_DISPLAY_PARAMS = ["model", "mode_error"] +# Modes whose health-check probe is a chat-style completion call and +# therefore accept `max_tokens`. Other modes (embedding, image_generation, +# audio_*, rerank, video_generation, ocr, search, moderation, ...) hit +# endpoints that reject unknown fields with 400 "Unknown parameter: +# 'max_tokens'". Allow-list so new modes are safe by default. +# Per-deployment override: `model_info.health_check_supports_max_tokens`. +_MAX_TOKEN_SUPPORT_MODES: frozenset = frozenset({"chat", "completion", "responses"}) + + +def _should_inject_health_check_max_tokens(model_info: dict) -> bool: + """ + Whether the health-check probe should include `max_tokens`. + + Order: + 1. `model_info.health_check_supports_max_tokens` (operator override). + 2. `_MAX_TOKEN_SUPPORT_MODES`. Missing `mode` is treated as `chat` + for backward compatibility. + """ + explicit = model_info.get("health_check_supports_max_tokens") + if explicit is not None: + return bool(explicit) + mode = model_info.get("mode") or "chat" + return mode in _MAX_TOKEN_SUPPORT_MODES + def _get_process_rss_mb() -> Optional[float]: """ @@ -362,14 +386,22 @@ def _update_litellm_params_for_health_check( Update the litellm params for health check. - gets a short `messages` param for health check + - adds a bounded `max_tokens` when the deployment is a chat-style mode + (`chat`, `completion`, `responses`) or the operator explicitly opts in + via `model_info.health_check_supports_max_tokens`. Non-chat endpoints + (image, embedding, audio_*, rerank, video, ocr, search, moderation, ...) + reject unknown fields with 400 "Unknown parameter: 'max_tokens'". - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID """ litellm_params["messages"] = _get_random_llm_message() - _resolved_max_tokens = _resolve_health_check_max_tokens(model_info, litellm_params) - if _resolved_max_tokens is not None: - litellm_params["max_tokens"] = _resolved_max_tokens + if _should_inject_health_check_max_tokens(model_info): + _resolved_max_tokens = _resolve_health_check_max_tokens( + model_info, litellm_params + ) + if _resolved_max_tokens is not None: + litellm_params["max_tokens"] = _resolved_max_tokens _health_check_model = model_info.get("health_check_model", None) if _health_check_model is not None: diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 09211b72c3eb..292ac771369e 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -225,3 +225,129 @@ def test_wildcard_ignores_reasoning_split_model_info(monkeypatch): litellm_params = {"model": "openai/*"} assert _resolve_health_check_max_tokens(model_info, litellm_params) is None + + +# --------------------------------------------------------------------------- +# image_generation must not receive max_tokens. +# +# _update_litellm_params_for_health_check injected `max_tokens` for every +# deployment. For `mode: image_generation` that leaked into OpenAI +# `/v1/images/generations`, which strictly rejects unknown fields with +# `400 "Unknown parameter: 'max_tokens'"`, marking dall-e-* and +# gpt-image-1 as permanently unhealthy even though their actual image +# calls succeed. `messages` still gets injected (downstream +# `_filter_model_params` already strips it for non-chat handlers). +# --------------------------------------------------------------------------- + + +def test_image_generation_mode_skips_max_tokens(): + """image_generation must not receive max_tokens.""" + model_info = {"mode": "image_generation"} + litellm_params = {"model": "openai/dall-e-3", "api_key": "sk-test"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated + # connection-level params must still pass through unchanged + assert updated["api_key"] == "sk-test" + + +def test_health_check_max_tokens_value_is_ignored_for_non_chat_modes(): + """A configured `health_check_max_tokens` *value* (the int that controls + how many tokens to inject) is still skipped when the mode is outside the + allow-list — the inject decision runs before value resolution, so the + value never reaches `_resolve_health_check_max_tokens`. Note this is + distinct from `health_check_supports_max_tokens` (the bool that toggles + injection on/off per deployment).""" + model_info = {"mode": "image_generation", "health_check_max_tokens": 50} + litellm_params = {"model": "openai/dall-e-3"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated + + +def test_chat_mode_still_injects_max_tokens(): + """Regression guard: the chat-style probe payload is unchanged.""" + model_info = {"mode": "chat"} + litellm_params = {"model": "gpt-4"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +def test_no_mode_still_injects_max_tokens(): + """Regression guard: model_info without `mode` keeps the legacy path.""" + model_info: dict = {} + litellm_params = {"model": "gpt-4"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +# --------------------------------------------------------------------------- +# Allow-list behavior: only chat-style modes (chat / completion / responses) +# receive max_tokens. Every other mode is skipped by default. +# +# Per-deployment override via `health_check_supports_max_tokens` lets the +# operator force injection on (e.g. a non-listed but max_tokens-capable +# endpoint where they want to bound probe token usage) or off (e.g. a +# chat-style provider with a strict schema that rejects unknown fields). +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("mode", ["chat", "completion", "responses"]) +def test_chat_style_modes_inject_max_tokens(mode): + updated = _update_litellm_params_for_health_check( + {"mode": mode}, {"model": f"openai/dummy-{mode}"} + ) + + assert updated["max_tokens"] == 5 + + +@pytest.mark.parametrize( + "mode", + [ + "embedding", + "image_generation", + "image_edit", + "audio_speech", + "audio_transcription", + "rerank", + "video_generation", + "ocr", + "search", + "moderation", + ], +) +def test_non_chat_modes_skip_max_tokens(mode): + updated = _update_litellm_params_for_health_check( + {"mode": mode}, {"model": f"openai/dummy-{mode}"} + ) + + assert "max_tokens" not in updated + + +def test_explicit_override_true_forces_injection_outside_allowlist(): + """Operator opts a non-listed deployment in to bound probe token usage.""" + model_info = { + "mode": "image_generation", + "health_check_supports_max_tokens": True, + } + litellm_params = {"model": "openai/some-future-image-model"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +def test_explicit_override_false_suppresses_injection_inside_allowlist(): + """Operator opts a chat-style deployment out (strict-schema provider).""" + model_info = {"mode": "chat", "health_check_supports_max_tokens": False} + litellm_params = {"model": "openai/strict-schema-chat"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated From 519cb4a5789a137ebf9a87497ae04936b07d1281 Mon Sep 17 00:00:00 2001 From: dawidkulpa <84176950+dawidkulpa@users.noreply.github.com> Date: Mon, 11 May 2026 19:37:34 +0200 Subject: [PATCH 4/4] fix(http_handler): handle RequestNotRead in MaskedHTTPStatusError for multipart uploads (#26718) Squash-merged by litellm-agent from dawidkulpa's PR. --- litellm/llms/custom_httpx/http_handler.py | 7 +++++- .../test_credential_leak_prevention.py | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 03d2af723290..cec051d8f9e3 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -387,11 +387,16 @@ def __init__( if k.lower() not in ("content-encoding", "content-length") } + try: + request_content = original_error.request.content + except httpx.RequestNotRead: + request_content = b"" + masked_request = httpx.Request( method=original_error.request.method, url=masked_url, headers=original_error.request.headers, - content=original_error.request.content, + content=request_content, ) super().__init__( diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py index 72b4da7b38be..0a3bf403bf8f 100644 --- a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py +++ b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py @@ -104,6 +104,31 @@ def test_response_request_is_set(self): # The attached request must be the masked one, not the original. assert "KEY_X" not in str(req.url) + def test_handles_streaming_request_content(self): + """MaskedHTTPStatusError must not crash when request body is streamed.""" + streaming_request = httpx.Request( + "POST", + "https://api.openai.com/v1/images/edits?key=SECRET_KEY", + stream=httpx.ByteStream(b"multipart-data"), + ) + response = httpx.Response( + 400, + request=streaming_request, + content=b'{"error": "bad request"}', + ) + orig = httpx.HTTPStatusError( + message="400 Bad Request", + request=streaming_request, + response=response, + ) + + masked = MaskedHTTPStatusError(orig) + + assert masked.status_code == 400 + assert masked.response.status_code == 400 + assert masked.response.request is not None + assert "SECRET_KEY" not in str(masked.request.url) + def test_strips_content_encoding_to_avoid_double_decode(self): """If the upstream response declared Content-Encoding (e.g. gzip), the rebuilt Response must not carry that header over — otherwise httpx