diff --git a/.github/run-eval/ADDINGMODEL.md b/.github/run-eval/ADDINGMODEL.md index 5b0dc36799..937f50a3d5 100644 --- a/.github/run-eval/ADDINGMODEL.md +++ b/.github/run-eval/ADDINGMODEL.md @@ -52,11 +52,20 @@ This file (`resolve_model_config.py`) defines models available for evaluation. M - `openhands-sdk/openhands/sdk/llm/utils/model_prompt_spec.py` - GPT models only (variant detection) - `openhands-sdk/openhands/sdk/llm/utils/verified_models.py` - Production-ready models - > ⚠️ **When editing `verified_models.py`**: If you add a model to `VERIFIED_OPENHANDS_MODELS`, - > you **must also** add it to its provider-specific list (e.g. `VERIFIED_ANTHROPIC_MODELS`, - > `VERIFIED_GEMINI_MODELS`, `VERIFIED_MOONSHOT_MODELS`, etc.). - > If no list exists for the provider yet, create one and add it to the `VERIFIED_MODELS` dict. - > This ensures the model appears under its actual provider in the UI, not just under "openhands". + > ⛔ **Do NOT add a model to `verified_models.py` unless explicitly asked to.** + > "Verified" means the model has been validated against the OpenHands integration + > test suite **and** an OpenHands maintainer has approved it for the production UI. + > A passing integration run is *necessary but not sufficient*. New models should be + > added to `MODELS` in `resolve_model_config.py` (and `model_features.py` if + > applicable) only — leave `verified_models.py` alone until a maintainer requests it + > in the PR. + > + > ⚠️ **When you are explicitly asked to edit `verified_models.py`**: If you add a + > model to `VERIFIED_OPENHANDS_MODELS`, you **must also** add it to its + > provider-specific list (e.g. `VERIFIED_ANTHROPIC_MODELS`, `VERIFIED_GEMINI_MODELS`, + > `VERIFIED_MOONSHOT_MODELS`, etc.). If no list exists for the provider yet, create + > one and add it to the `VERIFIED_MODELS` dict. This ensures the model appears under + > its actual provider in the UI, not just under "openhands". ## Step 1: Add to resolve_model_config.py diff --git a/.github/run-eval/resolve_model_config.py b/.github/run-eval/resolve_model_config.py index 936c1a4e09..a8b4094b3b 100755 --- a/.github/run-eval/resolve_model_config.py +++ b/.github/run-eval/resolve_model_config.py @@ -161,6 +161,14 @@ def _sigterm_handler(signum: int, _frame: object) -> None: "temperature": 0.0, }, }, + "gemini-3.5-flash": { + "id": "gemini-3.5-flash", + "display_name": "Gemini 3.5 Flash", + "llm_config": { + "model": "litellm_proxy/gemini-3.5-flash", + "temperature": 0.0, + }, + }, "gpt-5.2": { "id": "gpt-5.2", "display_name": "GPT-5.2", diff --git a/openhands-sdk/openhands/sdk/llm/exceptions/__init__.py b/openhands-sdk/openhands/sdk/llm/exceptions/__init__.py index 41e668dc6f..383e6580f6 100644 --- a/openhands-sdk/openhands/sdk/llm/exceptions/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/exceptions/__init__.py @@ -1,5 +1,6 @@ from .classifier import ( is_context_window_exceeded, + is_prompt_cache_too_small, looks_like_auth_error, looks_like_malformed_conversation_history_error, ) @@ -48,6 +49,7 @@ "OperationCancelled", # Helpers "is_context_window_exceeded", + "is_prompt_cache_too_small", "looks_like_auth_error", "looks_like_malformed_conversation_history_error", "map_provider_exception", diff --git a/openhands-sdk/openhands/sdk/llm/exceptions/classifier.py b/openhands-sdk/openhands/sdk/llm/exceptions/classifier.py index bdebcafdfe..92ae586903 100644 --- a/openhands-sdk/openhands/sdk/llm/exceptions/classifier.py +++ b/openhands-sdk/openhands/sdk/llm/exceptions/classifier.py @@ -76,6 +76,16 @@ def looks_like_malformed_conversation_history_error(exception: Exception) -> boo return any(p in s for p in MALFORMED_HISTORY_PATTERNS) +# Vertex AI (Gemini) rejects context-caching requests when the cached content +# is below the provider's minimum token threshold (currently 4096 tokens). +# Example error: "The cached content is of 1171 tokens. The minimum token +# count to start caching is 4096." — the `.lower()` comparison handles case +# variation across providers but won't match reworded messages; update this +# pattern if the API phrasing changes. +PROMPT_CACHE_TOO_SMALL_PATTERNS: list[str] = [ + "minimum token count to start caching", +] + AUTH_PATTERNS: list[str] = [ "invalid api key", "unauthorized", @@ -85,6 +95,20 @@ def looks_like_malformed_conversation_history_error(exception: Exception) -> boo ] +def is_prompt_cache_too_small(exception: Exception) -> bool: + """Return True if the error indicates the prompt cache content is too small. + + Vertex AI (Gemini) requires a minimum number of tokens (currently 4096) + to create a context cache. When the cached content is below this threshold, + the API returns a 400 error. The SDK should detect this and retry without + prompt caching markers. + """ + if not isinstance(exception, (BadRequestError, OpenAIError)): + return False + s = str(exception).lower() + return any(p in s for p in PROMPT_CACHE_TOO_SMALL_PATTERNS) + + def looks_like_auth_error(exception: Exception) -> bool: # Trust the typed exception when the provider/LiteLLM raised an explicit # 401/403 — its message text may not contain the heuristic patterns below. diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index d3060a6a10..b79ef70b12 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -87,6 +87,7 @@ from openhands.sdk.llm.exceptions import ( LLMContextWindowTooSmallError, LLMNoResponseError, + is_prompt_cache_too_small, map_provider_exception, ) @@ -1144,6 +1145,7 @@ def completion( removed_in="1.29.0", details=_RETURN_METRICS_DETAILS, ) + _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream if enable_streaming: if on_token is None: @@ -1183,6 +1185,22 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: try: return self._build_completion_result(_one_attempt()) except Exception as e: + # If the prompt cache content is too small for the provider's + # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), + # retry without prompt caching markers. + if is_prompt_cache_too_small(e) and self.is_caching_prompt_active(): + logger.warning( + "Prompt cache content too small for provider minimum, " + "retrying without prompt caching" + ) + no_cache_llm = self.model_copy(update={"caching_prompt": False}) + return no_cache_llm.completion( + messages, + tools, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + **_caller_kwargs, + ) return self._handle_error( e, lambda fb: fb.completion( @@ -1190,6 +1208,7 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + **_caller_kwargs, ), ) @@ -1217,6 +1236,7 @@ async def acompletion( removed_in="1.29.0", details=_RETURN_METRICS_DETAILS, ) + _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream if enable_streaming: if on_token is None: @@ -1256,6 +1276,22 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: try: return self._build_completion_result(await _one_attempt()) except Exception as e: + # If the prompt cache content is too small for the provider's + # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), + # retry without prompt caching markers. + if is_prompt_cache_too_small(e) and self.is_caching_prompt_active(): + logger.warning( + "Prompt cache content too small for provider minimum, " + "retrying without prompt caching" + ) + no_cache_llm = self.model_copy(update={"caching_prompt": False}) + return await no_cache_llm.acompletion( + messages, + tools, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + **_caller_kwargs, + ) # Fallback is synchronous; cast the token callback since the # fallback LLM's sync path accepts TokenCallbackType. _fb_token = cast("TokenCallbackType | None", on_token) @@ -1266,6 +1302,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=_fb_token, + **_caller_kwargs, ), ) @@ -1309,6 +1346,7 @@ def responses( removed_in="1.29.0", details=_RETURN_METRICS_DETAILS, ) + _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream if user_enable_streaming: # We allow on_token to be None for subscription mode @@ -1387,6 +1425,24 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: try: return self._build_responses_result(_one_attempt()) except Exception as e: + # If the prompt cache content is too small for the provider's + # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), + # retry without prompt caching markers. + if is_prompt_cache_too_small(e) and self.is_caching_prompt_active(): + logger.warning( + "Prompt cache content too small for provider minimum, " + "retrying without prompt caching" + ) + no_cache_llm = self.model_copy(update={"caching_prompt": False}) + return no_cache_llm.responses( + messages, + tools, + include, + store, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + **_caller_kwargs, + ) return self._handle_error( e, lambda fb: fb.responses( @@ -1396,6 +1452,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: store, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + **_caller_kwargs, ), ) @@ -1425,6 +1482,7 @@ async def aresponses( removed_in="1.29.0", details=_RETURN_METRICS_DETAILS, ) + _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream if user_enable_streaming: # We allow on_token to be None for subscription mode @@ -1506,6 +1564,24 @@ async def _one_attempt( try: return self._build_responses_result(await _one_attempt()) except Exception as e: + # If the prompt cache content is too small for the provider's + # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), + # retry without prompt caching markers. + if is_prompt_cache_too_small(e) and self.is_caching_prompt_active(): + logger.warning( + "Prompt cache content too small for provider minimum, " + "retrying without prompt caching" + ) + no_cache_llm = self.model_copy(update={"caching_prompt": False}) + return await no_cache_llm.aresponses( + messages, + tools, + include, + store, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + **_caller_kwargs, + ) _fb_token = cast("TokenCallbackType | None", on_token) return await self._ahandle_error( e, @@ -1516,6 +1592,7 @@ async def _one_attempt( store, add_security_risk_prediction=add_security_risk_prediction, on_token=_fb_token, + **_caller_kwargs, ), ) diff --git a/tests/cross/test_resolve_model_config.py b/tests/cross/test_resolve_model_config.py index 1cf4b6327b..fd6cdebce4 100644 --- a/tests/cross/test_resolve_model_config.py +++ b/tests/cross/test_resolve_model_config.py @@ -661,6 +661,16 @@ def test_deepseek_v4_flash_config(): assert model["llm_config"]["model"] == "litellm_proxy/deepseek/deepseek-v4-flash" +def test_gemini_3_5_flash_config(): + """Test that gemini-3.5-flash has correct configuration.""" + model = MODELS["gemini-3.5-flash"] + + assert model["id"] == "gemini-3.5-flash" + assert model["display_name"] == "Gemini 3.5 Flash" + assert model["llm_config"]["model"] == "litellm_proxy/gemini-3.5-flash" + assert model["llm_config"]["temperature"] == 0.0 + + def test_nemotron_3_ultra_550b_a55b_config(): """Test that nemotron-3-ultra-550b-a55b has correct configuration.""" model = MODELS["nemotron-3-ultra-550b-a55b"] diff --git a/tests/sdk/llm/test_exception_classifier.py b/tests/sdk/llm/test_exception_classifier.py index 3caffeea1e..bd39d7f067 100644 --- a/tests/sdk/llm/test_exception_classifier.py +++ b/tests/sdk/llm/test_exception_classifier.py @@ -6,6 +6,7 @@ from openhands.sdk.llm.exceptions import ( is_context_window_exceeded, + is_prompt_cache_too_small, looks_like_auth_error, looks_like_malformed_conversation_history_error, ) @@ -119,3 +120,35 @@ def test_looks_like_auth_error_negative(): looks_like_auth_error(BadRequestError("Something else", MODEL, PROVIDER)) is False ) + + +def test_is_prompt_cache_too_small_positive(): + """Vertex AI rejects caching when cached content is below minimum token count.""" + vertex_error = BadRequestError( + ( + "Vertex_aiException BadRequestError - " + '{"error":{"code":400,' + '"message":"The cached content is of 1171 tokens. ' + 'The minimum token count to start caching is 4096.",' + '"status":"INVALID_ARGUMENT"}}' + ), + MODEL, + PROVIDER, + ) + assert is_prompt_cache_too_small(vertex_error) is True + + +def test_is_prompt_cache_too_small_negative(): + assert ( + is_prompt_cache_too_small(BadRequestError("irrelevant", MODEL, PROVIDER)) + is False + ) + + +def test_is_prompt_cache_too_small_context_window_not_cache_too_small(): + """Context window exceeded is a different error from cache too small.""" + ctx_error = BadRequestError( + "The request exceeds the available context size", MODEL, PROVIDER + ) + assert is_prompt_cache_too_small(ctx_error) is False + assert is_context_window_exceeded(ctx_error) is True diff --git a/tests/sdk/llm/test_llm_completion.py b/tests/sdk/llm/test_llm_completion.py index 0b484e45ac..4203fb569e 100644 --- a/tests/sdk/llm/test_llm_completion.py +++ b/tests/sdk/llm/test_llm_completion.py @@ -3,7 +3,7 @@ import threading from collections.abc import Sequence from typing import Any, ClassVar -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm import ChatCompletionMessageToolCall, CustomStreamWrapper @@ -827,5 +827,146 @@ def test_llm_streaming_preserves_cache_read_tokens(mock_completion): ) -# This file focuses on LLM completion functionality, configuration options, -# and metrics tracking for the synchronous LLM implementation +@patch("openhands.sdk.llm.llm.litellm_completion") +def test_completion_retries_without_caching_on_prompt_cache_too_small( + mock_completion, +): + """When Vertex AI rejects caching due to small content, retry without cache.""" + from litellm.exceptions import BadRequestError + + # First call raises the "cache too small" error, second succeeds + cache_error = BadRequestError( + ( + "Vertex_aiException BadRequestError - " + '{"error":{"code":400,' + '"message":"The cached content is of 1171 tokens. ' + 'The minimum token count to start caching is 4096.",' + '"status":"INVALID_ARGUMENT"}}' + ), + model="gemini-3.5-flash", + llm_provider="vertex_ai", + ) + mock_response = create_mock_response("Retry succeeded") + mock_completion.side_effect = [cache_error, mock_response] + + llm = LLM( + model="claude-sonnet-4-20250514", + api_key=SecretStr("test_key"), + usage_id="test-llm", + caching_prompt=True, + num_retries=2, + retry_min_wait=1, + retry_max_wait=2, + ) + + messages = [Message(role="user", content=[TextContent(text="Hello")])] + # Pass a kwarg via **kwargs to verify _caller_kwargs preservation on retry. + response = llm.completion(messages=messages, metadata={"trace": "sync"}) + + # Should succeed after retry without caching + assert response.raw_response == mock_response + # Two calls: first with cache (fails), second without cache (succeeds) + assert mock_completion.call_count == 2 + + # The first call SHOULD have cache_control markers + first_call_kwargs = mock_completion.call_args_list[0].kwargs + first_messages = first_call_kwargs.get("messages", []) + first_has_cache = any( + "cache_control" in str(block) + for msg in first_messages + for block in ( + msg.get("content", []) if isinstance(msg.get("content"), list) else [] + ) + ) + assert first_has_cache, "First call should include cache_control markers" + + # The second call should NOT have cache_control markers + second_call_kwargs = mock_completion.call_args_list[1].kwargs + second_messages = second_call_kwargs.get("messages", []) + second_has_cache = any( + "cache_control" in str(block) + for msg in second_messages + for block in ( + msg.get("content", []) if isinstance(msg.get("content"), list) else [] + ) + ) + assert not second_has_cache, "Retry should not include cache_control markers" + + # Caller kwargs preserved on the retry — without _caller_kwargs the retry + # would silently drop them. + assert second_call_kwargs.get("metadata") == {"trace": "sync"} + + +@pytest.mark.asyncio +@patch("openhands.sdk.llm.llm.litellm_acompletion", new_callable=AsyncMock) +async def test_acompletion_retries_without_caching_on_prompt_cache_too_small( + mock_acompletion, +): + """Async version of the sync prompt-cache-too-small retry test. + + When Vertex AI rejects caching due to small content, acompletion() should + retry without prompt caching while preserving caller kwargs. Mirrors + test_completion_retries_without_caching_on_prompt_cache_too_small. + """ + from litellm.exceptions import BadRequestError + + cache_error = BadRequestError( + ( + "Vertex_aiException BadRequestError - " + '{"error":{"code":400,' + '"message":"The cached content is of 1171 tokens. ' + 'The minimum token count to start caching is 4096.",' + '"status":"INVALID_ARGUMENT"}}' + ), + model="gemini-3.5-flash", + llm_provider="vertex_ai", + ) + mock_response = create_mock_response("Retry succeeded") + mock_acompletion.side_effect = [cache_error, mock_response] + + llm = LLM( + model="claude-sonnet-4-20250514", + api_key=SecretStr("test_key"), + usage_id="test-llm", + caching_prompt=True, + num_retries=2, + retry_min_wait=1, + retry_max_wait=2, + ) + + messages = [Message(role="user", content=[TextContent(text="Hello")])] + # Pass a kwarg via **kwargs to verify _caller_kwargs preservation on retry. + response = await llm.acompletion(messages=messages, metadata={"trace": "abc"}) + + # Should succeed after retry without caching + assert response.raw_response == mock_response + # Two calls: first with cache (fails), second without cache (succeeds) + assert mock_acompletion.call_count == 2 + + # The first call SHOULD have cache_control markers + first_call_kwargs = mock_acompletion.call_args_list[0].kwargs + first_messages = first_call_kwargs.get("messages", []) + first_has_cache = any( + "cache_control" in str(block) + for msg in first_messages + for block in ( + msg.get("content", []) if isinstance(msg.get("content"), list) else [] + ) + ) + assert first_has_cache, "First call should include cache_control markers" + + # The second call should NOT have cache_control markers + second_call_kwargs = mock_acompletion.call_args_list[1].kwargs + second_messages = second_call_kwargs.get("messages", []) + second_has_cache = any( + "cache_control" in str(block) + for msg in second_messages + for block in ( + msg.get("content", []) if isinstance(msg.get("content"), list) else [] + ) + ) + assert not second_has_cache, "Retry should not include cache_control markers" + + # Caller kwargs preserved on the retry — without _caller_kwargs the retry + # would silently drop them. + assert second_call_kwargs.get("metadata") == {"trace": "abc"} diff --git a/tests/sdk/llm/test_responses_parsing_and_kwargs.py b/tests/sdk/llm/test_responses_parsing_and_kwargs.py index 6dde5ee8ef..ad4345a79c 100644 --- a/tests/sdk/llm/test_responses_parsing_and_kwargs.py +++ b/tests/sdk/llm/test_responses_parsing_and_kwargs.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest from litellm.types.llms.openai import ( @@ -12,6 +12,7 @@ ResponseReasoningItem, Summary, ) +from pydantic import SecretStr from openhands.sdk.llm import LLM from openhands.sdk.llm.message import Message, ReasoningItemModel, TextContent @@ -265,3 +266,154 @@ def test_responses_options_omits_prompt_cache_key_when_unset(): assert "prompt_cache_key" not in select_responses_options( llm, {}, include=None, store=None ) + + +@patch("openhands.sdk.llm.llm.litellm_responses") +def test_responses_retries_without_caching_on_prompt_cache_too_small(mock_responses): + """When Vertex AI rejects caching due to small content, responses() should + retry without prompt caching while preserving caller kwargs. + + Mirrors test_completion_retries_without_caching_on_prompt_cache_too_small in + test_llm_completion.py, but exercises the Responses API path. The two + methods differ in signature (``include``, ``store`` positional args) and in + how ``stream`` is resolved, so they need independent coverage. + """ + from litellm.exceptions import BadRequestError + + cache_error = BadRequestError( + ( + "Vertex_aiException BadRequestError - " + '{"error":{"code":400,' + '"message":"The cached content is of 1171 tokens. ' + 'The minimum token count to start caching is 4096.",' + '"status":"INVALID_ARGUMENT"}}' + ), + model="gemini-3-flash", + llm_provider="vertex_ai", + ) + + # Build a typed ResponsesAPIResponse for the successful retry + msg = build_responses_message_output(["Retry succeeded"]) + usage = ResponseAPIUsage(input_tokens=0, output_tokens=0, total_tokens=0) + success_resp = ResponsesAPIResponse( + id="r1", + created_at=0, + output=[msg], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=usage, + instructions="", + status="completed", + ) + mock_responses.side_effect = [cache_error, success_resp] + + # Pick a model that supports prompt caching so is_caching_prompt_active() + # is True and the retry branch is reachable on the responses() path. + llm = LLM( + model="gemini-3-flash", + api_key=SecretStr("test_key"), + usage_id="test-llm", + caching_prompt=True, + num_retries=2, + retry_min_wait=1, + retry_max_wait=2, + ) + + messages = [ + Message(role="system", content=[TextContent(text="sys")]), + Message(role="user", content=[TextContent(text="Hello")]), + ] + + # Pass caller kwargs that must survive the retry. ``metadata`` flows through + # ``**kwargs`` (not a named param), so it's the cleanest probe for the + # ``_caller_kwargs`` forwarding fix; ``store`` exercises the positional-arg + # path on the retry call signature. + response = llm.responses( + messages, + store=False, + metadata={"trace_id": "abc-123"}, + ) + + # Two calls: first with caching active (fails), second without (succeeds). + assert mock_responses.call_count == 2 + assert response.raw_response is success_resp + + # Caller kwargs preserved on the retry — without ``_caller_kwargs`` the + # retry would silently drop them. + second_kwargs = mock_responses.call_args_list[1].kwargs + assert second_kwargs.get("store") is False + assert second_kwargs.get("metadata") == {"trace_id": "abc-123"} + + +@pytest.mark.asyncio +@patch("openhands.sdk.llm.llm.litellm_aresponses", new_callable=AsyncMock) +async def test_aresponses_retries_without_caching_on_prompt_cache_too_small( + mock_aresponses, +): + """Async version of the sync responses prompt-cache-too-small retry test. + + Ensures aresponses() also retries without prompt caching when Vertex AI + rejects the request due to cache content below the minimum token + threshold, and preserves caller kwargs (positional ``store`` and + ``**kwargs`` metadata). Mirrors + test_responses_retries_without_caching_on_prompt_cache_too_small. + """ + from litellm.exceptions import BadRequestError + + cache_error = BadRequestError( + ( + "Vertex_aiException BadRequestError - " + '{"error":{"code":400,' + '"message":"The cached content is of 1171 tokens. ' + 'The minimum token count to start caching is 4096.",' + '"status":"INVALID_ARGUMENT"}}' + ), + model="gemini-3-flash", + llm_provider="vertex_ai", + ) + + msg = build_responses_message_output(["Retry succeeded"]) + usage = ResponseAPIUsage(input_tokens=0, output_tokens=0, total_tokens=0) + success_resp = ResponsesAPIResponse( + id="r1", + created_at=0, + output=[msg], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=usage, + instructions="", + status="completed", + ) + mock_aresponses.side_effect = [cache_error, success_resp] + + llm = LLM( + model="gemini-3-flash", + api_key=SecretStr("test_key"), + usage_id="test-llm", + caching_prompt=True, + num_retries=2, + retry_min_wait=1, + retry_max_wait=2, + ) + + messages = [ + Message(role="system", content=[TextContent(text="sys")]), + Message(role="user", content=[TextContent(text="Hello")]), + ] + + response = await llm.aresponses( + messages, + store=False, + metadata={"trace_id": "abc-123"}, + ) + + assert mock_aresponses.call_count == 2 + assert response.raw_response is success_resp + + second_kwargs = mock_aresponses.call_args_list[1].kwargs + assert second_kwargs.get("store") is False + assert second_kwargs.get("metadata") == {"trace_id": "abc-123"}