Skip to content

fix(llm): remove obsolete Vertex cache retry #3878

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
enyst wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix(llm): remove obsolete Vertex cache retry #3878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
39c5a1b
fix(llm): remove obsolete Vertex cache retry
enyst Jun 25, 2026
0f52426
Merge origin/main into cleanup/remove-obsolete-vertex-cache-retry
enyst Jul 9, 2026
589c235
Merge branch 'main' into cleanup/remove-obsolete-vertex-cache-retry
enyst Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions openhands-sdk/openhands/sdk/llm/exceptions/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from .classifier import (
is_content_policy_violation,
is_context_window_exceeded,
is_prompt_cache_too_small,
looks_like_auth_error,
looks_like_malformed_conversation_history_error,
)
Expand Down Expand Up @@ -53,7 +52,6 @@
# Helpers
"is_content_policy_violation",
"is_context_window_exceeded",
"is_prompt_cache_too_small",
"looks_like_auth_error",
"looks_like_malformed_conversation_history_error",
"map_provider_exception",
Expand Down
24 changes: 0 additions & 24 deletions openhands-sdk/openhands/sdk/llm/exceptions/classifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,6 @@
"failed to parse tool call arguments as json",
]

# Vertex AI (Gemini) rejects context-caching requests when the cached content
# is below the provider's minimum token threshold (currently 4096 tokens).
# Example error: "The cached content is of 1171 tokens. The minimum token
# count to start caching is 4096." — the `.lower()` comparison handles case
# variation across providers but won't match reworded messages; update this
# pattern if the API phrasing changes.
PROMPT_CACHE_TOO_SMALL_PATTERNS: Final[list[str]] = [
"minimum token count to start caching",
]

AUTH_PATTERNS: Final[list[str]] = [
"invalid api key",
"unauthorized",
Expand Down Expand Up @@ -113,20 +103,6 @@ def looks_like_malformed_conversation_history_error(exception: Exception) -> boo
return any(p in s for p in MALFORMED_HISTORY_PATTERNS)


def is_prompt_cache_too_small(exception: Exception) -> bool:
"""Return True if the error indicates the prompt cache content is too small.

Vertex AI (Gemini) requires a minimum number of tokens (currently 4096)
to create a context cache. When the cached content is below this threshold,
the API returns a 400 error. The SDK should detect this and retry without
prompt caching markers.
"""
if not isinstance(exception, (BadRequestError, OpenAIError)):
return False
s = str(exception).lower()
return any(p in s for p in PROMPT_CACHE_TOO_SMALL_PATTERNS)


def looks_like_auth_error(exception: Exception) -> bool:
# Trust the typed exception when the provider/LiteLLM raised an explicit
# 401/403 — its message text may not contain the heuristic patterns below.
Expand Down
69 changes: 0 additions & 69 deletions openhands-sdk/openhands/sdk/llm/llm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@
from openhands.sdk.llm.exceptions import (
LLMContextWindowTooSmallError,
LLMNoResponseError,
is_prompt_cache_too_small,
map_provider_exception,
)

Expand Down Expand Up @@ -1449,22 +1448,6 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse:
try:
return self._build_completion_result(_one_attempt())
except Exception as e:
# If the prompt cache content is too small for the provider's
# minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
# retry without prompt caching markers.
if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
logger.warning(
"Prompt cache content too small for provider minimum, "
"retrying without prompt caching"
)
no_cache_llm = self.model_copy(update={"caching_prompt": False})
return no_cache_llm.completion(
messages,
tools,
add_security_risk_prediction=add_security_risk_prediction,
on_token=on_token,
**_caller_kwargs,
)
return self._handle_error(
e,
lambda fb: fb.completion(
Expand Down Expand Up @@ -1545,22 +1528,6 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse:
try:
return self._build_completion_result(await _one_attempt())
except Exception as e:
# If the prompt cache content is too small for the provider's
# minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
# retry without prompt caching markers.
if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
logger.warning(
"Prompt cache content too small for provider minimum, "
"retrying without prompt caching"
)
no_cache_llm = self.model_copy(update={"caching_prompt": False})
return await no_cache_llm.acompletion(
messages,
tools,
add_security_risk_prediction=add_security_risk_prediction,
on_token=on_token,
**_caller_kwargs,
)
# Fallback is synchronous; cast the token callback since the
# fallback LLM's sync path accepts TokenCallbackType.
_fb_token = cast("TokenCallbackType | None", on_token)
Expand Down Expand Up @@ -1703,24 +1670,6 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse:
try:
return self._build_responses_result(_one_attempt())
except Exception as e:
# If the prompt cache content is too small for the provider's
# minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
# retry without prompt caching markers.
if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
logger.warning(
"Prompt cache content too small for provider minimum, "
"retrying without prompt caching"
)
no_cache_llm = self.model_copy(update={"caching_prompt": False})
return no_cache_llm.responses(
messages,
tools,
include,
store,
add_security_risk_prediction=add_security_risk_prediction,
on_token=on_token,
**_caller_kwargs,
)
return self._handle_error(
e,
lambda fb: fb.responses(
Expand Down Expand Up @@ -1876,24 +1825,6 @@ async def _one_attempt(
try:
return self._build_responses_result(await _one_attempt())
except Exception as e:
# If the prompt cache content is too small for the provider's
# minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
# retry without prompt caching markers.
if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
logger.warning(
"Prompt cache content too small for provider minimum, "
"retrying without prompt caching"
)
no_cache_llm = self.model_copy(update={"caching_prompt": False})
return await no_cache_llm.aresponses(
messages,
tools,
include,
store,
add_security_risk_prediction=add_security_risk_prediction,
on_token=on_token,
**_caller_kwargs,
)
_fb_token = cast("TokenCallbackType | None", on_token)
return await self._ahandle_error(
e,
Expand Down
33 changes: 0 additions & 33 deletions tests/sdk/llm/test_exception_classifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

from openhands.sdk.llm.exceptions import (
is_context_window_exceeded,
is_prompt_cache_too_small,
looks_like_auth_error,
looks_like_malformed_conversation_history_error,
)
Expand Down Expand Up @@ -148,35 +147,3 @@ def test_looks_like_auth_error_negative():
looks_like_auth_error(BadRequestError("Something else", MODEL, PROVIDER))
is False
)


def test_is_prompt_cache_too_small_positive():
"""Vertex AI rejects caching when cached content is below minimum token count."""
vertex_error = BadRequestError(
(
"Vertex_aiException BadRequestError - "
'{"error":{"code":400,'
'"message":"The cached content is of 1171 tokens. '
'The minimum token count to start caching is 4096.",'
'"status":"INVALID_ARGUMENT"}}'
),
MODEL,
PROVIDER,
)
assert is_prompt_cache_too_small(vertex_error) is True


def test_is_prompt_cache_too_small_negative():
assert (
is_prompt_cache_too_small(BadRequestError("irrelevant", MODEL, PROVIDER))
is False
)


def test_is_prompt_cache_too_small_context_window_not_cache_too_small():
"""Context window exceeded is a different error from cache too small."""
ctx_error = BadRequestError(
"The request exceeds the available context size", MODEL, PROVIDER
)
assert is_prompt_cache_too_small(ctx_error) is False
assert is_context_window_exceeded(ctx_error) is True
145 changes: 0 additions & 145 deletions tests/sdk/llm/test_llm_completion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -954,151 +954,6 @@ def test_llm_streaming_preserves_cache_read_tokens(mock_completion):
)


@patch("openhands.sdk.llm.llm.litellm_completion")
def test_completion_retries_without_caching_on_prompt_cache_too_small(
mock_completion,
):
"""When Vertex AI rejects caching due to small content, retry without cache."""
from litellm.exceptions import BadRequestError

# First call raises the "cache too small" error, second succeeds
cache_error = BadRequestError(
(
"Vertex_aiException BadRequestError - "
'{"error":{"code":400,'
'"message":"The cached content is of 1171 tokens. '
'The minimum token count to start caching is 4096.",'
'"status":"INVALID_ARGUMENT"}}'
),
model="gemini-3.5-flash",
llm_provider="vertex_ai",
)
mock_response = create_mock_response("Retry succeeded")
mock_completion.side_effect = [cache_error, mock_response]

llm = LLM(
model="claude-sonnet-4-20250514",
api_key=SecretStr("test_key"),
usage_id="test-llm",
caching_prompt=True,
num_retries=2,
retry_min_wait=1,
retry_max_wait=2,
)

messages = [Message(role="user", content=[TextContent(text="Hello")])]
# Pass a kwarg via **kwargs to verify _caller_kwargs preservation on retry.
response = llm.completion(messages=messages, metadata={"trace": "sync"})

# Should succeed after retry without caching
assert response.raw_response == mock_response
# Two calls: first with cache (fails), second without cache (succeeds)
assert mock_completion.call_count == 2

# The first call SHOULD have cache_control markers
first_call_kwargs = mock_completion.call_args_list[0].kwargs
first_messages = first_call_kwargs.get("messages", [])
first_has_cache = any(
"cache_control" in str(block)
for msg in first_messages
for block in (
msg.get("content", []) if isinstance(msg.get("content"), list) else []
)
)
assert first_has_cache, "First call should include cache_control markers"

# The second call should NOT have cache_control markers
second_call_kwargs = mock_completion.call_args_list[1].kwargs
second_messages = second_call_kwargs.get("messages", [])
second_has_cache = any(
"cache_control" in str(block)
for msg in second_messages
for block in (
msg.get("content", []) if isinstance(msg.get("content"), list) else []
)
)
assert not second_has_cache, "Retry should not include cache_control markers"

# Caller kwargs preserved on the retry — without _caller_kwargs the retry
# would silently drop them.
assert second_call_kwargs.get("metadata") == {"trace": "sync"}


@pytest.mark.asyncio
@patch("openhands.sdk.llm.llm.litellm_acompletion", new_callable=AsyncMock)
async def test_acompletion_retries_without_caching_on_prompt_cache_too_small(
mock_acompletion,
):
"""Async version of the sync prompt-cache-too-small retry test.

When Vertex AI rejects caching due to small content, acompletion() should
retry without prompt caching while preserving caller kwargs. Mirrors
test_completion_retries_without_caching_on_prompt_cache_too_small.
"""
from litellm.exceptions import BadRequestError

cache_error = BadRequestError(
(
"Vertex_aiException BadRequestError - "
'{"error":{"code":400,'
'"message":"The cached content is of 1171 tokens. '
'The minimum token count to start caching is 4096.",'
'"status":"INVALID_ARGUMENT"}}'
),
model="gemini-3.5-flash",
llm_provider="vertex_ai",
)
mock_response = create_mock_response("Retry succeeded")
mock_acompletion.side_effect = [cache_error, mock_response]

llm = LLM(
model="claude-sonnet-4-20250514",
api_key=SecretStr("test_key"),
usage_id="test-llm",
caching_prompt=True,
num_retries=2,
retry_min_wait=1,
retry_max_wait=2,
)

messages = [Message(role="user", content=[TextContent(text="Hello")])]
# Pass a kwarg via **kwargs to verify _caller_kwargs preservation on retry.
response = await llm.acompletion(messages=messages, metadata={"trace": "abc"})

# Should succeed after retry without caching
assert response.raw_response == mock_response
# Two calls: first with cache (fails), second without cache (succeeds)
assert mock_acompletion.call_count == 2

# The first call SHOULD have cache_control markers
first_call_kwargs = mock_acompletion.call_args_list[0].kwargs
first_messages = first_call_kwargs.get("messages", [])
first_has_cache = any(
"cache_control" in str(block)
for msg in first_messages
for block in (
msg.get("content", []) if isinstance(msg.get("content"), list) else []
)
)
assert first_has_cache, "First call should include cache_control markers"

# The second call should NOT have cache_control markers
second_call_kwargs = mock_acompletion.call_args_list[1].kwargs
second_messages = second_call_kwargs.get("messages", [])
second_has_cache = any(
"cache_control" in str(block)
for msg in second_messages
for block in (
msg.get("content", []) if isinstance(msg.get("content"), list) else []
)
)
assert not second_has_cache, "Retry should not include cache_control markers"

# Caller kwargs preserved on the retry — without _caller_kwargs the retry
# would silently drop them.
assert second_call_kwargs.get("metadata") == {"trace": "abc"}


# ---------------------------------------------------------------------------
# Streaming path tolerance: the SDK must accept any iterable of
# ``ModelResponseStream`` chunks, not only ``litellm.CustomStreamWrapper``.
Expand Down
5 changes: 2 additions & 3 deletions tests/sdk/llm/test_llm_fallback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,9 +442,8 @@ async def test_aresponses_maps_connection_error(mock_aresp):
def test_fallback_forwards_caller_kwargs(mock_comp):
"""Caller kwargs (e.g. ``metadata``) must reach the fallback LLM call.

Regression guard for the ``_caller_kwargs`` forwarding added alongside
the prompt-cache-too-small retry: the fallback path now receives the
same kwargs the caller passed to the primary's ``completion()``.
Regression guard that the fallback path receives the same kwargs the caller
passed to the primary's ``completion()``.
"""
primary_error = APIConnectionError(
message="connection reset", llm_provider="openai", model="gpt-4o"
Expand Down
Loading
Loading