feat(llm): retry without prompt caching when cached content is below provider minimum

openhands-sdk/openhands/sdk/llm/exceptions/__init__.py

@@ -1,5 +1,6 @@
 from .classifier import (
     is_context_window_exceeded,
+    is_prompt_cache_too_small,
     looks_like_auth_error,
     looks_like_malformed_conversation_history_error,
 )
@@ -48,6 +49,7 @@
     "OperationCancelled",
     # Helpers
     "is_context_window_exceeded",
+    "is_prompt_cache_too_small",
     "looks_like_auth_error",
     "looks_like_malformed_conversation_history_error",
     "map_provider_exception",

openhands-sdk/openhands/sdk/llm/exceptions/classifier.py

@@ -76,6 +76,16 @@ def looks_like_malformed_conversation_history_error(exception: Exception) -> boo
     return any(p in s for p in MALFORMED_HISTORY_PATTERNS)
 
 
+# Vertex AI (Gemini) rejects context-caching requests when the cached content
+# is below the provider's minimum token threshold (currently 4096 tokens).
+# Example error: "The cached content is of 1171 tokens. The minimum token
+# count to start caching is 4096." — the `.lower()` comparison handles case
+# variation across providers but won't match reworded messages; update this
+# pattern if the API phrasing changes.
+PROMPT_CACHE_TOO_SMALL_PATTERNS: list[str] = [
+    "minimum token count to start caching",
+]
+
 AUTH_PATTERNS: list[str] = [
     "invalid api key",
     "unauthorized",
@@ -85,6 +95,20 @@ def looks_like_malformed_conversation_history_error(exception: Exception) -> boo
 ]
 
 
+def is_prompt_cache_too_small(exception: Exception) -> bool:
+    """Return True if the error indicates the prompt cache content is too small.
+
+    Vertex AI (Gemini) requires a minimum number of tokens (currently 4096)
+    to create a context cache. When the cached content is below this threshold,
+    the API returns a 400 error. The SDK should detect this and retry without
+    prompt caching markers.
+    """
+    if not isinstance(exception, (BadRequestError, OpenAIError)):
+        return False
+    s = str(exception).lower()
+    return any(p in s for p in PROMPT_CACHE_TOO_SMALL_PATTERNS)
+
+
 def looks_like_auth_error(exception: Exception) -> bool:
     # Trust the typed exception when the provider/LiteLLM raised an explicit
     # 401/403 — its message text may not contain the heuristic patterns below.

openhands-sdk/openhands/sdk/llm/llm.py

@@ -87,6 +87,7 @@
 from openhands.sdk.llm.exceptions import (
     LLMContextWindowTooSmallError,
     LLMNoResponseError,
+    is_prompt_cache_too_small,
     map_provider_exception,
 )
 
@@ -1269,6 +1270,7 @@ def completion(
                 removed_in="1.29.0",
                 details=_RETURN_METRICS_DETAILS,
             )
+        _caller_kwargs = kwargs.copy()
         enable_streaming = bool(kwargs.get("stream", False)) or self.stream
         if enable_streaming:
             if on_token is None:
@@ -1308,13 +1310,30 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse:
         try:
             return self._build_completion_result(_one_attempt())
         except Exception as e:
+            # If the prompt cache content is too small for the provider's
+            # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
+            # retry without prompt caching markers.
+            if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
+                logger.warning(
+                    "Prompt cache content too small for provider minimum, "
+                    "retrying without prompt caching"
+                )
+                no_cache_llm = self.model_copy(update={"caching_prompt": False})
+                return no_cache_llm.completion(
+                    messages,
+                    tools,
+                    add_security_risk_prediction=add_security_risk_prediction,
+                    on_token=on_token,
+                    **_caller_kwargs,
+                )
             return self._handle_error(
                 e,
                 lambda fb: fb.completion(
                     messages,
                     tools,
                     add_security_risk_prediction=add_security_risk_prediction,
                     on_token=on_token,
+                    **_caller_kwargs,
                 ),
             )
 
@@ -1342,6 +1361,7 @@ async def acompletion(
                 removed_in="1.29.0",
                 details=_RETURN_METRICS_DETAILS,
             )
+        _caller_kwargs = kwargs.copy()
         enable_streaming = bool(kwargs.get("stream", False)) or self.stream
         if enable_streaming:
             if on_token is None:
@@ -1381,6 +1401,22 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse:
         try:
             return self._build_completion_result(await _one_attempt())
         except Exception as e:
+            # If the prompt cache content is too small for the provider's
+            # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
+            # retry without prompt caching markers.
+            if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
+                logger.warning(
+                    "Prompt cache content too small for provider minimum, "
+                    "retrying without prompt caching"
+                )
+                no_cache_llm = self.model_copy(update={"caching_prompt": False})
+                return await no_cache_llm.acompletion(
+                    messages,
+                    tools,
+                    add_security_risk_prediction=add_security_risk_prediction,
+                    on_token=on_token,
+                    **_caller_kwargs,
+                )
             # Fallback is synchronous; cast the token callback since the
             # fallback LLM's sync path accepts TokenCallbackType.
             _fb_token = cast("TokenCallbackType | None", on_token)
@@ -1391,6 +1427,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse:
                     tools,
                     add_security_risk_prediction=add_security_risk_prediction,
                     on_token=_fb_token,
+                    **_caller_kwargs,
                 ),
             )
 
@@ -1434,6 +1471,7 @@ def responses(
                 removed_in="1.29.0",
                 details=_RETURN_METRICS_DETAILS,
             )
+        _caller_kwargs = kwargs.copy()
         user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream
         if user_enable_streaming:
             # We allow on_token to be None for subscription mode
@@ -1512,6 +1550,24 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse:
         try:
             return self._build_responses_result(_one_attempt())
         except Exception as e:
+            # If the prompt cache content is too small for the provider's
+            # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
+            # retry without prompt caching markers.
+            if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
+                logger.warning(
+                    "Prompt cache content too small for provider minimum, "
+                    "retrying without prompt caching"
+                )
+                no_cache_llm = self.model_copy(update={"caching_prompt": False})
+                return no_cache_llm.responses(
+                    messages,
+                    tools,
+                    include,
+                    store,
+                    add_security_risk_prediction=add_security_risk_prediction,
+                    on_token=on_token,
+                    **_caller_kwargs,
+                )
             return self._handle_error(
                 e,
                 lambda fb: fb.responses(
@@ -1521,6 +1577,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse:
                     store,
                     add_security_risk_prediction=add_security_risk_prediction,
                     on_token=on_token,
+                    **_caller_kwargs,
                 ),
             )
 
@@ -1550,6 +1607,7 @@ async def aresponses(
                 removed_in="1.29.0",
                 details=_RETURN_METRICS_DETAILS,
             )
+        _caller_kwargs = kwargs.copy()
         user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream
         if user_enable_streaming:
             # We allow on_token to be None for subscription mode
@@ -1631,6 +1689,24 @@ async def _one_attempt(
         try:
             return self._build_responses_result(await _one_attempt())
         except Exception as e:
+            # If the prompt cache content is too small for the provider's
+            # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens),
+            # retry without prompt caching markers.
+            if is_prompt_cache_too_small(e) and self.is_caching_prompt_active():
+                logger.warning(
+                    "Prompt cache content too small for provider minimum, "
+                    "retrying without prompt caching"
+                )
+                no_cache_llm = self.model_copy(update={"caching_prompt": False})
+                return await no_cache_llm.aresponses(
+                    messages,
+                    tools,
+                    include,
+                    store,
+                    add_security_risk_prediction=add_security_risk_prediction,
+                    on_token=on_token,
+                    **_caller_kwargs,
+                )
             _fb_token = cast("TokenCallbackType | None", on_token)
             return await self._ahandle_error(
                 e,
@@ -1641,6 +1717,7 @@ async def _one_attempt(
                     store,
                     add_security_risk_prediction=add_security_risk_prediction,
                     on_token=_fb_token,
+                    **_caller_kwargs,
                 ),
             )
 

tests/sdk/llm/test_exception_classifier.py

@@ -6,6 +6,7 @@
 
 from openhands.sdk.llm.exceptions import (
     is_context_window_exceeded,
+    is_prompt_cache_too_small,
     looks_like_auth_error,
     looks_like_malformed_conversation_history_error,
 )
@@ -119,3 +120,35 @@ def test_looks_like_auth_error_negative():
         looks_like_auth_error(BadRequestError("Something else", MODEL, PROVIDER))
         is False
     )
+
+
+def test_is_prompt_cache_too_small_positive():
+    """Vertex AI rejects caching when cached content is below minimum token count."""
+    vertex_error = BadRequestError(
+        (
+            "Vertex_aiException BadRequestError - "
+            '{"error":{"code":400,'
+            '"message":"The cached content is of 1171 tokens. '
+            'The minimum token count to start caching is 4096.",'
+            '"status":"INVALID_ARGUMENT"}}'
+        ),
+        MODEL,
+        PROVIDER,
+    )
+    assert is_prompt_cache_too_small(vertex_error) is True
+
+
+def test_is_prompt_cache_too_small_negative():
+    assert (
+        is_prompt_cache_too_small(BadRequestError("irrelevant", MODEL, PROVIDER))
+        is False
+    )
+
+
+def test_is_prompt_cache_too_small_context_window_not_cache_too_small():
+    """Context window exceeded is a different error from cache too small."""
+    ctx_error = BadRequestError(
+        "The request exceeds the available context size", MODEL, PROVIDER
+    )
+    assert is_prompt_cache_too_small(ctx_error) is False
+    assert is_context_window_exceeded(ctx_error) is True