fix(classifier): retry provider-injected parameter 400s instead of aborting the turn

agent/error_classifier.py

@@ -459,6 +459,59 @@ def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
     "unsupported_parameter",
 ]
 
+# Request parameters that Hermes sends on SOME routes only, paired with the
+# providers/hosts where sending them is deliberate.
+#
+# When a host that is NOT in the allowed set rejects one of these fields, the
+# client never put it in the body — the provider's own gateway injected it —
+# so the 400 is a server-side flake rather than a deterministic request-shape
+# error.  See ``_is_server_injected_param_rejection`` and the branch in
+# ``_classify_400``.
+#
+# ``prompt_cache_retention`` is only sent for api.meta.ai and bedrock-mantle
+# hosts (agent/transports/codex.py::_default_prompt_cache_retention_for_request).
+# The Codex OAuth backend rejects it spontaneously on requests that provably
+# never carried it.
+_SERVER_INJECTED_PARAM_SENDERS: Dict[str, tuple] = {
+    "prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
+}
+
+
+def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
+    """True when a 400 blames a parameter this route never sends.
+
+    ``error_msg`` is the lowercased, concatenated message text; ``provider`` is
+    the lowercased provider slug.  A match means the rejection cannot be
+    attributed to our own request shape, so the error is transient and retrying
+    the identical request is the correct recovery.
+
+    Deliberately conservative: it fires only for known one-route-only
+    parameters AND only when the current provider is not one of the routes that
+    actually sends them, so a genuine client-side bad parameter (``max_tokens``
+    on a GPT-5 model) still fails fast as a ``format_error``.
+    """
+    if not error_msg:
+        return False
+    provider_slug = (provider or "").strip().lower()
+    for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
+        if param not in error_msg:
+            continue
+        # Require the message to actually be a rejection of that parameter,
+        # not an incidental mention.
+        if not (
+            "not supported" in error_msg
+            or "unsupported" in error_msg
+            or "unknown" in error_msg
+            or "unrecognized" in error_msg
+        ):
+            continue
+        if any(sender in provider_slug for sender in senders):
+            # This route sends the field on purpose — a real request error.
+            return False
+        return True
+    return False
+
+
 # OpenRouter aggregator policy-block patterns.
 #
 # When a user's OpenRouter account privacy setting (or a per-request
@@ -1310,11 +1363,16 @@ def _classify_by_status(
         # server_error" rule turns one bad request into a retry flood.
         # Detect the unambiguous request-validation signals (in either the
         # message text or the structured error code) and fail fast.
+        #
+        # Exception: a parameter WE never sent on this route was injected by
+        # the provider/proxy itself, so the rejection is not deterministic and
+        # the generic retryable-5xx handling is correct. Mirrors the guard in
+        # _classify_400 — see _is_server_injected_param_rejection.
         if (
             any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS)
             or error_code.lower() in {"invalid_request_error", "unknown_parameter",
                                       "unsupported_parameter"}
-        ):
+        ) and not _is_server_injected_param_rejection(error_msg, provider):
             return result_fn(
                 FailoverReason.format_error,
                 retryable=False,
@@ -1473,6 +1531,25 @@ def _classify_400(
             should_fallback=False,
         )
 
+    # Server-injected parameter rejection: a 400 blaming a request field the
+    # client never sent.  MUST be checked BEFORE the request-validation branch
+    # below, which would otherwise class it as a deterministic format_error and
+    # abort the turn.
+    #
+    # Observed live on the Codex OAuth backend (chatgpt.com/backend-api/codex):
+    # it intermittently adds ``prompt_cache_retention`` to its own upstream
+    # call and then rejects it, so a byte-identical request succeeds on retry
+    # (measured ~20% failure over n=20 on a minimal 1-message request that
+    # provably carried no cache parameters).  Retrying is the correct and only
+    # recovery; failing fast burnt an entire large-context request per attempt.
+    if _is_server_injected_param_rejection(error_msg, provider):
+        return result_fn(
+            FailoverReason.server_error,
+            retryable=True,
+            # The request shape was fine — never route this into compression.
+            should_compress=False,
+        )
+
     # Request-validation errors (unsupported / unknown parameter) MUST be
     # checked BEFORE context_overflow.  A GPT-5 model rejecting max_tokens
     # returns:

tests/agent/test_error_classifier.py

@@ -1292,3 +1292,152 @@ def test_longer_than_context_length_still_overflow(self):
         assert result.reason == FailoverReason.context_overflow
 
 
+class TestServerInjectedParameterRejection:
+    """A 400 blaming a parameter the client never sent is a server-side flake.
+
+    The Codex backend (chatgpt.com/backend-api/codex) intermittently adds
+    ``prompt_cache_retention`` to its own upstream call and then rejects it,
+    so an identical request succeeds on retry ~80% of the time.  Hermes never
+    sends that field on this route, so the 400 is not a deterministic
+    request-shape error and must stay retryable instead of aborting the turn.
+    """
+
+    RETENTION_BODY = {
+        "message": "prompt_cache_retention is not supported on this model",
+        "type": "invalid_request_error",
+        "param": "prompt_cache_retention",
+        "code": "invalid_parameter",
+    }
+
+    def test_codex_retention_400_is_retryable_server_error(self):
+        e = MockAPIError(
+            "Error code: 400 - {'error': {'message': 'prompt_cache_retention "
+            "is not supported on this model', 'type': 'invalid_request_error', "
+            "'param': 'prompt_cache_retention', 'code': 'invalid_parameter'}}",
+            status_code=400,
+            body=dict(self.RETENTION_BODY),
+        )
+        result = classify_api_error(
+            e,
+            provider="openai-codex",
+            model="gpt-5.6-sol",
+            approx_tokens=546912,
+            context_length=272000,
+            num_messages=576,
+        )
+        assert result.reason == FailoverReason.server_error
+        assert result.retryable is True
+        # Retrying the identical request is the recovery — do NOT enter the
+        # compression loop (the context was never the problem).
+        assert result.should_compress is False
+
+    def test_codex_retention_400_nested_error_body_is_retryable(self):
+        """The same rejection arrives wrapped in an ``error`` envelope too."""
+        e = MockAPIError(
+            "prompt_cache_retention is not supported on this model",
+            status_code=400,
+            body={"error": dict(self.RETENTION_BODY)},
+        )
+        result = classify_api_error(
+            e, provider="openai-codex", model="gpt-5.6-sol",
+        )
+        assert result.reason == FailoverReason.server_error
+        assert result.retryable is True
+
+    def test_codex_gateway_terse_retention_400_is_retryable(self):
+        """The Codex gateway's own validator uses a bare ``detail`` body."""
+        e = MockAPIError(
+            "Unsupported parameter: prompt_cache_retention",
+            status_code=400,
+            body={"detail": "Unsupported parameter: prompt_cache_retention"},
+        )
+        result = classify_api_error(
+            e, provider="openai-codex", model="gpt-5.6-sol",
+        )
+        assert result.reason == FailoverReason.server_error
+        assert result.retryable is True
+
+    def test_small_session_retention_400_is_still_retryable(self):
+        """Must not depend on the context-size heuristic — a tiny request
+        gets the identical spontaneous rejection (reproduced live)."""
+        e = MockAPIError(
+            "prompt_cache_retention is not supported on this model",
+            status_code=400,
+            body=dict(self.RETENTION_BODY),
+        )
+        result = classify_api_error(
+            e,
+            provider="openai-codex",
+            model="gpt-5.6-sol",
+            approx_tokens=50,
+            num_messages=1,
+        )
+        assert result.reason == FailoverReason.server_error
+        assert result.retryable is True
+
+    def test_other_unsupported_parameter_400_stays_non_retryable(self):
+        """Boundary: a genuine client-sent bad parameter is deterministic and
+        must keep failing fast as a format_error (the existing behaviour)."""
+        e = MockAPIError(
+            "Unsupported parameter: 'max_tokens' is not supported with this "
+            "model. Use 'max_completion_tokens' instead.",
+            status_code=400,
+            body={
+                "message": "Unsupported parameter: 'max_tokens' is not supported.",
+                "type": "invalid_request_error",
+                "param": "max_tokens",
+                "code": "unsupported_parameter",
+            },
+        )
+        result = classify_api_error(
+            e, provider="openai-codex", model="gpt-5.6-sol",
+        )
+        assert result.reason == FailoverReason.format_error
+        assert result.retryable is False
+
+    def test_retention_rejection_from_meta_host_stays_non_retryable(self):
+        """Boundary: on api.meta.ai / Bedrock Mantle Hermes DOES send
+        ``prompt_cache_retention`` deliberately, so a rejection there is a
+        real client-side request error and must not be retried blindly."""
+        e = MockAPIError(
+            "prompt_cache_retention is not supported on this model",
+            status_code=400,
+            body=dict(self.RETENTION_BODY),
+        )
+        result = classify_api_error(
+            e, provider="meta-ai", model="muse-spark-1.2",
+        )
+        assert result.reason == FailoverReason.format_error
+        assert result.retryable is False
+
+    @pytest.mark.parametrize("status_code", [500, 502])
+    def test_retention_rejection_via_5xx_proxy_is_retryable(self, status_code):
+        """Sibling path: a proxy in front of the route can surface the same
+        injected-parameter rejection as 5xx, where the request-validation
+        guard would also wrongly fail it fast as a format_error."""
+        e = MockAPIError(
+            "Unsupported parameter: prompt_cache_retention",
+            status_code=status_code,
+            body={"error": dict(self.RETENTION_BODY)},
+        )
+        result = classify_api_error(
+            e, provider="openai-codex", model="gpt-5.6-sol",
+        )
+        assert result.reason == FailoverReason.server_error
+        assert result.retryable is True
+
+    @pytest.mark.parametrize("status_code", [500, 502])
+    def test_other_bad_parameter_via_5xx_stays_non_retryable(self, status_code):
+        """Boundary for the sibling path: the codex.nekos.me 502-on-bad-param
+        behaviour must keep failing fast (regression guard for that fix)."""
+        e = MockAPIError(
+            "Unknown parameter: 'frequency_penalty'",
+            status_code=status_code,
+            body={"error": {"message": "Unknown parameter: 'frequency_penalty'",
+                            "code": "unknown_parameter"}},
+        )
+        result = classify_api_error(e, provider="custom", model="m")
+        assert result.reason == FailoverReason.format_error
+        assert result.retryable is False
+
+