From 619397f970e7b0b55b476d5e7e37991ee3ce8d19 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Tue, 25 Aug 2026 19:55:37 +0000 Subject: [PATCH 1/2] fix(gateway): default an Anthropic client timeout so the non-streaming pre-flight guard does not 502 Anthropic's SDK raises a status-less ValueError before any request is sent when a non-streaming call's max_tokens implies more than 10 minutes and the client's timeout is untouched. build_attempt_client_args never set one, so both the bare no-extra_params path and the hybrid-mode extra_params path built an Anthropic client with the SDK's own default, and classify_provider_error returned None for the status-less error, collapsing it to a generic 502. Give Anthropic clients an explicit default timeout (the SDK's own default value, so this only defuses the guard rather than changing behavior) unless an operator already configured one, and add a classifier branch so any occurrence that still reaches it maps to a 400 naming the cause instead of a generic 502. Fixes #533 --- src/gateway/api/routes/_pipeline.py | 22 ++++++ src/gateway/api/routes/_platform.py | 16 +++-- src/gateway/services/provider_kwargs.py | 25 +++++++ tests/unit/test_hybrid_client_args_routing.py | 71 +++++++++++++++++++ .../test_provider_error_classification.py | 60 +++++++++++++++- tests/unit/test_provider_instances.py | 43 +++++++++++ 6 files changed, 230 insertions(+), 7 deletions(-) diff --git a/src/gateway/api/routes/_pipeline.py b/src/gateway/api/routes/_pipeline.py index 9451961657..672459dd69 100644 --- a/src/gateway/api/routes/_pipeline.py +++ b/src/gateway/api/routes/_pipeline.py @@ -237,6 +237,10 @@ "Top up the provider account, or route this model to a provider that has credit." ) PROVIDER_RATE_LIMITED_DETAIL = "The provider rate-limited this request" +PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL = ( + "max_tokens is too large for a non-streaming request to this provider; lower max_tokens " + "or set stream: true" +) ALL_PROVIDERS_FAILED_DETAIL = "All upstream providers failed" ALL_PROVIDERS_TIMED_OUT_DETAIL = "All upstream providers timed out" SANDBOX_NOT_CONFIGURED_DETAIL = ( @@ -343,6 +347,20 @@ def _unsupported_feature_detail(exc: BaseException) -> str: return _caller_fault_detail(exc, PROVIDER_BAD_REQUEST_DETAIL) +# The anthropic SDK's non-streaming pre-flight guard message (otari#533); status-less, +# so this classifier has to name it explicitly or it falls through to a generic 502. +# Matched as a substring: the SDK message also links a docs anchor that has moved. +_ANTHROPIC_NONSTREAMING_TIMEOUT_GUARD_MESSAGE = "Streaming is required for operations that may take longer" + + +def _is_anthropic_nonstreaming_timeout_guard(exc: BaseException) -> bool: + """True when a status-less ``ValueError`` is the anthropic SDK's own pre-flight guard.""" + return any( + isinstance(candidate, ValueError) and _ANTHROPIC_NONSTREAMING_TIMEOUT_GUARD_MESSAGE in str(candidate) + for candidate in upstream_exception_chain(exc) + ) + + def _redacted_caller_fault_detail(message: str, fallback: str) -> str: """Return redacted explanatory text, or a fixed detail when none remains.""" redacted = redact_upstream_message(message) @@ -403,6 +421,10 @@ def classify_provider_error(exc: BaseException) -> ProviderErrorMapping | None: # whole signal here, and its message names the unsupported feature. if _is_unsupported_feature_error(exc): return ProviderErrorMapping(status.HTTP_400_BAD_REQUEST, _unsupported_feature_detail(exc)) + # Defense in depth for otari#533: an operator-supplied timeout equal to the + # SDK's own default re-arms this same status-less guard. + if _is_anthropic_nonstreaming_timeout_guard(exc): + return ProviderErrorMapping(status.HTTP_400_BAD_REQUEST, PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL) if status_code is None: return None # Account billing exhaustion, which several providers report as a 400/422 diff --git a/src/gateway/api/routes/_platform.py b/src/gateway/api/routes/_platform.py index f6ebb43267..77e8cbd84d 100644 --- a/src/gateway/api/routes/_platform.py +++ b/src/gateway/api/routes/_platform.py @@ -34,6 +34,7 @@ from gateway.models.mcp import McpServerConfig from gateway.services.bedrock_gateway_auth import build_bedrock_client_args from gateway.services.mcp_loop import MaxToolIterationsExceeded +from gateway.services.provider_kwargs import with_anthropic_default_timeout from gateway.services.sandbox_backend import SandboxNotReachableError from gateway.services.web_search_backend import WebSearchNotReachableError @@ -173,7 +174,8 @@ class _AttemptFailure(NamedTuple): def build_attempt_client_args(attempt: ResolvedAttempt) -> dict[str, Any] | None: """Build the ``client_args`` any-llm needs to construct this attempt's - provider client, or ``None`` when the attempt carries no ``extra_params``. + provider client, or ``None`` when the attempt carries no ``extra_params`` + and needs none injected (see ``with_anthropic_default_timeout``). This *must* be a separate ``client_args`` dict, not merged flat into the completion kwargs: any-llm's ``acompletion()`` only forwards a @@ -189,14 +191,16 @@ def build_attempt_client_args(attempt: ResolvedAttempt) -> dict[str, Any] | None Bedrock gets dedicated handling (see :mod:`gateway.services.bedrock_gateway_auth`) because it also needs its secret aliased to a different boto3 kwarg name and, for the bearer-token ("Bedrock API key") credential shape, a custom - pre-built client. Every other provider's ``extra_params`` is forwarded - as-is. + pre-built client. Anthropic gets a default ``timeout`` filled in when the + attempt carries none (see :func:`gateway.services.provider_kwargs.with_anthropic_default_timeout`, + otari#533). Every other provider's ``extra_params`` is forwarded as-is. """ + provider = LLMProvider(attempt.provider) if not attempt.extra_params: - return None - if LLMProvider(attempt.provider) == LLMProvider.BEDROCK: + return with_anthropic_default_timeout(provider, None) + if provider == LLMProvider.BEDROCK: return build_bedrock_client_args(attempt.api_key, attempt.extra_params) - return dict(attempt.extra_params) + return with_anthropic_default_timeout(provider, dict(attempt.extra_params)) def default_attempt_kwargs( diff --git a/src/gateway/services/provider_kwargs.py b/src/gateway/services/provider_kwargs.py index 165c0428c0..9aa8adac34 100644 --- a/src/gateway/services/provider_kwargs.py +++ b/src/gateway/services/provider_kwargs.py @@ -91,6 +91,28 @@ # and no key takes the keyless placeholder, so it never reaches a caller alone. _NON_CREDENTIAL_KWARGS = frozenset({"client_args"}) +# The anthropic SDK's non-streaming pre-flight guard fires for a large max_tokens +# only while the client's timeout is untouched (otari#533); this is also the +# guard's own fallback value, so setting it explicitly disables the guard alone. +ANTHROPIC_DEFAULT_TIMEOUT_SECONDS = 600.0 + + +def with_anthropic_default_timeout( + provider: LLMProvider, client_args: dict[str, Any] | None +) -> dict[str, Any] | None: + """Ensure an Anthropic ``client_args`` carries an explicit ``timeout``. + + A no-op for every other provider. An operator's own ``timeout`` (set via + ``client_args`` on the provider config or a platform attempt's + ``extra_params``) is left untouched; this only fills the gap when nothing + was configured. + """ + if provider != LLMProvider.ANTHROPIC: + return client_args + merged = dict(client_args) if client_args else {} + merged.setdefault("timeout", ANTHROPIC_DEFAULT_TIMEOUT_SECONDS) + return merged + def _kwargs_carry_a_credential(kwargs: dict[str, Any]) -> bool: """Whether anything in a resolved provider's kwargs could authenticate a call. @@ -227,6 +249,9 @@ def get_provider_kwargs( if placeholder is not None: kwargs["api_key"] = placeholder + if provider == LLMProvider.ANTHROPIC: + kwargs["client_args"] = with_anthropic_default_timeout(provider, kwargs.get("client_args")) + return kwargs diff --git a/tests/unit/test_hybrid_client_args_routing.py b/tests/unit/test_hybrid_client_args_routing.py index 7bda9e0e74..c3e270d89e 100644 --- a/tests/unit/test_hybrid_client_args_routing.py +++ b/tests/unit/test_hybrid_client_args_routing.py @@ -20,6 +20,7 @@ from unittest.mock import patch import pytest +from anthropic import APIConnectionError from any_llm import acompletion from gateway.api.routes._platform import ResolvedAttempt, default_attempt_kwargs @@ -98,3 +99,73 @@ def fake_converse(**_call_kwargs: Any) -> Any: # client and skips constructing its own; Session.client() is # untouched for the runtime client in this shape. fake_boto3_client.assert_not_called() + + +@pytest.mark.asyncio +async def test_anthropic_default_timeout_reaches_the_real_sdk_client() -> None: + """otari#533: with no extra_params, the anthropic client used to be built + with the SDK's own default timeout, which makes AsyncAnthropic's + non-streaming pre-flight guard raise a bare ValueError for a large + max_tokens before any request is even attempted. default_attempt_kwargs now + carries an explicit timeout through client_args, which the real SDK's guard + treats identically to an operator-supplied one and skips.""" + attempt = ResolvedAttempt( + attempt_id="a0", + position=0, + provider="anthropic", + model="claude-opus-5", + api_key="sk-test", + managed=False, + extra_params=None, + ) + kwargs = default_attempt_kwargs( + attempt, + {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 65536, "stream": False}, + ) + + class _Sentinel(Exception): + pass + + async def fake_send(*_args: Any, **_kwargs: Any) -> Any: + raise _Sentinel + + # any-llm/anthropic wraps a non-HTTP send failure in APIConnectionError after + # retries; reaching that (rather than the pre-flight ValueError) is what + # proves the guard was skipped and the request reached the transport layer. + with patch("httpx.AsyncClient.send", side_effect=fake_send): + with pytest.raises(APIConnectionError): + await acompletion(**kwargs) + + +@pytest.mark.asyncio +async def test_anthropic_default_timeout_reaches_the_real_sdk_client_in_hybrid_mode() -> None: + """otari#533 point 1: an attempt with its own extra_params (the hybrid-mode + shape) took a different branch of build_attempt_client_args than the bare + no-extra_params case above, and that branch never filled in a default + timeout, so a platform-routed Anthropic attempt with any extra_params at + all (e.g. a base_url override) still hit the pre-flight guard.""" + attempt = ResolvedAttempt( + attempt_id="a0", + position=0, + provider="anthropic", + model="claude-opus-5", + api_key="sk-test", + managed=False, + extra_params={"auth_token": "operator-supplied-bearer-token"}, + ) + kwargs = default_attempt_kwargs( + attempt, + {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 65536, "stream": False}, + ) + assert kwargs["client_args"]["timeout"] == 600.0 + assert kwargs["client_args"]["auth_token"] == "operator-supplied-bearer-token" + + class _Sentinel(Exception): + pass + + async def fake_send(*_args: Any, **_kwargs: Any) -> Any: + raise _Sentinel + + with patch("httpx.AsyncClient.send", side_effect=fake_send): + with pytest.raises(APIConnectionError): + await acompletion(**kwargs) diff --git a/tests/unit/test_provider_error_classification.py b/tests/unit/test_provider_error_classification.py index 3581025201..d9012efa01 100644 --- a/tests/unit/test_provider_error_classification.py +++ b/tests/unit/test_provider_error_classification.py @@ -16,7 +16,7 @@ import httpx import pytest from anthropic import APITimeoutError as AnthropicAPITimeoutError -from any_llm.exceptions import UnsupportedParameterError +from any_llm.exceptions import InvalidRequestError, UnsupportedParameterError from openai import APITimeoutError as OpenAIAPITimeoutError from gateway.api.routes._pipeline import ( @@ -24,6 +24,7 @@ PROVIDER_BILLING_DETAIL, PROVIDER_CREDENTIALS_DETAIL, PROVIDER_MODEL_NOT_FOUND_DETAIL, + PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL, PROVIDER_RATE_LIMITED_DETAIL, PROVIDER_TIMEOUT_DETAIL, classify_provider_error, @@ -348,6 +349,63 @@ def test_unsupported_feature_is_recorded_as_400_on_the_usage_log() -> None: assert failure_status_code(NotImplementedError(_CONTEXT_MANAGEMENT_MSG)) == 400 +_ANTHROPIC_NONSTREAMING_TIMEOUT_MSG = ( + "Streaming is required for operations that may take longer than 10 minutes. See " + "https://github.com/anthropics/anthropic-sdk-python#long-requests for more details" +) + + +def test_anthropic_nonstreaming_timeout_guard_maps_to_400() -> None: + """otari#533: the anthropic SDK's own pre-flight guard for a non-streaming + request whose max_tokens implies over 10 minutes raises a bare, status-less + ValueError, which used to fall through to a generic 502 telling the caller + nothing about the client-side limit it hit.""" + mapping = classify_provider_error(ValueError(_ANTHROPIC_NONSTREAMING_TIMEOUT_MSG)) + assert mapping is not None + assert mapping.status_code == 400 + assert mapping.detail == PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL + + +def test_anthropic_nonstreaming_timeout_guard_survives_the_unified_exception_wrapper() -> None: + """The message match still fires once ANY_LLM_UNIFIED_EXCEPTIONS=1 wraps the + raw error, because the wrapper keeps it on ``original_exception``.""" + mapping = classify_provider_error(_WrappedError(500, ValueError(_ANTHROPIC_NONSTREAMING_TIMEOUT_MSG))) + assert mapping is not None + assert mapping.status_code == 400 + assert mapping.detail == PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL + + +def test_anthropic_nonstreaming_timeout_guard_is_recorded_as_400_on_the_usage_log() -> None: + assert failure_status_code(ValueError(_ANTHROPIC_NONSTREAMING_TIMEOUT_MSG)) == 400 + + +def test_unrelated_value_error_is_not_mistaken_for_the_timeout_guard() -> None: + """The match is a specific substring of the SDK's own wording, not any + ValueError: an unrelated caller-code or config ValueError must still return + None so it stays on the generic 502 rather than a wrong client-facing 400.""" + assert classify_provider_error(ValueError("invalid literal for int() with base 10")) is None + + +def test_anthropic_nonstreaming_timeout_guard_via_the_real_any_llm_wrapper() -> None: + """any-llm-sdk 1.25 already translates the raw ValueError into its own + InvalidRequestError (any_llm/providers/anthropic/base.py's + _translating_nonstreaming_guard), which is what actually reaches this + classifier through acompletion(); it is not a bare ValueError, but it does + carry the original one on .original_exception, which + upstream_exception_chain walks.""" + original = ValueError(_ANTHROPIC_NONSTREAMING_TIMEOUT_MSG) + exc = InvalidRequestError( + "A non-streaming request with max_tokens=65536 can exceed the default per-request time " + "limit, so it needs an explicit timeout. Pass a `timeout` (in seconds) or use `stream=True`.", + original_exception=original, + provider_name="anthropic", + ) + mapping = classify_provider_error(exc) + assert mapping is not None + assert mapping.status_code == 400 + assert mapping.detail == PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL + + def test_redaction_keeps_the_part_that_explains_the_rejection() -> None: """Redaction targets secret shapes, not meaning: the text that tells the caller what to change has to survive.""" diff --git a/tests/unit/test_provider_instances.py b/tests/unit/test_provider_instances.py index 6887320dfe..31bd5df564 100644 --- a/tests/unit/test_provider_instances.py +++ b/tests/unit/test_provider_instances.py @@ -17,12 +17,14 @@ _AMBIENT_CREDENTIAL_PROVIDERS, _KEYLESS_PLACEHOLDER_API_KEY, _KEYLESS_SELF_HOSTED_PROVIDERS, + ANTHROPIC_DEFAULT_TIMEOUT_SECONDS, get_provider_kwargs, keyless_placeholder_api_key, normalize_pricing_key, provider_key, resolve_provider_selector, split_selector, + with_anthropic_default_timeout, ) # --------------------------------------------------------------------------- @@ -405,6 +407,47 @@ def test_get_provider_kwargs_strips_provider_type_and_models() -> None: assert kwargs == {"api_base": "http://x/v1", "api_key": "k"} +# --------------------------------------------------------------------------- +# get_provider_kwargs / with_anthropic_default_timeout (otari#533) +# --------------------------------------------------------------------------- + + +def test_get_provider_kwargs_fills_a_default_anthropic_timeout() -> None: + config = GatewayConfig(providers={"anthropic": {"api_key": "sk-test"}}) + kwargs = get_provider_kwargs(config, LLMProvider.ANTHROPIC, instance="anthropic") + assert kwargs["client_args"] == {"timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS} + + +def test_get_provider_kwargs_preserves_an_operator_configured_anthropic_timeout() -> None: + config = GatewayConfig(providers={"anthropic": {"api_key": "sk-test", "client_args": {"timeout": 900}}}) + kwargs = get_provider_kwargs(config, LLMProvider.ANTHROPIC, instance="anthropic") + assert kwargs["client_args"] == {"timeout": 900} + + +def test_get_provider_kwargs_leaves_other_providers_client_args_untouched() -> None: + config = GatewayConfig(providers={"openai": {"api_key": "sk-test"}}) + kwargs = get_provider_kwargs(config, LLMProvider.OPENAI, instance="openai") + assert "client_args" not in kwargs + + +def test_with_anthropic_default_timeout_is_a_no_op_for_other_providers() -> None: + assert with_anthropic_default_timeout(LLMProvider.OPENAI, None) is None + assert with_anthropic_default_timeout(LLMProvider.OPENAI, {"timeout": 5}) == {"timeout": 5} + + +def test_with_anthropic_default_timeout_fills_a_missing_default() -> None: + assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, None) == { + "timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS + } + assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, {}) == { + "timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS + } + + +def test_with_anthropic_default_timeout_never_overrides_an_explicit_value() -> None: + assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, {"timeout": 42}) == {"timeout": 42} + + # --------------------------------------------------------------------------- # get_provider_kwargs: keyless custom-endpoint placeholder (mozilla-ai/otari#421) # --------------------------------------------------------------------------- From 4bdef8c83f0d2ebc178c6593cdbd7fb102e6a6cf Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Thu, 3 Sep 2026 06:43:19 +0000 Subject: [PATCH 2/2] fix(gateway): keep the anthropic default timeout from widening connect A flat float timeout disarms the anthropic SDK's non-streaming pre-flight guard, but the SDK expands a flat float into an httpx.Timeout that sets every dimension including connect, raising it from 5s to 600s on every Anthropic request. Inject an explicit httpx.Timeout with connect pinned at the SDK's own default instead. Also add a drift assertion on the SDK's own default, reword an inaccurate defense-in-depth comment, pin the credential ladder against a default-filled client_args, and update the two docs that were never touched for the original fix (otari#799 review). --- docs/configuration.md | 5 ++ docs/hybrid-mode-protocol.md | 11 ++++ src/gateway/api/routes/_pipeline.py | 12 ++++- src/gateway/services/provider_kwargs.py | 26 ++++++++-- tests/unit/test_hybrid_client_args_routing.py | 31 +++++++++-- tests/unit/test_provider_instances.py | 51 +++++++++++++++++-- 6 files changed, 123 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5b7c19fdca..0ca48a95e2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -396,6 +396,11 @@ Each provider entry in the config file supports: | `api_base` | Custom API base URL (optional) | | `client_args` | Extra client options: `custom_headers`, `timeout` (optional) | +For an `anthropic` instance, a `timeout` is filled in automatically when +`client_args` does not set one, so the anthropic SDK's non-streaming +pre-flight guard does not turn a large `max_tokens` into an opaque 502 +(otari#533). An explicit `client_args.timeout` on the instance is left alone. + ### Vertex AI Vertex AI requires additional fields instead of a simple API key: diff --git a/docs/hybrid-mode-protocol.md b/docs/hybrid-mode-protocol.md index a3c50605e9..7b00126ba7 100644 --- a/docs/hybrid-mode-protocol.md +++ b/docs/hybrid-mode-protocol.md @@ -155,6 +155,17 @@ boto3 client and AWS has two distinct credential shapes: client, safe under concurrent load. See `src/gateway/services/bedrock_gateway_auth.py`. +Anthropic gets a smaller piece of handling on top of the same generic nesting: +when a resolved `client_args` carries no `timeout` (neither an +operator-configured one nor one already present in `extra_params`), a default +is filled in (otari#533). This exists because the anthropic SDK's own +non-streaming pre-flight guard raises a bare, status-less error for a large +`max_tokens` unless the client's timeout differs from the SDK's own default; +left unset, that turns into an opaque 502. The value filled in is an +`httpx.Timeout` object rather than a plain number, specifically so it changes +only the read/write/pool dimensions and leaves `connect` at the SDK's own +default. See `gateway.services.provider_kwargs.with_anthropic_default_timeout`. + `request_id` groups every `attempt_id` from the same resolve call so the platform can attribute spend, render trace timelines, and emit fallback events. Otari also surfaces it as the `X-Otari-Request-ID` response header. diff --git a/src/gateway/api/routes/_pipeline.py b/src/gateway/api/routes/_pipeline.py index 1178a99fd2..44c9a77b7a 100644 --- a/src/gateway/api/routes/_pipeline.py +++ b/src/gateway/api/routes/_pipeline.py @@ -503,8 +503,16 @@ def classify_provider_error(exc: BaseException) -> ProviderErrorMapping | None: # whole signal here, and its message names the unsupported feature. if _is_unsupported_feature_error(exc): return ProviderErrorMapping(status.HTTP_400_BAD_REQUEST, _unsupported_feature_detail(exc)) - # Defense in depth for otari#533: an operator-supplied timeout equal to the - # SDK's own default re-arms this same status-less guard. + # Defense in depth for otari#533, not coverage for a config scenario: an + # operator-supplied client_args timeout can't reproduce the SDK's own + # DEFAULT_TIMEOUT, because config only ever supplies a flat float and a flat + # float sets every httpx.Timeout dimension including connect, so it can only + # equal DEFAULT_TIMEOUT (connect=5.0, read/write/pool=600) by also setting + # connect to 5.0, at which point read/write/pool are 5.0 too, not 600. What + # this still catches is a client built without with_anthropic_default_timeout + # (any future call path that does not route through get_provider_kwargs or + # build_attempt_client_args), which is left at the SDK's real default and can + # still trip the guard for a large max_tokens. if _is_anthropic_nonstreaming_timeout_guard(exc): return ProviderErrorMapping(status.HTTP_400_BAD_REQUEST, PROVIDER_NONSTREAMING_MAX_TOKENS_DETAIL) # The same shape one layer down: a param any-llm forwards that the resolved diff --git a/src/gateway/services/provider_kwargs.py b/src/gateway/services/provider_kwargs.py index 9aa8adac34..bf84b1affd 100644 --- a/src/gateway/services/provider_kwargs.py +++ b/src/gateway/services/provider_kwargs.py @@ -33,6 +33,7 @@ from dataclasses import dataclass from typing import Any +import httpx from any_llm import AnyLLM, LLMProvider from any_llm.exceptions import AnyLLMError @@ -92,9 +93,19 @@ _NON_CREDENTIAL_KWARGS = frozenset({"client_args"}) # The anthropic SDK's non-streaming pre-flight guard fires for a large max_tokens -# only while the client's timeout is untouched (otari#533); this is also the -# guard's own fallback value, so setting it explicitly disables the guard alone. -ANTHROPIC_DEFAULT_TIMEOUT_SECONDS = 600.0 +# only while the client's timeout compares equal to the SDK's own DEFAULT_TIMEOUT +# (anthropic._constants.DEFAULT_TIMEOUT == Timeout(connect=5.0, read=600, write=600, +# pool=600), checked in anthropic/resources/messages/messages.py). A flat float +# disarms the guard but is not a narrow disarm: the anthropic SDK expands a flat +# float into httpx.Timeout(timeout=), which sets *every* dimension +# including connect, so client_args={"timeout": 600.0} silently raises the +# client's connect timeout from 5s to 600s on every Anthropic request, streaming +# included (otari#799 review). Building the httpx.Timeout explicitly, with +# read/write/pool one second above the SDK's own default and connect pinned at +# the SDK's own default, disarms the guard (unequal to DEFAULT_TIMEOUT on +# read/write/pool) without touching connect at all. +ANTHROPIC_DEFAULT_TIMEOUT_SECONDS = 601.0 +ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS = 5.0 def with_anthropic_default_timeout( @@ -106,11 +117,18 @@ def with_anthropic_default_timeout( ``client_args`` on the provider config or a platform attempt's ``extra_params``) is left untouched; this only fills the gap when nothing was configured. + + The injected value is an ``httpx.Timeout``, not a flat float: a flat float + would disarm the pre-flight guard by changing the client's *connect* + timeout too (see the module-level comment above), which nothing here wants. """ if provider != LLMProvider.ANTHROPIC: return client_args merged = dict(client_args) if client_args else {} - merged.setdefault("timeout", ANTHROPIC_DEFAULT_TIMEOUT_SECONDS) + merged.setdefault( + "timeout", + httpx.Timeout(ANTHROPIC_DEFAULT_TIMEOUT_SECONDS, connect=ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS), + ) return merged diff --git a/tests/unit/test_hybrid_client_args_routing.py b/tests/unit/test_hybrid_client_args_routing.py index c3e270d89e..5829dcfa70 100644 --- a/tests/unit/test_hybrid_client_args_routing.py +++ b/tests/unit/test_hybrid_client_args_routing.py @@ -19,11 +19,14 @@ from typing import Any from unittest.mock import patch +import anthropic +import httpx import pytest from anthropic import APIConnectionError from any_llm import acompletion from gateway.api.routes._platform import ResolvedAttempt, default_attempt_kwargs +from gateway.services.provider_kwargs import ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS @pytest.mark.asyncio @@ -126,15 +129,32 @@ async def test_anthropic_default_timeout_reaches_the_real_sdk_client() -> None: class _Sentinel(Exception): pass + captured_client_timeout: list[Any] = [] + + def fake_client_init(self: Any, *args: Any, **client_kwargs: Any) -> None: + # otari#799 review, finding 1: capture the *real* httpx client's timeout + # at construction time, before any request goes out, so this proves the + # end-to-end value the real SDK ends up with rather than only what our + # own dict construction says. + real_init(self, *args, **client_kwargs) + captured_client_timeout.append(self.timeout) + + real_init = httpx.AsyncClient.__init__ + async def fake_send(*_args: Any, **_kwargs: Any) -> Any: raise _Sentinel # any-llm/anthropic wraps a non-HTTP send failure in APIConnectionError after # retries; reaching that (rather than the pre-flight ValueError) is what # proves the guard was skipped and the request reached the transport layer. - with patch("httpx.AsyncClient.send", side_effect=fake_send): - with pytest.raises(APIConnectionError): - await acompletion(**kwargs) + with patch("httpx.AsyncClient.__init__", fake_client_init): + with patch("httpx.AsyncClient.send", side_effect=fake_send): + with pytest.raises(APIConnectionError): + await acompletion(**kwargs) + + assert captured_client_timeout + assert all(t.connect == ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS for t in captured_client_timeout) + assert all(t != anthropic.DEFAULT_TIMEOUT for t in captured_client_timeout) @pytest.mark.asyncio @@ -157,7 +177,10 @@ async def test_anthropic_default_timeout_reaches_the_real_sdk_client_in_hybrid_m attempt, {"messages": [{"role": "user", "content": "hi"}], "max_tokens": 65536, "stream": False}, ) - assert kwargs["client_args"]["timeout"] == 600.0 + # otari#799 review, finding 1: this must not be a flat float, or the SDK + # expands it into an httpx.Timeout that also raises connect from 5s to + # 600s. connect must stay at the SDK's own default. + assert kwargs["client_args"]["timeout"].connect == ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS assert kwargs["client_args"]["auth_token"] == "operator-supplied-bearer-token" class _Sentinel(Exception): diff --git a/tests/unit/test_provider_instances.py b/tests/unit/test_provider_instances.py index 31bd5df564..76bce4bafe 100644 --- a/tests/unit/test_provider_instances.py +++ b/tests/unit/test_provider_instances.py @@ -4,6 +4,8 @@ import logging +import anthropic +import httpx import pytest import yaml from any_llm import AnyLLM, LLMProvider @@ -17,7 +19,9 @@ _AMBIENT_CREDENTIAL_PROVIDERS, _KEYLESS_PLACEHOLDER_API_KEY, _KEYLESS_SELF_HOSTED_PROVIDERS, + ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS, ANTHROPIC_DEFAULT_TIMEOUT_SECONDS, + credential_ladder_exhausted, get_provider_kwargs, keyless_placeholder_api_key, normalize_pricing_key, @@ -412,10 +416,30 @@ def test_get_provider_kwargs_strips_provider_type_and_models() -> None: # --------------------------------------------------------------------------- +def _expected_default_anthropic_timeout() -> httpx.Timeout: + return httpx.Timeout(ANTHROPIC_DEFAULT_TIMEOUT_SECONDS, connect=ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS) + + +def test_anthropic_default_timeout_constants_track_the_sdks_own_default() -> None: + """Pin the two facts the injected timeout depends on not drifting silently. + + ``ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS`` exists to keep the client's + connect timeout at whatever the anthropic SDK itself ships as a default; if + the SDK ever changes that value, this constant has silently gone stale and + the fix for otari#533 quietly stops preserving connect. And the timeout this + module builds must stay unequal to the SDK's own ``DEFAULT_TIMEOUT`` (that + inequality is exactly what disarms the pre-flight guard), so a future SDK + release choosing read/write/pool=601 would silently re-arm the guard we mean + to disable. + """ + assert anthropic.DEFAULT_TIMEOUT.connect == ANTHROPIC_DEFAULT_CONNECT_TIMEOUT_SECONDS + assert _expected_default_anthropic_timeout() != anthropic.DEFAULT_TIMEOUT + + def test_get_provider_kwargs_fills_a_default_anthropic_timeout() -> None: config = GatewayConfig(providers={"anthropic": {"api_key": "sk-test"}}) kwargs = get_provider_kwargs(config, LLMProvider.ANTHROPIC, instance="anthropic") - assert kwargs["client_args"] == {"timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS} + assert kwargs["client_args"] == {"timeout": _expected_default_anthropic_timeout()} def test_get_provider_kwargs_preserves_an_operator_configured_anthropic_timeout() -> None: @@ -437,10 +461,10 @@ def test_with_anthropic_default_timeout_is_a_no_op_for_other_providers() -> None def test_with_anthropic_default_timeout_fills_a_missing_default() -> None: assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, None) == { - "timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS + "timeout": _expected_default_anthropic_timeout() } assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, {}) == { - "timeout": ANTHROPIC_DEFAULT_TIMEOUT_SECONDS + "timeout": _expected_default_anthropic_timeout() } @@ -448,6 +472,27 @@ def test_with_anthropic_default_timeout_never_overrides_an_explicit_value() -> N assert with_anthropic_default_timeout(LLMProvider.ANTHROPIC, {"timeout": 42}) == {"timeout": 42} +def test_with_anthropic_default_timeout_keeps_the_sdks_own_connect_timeout() -> None: + """The regression otari#799's review caught: a flat float disarms the guard + by also raising connect from 5s to 600s. The injected value must disarm the + guard (unequal to DEFAULT_TIMEOUT) while leaving connect untouched.""" + injected = with_anthropic_default_timeout(LLMProvider.ANTHROPIC, None) + assert injected is not None + timeout = injected["timeout"] + assert isinstance(timeout, httpx.Timeout) + assert timeout.connect == anthropic.DEFAULT_TIMEOUT.connect + assert timeout != anthropic.DEFAULT_TIMEOUT + + +def test_a_default_anthropic_timeout_does_not_look_like_a_credential() -> None: + """``client_args`` is already in ``_NON_CREDENTIAL_KWARGS``, but nothing else + pinned that a default-filled Anthropic timeout keeps reporting the ladder as + exhausted; this is load-bearing for the organization-key and fallback-rung + paths (otari#799 review).""" + kwargs = get_provider_kwargs(GatewayConfig(), LLMProvider.ANTHROPIC) + assert credential_ladder_exhausted(LLMProvider.ANTHROPIC, kwargs) is True + + # --------------------------------------------------------------------------- # get_provider_kwargs: keyless custom-endpoint placeholder (mozilla-ai/otari#421) # ---------------------------------------------------------------------------