From 2b4f958dfd11b8e21bcbc55ce035e07cb9d9a8e0 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:43:31 +0800 Subject: [PATCH 01/10] fix(llm): preserve fallback for native provider errors --- docs/notes.md | 2 + tests/test_any_llm_provider.py | 152 +++++++++++++++++++++++++++++++- tests/test_llm.py | 4 +- tests/test_llm_fallback.py | 16 +++- weather_briefing/llm/any_llm.py | 102 +++++++++++++-------- 5 files changed, 228 insertions(+), 48 deletions(-) diff --git a/docs/notes.md b/docs/notes.md index 3b7f4265..bb572ddc 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -58,6 +58,8 @@ 这是一个有意保留的自定义外部服务集成。选择 `FallbackLLMProvider` 的原因是 any-llm 只统一调用单个 provider，不编排跨 provider 的故障切换。包装器捕获主适配器的 `LLMRequestError`，切换后在剩余生命周期内固定使用备用适配器，使同一进程里的契约修复不会回到刚刚失败的主服务。 +适配器把真实 AnyLLM provider client 在 completion 调用中抛出的厂商原生异常归一化为 `LLMRequestError`，使配置正确的 fallback 能够启动。结构化输出的 Pydantic 校验错误和直接注入的协议实现所抛出的编程错误继续原样传播。 + 它替代的是每个调用点手写的故障切换分支，不替代 any-llm 的厂商适配器，也不接管 SDK 凭据、请求重试或输出验证。 这个选择成立的条件是一次主服务请求失败足以让当前模型对象的后续调用继续使用备用服务，恢复主服务交给下次创建模型对象。如果 any-llm 提供可观察的跨 provider 路由，或者常驻进程需要在不重启的情况下探测并恢复主服务，就重新评估并用带健康状态和冷却时间的路由替换当前粘性开关。 diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 91808ccd..4e9adda2 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -1,20 +1,26 @@ import json import logging -from collections.abc import Mapping +import os +from collections.abc import Callable, Mapping from types import SimpleNamespace +from typing import Any from unittest.mock import AsyncMock, Mock import httpx import pytest +from anthropic import BadRequestError as AnthropicBadRequestError from any_llm import AnyLLM from any_llm.providers.openai.base import BaseOpenAIProvider -from openai import AsyncOpenAI -from pydantic import BaseModel +from any_llm.types.completion import ChatCompletionMessage +from openai import AsyncOpenAI, BadRequestError +from pydantic import BaseModel, ValidationError from weather_briefing.api_client import LoggedAsyncClient from weather_briefing.llm import ( AnyLLMStructuredProvider, + FallbackLLMProvider, LazyServiceStatusLLM, + LLMRequestError, LLMStructuredOutput, create_any_llm_provider, ) @@ -31,7 +37,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, str]], + messages: list[dict[str, Any] | ChatCompletionMessage], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -48,6 +54,14 @@ async def acompletion( return self._response +def _openai_bad_request(response: httpx.Response) -> Exception: + return BadRequestError("Upstream request failed", response=response, body={"error": "upstream"}) + + +def _anthropic_bad_request(response: httpx.Response) -> Exception: + return AnthropicBadRequestError("Upstream request failed", response=response, body={"error": "upstream"}) + + async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -176,6 +190,136 @@ async def test_any_llm_provider_assesses_notification_value_with_a_narrow_schema assert client.calls[0]["max_tokens"] == 256 +@pytest.mark.parametrize( + ("provider_name", "error_factory", "operation", "args", "message"), + ( + ( + "openai", + _openai_bad_request, + "summarize", + ("Return JSON", {"input": "data"}), + "LLM request failed", + ), + ( + "anthropic", + _anthropic_bad_request, + "summarize", + ("Return JSON", {"input": "data"}), + "LLM request failed", + ), + ( + "openai", + _openai_bad_request, + "assess_notification", + ({"current": {"status": "operational"}},), + "LLM notification decision request failed", + ), + ( + "openai", + _openai_bad_request, + "translate_service_status", + ("Incident", "Elevated errors", "en"), + "LLM translation request failed", + ), + ), +) +async def test_factory_normalizes_provider_native_request_errors( + monkeypatch, + provider_name: str, + error_factory: Callable[[httpx.Response], Exception], + operation: str, + args: tuple[object, ...], + message: str, +) -> None: + request = httpx.Request("POST", "https://api.example.invalid/chat/completions") + response = httpx.Response(400, request=request) + error = error_factory(response) + client = AsyncMock(spec=AnyLLM) + client.acompletion.side_effect = error + monkeypatch.delenv("ANY_LLM_UNIFIED_EXCEPTIONS", raising=False) + monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: client) + provider = create_any_llm_provider(provider_name, "requested-model", 4096) + + with pytest.raises(LLMRequestError, match=f"^{message}$") as exc_info: + await getattr(provider, operation)(*args) + + assert exc_info.value.__cause__ is error + assert "ANY_LLM_UNIFIED_EXCEPTIONS" not in os.environ + + +async def test_factory_preserves_sdk_output_validation_errors(monkeypatch) -> None: + with pytest.raises(ValidationError) as validation: + LLMStructuredOutput.model_validate({}) + client = AsyncMock(spec=AnyLLM) + client.acompletion.side_effect = validation.value + monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: client) + provider = create_any_llm_provider("openai", "requested-model", 4096) + + with pytest.raises(ValidationError) as propagated: + await provider.summarize("Return JSON", {"input": "data"}) + + assert propagated.value is validation.value + + +async def test_any_llm_client_does_not_mask_payload_serialization_errors() -> None: + client = AsyncMock(spec=AnyLLM) + provider = AnyLLMStructuredProvider( + client, + provider="openai", + model="requested-model", + max_output_tokens=4096, + ) + + with pytest.raises(TypeError, match="not JSON serializable"): + await provider.summarize("Return JSON", {"input": object()}) + + client.acompletion.assert_not_awaited() + + +async def test_provider_native_request_error_switches_to_fallback(monkeypatch) -> None: + request = httpx.Request("POST", "https://api.example.invalid/chat/completions") + response = httpx.Response(400, request=request) + error = BadRequestError( + "Upstream request failed", + response=response, + body={"error": "upstream"}, + ) + primary_client = AsyncMock(spec=AnyLLM) + primary_client.acompletion.side_effect = error + monkeypatch.delenv("ANY_LLM_UNIFIED_EXCEPTIONS", raising=False) + monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: primary_client) + fallback_result = { + "headline": "Fallback briefing", + "headline_source_ids": ["source"], + "conclusions": [], + "active_warnings": [], + "resolved_warning_ids": [], + "advice": [], + "disaster_tracking": [], + "should_publish": True, + } + fallback_client = _CompletionClientStub( + SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=json.dumps(fallback_result)))]) + ) + provider = FallbackLLMProvider( + create_any_llm_provider("openai", "primary-model", 4096), + AnyLLMStructuredProvider( + fallback_client, + provider="openai", + model="fallback-model", + max_output_tokens=4096, + ), + primary_name="openai/primary-model", + fallback_name="openai/fallback-model", + ) + + result = await provider.summarize("Return JSON", {"input": "data"}) + + assert result == fallback_result + primary_client.acompletion.assert_awaited_once() + assert len(fallback_client.calls) == 1 + + async def test_factory_accepts_every_any_llm_completion_provider(monkeypatch) -> None: created: list[tuple[str, dict[str, object]]] = [] diff --git a/tests/test_llm.py b/tests/test_llm.py index e940b577..0ece603a 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -7,7 +7,7 @@ import pendulum import pytest from any_llm.exceptions import LengthFinishReasonError, ProviderError -from any_llm.types.completion import ParsedChatCompletion +from any_llm.types.completion import ChatCompletionMessage, ParsedChatCompletion from pydantic import BaseModel from weather_briefing.llm import ( @@ -36,7 +36,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, str]], + messages: list[dict[str, Any] | ChatCompletionMessage], response_format: type[BaseModel], temperature: float, max_tokens: int, diff --git a/tests/test_llm_fallback.py b/tests/test_llm_fallback.py index 04ddb052..096b04d7 100644 --- a/tests/test_llm_fallback.py +++ b/tests/test_llm_fallback.py @@ -163,8 +163,14 @@ async def test_output_contract_failure_does_not_use_fallback() -> None: async def test_fallback_failure_preserves_primary_as_context() -> None: primary = _provider() fallback = _provider() - primary.summarize.side_effect = LLMRequestError("primary unavailable") - fallback.summarize.side_effect = LLMRequestError("fallback unavailable") + primary_cause = TimeoutError("primary unavailable") + primary_error = LLMRequestError("primary request failed") + primary_error.__cause__ = primary_cause + fallback_cause = ConnectionError("fallback unavailable") + fallback_error = LLMRequestError("fallback request failed") + fallback_error.__cause__ = fallback_cause + primary.summarize.side_effect = primary_error + fallback.summarize.side_effect = fallback_error provider = FallbackLLMProvider( primary, fallback, @@ -172,11 +178,13 @@ async def test_fallback_failure_preserves_primary_as_context() -> None: fallback_name="fallback", ) - with pytest.raises(LLMRequestError, match="fallback unavailable") as exc_info: + with pytest.raises(LLMRequestError, match="fallback request failed") as exc_info: await provider.summarize("system", {"input": "value"}) + assert exc_info.value.__cause__ is fallback_cause assert isinstance(exc_info.value.__context__, LLMRequestError) - assert str(exc_info.value.__context__) == "primary unavailable" + assert str(exc_info.value.__context__) == "primary request failed" + assert exc_info.value.__context__.__cause__ is primary_cause async def test_fallback_log_excludes_exception_details(caplog) -> None: diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index e00cd40a..0c4197eb 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -3,13 +3,15 @@ from __future__ import annotations import logging -from collections.abc import Mapping +from collections.abc import Iterator, Mapping +from contextlib import contextmanager from inspect import isawaitable -from typing import Protocol +from typing import Any, Protocol from any_llm import AnyLLM from any_llm.exceptions import AnyLLMError, LengthFinishReasonError -from pydantic import BaseModel +from any_llm.types.completion import ChatCompletionMessage +from pydantic import BaseModel, ValidationError from ..api_client import api_call_context from ..data.any_llm_compatibility import UNSUPPORTED_DEFAULT_HEADER_PROVIDERS @@ -35,7 +37,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, str]], + messages: list[dict[str, Any] | ChatCompletionMessage], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -44,6 +46,24 @@ async def acompletion( ... +@contextmanager +def _normalize_request_errors( + client: AnyLLM | LLMCompletionClient, + message: str, +) -> Iterator[None]: + """Normalize request failures only for any-llm SDK clients.""" + try: + yield + except (LengthFinishReasonError, ValidationError): + raise + except AnyLLMError as exc: + raise LLMRequestError(message) from exc + except Exception as exc: + if isinstance(client, AnyLLM): + raise LLMRequestError(message) from exc + raise + + class AnyLLMStructuredProvider: """Adapt an any-llm provider to the application's structured LLM boundary.""" @@ -87,14 +107,18 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic system_prompt, payload, ) + messages: list[dict[str, Any] | ChatCompletionMessage] = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": serialize_llm_payload(payload)}, + ] try: - with api_call_context(self._provider, "chat-completions"): + with ( + _normalize_request_errors(self._client, "LLM request failed"), + api_call_context(self._provider, "chat-completions"), + ): response = await self._client.acompletion( model=self._model, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": serialize_llm_payload(payload)}, - ], + messages=messages, response_format=LLMStructuredOutput, temperature=0.2, max_tokens=self._max_output_tokens, @@ -108,8 +132,6 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic type(exc).__name__, ) raise LLMOutputLimitError("LLM response reached output token limit") from exc - except AnyLLMError as exc: - raise LLMRequestError("LLM request failed") from exc result_payload = decode_structured_response(response).model_dump(mode="json") if log_sensitive: _LOGGER.debug( @@ -122,17 +144,21 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic async def assess_notification(self, payload: dict[str, object]) -> NotificationDecision: """Evaluate notification value independently from content generation.""" + messages: list[dict[str, Any] | ChatCompletionMessage] = [ + { + "role": "system", + "content": (f"{NOTIFICATION_POLICY}\n根据输入返回 should_notify。只返回请求的 JSON 对象。"), + }, + {"role": "user", "content": serialize_llm_payload(payload)}, + ] try: - with api_call_context(self._provider, "chat-completions"): + with ( + _normalize_request_errors(self._client, "LLM notification decision request failed"), + api_call_context(self._provider, "chat-completions"), + ): response = await self._client.acompletion( model=self._model, - messages=[ - { - "role": "system", - "content": (f"{NOTIFICATION_POLICY}\n根据输入返回 should_notify。只返回请求的 JSON 对象。"), - }, - {"role": "user", "content": serialize_llm_payload(payload)}, - ], + messages=messages, response_format=NotificationDecisionOutput, temperature=0.0, max_tokens=min(self._max_output_tokens, 256), @@ -147,8 +173,6 @@ async def assess_notification(self, payload: dict[str, object]) -> NotificationD type(exc).__name__, ) raise LLMOutputLimitError("LLM notification decision reached output token limit") from exc - except AnyLLMError as exc: - raise LLMRequestError("LLM notification decision request failed") from exc return NotificationDecision(should_notify=decode_notification_decision(response)) async def translate_service_status( @@ -165,24 +189,28 @@ async def translate_service_status( }.get(target_language) if language_name is None: raise ValueError(f"Unsupported service-status translation language: {target_language}") + messages: list[dict[str, Any] | ChatCompletionMessage] = [ + { + "role": "system", + "content": ( + f"Translate the official service-incident explanation into concise {language_name}. " + "Preserve product names, incident facts, status, times, and technical terms. " + "Do not add analysis, advice, or facts. Return only the requested JSON object." + ), + }, + { + "role": "user", + "content": serialize_llm_payload({"title": title, "body": body}), + }, + ] try: - with api_call_context(self._provider, "chat-completions"): + with ( + _normalize_request_errors(self._client, "LLM translation request failed"), + api_call_context(self._provider, "chat-completions"), + ): response = await self._client.acompletion( model=self._model, - messages=[ - { - "role": "system", - "content": ( - f"Translate the official service-incident explanation into concise {language_name}. " - "Preserve product names, incident facts, status, times, and technical terms. " - "Do not add analysis, advice, or facts. Return only the requested JSON object." - ), - }, - { - "role": "user", - "content": serialize_llm_payload({"title": title, "body": body}), - }, - ], + messages=messages, response_format=ServiceStatusTranslationOutput, temperature=0.0, max_tokens=min(self._max_output_tokens, 2048), @@ -196,8 +224,6 @@ async def translate_service_status( type(exc).__name__, ) raise LLMOutputLimitError("LLM translation reached output token limit") from exc - except AnyLLMError as exc: - raise LLMRequestError("LLM translation request failed") from exc translated = decode_service_status_translation(response) return translated.title, translated.body From 26d5701705315302b3ebea5bf63593b8101723f8 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:03:30 +0800 Subject: [PATCH 02/10] fix(llm): narrow native error normalization --- docs/notes.md | 2 +- tests/test_any_llm_provider.py | 53 +++++++++++++++++++++++++++++++++ weather_briefing/llm/any_llm.py | 18 ++++++++++- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/notes.md b/docs/notes.md index bb572ddc..a2eb14c7 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -58,7 +58,7 @@ 这是一个有意保留的自定义外部服务集成。选择 `FallbackLLMProvider` 的原因是 any-llm 只统一调用单个 provider，不编排跨 provider 的故障切换。包装器捕获主适配器的 `LLMRequestError`，切换后在剩余生命周期内固定使用备用适配器，使同一进程里的契约修复不会回到刚刚失败的主服务。 -适配器把真实 AnyLLM provider client 在 completion 调用中抛出的厂商原生异常归一化为 `LLMRequestError`，使配置正确的 fallback 能够启动。结构化输出的 Pydantic 校验错误和直接注入的协议实现所抛出的编程错误继续原样传播。 +适配器把真实 AnyLLM provider client 在 completion 调用中抛出的 HTTP 传输错误，以及带请求或响应状态的厂商原生错误归一化为 `LLMRequestError`，使配置正确的 fallback 能够启动。结构化输出的 Pydantic 校验错误和 SDK 或直接注入协议实现所抛出的编程错误继续原样传播。 它替代的是每个调用点手写的故障切换分支，不替代 any-llm 的厂商适配器，也不接管 SDK 凭据、请求重试或输出验证。 diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 4e9adda2..8d8dd497 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -62,6 +62,20 @@ def _anthropic_bad_request(response: httpx.Response) -> Exception: return AnthropicBadRequestError("Upstream request failed", response=response, body={"error": "upstream"}) +def _httpx_connection_error(response: httpx.Response) -> Exception: + return httpx.ConnectError("Upstream connection failed", request=response.request) + + +class _ProviderStatusError(Exception): + def __init__(self, response: httpx.Response) -> None: + super().__init__("Upstream request failed") + self.response = response + + +def _provider_status_error(response: httpx.Response) -> Exception: + return _ProviderStatusError(response) + + async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -207,6 +221,20 @@ async def test_any_llm_provider_assesses_notification_value_with_a_narrow_schema ("Return JSON", {"input": "data"}), "LLM request failed", ), + ( + "openrouter", + _httpx_connection_error, + "summarize", + ("Return JSON", {"input": "data"}), + "LLM request failed", + ), + ( + "gemini", + _provider_status_error, + "summarize", + ("Return JSON", {"input": "data"}), + "LLM request failed", + ), ( "openai", _openai_bad_request, @@ -276,6 +304,31 @@ async def test_any_llm_client_does_not_mask_payload_serialization_errors() -> No client.acompletion.assert_not_awaited() +async def test_completion_programming_error_does_not_switch_to_fallback(monkeypatch) -> None: + error = TypeError("SDK programming failure") + primary_client = AsyncMock(spec=AnyLLM) + primary_client.acompletion.side_effect = error + monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: primary_client) + fallback_client = AsyncMock(spec=AnyLLM) + provider = FallbackLLMProvider( + create_any_llm_provider("openai", "primary-model", 4096), + AnyLLMStructuredProvider( + fallback_client, + provider="anthropic", + model="fallback-model", + max_output_tokens=4096, + ), + primary_name="openai/primary-model", + fallback_name="anthropic/fallback-model", + ) + + with pytest.raises(TypeError, match="SDK programming failure") as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value is error + fallback_client.acompletion.assert_not_awaited() + + async def test_provider_native_request_error_switches_to_fallback(monkeypatch) -> None: request = httpx.Request("POST", "https://api.example.invalid/chat/completions") response = httpx.Response(400, request=request) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index 0c4197eb..c56bf411 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -8,6 +8,7 @@ from inspect import isawaitable from typing import Any, Protocol +import httpx from any_llm import AnyLLM from any_llm.exceptions import AnyLLMError, LengthFinishReasonError from any_llm.types.completion import ChatCompletionMessage @@ -30,6 +31,21 @@ _LOGGER = logging.getLogger("weather_briefing.llm") +def _has_http_status(value: object) -> bool: + """Return whether an exception or response exposes a concrete HTTP status.""" + return any(isinstance(getattr(value, name, None), int) for name in ("status_code", "status", "code")) + + +def _is_provider_request_error(exc: Exception) -> bool: + """Recognize transport and provider errors without binding to each vendor SDK.""" + if isinstance(exc, httpx.HTTPError): + return True + if isinstance(getattr(exc, "request", None), httpx.Request): + return True + response = getattr(exc, "response", None) + return response is not None and (_has_http_status(response) or _has_http_status(exc)) + + class LLMCompletionClient(Protocol): """Expose the any-llm completion operation used by the application adapter.""" @@ -59,7 +75,7 @@ def _normalize_request_errors( except AnyLLMError as exc: raise LLMRequestError(message) from exc except Exception as exc: - if isinstance(client, AnyLLM): + if isinstance(client, AnyLLM) and _is_provider_request_error(exc): raise LLMRequestError(message) from exc raise From 1ebc5f5891591b0460b68ef37483a7a409cdebe6 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:15:39 +0800 Subject: [PATCH 03/10] fix(llm): normalize protocol client request errors --- tests/test_any_llm_provider.py | 18 ++++++++++++++++++ weather_briefing/llm/any_llm.py | 15 ++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 8d8dd497..f6b7ca40 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -304,6 +304,24 @@ async def test_any_llm_client_does_not_mask_payload_serialization_errors() -> No client.acompletion.assert_not_awaited() +async def test_protocol_client_normalizes_transport_errors() -> None: + request = httpx.Request("POST", "https://api.example.invalid/chat/completions") + error = httpx.ConnectError("Upstream connection failed", request=request) + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + ) + + with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value.__cause__ is error + + async def test_completion_programming_error_does_not_switch_to_fallback(monkeypatch) -> None: error = TypeError("SDK programming failure") primary_client = AsyncMock(spec=AnyLLM) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index c56bf411..95a1e06c 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -63,11 +63,8 @@ async def acompletion( @contextmanager -def _normalize_request_errors( - client: AnyLLM | LLMCompletionClient, - message: str, -) -> Iterator[None]: - """Normalize request failures only for any-llm SDK clients.""" +def _normalize_request_errors(message: str) -> Iterator[None]: + """Normalize recognized request failures at the completion boundary.""" try: yield except (LengthFinishReasonError, ValidationError): @@ -75,7 +72,7 @@ def _normalize_request_errors( except AnyLLMError as exc: raise LLMRequestError(message) from exc except Exception as exc: - if isinstance(client, AnyLLM) and _is_provider_request_error(exc): + if _is_provider_request_error(exc): raise LLMRequestError(message) from exc raise @@ -129,7 +126,7 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic ] try: with ( - _normalize_request_errors(self._client, "LLM request failed"), + _normalize_request_errors("LLM request failed"), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( @@ -169,7 +166,7 @@ async def assess_notification(self, payload: dict[str, object]) -> NotificationD ] try: with ( - _normalize_request_errors(self._client, "LLM notification decision request failed"), + _normalize_request_errors("LLM notification decision request failed"), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( @@ -221,7 +218,7 @@ async def translate_service_status( ] try: with ( - _normalize_request_errors(self._client, "LLM translation request failed"), + _normalize_request_errors("LLM translation request failed"), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( From 681514f4da722efdce89b48bd4979e9a0ec4a189 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:24:21 +0800 Subject: [PATCH 04/10] fix(llm): recognize request metadata errors --- tests/test_any_llm_provider.py | 23 +++++++++++++++++++++++ weather_briefing/llm/any_llm.py | 12 +++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index f6b7ca40..755514a3 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -76,6 +76,12 @@ def _provider_status_error(response: httpx.Response) -> Exception: return _ProviderStatusError(response) +class _RequestMetadataTransportError(Exception): + def __init__(self) -> None: + super().__init__("Upstream connection failed") + self.request = SimpleNamespace(method="POST", url="https://api.example.invalid/chat/completions") + + async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -322,6 +328,23 @@ async def test_protocol_client_normalizes_transport_errors() -> None: assert exc_info.value.__cause__ is error +async def test_protocol_client_normalizes_request_metadata_errors() -> None: + error = _RequestMetadataTransportError() + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + ) + + with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value.__cause__ is error + + async def test_completion_programming_error_does_not_switch_to_fallback(monkeypatch) -> None: error = TypeError("SDK programming failure") primary_client = AsyncMock(spec=AnyLLM) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index 95a1e06c..5a647e67 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -36,11 +36,21 @@ def _has_http_status(value: object) -> bool: return any(isinstance(getattr(value, name, None), int) for name in ("status_code", "status", "code")) +def _is_request_context(value: object) -> bool: + """Recognize request metadata shared by common HTTP client libraries.""" + return isinstance(getattr(value, "method", None), str) and any( + getattr(value, name, None) is not None for name in ("url", "real_url") + ) + + def _is_provider_request_error(exc: Exception) -> bool: """Recognize transport and provider errors without binding to each vendor SDK.""" if isinstance(exc, httpx.HTTPError): return True - if isinstance(getattr(exc, "request", None), httpx.Request): + request = getattr(exc, "request", None) + if request is None: + request = getattr(exc, "request_info", None) + if request is not None and _is_request_context(request): return True response = getattr(exc, "response", None) return response is not None and (_has_http_status(response) or _has_http_status(exc)) From d7ebed68fb79509623bed90d3c9ea2a889f259ca Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:31:32 +0800 Subject: [PATCH 05/10] fix(llm): gate native error normalization explicitly --- tests/test_any_llm_provider.py | 64 ++++++++++++++++++++++++++++++++- weather_briefing/llm/any_llm.py | 37 ++++++++++++++----- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 755514a3..62f374af 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -77,9 +77,15 @@ def _provider_status_error(response: httpx.Response) -> Exception: class _RequestMetadataTransportError(Exception): + def __init__(self, method: str = "POST") -> None: + super().__init__("Upstream connection failed") + self.request = SimpleNamespace(method=method, url="https://api.example.invalid/chat/completions") + + +class _RequestInfoTransportError(Exception): def __init__(self) -> None: super().__init__("Upstream connection failed") - self.request = SimpleNamespace(method="POST", url="https://api.example.invalid/chat/completions") + self.request_info = SimpleNamespace(method="GET", real_url="https://api.example.invalid/models") async def test_service_status_llm_is_created_only_on_first_operation() -> None: @@ -320,6 +326,7 @@ async def test_protocol_client_normalizes_transport_errors() -> None: provider="wrapped-provider", model="requested-model", max_output_tokens=4096, + normalize_native_errors=True, ) with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: @@ -337,6 +344,25 @@ async def test_protocol_client_normalizes_request_metadata_errors() -> None: provider="wrapped-provider", model="requested-model", max_output_tokens=4096, + normalize_native_errors=True, + ) + + with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value.__cause__ is error + + +async def test_protocol_client_normalizes_request_info_errors() -> None: + error = _RequestInfoTransportError() + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + normalize_native_errors=True, ) with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: @@ -345,6 +371,42 @@ async def test_protocol_client_normalizes_request_metadata_errors() -> None: assert exc_info.value.__cause__ is error +async def test_protocol_client_preserves_non_http_request_metadata() -> None: + error = _RequestMetadataTransportError(method="FETCH") + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + normalize_native_errors=True, + ) + + with pytest.raises(_RequestMetadataTransportError) as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value is error + + +async def test_protocol_client_preserves_native_errors_without_opt_in() -> None: + request = httpx.Request("POST", "https://api.example.invalid/chat/completions") + error = httpx.ConnectError("Upstream connection failed", request=request) + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + ) + + with pytest.raises(httpx.ConnectError) as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value is error + + async def test_completion_programming_error_does_not_switch_to_fallback(monkeypatch) -> None: error = TypeError("SDK programming failure") primary_client = AsyncMock(spec=AnyLLM) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index 5a647e67..a2a9eefc 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -29,6 +29,7 @@ ) _LOGGER = logging.getLogger("weather_briefing.llm") +_HTTP_METHODS = frozenset({"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"}) def _has_http_status(value: object) -> bool: @@ -38,9 +39,13 @@ def _has_http_status(value: object) -> bool: def _is_request_context(value: object) -> bool: """Recognize request metadata shared by common HTTP client libraries.""" - return isinstance(getattr(value, "method", None), str) and any( - getattr(value, name, None) is not None for name in ("url", "real_url") - ) + method = getattr(value, "method", None) + if not isinstance(method, str) or method.upper() not in _HTTP_METHODS: + return False + url = getattr(value, "url", None) + if url is None: + url = getattr(value, "real_url", None) + return str(url).lower().startswith(("http://", "https://")) def _is_provider_request_error(exc: Exception) -> bool: @@ -73,7 +78,11 @@ async def acompletion( @contextmanager -def _normalize_request_errors(message: str) -> Iterator[None]: +def _normalize_request_errors( + message: str, + *, + normalize_native_errors: bool, +) -> Iterator[None]: """Normalize recognized request failures at the completion boundary.""" try: yield @@ -82,7 +91,7 @@ def _normalize_request_errors(message: str) -> Iterator[None]: except AnyLLMError as exc: raise LLMRequestError(message) from exc except Exception as exc: - if _is_provider_request_error(exc): + if normalize_native_errors and _is_provider_request_error(exc): raise LLMRequestError(message) from exc raise @@ -99,6 +108,7 @@ def __init__( max_output_tokens: int, diagnostics: SensitiveLLMDiagnostics | None = None, owns_client: bool = False, + normalize_native_errors: bool = False, ) -> None: """Configure a reusable any-llm client and output limit.""" self._client = client @@ -107,6 +117,7 @@ def __init__( self._max_output_tokens = max_output_tokens self._diagnostics = diagnostics self._owns_client = owns_client + self._normalize_native_errors = normalize_native_errors @property def provider(self) -> str: @@ -136,7 +147,10 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic ] try: with ( - _normalize_request_errors("LLM request failed"), + _normalize_request_errors( + "LLM request failed", + normalize_native_errors=self._normalize_native_errors, + ), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( @@ -176,7 +190,10 @@ async def assess_notification(self, payload: dict[str, object]) -> NotificationD ] try: with ( - _normalize_request_errors("LLM notification decision request failed"), + _normalize_request_errors( + "LLM notification decision request failed", + normalize_native_errors=self._normalize_native_errors, + ), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( @@ -228,7 +245,10 @@ async def translate_service_status( ] try: with ( - _normalize_request_errors("LLM translation request failed"), + _normalize_request_errors( + "LLM translation request failed", + normalize_native_errors=self._normalize_native_errors, + ), api_call_context(self._provider, "chat-completions"), ): response = await self._client.acompletion( @@ -331,4 +351,5 @@ def create_any_llm_provider( max_output_tokens=max_output_tokens, diagnostics=diagnostics, owns_client=True, + normalize_native_errors=True, ) From 4da2b8220afa835239266a9044181c6df11fc018 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:37:52 +0800 Subject: [PATCH 06/10] fix(llm): preserve errors when classification fails --- tests/test_any_llm_provider.py | 24 ++++++++++++++++++++++++ weather_briefing/llm/any_llm.py | 21 ++++++++++++--------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 62f374af..c0d75c53 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -88,6 +88,12 @@ def __init__(self) -> None: self.request_info = SimpleNamespace(method="GET", real_url="https://api.example.invalid/models") +class _BrokenRequestMetadataError(Exception): + @property + def request(self) -> object: + raise RuntimeError("broken request metadata") + + async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -389,6 +395,24 @@ async def test_protocol_client_preserves_non_http_request_metadata() -> None: assert exc_info.value is error +async def test_protocol_client_preserves_error_when_metadata_inspection_fails() -> None: + error = _BrokenRequestMetadataError("Upstream connection failed") + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + normalize_native_errors=True, + ) + + with pytest.raises(_BrokenRequestMetadataError) as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value is error + + async def test_protocol_client_preserves_native_errors_without_opt_in() -> None: request = httpx.Request("POST", "https://api.example.invalid/chat/completions") error = httpx.ConnectError("Upstream connection failed", request=request) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index a2a9eefc..483e6ab4 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -50,15 +50,18 @@ def _is_request_context(value: object) -> bool: def _is_provider_request_error(exc: Exception) -> bool: """Recognize transport and provider errors without binding to each vendor SDK.""" - if isinstance(exc, httpx.HTTPError): - return True - request = getattr(exc, "request", None) - if request is None: - request = getattr(exc, "request_info", None) - if request is not None and _is_request_context(request): - return True - response = getattr(exc, "response", None) - return response is not None and (_has_http_status(response) or _has_http_status(exc)) + try: + if isinstance(exc, httpx.HTTPError): + return True + request = getattr(exc, "request", None) + if request is None: + request = getattr(exc, "request_info", None) + if request is not None and _is_request_context(request): + return True + response = getattr(exc, "response", None) + return response is not None and (_has_http_status(response) or _has_http_status(exc)) + except Exception: + return False class LLMCompletionClient(Protocol): From a051749ee71953f75814d28f826b5eb0af5873e5 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:50:04 +0800 Subject: [PATCH 07/10] fix(llm): inspect error metadata independently --- tests/test_any_llm_provider.py | 53 +++++++++++++++++++++++++++++++++ weather_briefing/llm/any_llm.py | 42 +++++++++++++++----------- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index c0d75c53..06e2ad77 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -89,11 +89,26 @@ def __init__(self) -> None: class _BrokenRequestMetadataError(Exception): + def __init__(self, message: str, *, response: object | None = None) -> None: + super().__init__(message) + self.response = response + @property def request(self) -> object: raise RuntimeError("broken request metadata") +class _BrokenURL: + def __str__(self) -> str: + raise RuntimeError("broken URL metadata") + + +class _BrokenURLMetadataError(Exception): + def __init__(self) -> None: + super().__init__("Upstream connection failed") + self.request = SimpleNamespace(method="POST", url=_BrokenURL()) + + async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -413,6 +428,44 @@ async def test_protocol_client_preserves_error_when_metadata_inspection_fails() assert exc_info.value is error +async def test_protocol_client_uses_response_when_request_metadata_fails() -> None: + request = httpx.Request("POST", "https://api.example.invalid/chat/completions") + response = httpx.Response(503, request=request) + error = _BrokenRequestMetadataError("Upstream request failed", response=response) + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + normalize_native_errors=True, + ) + + with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value.__cause__ is error + + +async def test_protocol_client_preserves_error_when_url_conversion_fails() -> None: + error = _BrokenURLMetadataError() + client = AsyncMock() + client.acompletion.side_effect = error + provider = AnyLLMStructuredProvider( + client, + provider="wrapped-provider", + model="requested-model", + max_output_tokens=4096, + normalize_native_errors=True, + ) + + with pytest.raises(_BrokenURLMetadataError) as exc_info: + await provider.summarize("Return JSON", {"input": "data"}) + + assert exc_info.value is error + + async def test_protocol_client_preserves_native_errors_without_opt_in() -> None: request = httpx.Request("POST", "https://api.example.invalid/chat/completions") error = httpx.ConnectError("Upstream connection failed", request=request) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index 483e6ab4..b3490c08 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -32,36 +32,44 @@ _HTTP_METHODS = frozenset({"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"}) +def _read_error_metadata(value: object, name: str) -> object | None: + """Read one optional exception field without masking the active failure.""" + try: + return getattr(value, name, None) + except Exception: + return None + + def _has_http_status(value: object) -> bool: """Return whether an exception or response exposes a concrete HTTP status.""" - return any(isinstance(getattr(value, name, None), int) for name in ("status_code", "status", "code")) + return any(isinstance(_read_error_metadata(value, name), int) for name in ("status_code", "status", "code")) def _is_request_context(value: object) -> bool: """Recognize request metadata shared by common HTTP client libraries.""" - method = getattr(value, "method", None) + method = _read_error_metadata(value, "method") if not isinstance(method, str) or method.upper() not in _HTTP_METHODS: return False - url = getattr(value, "url", None) + url = _read_error_metadata(value, "url") if url is None: - url = getattr(value, "real_url", None) - return str(url).lower().startswith(("http://", "https://")) + url = _read_error_metadata(value, "real_url") + try: + return str(url).lower().startswith(("http://", "https://")) + except Exception: + return False def _is_provider_request_error(exc: Exception) -> bool: """Recognize transport and provider errors without binding to each vendor SDK.""" - try: - if isinstance(exc, httpx.HTTPError): - return True - request = getattr(exc, "request", None) - if request is None: - request = getattr(exc, "request_info", None) - if request is not None and _is_request_context(request): - return True - response = getattr(exc, "response", None) - return response is not None and (_has_http_status(response) or _has_http_status(exc)) - except Exception: - return False + if isinstance(exc, httpx.HTTPError): + return True + request = _read_error_metadata(exc, "request") + if request is None: + request = _read_error_metadata(exc, "request_info") + if request is not None and _is_request_context(request): + return True + response = _read_error_metadata(exc, "response") + return response is not None and (_has_http_status(response) or _has_http_status(exc)) class LLMCompletionClient(Protocol): From b9daec8c7a95020fa32e56f08532567837463592 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:05:41 +0800 Subject: [PATCH 08/10] refactor(llm): share completion request boundary --- weather_briefing/llm/any_llm.py | 87 +++++++++++++++++---------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index b3490c08..fe48ac53 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -135,6 +135,30 @@ def provider(self) -> str: """Return the application-facing provider name used for diagnostics.""" return self._provider + async def _complete( + self, + messages: list[dict[str, Any] | ChatCompletionMessage], + *, + response_format: type[BaseModel], + temperature: float, + max_tokens: int, + request_error_message: str, + ) -> object: + with ( + _normalize_request_errors( + request_error_message, + normalize_native_errors=self._normalize_native_errors, + ), + api_call_context(self._provider, "chat-completions"), + ): + return await self._client.acompletion( + model=self._model, + messages=messages, + response_format=response_format, + temperature=temperature, + max_tokens=max_tokens, + ) + async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dict[str, object]: """Request and decode one structured JSON response.""" log_sensitive = _sensitive_llm_diagnostics_enabled(self._diagnostics) @@ -157,20 +181,13 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic {"role": "user", "content": serialize_llm_payload(payload)}, ] try: - with ( - _normalize_request_errors( - "LLM request failed", - normalize_native_errors=self._normalize_native_errors, - ), - api_call_context(self._provider, "chat-completions"), - ): - response = await self._client.acompletion( - model=self._model, - messages=messages, - response_format=LLMStructuredOutput, - temperature=0.2, - max_tokens=self._max_output_tokens, - ) + response = await self._complete( + messages, + response_format=LLMStructuredOutput, + temperature=0.2, + max_tokens=self._max_output_tokens, + request_error_message="LLM request failed", + ) except LengthFinishReasonError as exc: _LOGGER.warning( "LLM response reached output token limit: provider=%s model=%r max_output_tokens=%d error_type=%s", @@ -200,20 +217,13 @@ async def assess_notification(self, payload: dict[str, object]) -> NotificationD {"role": "user", "content": serialize_llm_payload(payload)}, ] try: - with ( - _normalize_request_errors( - "LLM notification decision request failed", - normalize_native_errors=self._normalize_native_errors, - ), - api_call_context(self._provider, "chat-completions"), - ): - response = await self._client.acompletion( - model=self._model, - messages=messages, - response_format=NotificationDecisionOutput, - temperature=0.0, - max_tokens=min(self._max_output_tokens, 256), - ) + response = await self._complete( + messages, + response_format=NotificationDecisionOutput, + temperature=0.0, + max_tokens=min(self._max_output_tokens, 256), + request_error_message="LLM notification decision request failed", + ) except LengthFinishReasonError as exc: _LOGGER.warning( "LLM notification decision reached output token limit: " @@ -255,20 +265,13 @@ async def translate_service_status( }, ] try: - with ( - _normalize_request_errors( - "LLM translation request failed", - normalize_native_errors=self._normalize_native_errors, - ), - api_call_context(self._provider, "chat-completions"), - ): - response = await self._client.acompletion( - model=self._model, - messages=messages, - response_format=ServiceStatusTranslationOutput, - temperature=0.0, - max_tokens=min(self._max_output_tokens, 2048), - ) + response = await self._complete( + messages, + response_format=ServiceStatusTranslationOutput, + temperature=0.0, + max_tokens=min(self._max_output_tokens, 2048), + request_error_message="LLM translation request failed", + ) except LengthFinishReasonError as exc: _LOGGER.warning( "LLM translation reached output token limit: provider=%s model=%r max_output_tokens=%d error_type=%s", From e9f8a36cd546e475af225eb07f0fdb138aed4348 Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:53:35 +0800 Subject: [PATCH 09/10] refactor(llm): simplify completion error boundary --- docs/notes.md | 2 +- tests/test_any_llm_provider.py | 197 +------------------------------- weather_briefing/llm/any_llm.py | 71 +++--------- 3 files changed, 20 insertions(+), 250 deletions(-) diff --git a/docs/notes.md b/docs/notes.md index a2eb14c7..02aa284f 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -58,7 +58,7 @@ 这是一个有意保留的自定义外部服务集成。选择 `FallbackLLMProvider` 的原因是 any-llm 只统一调用单个 provider，不编排跨 provider 的故障切换。包装器捕获主适配器的 `LLMRequestError`，切换后在剩余生命周期内固定使用备用适配器，使同一进程里的契约修复不会回到刚刚失败的主服务。 -适配器把真实 AnyLLM provider client 在 completion 调用中抛出的 HTTP 传输错误，以及带请求或响应状态的厂商原生错误归一化为 `LLMRequestError`，使配置正确的 fallback 能够启动。结构化输出的 Pydantic 校验错误和 SDK 或直接注入协议实现所抛出的编程错误继续原样传播。 +适配器把应用工厂创建的 AnyLLM provider client 在 completion 调用中抛出的异常归一化为 `LLMRequestError`，使配置正确的 fallback 能够启动。请求消息在进入该边界前完成序列化，因此应用自身的序列化错误不会触发 fallback；结构化输出的 Pydantic 校验错误和直接注入协议实现所抛出的异常继续原样传播。 它替代的是每个调用点手写的故障切换分支，不替代 any-llm 的厂商适配器，也不接管 SDK 凭据、请求重试或输出验证。 diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 06e2ad77..6f82f517 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -3,7 +3,6 @@ import os from collections.abc import Callable, Mapping from types import SimpleNamespace -from typing import Any from unittest.mock import AsyncMock, Mock import httpx @@ -37,7 +36,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, Any] | ChatCompletionMessage], + messages: list[dict[str, object] | ChatCompletionMessage], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -76,39 +75,6 @@ def _provider_status_error(response: httpx.Response) -> Exception: return _ProviderStatusError(response) -class _RequestMetadataTransportError(Exception): - def __init__(self, method: str = "POST") -> None: - super().__init__("Upstream connection failed") - self.request = SimpleNamespace(method=method, url="https://api.example.invalid/chat/completions") - - -class _RequestInfoTransportError(Exception): - def __init__(self) -> None: - super().__init__("Upstream connection failed") - self.request_info = SimpleNamespace(method="GET", real_url="https://api.example.invalid/models") - - -class _BrokenRequestMetadataError(Exception): - def __init__(self, message: str, *, response: object | None = None) -> None: - super().__init__(message) - self.response = response - - @property - def request(self) -> object: - raise RuntimeError("broken request metadata") - - -class _BrokenURL: - def __str__(self) -> str: - raise RuntimeError("broken URL metadata") - - -class _BrokenURLMetadataError(Exception): - def __init__(self) -> None: - super().__init__("Upstream connection failed") - self.request = SimpleNamespace(method="POST", url=_BrokenURL()) - - async def test_service_status_llm_is_created_only_on_first_operation() -> None: provider = AsyncMock() provider.assess_notification.return_value = NotificationDecision(True) @@ -337,45 +303,8 @@ async def test_any_llm_client_does_not_mask_payload_serialization_errors() -> No client.acompletion.assert_not_awaited() -async def test_protocol_client_normalizes_transport_errors() -> None: - request = httpx.Request("POST", "https://api.example.invalid/chat/completions") - error = httpx.ConnectError("Upstream connection failed", request=request) - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value.__cause__ is error - - -async def test_protocol_client_normalizes_request_metadata_errors() -> None: - error = _RequestMetadataTransportError() - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value.__cause__ is error - - -async def test_protocol_client_normalizes_request_info_errors() -> None: - error = _RequestInfoTransportError() +async def test_protocol_client_preserves_completion_errors() -> None: + error = RuntimeError("Injected client failed") client = AsyncMock() client.acompletion.side_effect = error provider = AnyLLMStructuredProvider( @@ -383,130 +312,12 @@ async def test_protocol_client_normalizes_request_info_errors() -> None: provider="wrapped-provider", model="requested-model", max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value.__cause__ is error - - -async def test_protocol_client_preserves_non_http_request_metadata() -> None: - error = _RequestMetadataTransportError(method="FETCH") - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(_RequestMetadataTransportError) as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value is error - - -async def test_protocol_client_preserves_error_when_metadata_inspection_fails() -> None: - error = _BrokenRequestMetadataError("Upstream connection failed") - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(_BrokenRequestMetadataError) as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value is error - - -async def test_protocol_client_uses_response_when_request_metadata_fails() -> None: - request = httpx.Request("POST", "https://api.example.invalid/chat/completions") - response = httpx.Response(503, request=request) - error = _BrokenRequestMetadataError("Upstream request failed", response=response) - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(LLMRequestError, match="^LLM request failed$") as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value.__cause__ is error - - -async def test_protocol_client_preserves_error_when_url_conversion_fails() -> None: - error = _BrokenURLMetadataError() - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - normalize_native_errors=True, - ) - - with pytest.raises(_BrokenURLMetadataError) as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value is error - - -async def test_protocol_client_preserves_native_errors_without_opt_in() -> None: - request = httpx.Request("POST", "https://api.example.invalid/chat/completions") - error = httpx.ConnectError("Upstream connection failed", request=request) - client = AsyncMock() - client.acompletion.side_effect = error - provider = AnyLLMStructuredProvider( - client, - provider="wrapped-provider", - model="requested-model", - max_output_tokens=4096, - ) - - with pytest.raises(httpx.ConnectError) as exc_info: - await provider.summarize("Return JSON", {"input": "data"}) - - assert exc_info.value is error - - -async def test_completion_programming_error_does_not_switch_to_fallback(monkeypatch) -> None: - error = TypeError("SDK programming failure") - primary_client = AsyncMock(spec=AnyLLM) - primary_client.acompletion.side_effect = error - monkeypatch.setattr(AnyLLM, "create", lambda *args, **kwargs: primary_client) - fallback_client = AsyncMock(spec=AnyLLM) - provider = FallbackLLMProvider( - create_any_llm_provider("openai", "primary-model", 4096), - AnyLLMStructuredProvider( - fallback_client, - provider="anthropic", - model="fallback-model", - max_output_tokens=4096, - ), - primary_name="openai/primary-model", - fallback_name="anthropic/fallback-model", ) - with pytest.raises(TypeError, match="SDK programming failure") as exc_info: + with pytest.raises(RuntimeError, match="Injected client failed") as exc_info: await provider.summarize("Return JSON", {"input": "data"}) assert exc_info.value is error - fallback_client.acompletion.assert_not_awaited() async def test_provider_native_request_error_switches_to_fallback(monkeypatch) -> None: diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index fe48ac53..0d9c1c53 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -6,9 +6,8 @@ from collections.abc import Iterator, Mapping from contextlib import contextmanager from inspect import isawaitable -from typing import Any, Protocol +from typing import Any, Protocol, TypeAlias -import httpx from any_llm import AnyLLM from any_llm.exceptions import AnyLLMError, LengthFinishReasonError from any_llm.types.completion import ChatCompletionMessage @@ -29,47 +28,7 @@ ) _LOGGER = logging.getLogger("weather_briefing.llm") -_HTTP_METHODS = frozenset({"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"}) - - -def _read_error_metadata(value: object, name: str) -> object | None: - """Read one optional exception field without masking the active failure.""" - try: - return getattr(value, name, None) - except Exception: - return None - - -def _has_http_status(value: object) -> bool: - """Return whether an exception or response exposes a concrete HTTP status.""" - return any(isinstance(_read_error_metadata(value, name), int) for name in ("status_code", "status", "code")) - - -def _is_request_context(value: object) -> bool: - """Recognize request metadata shared by common HTTP client libraries.""" - method = _read_error_metadata(value, "method") - if not isinstance(method, str) or method.upper() not in _HTTP_METHODS: - return False - url = _read_error_metadata(value, "url") - if url is None: - url = _read_error_metadata(value, "real_url") - try: - return str(url).lower().startswith(("http://", "https://")) - except Exception: - return False - - -def _is_provider_request_error(exc: Exception) -> bool: - """Recognize transport and provider errors without binding to each vendor SDK.""" - if isinstance(exc, httpx.HTTPError): - return True - request = _read_error_metadata(exc, "request") - if request is None: - request = _read_error_metadata(exc, "request_info") - if request is not None and _is_request_context(request): - return True - response = _read_error_metadata(exc, "response") - return response is not None and (_has_http_status(response) or _has_http_status(exc)) +_CompletionMessage: TypeAlias = dict[str, Any] | ChatCompletionMessage class LLMCompletionClient(Protocol): @@ -79,7 +38,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, Any] | ChatCompletionMessage], + messages: list[_CompletionMessage], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -92,9 +51,9 @@ async def acompletion( def _normalize_request_errors( message: str, *, - normalize_native_errors: bool, + normalize_completion_errors: bool, ) -> Iterator[None]: - """Normalize recognized request failures at the completion boundary.""" + """Normalize failures escaping an application-owned completion client.""" try: yield except (LengthFinishReasonError, ValidationError): @@ -102,7 +61,7 @@ def _normalize_request_errors( except AnyLLMError as exc: raise LLMRequestError(message) from exc except Exception as exc: - if normalize_native_errors and _is_provider_request_error(exc): + if normalize_completion_errors: raise LLMRequestError(message) from exc raise @@ -119,7 +78,7 @@ def __init__( max_output_tokens: int, diagnostics: SensitiveLLMDiagnostics | None = None, owns_client: bool = False, - normalize_native_errors: bool = False, + normalize_completion_errors: bool = False, ) -> None: """Configure a reusable any-llm client and output limit.""" self._client = client @@ -128,7 +87,7 @@ def __init__( self._max_output_tokens = max_output_tokens self._diagnostics = diagnostics self._owns_client = owns_client - self._normalize_native_errors = normalize_native_errors + self._normalize_completion_errors = normalize_completion_errors @property def provider(self) -> str: @@ -137,7 +96,7 @@ def provider(self) -> str: async def _complete( self, - messages: list[dict[str, Any] | ChatCompletionMessage], + messages: list[_CompletionMessage], *, response_format: type[BaseModel], temperature: float, @@ -145,11 +104,11 @@ async def _complete( request_error_message: str, ) -> object: with ( + api_call_context(self._provider, "chat-completions"), _normalize_request_errors( request_error_message, - normalize_native_errors=self._normalize_native_errors, + normalize_completion_errors=self._normalize_completion_errors, ), - api_call_context(self._provider, "chat-completions"), ): return await self._client.acompletion( model=self._model, @@ -176,7 +135,7 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic system_prompt, payload, ) - messages: list[dict[str, Any] | ChatCompletionMessage] = [ + messages: list[_CompletionMessage] = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": serialize_llm_payload(payload)}, ] @@ -209,7 +168,7 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic async def assess_notification(self, payload: dict[str, object]) -> NotificationDecision: """Evaluate notification value independently from content generation.""" - messages: list[dict[str, Any] | ChatCompletionMessage] = [ + messages: list[_CompletionMessage] = [ { "role": "system", "content": (f"{NOTIFICATION_POLICY}\n根据输入返回 should_notify。只返回请求的 JSON 对象。"), @@ -250,7 +209,7 @@ async def translate_service_status( }.get(target_language) if language_name is None: raise ValueError(f"Unsupported service-status translation language: {target_language}") - messages: list[dict[str, Any] | ChatCompletionMessage] = [ + messages: list[_CompletionMessage] = [ { "role": "system", "content": ( @@ -365,5 +324,5 @@ def create_any_llm_provider( max_output_tokens=max_output_tokens, diagnostics=diagnostics, owns_client=True, - normalize_native_errors=True, + normalize_completion_errors=True, ) From 99e4751a91089b48b0211c02ce7879eb6a50e5cd Mon Sep 17 00:00:00 2001 From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:13:06 +0800 Subject: [PATCH 10/10] refactor(llm): keep completion messages narrowly typed --- tests/test_any_llm_provider.py | 3 +-- tests/test_llm.py | 7 +++---- weather_briefing/llm/any_llm.py | 16 +++++++--------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/test_any_llm_provider.py b/tests/test_any_llm_provider.py index 6f82f517..859b67b6 100644 --- a/tests/test_any_llm_provider.py +++ b/tests/test_any_llm_provider.py @@ -10,7 +10,6 @@ from anthropic import BadRequestError as AnthropicBadRequestError from any_llm import AnyLLM from any_llm.providers.openai.base import BaseOpenAIProvider -from any_llm.types.completion import ChatCompletionMessage from openai import AsyncOpenAI, BadRequestError from pydantic import BaseModel, ValidationError @@ -36,7 +35,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, object] | ChatCompletionMessage], + messages: list[dict[str, str]], response_format: type[BaseModel], temperature: float, max_tokens: int, diff --git a/tests/test_llm.py b/tests/test_llm.py index 0ece603a..900bb827 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -2,12 +2,11 @@ import logging from copy import deepcopy from types import SimpleNamespace -from typing import Any import pendulum import pytest from any_llm.exceptions import LengthFinishReasonError, ProviderError -from any_llm.types.completion import ChatCompletionMessage, ParsedChatCompletion +from any_llm.types.completion import ParsedChatCompletion from pydantic import BaseModel from weather_briefing.llm import ( @@ -36,7 +35,7 @@ async def acompletion( self, *, model: str, - messages: list[dict[str, Any] | ChatCompletionMessage], + messages: list[dict[str, str]], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -80,7 +79,7 @@ async def test_completion_client_stub_requires_a_configured_response() -> None: ) -def _valid_payload() -> dict[str, Any]: +def _valid_payload() -> dict[str, object]: return { "headline": "Briefing", "headline_source_ids": ["source"], diff --git a/weather_briefing/llm/any_llm.py b/weather_briefing/llm/any_llm.py index 0d9c1c53..0afd445f 100644 --- a/weather_briefing/llm/any_llm.py +++ b/weather_briefing/llm/any_llm.py @@ -6,11 +6,10 @@ from collections.abc import Iterator, Mapping from contextlib import contextmanager from inspect import isawaitable -from typing import Any, Protocol, TypeAlias +from typing import Protocol from any_llm import AnyLLM from any_llm.exceptions import AnyLLMError, LengthFinishReasonError -from any_llm.types.completion import ChatCompletionMessage from pydantic import BaseModel, ValidationError from ..api_client import api_call_context @@ -28,7 +27,6 @@ ) _LOGGER = logging.getLogger("weather_briefing.llm") -_CompletionMessage: TypeAlias = dict[str, Any] | ChatCompletionMessage class LLMCompletionClient(Protocol): @@ -38,7 +36,7 @@ async def acompletion( self, *, model: str, - messages: list[_CompletionMessage], + messages: list[dict[str, str]], response_format: type[BaseModel], temperature: float, max_tokens: int, @@ -96,7 +94,7 @@ def provider(self) -> str: async def _complete( self, - messages: list[_CompletionMessage], + messages: list[dict[str, str]], *, response_format: type[BaseModel], temperature: float, @@ -112,7 +110,7 @@ async def _complete( ): return await self._client.acompletion( model=self._model, - messages=messages, + messages=[*messages], response_format=response_format, temperature=temperature, max_tokens=max_tokens, @@ -135,7 +133,7 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic system_prompt, payload, ) - messages: list[_CompletionMessage] = [ + messages: list[dict[str, str]] = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": serialize_llm_payload(payload)}, ] @@ -168,7 +166,7 @@ async def summarize(self, system_prompt: str, payload: dict[str, object]) -> dic async def assess_notification(self, payload: dict[str, object]) -> NotificationDecision: """Evaluate notification value independently from content generation.""" - messages: list[_CompletionMessage] = [ + messages: list[dict[str, str]] = [ { "role": "system", "content": (f"{NOTIFICATION_POLICY}\n根据输入返回 should_notify。只返回请求的 JSON 对象。"), @@ -209,7 +207,7 @@ async def translate_service_status( }.get(target_language) if language_name is None: raise ValueError(f"Unsupported service-status translation language: {target_language}") - messages: list[_CompletionMessage] = [ + messages: list[dict[str, str]] = [ { "role": "system", "content": (