From 5c92daecc0e60df28bfecf66d96cd3509505af76 Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 11:08:31 -0700 Subject: [PATCH 1/9] Merge model settings into fallback request parameters --- pydantic_ai_slim/pydantic_ai/models/fallback.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/fallback.py b/pydantic_ai_slim/pydantic_ai/models/fallback.py index 498d8e1bd4..82f8784798 100644 --- a/pydantic_ai_slim/pydantic_ai/models/fallback.py +++ b/pydantic_ai_slim/pydantic_ai/models/fallback.py @@ -65,8 +65,9 @@ async def request( for model in self.models: customized_model_request_parameters = model.customize_request_parameters(model_request_parameters) + merged_settings = merge_model_settings(model.settings, model_settings) try: - response = await model.request(messages, model_settings, customized_model_request_parameters) + response = await model.request(messages, merged_settings, customized_model_request_parameters) except Exception as exc: if self._fallback_on(exc): exceptions.append(exc) @@ -91,10 +92,11 @@ async def request_stream( for model in self.models: customized_model_request_parameters = model.customize_request_parameters(model_request_parameters) + merged_settings = merge_model_settings(model.settings, model_settings) async with AsyncExitStack() as stack: try: response = await stack.enter_async_context( - model.request_stream(messages, model_settings, customized_model_request_parameters, run_context) + model.request_stream(messages, merged_settings, customized_model_request_parameters, run_context) ) except Exception as exc: if self._fallback_on(exc): From 43834c4fc642a0f89403b49649e67a5d1636b9d0 Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 11:12:33 -0700 Subject: [PATCH 2/9] Fix imports --- pydantic_ai_slim/pydantic_ai/models/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/fallback.py b/pydantic_ai_slim/pydantic_ai/models/fallback.py index 82f8784798..4bca34aabb 100644 --- a/pydantic_ai_slim/pydantic_ai/models/fallback.py +++ b/pydantic_ai_slim/pydantic_ai/models/fallback.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from ..messages import ModelMessage, ModelResponse - from ..settings import ModelSettings + from ..settings import ModelSettings, merge_model_settings @dataclass(init=False) From 249583c33f4ffc9061390ca358ba693ec42f4a7a Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 11:16:58 -0700 Subject: [PATCH 3/9] Fix formatting --- pydantic_ai_slim/pydantic_ai/models/fallback.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/fallback.py b/pydantic_ai_slim/pydantic_ai/models/fallback.py index 4bca34aabb..9066071cf9 100644 --- a/pydantic_ai_slim/pydantic_ai/models/fallback.py +++ b/pydantic_ai_slim/pydantic_ai/models/fallback.py @@ -96,7 +96,9 @@ async def request_stream( async with AsyncExitStack() as stack: try: response = await stack.enter_async_context( - model.request_stream(messages, merged_settings, customized_model_request_parameters, run_context) + model.request_stream( + messages, merged_settings, customized_model_request_parameters, run_context + ) ) except Exception as exc: if self._fallback_on(exc): From eb305c9c331b468d07d4f4c4cbe9dee6e1b8f09f Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 11:21:18 -0700 Subject: [PATCH 4/9] Fix imports --- pydantic_ai_slim/pydantic_ai/models/fallback.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/fallback.py b/pydantic_ai_slim/pydantic_ai/models/fallback.py index 9066071cf9..fd4bf8c6ac 100644 --- a/pydantic_ai_slim/pydantic_ai/models/fallback.py +++ b/pydantic_ai_slim/pydantic_ai/models/fallback.py @@ -11,11 +11,12 @@ from pydantic_ai.models.instrumented import InstrumentedModel from ..exceptions import FallbackExceptionGroup, ModelHTTPError +from ..settings import merge_model_settings from . import KnownModelName, Model, ModelRequestParameters, StreamedResponse, infer_model if TYPE_CHECKING: from ..messages import ModelMessage, ModelResponse - from ..settings import ModelSettings, merge_model_settings + from ..settings import ModelSettings @dataclass(init=False) From e361b5684240e70c9993ce753e88a505eed96b5c Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 12:14:09 -0700 Subject: [PATCH 5/9] Add tests --- tests/test_agent.py | 68 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/test_agent.py b/tests/test_agent.py index bbe91d74c4..03f12996ec 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -2550,6 +2550,74 @@ def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: assert (await my_agent.run('Hello', model_settings={'temperature': 0.5})).output == IsJson({'temperature': 0.5}) +async def test_fallback_model_settings_merge(): + """Test that FallbackModel properly merges model settings from wrapped model and runtime settings.""" + from pydantic_ai.models.fallback import FallbackModel + + def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) + + base_model = FunctionModel(return_settings, settings={'temperature': 0.1, 'base_setting': 'base_value'}) + fallback_model = FallbackModel(base_model) + + # Test that base model settings are preserved when no additional settings are provided + agent = Agent(fallback_model) + result = await agent.run('Hello') + assert result.output == IsJson({'base_setting': 'base_value', 'temperature': 0.1}) + + # Test that runtime model_settings are merged with base settings + agent_with_settings = Agent(fallback_model, model_settings={'temperature': 0.5, 'new_setting': 'new_value'}) + result = await agent_with_settings.run('Hello') + expected = {'base_setting': 'base_value', 'temperature': 0.5, 'new_setting': 'new_value'} + assert result.output == IsJson(expected) + + # Test that run-time model_settings override both base and agent settings + result = await agent_with_settings.run( + 'Hello', model_settings={'temperature': 0.9, 'runtime_setting': 'runtime_value'} + ) + expected = { + 'base_setting': 'base_value', + 'temperature': 0.9, + 'new_setting': 'new_value', + 'runtime_setting': 'runtime_value', + } + assert result.output == IsJson(expected) + + +async def test_fallback_model_settings_merge_streaming(): + """Test that FallbackModel properly merges model settings in streaming mode.""" + from pydantic_ai.models.fallback import FallbackModel + + def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) + + async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): + # Yield the merged settings as JSON to verify they were properly combined + yield to_json(info.model_settings).decode() + + base_model = FunctionModel( + return_settings, + stream_function=return_settings_stream, + settings={'base_setting': 'base_value', 'temperature': 0.1}, + ) + fallback_model = FallbackModel(base_model) + + # Test that base model settings are preserved in streaming mode + agent = Agent(fallback_model) + async with agent.run_stream('Hello') as result: + output = await result.get_output() + + assert json.loads(output) == {'base_setting': 'base_value', 'temperature': 0.1} + + # Test that runtime model_settings are merged with base settings in streaming mode + agent_with_settings = Agent(fallback_model, model_settings={'temperature': 0.5, 'new_setting': 'new_value'}) + async with agent_with_settings.run_stream('Hello') as result: + output = await result.get_output() + + expected = {'base_setting': 'base_value', 'temperature': 0.5, 'new_setting': 'new_value'} + assert json.loads(output) == expected + + async def test_empty_text_part(): def return_empty_text(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: assert info.output_tools is not None From 0b226eff44208f700b44076496d028f325870cf8 Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 12:26:30 -0700 Subject: [PATCH 6/9] Address CI issues --- tests/test_agent.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/test_agent.py b/tests/test_agent.py index 03f12996ec..baca4e7f42 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -49,6 +49,7 @@ from pydantic_ai.output import StructuredDict, ToolOutput from pydantic_ai.profiles import ModelProfile from pydantic_ai.result import Usage +from pydantic_ai.settings import ModelSettings from pydantic_ai.tools import ToolDefinition from pydantic_ai.toolsets.abstract import AbstractToolset from pydantic_ai.toolsets.combined import CombinedToolset @@ -2557,29 +2558,31 @@ async def test_fallback_model_settings_merge(): def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) - base_model = FunctionModel(return_settings, settings={'temperature': 0.1, 'base_setting': 'base_value'}) + base_model = FunctionModel(return_settings, settings=ModelSettings(temperature=0.1, max_tokens=1024)) fallback_model = FallbackModel(base_model) # Test that base model settings are preserved when no additional settings are provided agent = Agent(fallback_model) result = await agent.run('Hello') - assert result.output == IsJson({'base_setting': 'base_value', 'temperature': 0.1}) + assert result.output == IsJson({'max_tokens': 1024, 'temperature': 0.1}) # Test that runtime model_settings are merged with base settings - agent_with_settings = Agent(fallback_model, model_settings={'temperature': 0.5, 'new_setting': 'new_value'}) + agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5, parallel_tool_calls=True)) result = await agent_with_settings.run('Hello') - expected = {'base_setting': 'base_value', 'temperature': 0.5, 'new_setting': 'new_value'} + expected = {'max_tokens': 1024, 'temperature': 0.5, 'parallel_tool_calls': True} assert result.output == IsJson(expected) # Test that run-time model_settings override both base and agent settings result = await agent_with_settings.run( - 'Hello', model_settings={'temperature': 0.9, 'runtime_setting': 'runtime_value'} + 'Hello', model_settings=ModelSettings(temperature=0.9, extra_headers={'runtime_setting': 'runtime_value'}) ) expected = { - 'base_setting': 'base_value', + 'max_tokens': 1024, 'temperature': 0.9, - 'new_setting': 'new_value', - 'runtime_setting': 'runtime_value', + 'parallel_tool_calls': True, + 'extra_headers': { + 'runtime_setting': 'runtime_value', + } } assert result.output == IsJson(expected) @@ -2598,7 +2601,7 @@ async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): base_model = FunctionModel( return_settings, stream_function=return_settings_stream, - settings={'base_setting': 'base_value', 'temperature': 0.1}, + settings=ModelSettings(temperature=0.1, extra_headers={'anthropic-beta': 'context-1m-2025-08-07'}), ) fallback_model = FallbackModel(base_model) @@ -2607,14 +2610,24 @@ async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): async with agent.run_stream('Hello') as result: output = await result.get_output() - assert json.loads(output) == {'base_setting': 'base_value', 'temperature': 0.1} + assert json.loads(output) == { + 'extra_headers': { + 'anthropic-beta': 'context-1m-2025-08-07' + }, + 'temperature': 0.1 + } # Test that runtime model_settings are merged with base settings in streaming mode - agent_with_settings = Agent(fallback_model, model_settings={'temperature': 0.5, 'new_setting': 'new_value'}) + agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5)) async with agent_with_settings.run_stream('Hello') as result: output = await result.get_output() - expected = {'base_setting': 'base_value', 'temperature': 0.5, 'new_setting': 'new_value'} + expected = { + 'extra_headers': { + 'anthropic-beta': 'context-1m-2025-08-07' + }, + 'temperature': 0.5 + } assert json.loads(output) == expected From 9013d69a704636d47f1347bd699309c9e64f10c5 Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 12:33:17 -0700 Subject: [PATCH 7/9] Fix remaining CI errors --- tests/test_agent.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/test_agent.py b/tests/test_agent.py index baca4e7f42..d6d90bac27 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -2582,7 +2582,7 @@ def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: 'parallel_tool_calls': True, 'extra_headers': { 'runtime_setting': 'runtime_value', - } + }, } assert result.output == IsJson(expected) @@ -2591,15 +2591,11 @@ async def test_fallback_model_settings_merge_streaming(): """Test that FallbackModel properly merges model settings in streaming mode.""" from pydantic_ai.models.fallback import FallbackModel - def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: - return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) - async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): # Yield the merged settings as JSON to verify they were properly combined yield to_json(info.model_settings).decode() base_model = FunctionModel( - return_settings, stream_function=return_settings_stream, settings=ModelSettings(temperature=0.1, extra_headers={'anthropic-beta': 'context-1m-2025-08-07'}), ) @@ -2610,24 +2606,14 @@ async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): async with agent.run_stream('Hello') as result: output = await result.get_output() - assert json.loads(output) == { - 'extra_headers': { - 'anthropic-beta': 'context-1m-2025-08-07' - }, - 'temperature': 0.1 - } + assert json.loads(output) == {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.1} # Test that runtime model_settings are merged with base settings in streaming mode agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5)) async with agent_with_settings.run_stream('Hello') as result: output = await result.get_output() - expected = { - 'extra_headers': { - 'anthropic-beta': 'context-1m-2025-08-07' - }, - 'temperature': 0.5 - } + expected = {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.5} assert json.loads(output) == expected From 01311f5d834be31b6b3877d4f62a0bcba2ea2ca7 Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 13:11:53 -0700 Subject: [PATCH 8/9] Move tests to `test_fallback` --- tests/models/test_fallback.py | 68 +++++++++++++++++++++++++++++++++++ tests/test_agent.py | 66 ---------------------------------- 2 files changed, 68 insertions(+), 66 deletions(-) diff --git a/tests/models/test_fallback.py b/tests/models/test_fallback.py index 597f34ddce..4e66ce6726 100644 --- a/tests/models/test_fallback.py +++ b/tests/models/test_fallback.py @@ -1,17 +1,21 @@ from __future__ import annotations +import json import sys from collections.abc import AsyncIterator from datetime import timezone from typing import Any import pytest +from dirty_equals import IsJson from inline_snapshot import snapshot +from pydantic_core import to_json from pydantic_ai import Agent, ModelHTTPError from pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse, TextPart, UserPromptPart from pydantic_ai.models.fallback import FallbackModel from pydantic_ai.models.function import AgentInfo, FunctionModel +from pydantic_ai.settings import ModelSettings from pydantic_ai.usage import Usage from ..conftest import IsNow, try_import @@ -445,3 +449,67 @@ async def test_fallback_condition_tuple() -> None: response = await agent.run('hello') assert response.output == 'success' + + +async def test_fallback_model_settings_merge(): + """Test that FallbackModel properly merges model settings from wrapped model and runtime settings.""" + + def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) + + base_model = FunctionModel(return_settings, settings=ModelSettings(temperature=0.1, max_tokens=1024)) + fallback_model = FallbackModel(base_model) + + # Test that base model settings are preserved when no additional settings are provided + agent = Agent(fallback_model) + result = await agent.run('Hello') + assert result.output == IsJson({'max_tokens': 1024, 'temperature': 0.1}) + + # Test that runtime model_settings are merged with base settings + agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5, parallel_tool_calls=True)) + result = await agent_with_settings.run('Hello') + expected = {'max_tokens': 1024, 'temperature': 0.5, 'parallel_tool_calls': True} + assert result.output == IsJson(expected) + + # Test that run-time model_settings override both base and agent settings + result = await agent_with_settings.run( + 'Hello', model_settings=ModelSettings(temperature=0.9, extra_headers={'runtime_setting': 'runtime_value'}) + ) + expected = { + 'max_tokens': 1024, + 'temperature': 0.9, + 'parallel_tool_calls': True, + 'extra_headers': { + 'runtime_setting': 'runtime_value', + }, + } + assert result.output == IsJson(expected) + + +async def test_fallback_model_settings_merge_streaming(): + """Test that FallbackModel properly merges model settings in streaming mode.""" + + async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): + # Yield the merged settings as JSON to verify they were properly combined + yield to_json(info.model_settings).decode() + + base_model = FunctionModel( + stream_function=return_settings_stream, + settings=ModelSettings(temperature=0.1, extra_headers={'anthropic-beta': 'context-1m-2025-08-07'}), + ) + fallback_model = FallbackModel(base_model) + + # Test that base model settings are preserved in streaming mode + agent = Agent(fallback_model) + async with agent.run_stream('Hello') as result: + output = await result.get_output() + + assert json.loads(output) == {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.1} + + # Test that runtime model_settings are merged with base settings in streaming mode + agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5)) + async with agent_with_settings.run_stream('Hello') as result: + output = await result.get_output() + + expected = {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.5} + assert json.loads(output) == expected diff --git a/tests/test_agent.py b/tests/test_agent.py index d6d90bac27..56e4122575 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -2551,72 +2551,6 @@ def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: assert (await my_agent.run('Hello', model_settings={'temperature': 0.5})).output == IsJson({'temperature': 0.5}) -async def test_fallback_model_settings_merge(): - """Test that FallbackModel properly merges model settings from wrapped model and runtime settings.""" - from pydantic_ai.models.fallback import FallbackModel - - def return_settings(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: - return ModelResponse(parts=[TextPart(to_json(info.model_settings).decode())]) - - base_model = FunctionModel(return_settings, settings=ModelSettings(temperature=0.1, max_tokens=1024)) - fallback_model = FallbackModel(base_model) - - # Test that base model settings are preserved when no additional settings are provided - agent = Agent(fallback_model) - result = await agent.run('Hello') - assert result.output == IsJson({'max_tokens': 1024, 'temperature': 0.1}) - - # Test that runtime model_settings are merged with base settings - agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5, parallel_tool_calls=True)) - result = await agent_with_settings.run('Hello') - expected = {'max_tokens': 1024, 'temperature': 0.5, 'parallel_tool_calls': True} - assert result.output == IsJson(expected) - - # Test that run-time model_settings override both base and agent settings - result = await agent_with_settings.run( - 'Hello', model_settings=ModelSettings(temperature=0.9, extra_headers={'runtime_setting': 'runtime_value'}) - ) - expected = { - 'max_tokens': 1024, - 'temperature': 0.9, - 'parallel_tool_calls': True, - 'extra_headers': { - 'runtime_setting': 'runtime_value', - }, - } - assert result.output == IsJson(expected) - - -async def test_fallback_model_settings_merge_streaming(): - """Test that FallbackModel properly merges model settings in streaming mode.""" - from pydantic_ai.models.fallback import FallbackModel - - async def return_settings_stream(_: list[ModelMessage], info: AgentInfo): - # Yield the merged settings as JSON to verify they were properly combined - yield to_json(info.model_settings).decode() - - base_model = FunctionModel( - stream_function=return_settings_stream, - settings=ModelSettings(temperature=0.1, extra_headers={'anthropic-beta': 'context-1m-2025-08-07'}), - ) - fallback_model = FallbackModel(base_model) - - # Test that base model settings are preserved in streaming mode - agent = Agent(fallback_model) - async with agent.run_stream('Hello') as result: - output = await result.get_output() - - assert json.loads(output) == {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.1} - - # Test that runtime model_settings are merged with base settings in streaming mode - agent_with_settings = Agent(fallback_model, model_settings=ModelSettings(temperature=0.5)) - async with agent_with_settings.run_stream('Hello') as result: - output = await result.get_output() - - expected = {'extra_headers': {'anthropic-beta': 'context-1m-2025-08-07'}, 'temperature': 0.5} - assert json.loads(output) == expected - - async def test_empty_text_part(): def return_empty_text(_: list[ModelMessage], info: AgentInfo) -> ModelResponse: assert info.output_tools is not None From c1c37d1c506e468ac3cd7478764ff1029d26554d Mon Sep 17 00:00:00 2001 From: Jerry Lin Date: Wed, 13 Aug 2025 13:15:44 -0700 Subject: [PATCH 9/9] Cleanup --- tests/test_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_agent.py b/tests/test_agent.py index 56e4122575..bbe91d74c4 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -49,7 +49,6 @@ from pydantic_ai.output import StructuredDict, ToolOutput from pydantic_ai.profiles import ModelProfile from pydantic_ai.result import Usage -from pydantic_ai.settings import ModelSettings from pydantic_ai.tools import ToolDefinition from pydantic_ai.toolsets.abstract import AbstractToolset from pydantic_ai.toolsets.combined import CombinedToolset