Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 47 additions & 30 deletions litellm/integrations/websearch_interception/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1031,36 +1031,12 @@ async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchRespons
)
llm_router = None

# Determine search provider from router's search_tools
search_tool = self._select_search_tool_from_router(llm_router=llm_router)
search_provider: Optional[str] = None
if llm_router is not None and hasattr(llm_router, "search_tools"):
if self.search_tool_name:
# Find specific search tool by name
matching_tools = [
tool
for tool in llm_router.search_tools
if tool.get("search_tool_name") == self.search_tool_name
]
if matching_tools:
search_tool = matching_tools[0]
search_provider = search_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"with provider '{search_provider}'"
)
else:
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
"falling back to first available or perplexity"
)

# If no specific tool or not found, use first available
if not search_provider and llm_router.search_tools:
first_tool = llm_router.search_tools[0]
search_provider = first_tool.get("litellm_params", {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
)
search_litellm_params: Dict[str, Any] = {}
if search_tool is not None:
search_litellm_params = dict(search_tool.get("litellm_params", {}) or {})
search_provider = search_litellm_params.get("search_provider")

# Fallback to perplexity if no router or no search tools configured
if not search_provider:
Expand All @@ -1073,7 +1049,12 @@ async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchRespons
verbose_logger.debug(
f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
)
result = await litellm.asearch(query=query, search_provider=search_provider)
search_kwargs = {
key: value
for key, value in search_litellm_params.items()
if key != "search_provider" and value is not None
}
result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs)
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved

# Format using transformation function
search_result_text = WebSearchTransformation.format_search_response(result)
Expand All @@ -1086,6 +1067,42 @@ async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchRespons
verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}")
raise

def _select_search_tool_from_router(self, llm_router: Any) -> Optional[Dict[str, Any]]:
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools = list(getattr(llm_router, "search_tools") or [])
return self._select_search_tool_from_list(search_tools=search_tools, source="router")

def _select_search_tool_from_list(
self,
search_tools: List[Dict[str, Any]],
source: str,
) -> Optional[Dict[str, Any]]:
if self.search_tool_name:
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
if matching_tools:
search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
f"from {source} with provider '{search_provider}'"
)
return matching_tools[0]
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, "
"falling back to first available or perplexity"
)

if search_tools:
first_tool = search_tools[0]
search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider")
verbose_logger.debug(
f"WebSearchInterception: Using first available search tool from {source} "
f"with provider '{search_provider}'"
)
return first_tool

return None

async def _execute_chat_completion_agentic_loop(
self,
model: str,
Expand Down
55 changes: 38 additions & 17 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -6382,7 +6382,6 @@ async def _init_agents_in_db(self, prisma_client: PrismaClient):
async def _init_search_tools_in_db(self, prisma_client: PrismaClient):
"""
Initialize search tools from database into the router on startup.
Only updates router if there are tools in the database, otherwise preserves config-loaded tools.
"""
global llm_router

Expand All @@ -6392,33 +6391,55 @@ async def _init_search_tools_in_db(self, prisma_client: PrismaClient):
from litellm.router_utils.search_api_router import SearchAPIRouter

try:
search_tools = await SearchToolRegistry.get_all_search_tools_from_db(prisma_client=prisma_client)
db_search_tools = await SearchToolRegistry.get_all_search_tools_from_db(prisma_client=prisma_client)

verbose_proxy_logger.info(f"Loading {len(search_tools)} search tool(s) from database into router")
config_search_tools = []
try:
config = await self.get_config()
parsed_tools = self.parse_search_tools(config)
if parsed_tools:
config_search_tools = parsed_tools
except Exception as e:
verbose_proxy_logger.debug(f"Could not get config-defined search tools: {e}")
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve config search tools when config reload fails

In the DB-enabled sync path (add_deployment -> _init_non_llm_objects_in_db), if get_config() raises here, config_search_tools stays empty and the later update_router_search_tools(...) call overwrites llm_router.search_tools with only DB tools, or an empty list when the DB has none. For proxies that already loaded search tools from config, a transient remote config read failure or missing config file during this periodic sync will evict the working config-defined search tools instead of preserving the last known router state.

Useful? React with 👍 / 👎.


# Only update router if there are tools in the database
# This prevents overwriting config-loaded tools with an empty list
if len(search_tools) > 0:
if llm_router is not None:
# Add search tools to the router
await SearchAPIRouter.update_router_search_tools(
router_instance=llm_router, search_tools=search_tools
)
verbose_proxy_logger.info(f"Successfully loaded {len(search_tools)} search tool(s) into router")
else:
verbose_proxy_logger.debug(
"Router not initialized yet, search tools will be added when router is created"
)
search_tools = self._merge_config_and_db_search_tools(
config_search_tools=config_search_tools,
db_search_tools=[dict(tool) for tool in db_search_tools],
)

verbose_proxy_logger.info(
f"Loading {len(search_tools)} search tool(s) into router "
f"({len(config_search_tools)} from config, {len(db_search_tools)} from database)"
)

if llm_router is not None:
await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools)
verbose_proxy_logger.info(f"Successfully loaded {len(search_tools)} search tool(s) into router")
else:
verbose_proxy_logger.debug(
"No search tools found in database, keeping config-loaded search tools (if any)"
"Router not initialized yet, search tools will be added when router is created"
)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

except Exception as e:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.py::ProxyConfig:_init_search_tools_in_db - {}".format(str(e))
)

@staticmethod
def _merge_config_and_db_search_tools(
config_search_tools: List[SearchToolTypedDict],
db_search_tools: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
db_tool_names = {tool.get("search_tool_name") for tool in db_search_tools}
return [
*[
dict(config_search_tool)
for config_search_tool in config_search_tools
if config_search_tool.get("search_tool_name") not in db_tool_names
],
*db_search_tools,
]

async def _init_pass_through_endpoints_in_db(self):
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
initialize_pass_through_endpoints_in_db,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from litellm.integrations.websearch_interception.handler import (
WebSearchInterceptionLogger,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.utils import LlmProviders


Expand Down Expand Up @@ -56,9 +57,7 @@ def test_initialize_from_proxy_config_honors_dict_callback_specific_params():
"""A valid dict under callback_settings.websearch_interception is applied."""
logger = WebSearchInterceptionLogger.initialize_from_proxy_config(
litellm_settings={},
callback_specific_params={
"websearch_interception": {"search_tool_name": "ws-tool"}
},
callback_specific_params={"websearch_interception": {"search_tool_name": "ws-tool"}},
)

assert logger.search_tool_name == "ws-tool"
Expand Down Expand Up @@ -119,9 +118,7 @@ async def test_async_build_agentic_loop_plan_returns_request_patch():
"response_format": "anthropic",
}
logging_obj = MagicMock()
logging_obj.model_call_details = {
"agentic_loop_params": {"model": "bedrock/invoke/claude-3-5-sonnet"}
}
logging_obj.model_call_details = {"agentic_loop_params": {"model": "bedrock/invoke/claude-3-5-sonnet"}}
kwargs = {
"temperature": 0.2,
"_websearch_interception_converted_stream": True,
Expand Down Expand Up @@ -162,8 +159,6 @@ async def test_internal_flags_filtered_from_followup_kwargs():
to the follow-up LLM request, causing "Extra inputs are not permitted" errors
from providers like Bedrock that use strict parameter validation.
"""
logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])

# Simulate kwargs that would be passed during agentic loop execution
kwargs_with_internal_flags = {
"_websearch_interception_converted_stream": True,
Expand All @@ -174,9 +169,7 @@ async def test_internal_flags_filtered_from_followup_kwargs():

# Apply the same filtering logic used in _execute_agentic_loop
kwargs_for_followup = {
k: v
for k, v in kwargs_with_internal_flags.items()
if not k.startswith("_websearch_interception")
k: v for k, v in kwargs_with_internal_flags.items() if not k.startswith("_websearch_interception")
}

# Verify internal flags are filtered out
Expand All @@ -188,6 +181,46 @@ async def test_internal_flags_filtered_from_followup_kwargs():
assert kwargs_for_followup["max_tokens"] == 1024


@pytest.mark.asyncio
async def test_execute_search_passes_selected_search_tool_litellm_params(monkeypatch):
import litellm
from litellm.proxy import proxy_server

logger = WebSearchInterceptionLogger(
enabled_providers=["bedrock"],
search_tool_name="ui-tavily",
)
router = MagicMock()
router.search_tools = [
{
"search_tool_name": "ui-tavily",
"litellm_params": {
"search_provider": "tavily",
"api_key": "fake-ui-key",
"api_base": "https://api.tavily.com",
"timeout": 10.0,
"max_retries": 2,
"country": None,
},
}
]
mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[]))

monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(litellm, "asearch", mock_asearch)

await logger._execute_search("what is litellm")

mock_asearch.assert_awaited_once_with(
query="what is litellm",
search_provider="tavily",
api_key="fake-ui-key",
api_base="https://api.tavily.com",
timeout=10.0,
max_retries=2,
)


@pytest.mark.asyncio
async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs():
"""Test that async_pre_call_deployment_hook finds custom_llm_provider at top-level kwargs.
Expand Down Expand Up @@ -216,15 +249,12 @@ async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs():
assert result is not None
# The web_search tool should be converted to litellm_web_search (OpenAI format)
assert any(
t.get("type") == "function"
and t.get("function", {}).get("name") == "litellm_web_search"
t.get("type") == "function" and t.get("function", {}).get("name") == "litellm_web_search"
for t in result["tools"]
)
# The non-web-search tool should be preserved
assert any(
t.get("type") == "function"
and t.get("function", {}).get("name") == "other_tool"
for t in result["tools"]
t.get("type") == "function" and t.get("function", {}).get("name") == "other_tool" for t in result["tools"]
)


Expand Down Expand Up @@ -261,8 +291,7 @@ async def test_async_pre_call_deployment_hook_returns_full_kwargs():
assert result["custom_llm_provider"] == "openai"
# Tools should be converted
assert any(
t.get("type") == "function"
and t.get("function", {}).get("name") == "litellm_web_search"
t.get("type") == "function" and t.get("function", {}).get("name") == "litellm_web_search"
for t in result["tools"]
)

Expand Down Expand Up @@ -323,8 +352,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback():

assert result is not None
assert any(
t.get("type") == "function"
and t.get("function", {}).get("name") == "litellm_web_search"
t.get("type") == "function" and t.get("function", {}).get("name") == "litellm_web_search"
for t in result["tools"]
)
# Full kwargs preserved
Expand Down Expand Up @@ -357,8 +385,7 @@ async def test_async_pre_call_deployment_hook_provider_derived_from_model_name()
# Should NOT be None — the hook should derive "openai" from "openai/gpt-4o-mini"
assert result is not None
assert any(
t.get("type") == "function"
and t.get("function", {}).get("name") == "litellm_web_search"
t.get("type") == "function" and t.get("function", {}).get("name") == "litellm_web_search"
for t in result["tools"]
)
# Full kwargs preserved
Expand Down Expand Up @@ -478,9 +505,7 @@ def test_sync_forced_tool_choice_leaves_non_forced_untouched(tool_choice):
and None pass through unchanged."""
converted_tools = [{"name": LITELLM_WEB_SEARCH_TOOL_NAME}]

result = WebSearchInterceptionLogger._sync_forced_tool_choice(
tool_choice, converted_tools
)
result = WebSearchInterceptionLogger._sync_forced_tool_choice(tool_choice, converted_tools)

assert result == tool_choice

Expand Down
Loading
Loading