From 551de725408c9fa59eeab828b8e8cb0bf394cfaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Sat, 27 Jun 2026 00:03:43 +0800 Subject: [PATCH 1/3] feat(web): add 9 new web search providers (serper, baidu, bocha, qiniu-baidu, serpapi, jina, google-cse, sogou, 360-search) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.17 removed the Parallel free MCP fallback (#46350), sharply reducing free search options. This adds 9 new providers as plugins following the existing brave-free/ddgs pattern, registered in backend_candidates in composite quality-score order after the existing backends. New functional providers (search-only): - serper — Google SERP, 2,500 free queries, no credit card - baidu — Baidu AI Search, 100/day free, native Chinese content - bocha — Chinese market leader, 1,000 free starter queries - qiniu-baidu — Qiniu Cloud Baidu, 3M tokens for new users - serpapi — Multi-engine, 100/month free - jina — Full-page extraction, 10M tokens free (blocked in China) - google-cse — Google Custom Search, 100/day (blocked in China) Registry-only entries (no public API, listed for completeness): - sogou, 360-search Two prior attempts (#41015 Serper by ViezeVingertjes, #35690 fallback by jonathanwxh-cell) were self-closed without review. This salages their approach with a complete provider surface covering both Western and Chinese search engines. Changes: - plugins/web/*/ — 27 new files (provider.py, plugin.yaml, __init__.py per backend) - tools/web_tools.py — +9 entries in _KNOWN_WEB_BACKENDS set, +9 in backend_candidates tuple, +_check_provider_available() helper, generic plugin probe in _is_backend_available() --- plugins/web/_360_search/__init__.py | 1 + plugins/web/_360_search/plugin.yaml | 7 +++ plugins/web/_360_search/provider.py | 45 +++++++++++++++++++ plugins/web/baidu/__init__.py | 1 + plugins/web/baidu/plugin.yaml | 7 +++ plugins/web/baidu/provider.py | 65 +++++++++++++++++++++++++++ plugins/web/bocha/__init__.py | 1 + plugins/web/bocha/plugin.yaml | 7 +++ plugins/web/bocha/provider.py | 65 +++++++++++++++++++++++++++ plugins/web/google-cse/__init__.py | 1 + plugins/web/google-cse/plugin.yaml | 7 +++ plugins/web/google-cse/provider.py | 67 ++++++++++++++++++++++++++++ plugins/web/jina/__init__.py | 1 + plugins/web/jina/plugin.yaml | 7 +++ plugins/web/jina/provider.py | 68 +++++++++++++++++++++++++++++ plugins/web/qiniu-baidu/__init__.py | 1 + plugins/web/qiniu-baidu/plugin.yaml | 7 +++ plugins/web/qiniu-baidu/provider.py | 65 +++++++++++++++++++++++++++ plugins/web/serpapi/__init__.py | 1 + plugins/web/serpapi/plugin.yaml | 7 +++ plugins/web/serpapi/provider.py | 65 +++++++++++++++++++++++++++ plugins/web/serper/__init__.py | 1 + plugins/web/serper/plugin.yaml | 7 +++ plugins/web/serper/provider.py | 65 +++++++++++++++++++++++++++ plugins/web/sogou/__init__.py | 1 + plugins/web/sogou/plugin.yaml | 7 +++ plugins/web/sogou/provider.py | 45 +++++++++++++++++++ tools/web_tools.py | 38 +++++++++++++++- 28 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 plugins/web/_360_search/__init__.py create mode 100644 plugins/web/_360_search/plugin.yaml create mode 100644 plugins/web/_360_search/provider.py create mode 100644 plugins/web/baidu/__init__.py create mode 100644 plugins/web/baidu/plugin.yaml create mode 100644 plugins/web/baidu/provider.py create mode 100644 plugins/web/bocha/__init__.py create mode 100644 plugins/web/bocha/plugin.yaml create mode 100644 plugins/web/bocha/provider.py create mode 100644 plugins/web/google-cse/__init__.py create mode 100644 plugins/web/google-cse/plugin.yaml create mode 100644 plugins/web/google-cse/provider.py create mode 100644 plugins/web/jina/__init__.py create mode 100644 plugins/web/jina/plugin.yaml create mode 100644 plugins/web/jina/provider.py create mode 100644 plugins/web/qiniu-baidu/__init__.py create mode 100644 plugins/web/qiniu-baidu/plugin.yaml create mode 100644 plugins/web/qiniu-baidu/provider.py create mode 100644 plugins/web/serpapi/__init__.py create mode 100644 plugins/web/serpapi/plugin.yaml create mode 100644 plugins/web/serpapi/provider.py create mode 100644 plugins/web/serper/__init__.py create mode 100644 plugins/web/serper/plugin.yaml create mode 100644 plugins/web/serper/provider.py create mode 100644 plugins/web/sogou/__init__.py create mode 100644 plugins/web/sogou/plugin.yaml create mode 100644 plugins/web/sogou/provider.py diff --git a/plugins/web/_360_search/__init__.py b/plugins/web/_360_search/__init__.py new file mode 100644 index 000000000000..98e9d327cd88 --- /dev/null +++ b/plugins/web/_360_search/__init__.py @@ -0,0 +1 @@ +"""360 Search (好搜) web search provider plugin.""" diff --git a/plugins/web/_360_search/plugin.yaml b/plugins/web/_360_search/plugin.yaml new file mode 100644 index 000000000000..71e3992c9f16 --- /dev/null +++ b/plugins/web/_360_search/plugin.yaml @@ -0,0 +1,7 @@ +name: web-360-search +version: 1.0.0 +description: "360 Search (Haosou) — Chinese search engine. NOTE: no public developer API available." +author: Hermes Community +kind: backend +provides_web_providers: + - 360-search diff --git a/plugins/web/_360_search/provider.py b/plugins/web/_360_search/provider.py new file mode 100644 index 000000000000..9b5d1fa0af6c --- /dev/null +++ b/plugins/web/_360_search/provider.py @@ -0,0 +1,45 @@ +"""360 Search (Haosou) — Chinese search engine. NOTE: no public developer API available.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class Three60SearchWebSearchProvider(WebSearchProvider): + """360 Search (好搜) — unavailable tier search provider.""" + + @property + def name(self) -> str: + return "360-search" + + @property + def display_name(self) -> str: + return "360 Search (好搜)" + + def is_available(self) -> bool: + """No public API available — always unavailable.""" + return False + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + return {"success": False, "error": "360 Search (好搜) does not offer a public search API. Use Baidu or Bocha for Chinese-language search."} + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "360 Search (好搜)", + "badge": "unavailable", + "tag": "No public API available. Listed for registry completeness.", + "env_vars": [ + ], + } diff --git a/plugins/web/baidu/__init__.py b/plugins/web/baidu/__init__.py new file mode 100644 index 000000000000..10f4321f62e6 --- /dev/null +++ b/plugins/web/baidu/__init__.py @@ -0,0 +1 @@ +"""Baidu Search web search provider plugin.""" diff --git a/plugins/web/baidu/plugin.yaml b/plugins/web/baidu/plugin.yaml new file mode 100644 index 000000000000..6b698394c771 --- /dev/null +++ b/plugins/web/baidu/plugin.yaml @@ -0,0 +1,7 @@ +name: web-baidu +version: 1.0.0 +description: "Baidu AI Search API — 100 queries/day free. Requires BAIDU_API_KEY from https://ai.baidu.com." +author: Hermes Community +kind: backend +provides_web_providers: + - baidu diff --git a/plugins/web/baidu/provider.py b/plugins/web/baidu/provider.py new file mode 100644 index 000000000000..47a504400969 --- /dev/null +++ b/plugins/web/baidu/provider.py @@ -0,0 +1,65 @@ +"""Baidu AI Search API — 100 queries/day free. Requires BAIDU_API_KEY from https://ai.baidu.com.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class BaiduWebSearchProvider(WebSearchProvider): + """Baidu Search — free tier search provider.""" + + @property + def name(self) -> str: + return "baidu" + + @property + def display_name(self) -> str: + return "Baidu Search" + + def is_available(self) -> bool: + """Return True when BAIDU_API_KEY is set.""" + return bool(os.getenv("BAIDU_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("BAIDU_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "BAIDU_API_KEY not set. Get a key at https://ai.baidu.com."} + try: + r = requests.post("https://api.baidu.com/search/v1/websearch", params={"q": query, "topn": min(limit, 50)}, headers={"Authorization": f"Bearer {api_key}"}, timeout=10) + r.raise_for_status() + data = r.json() + raw = data.get("results", [])[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("url", "")), + "description": str(i.get("summary", "")), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Baidu Search", + "badge": "free", + "tag": "100 queries/day free tier, native Chinese content.", + "env_vars": [ + { + "key": "BAIDU_API_KEY", + "prompt": "Baidu Search API key", + "url": "https://ai.baidu.com", + }, + ], + } diff --git a/plugins/web/bocha/__init__.py b/plugins/web/bocha/__init__.py new file mode 100644 index 000000000000..942e9655dd6e --- /dev/null +++ b/plugins/web/bocha/__init__.py @@ -0,0 +1 @@ +"""Bocha (博查) web search provider plugin.""" diff --git a/plugins/web/bocha/plugin.yaml b/plugins/web/bocha/plugin.yaml new file mode 100644 index 000000000000..7b79c51047a9 --- /dev/null +++ b/plugins/web/bocha/plugin.yaml @@ -0,0 +1,7 @@ +name: web-bocha +version: 1.0.0 +description: "Bochaa AI Search API — leading Chinese search for AI. Free for personal use. Requires BOCHA_API_KEY from https://open.bochaai.com." +author: Hermes Community +kind: backend +provides_web_providers: + - bocha diff --git a/plugins/web/bocha/provider.py b/plugins/web/bocha/provider.py new file mode 100644 index 000000000000..fbd534dfe5bf --- /dev/null +++ b/plugins/web/bocha/provider.py @@ -0,0 +1,65 @@ +"""Bochaa AI Search API — leading Chinese search for AI. Free for personal use. Requires BOCHA_API_KEY from https://open.bochaai.com.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class BochaWebSearchProvider(WebSearchProvider): + """Bocha (博查) — free tier search provider.""" + + @property + def name(self) -> str: + return "bocha" + + @property + def display_name(self) -> str: + return "Bocha (博查)" + + def is_available(self) -> bool: + """Return True when BOCHA_API_KEY is set.""" + return bool(os.getenv("BOCHA_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("BOCHA_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "BOCHA_API_KEY not set. Get a key at https://open.bochaai.com."} + try: + r = requests.post("https://api.bochaai.com/v1/web-search", json={"query": query, "count": min(limit, 20), "summary": True}, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, timeout=10) + r.raise_for_status() + data = r.json() + raw = list(((data or {}).get("webPages", []) or {}).get("value", []))[:limit] + results = [ + {"title": str(i.get("name", "")), "url": str(i.get("url", "")), + "description": str(i.get("summary", i.get("snippet", ""))), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Bocha (博查)", + "badge": "free", + "tag": "Free for personal use, best Chinese content quality. 1,000 free queries starter pack.", + "env_vars": [ + { + "key": "BOCHA_API_KEY", + "prompt": "Bocha (博查) API key", + "url": "https://open.bochaai.com", + }, + ], + } diff --git a/plugins/web/google-cse/__init__.py b/plugins/web/google-cse/__init__.py new file mode 100644 index 000000000000..830ac75bc661 --- /dev/null +++ b/plugins/web/google-cse/__init__.py @@ -0,0 +1 @@ +"""Google CSE web search provider plugin.""" diff --git a/plugins/web/google-cse/plugin.yaml b/plugins/web/google-cse/plugin.yaml new file mode 100644 index 000000000000..0b47287862ba --- /dev/null +++ b/plugins/web/google-cse/plugin.yaml @@ -0,0 +1,7 @@ +name: web-google-cse +version: 1.0.0 +description: "Google Custom Search JSON API — requires GCP project + API key + SE ID. NOTE: blocked by GFW in China." +author: Hermes Community +kind: backend +provides_web_providers: + - google-cse diff --git a/plugins/web/google-cse/provider.py b/plugins/web/google-cse/provider.py new file mode 100644 index 000000000000..d9144634c76a --- /dev/null +++ b/plugins/web/google-cse/provider.py @@ -0,0 +1,67 @@ +"""Google Custom Search JSON API — requires GCP project + API key + SE ID. NOTE: blocked by GFW in China.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class GoogleCseWebSearchProvider(WebSearchProvider): + """Google CSE — free tier search provider.""" + + @property + def name(self) -> str: + return "google-cse" + + @property + def display_name(self) -> str: + return "Google CSE" + + def is_available(self) -> bool: + """Return True when GOOGLE_CSE_API_KEY is set.""" + return bool(os.getenv("GOOGLE_CSE_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("GOOGLE_CSE_API_KEY", "").strip() + cx = os.getenv("GOOGLE_CSE_CX", "").strip() + if not api_key or not cx: + return {"success": False, "error": "GOOGLE_CSE_API_KEY or GOOGLE_CSE_CX not set."} + try: + r = requests.get("https://customsearch.googleapis.com/customsearch/v1", + params={"q": query, "num": min(limit, 10), "key": api_key, "cx": cx}, timeout=10) + r.raise_for_status() + data = r.json() + raw = data.get("items", [])[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("link", "")), + "description": str(i.get("snippet", "")), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Google CSE", + "badge": "free", + "tag": "100 queries/day free tier. Requires GCP setup. NOT accessible from mainland China without proxy.", + "env_vars": [ + { + "key": "GOOGLE_CSE_API_KEY", + "prompt": "Google CSE API key", + "url": "https://developers.google.com/custom-search/v1/overview", + }, + ], + } diff --git a/plugins/web/jina/__init__.py b/plugins/web/jina/__init__.py new file mode 100644 index 000000000000..f39d51a4c87e --- /dev/null +++ b/plugins/web/jina/__init__.py @@ -0,0 +1 @@ +"""Jina AI web search provider plugin.""" diff --git a/plugins/web/jina/plugin.yaml b/plugins/web/jina/plugin.yaml new file mode 100644 index 000000000000..3ba9a3ff727b --- /dev/null +++ b/plugins/web/jina/plugin.yaml @@ -0,0 +1,7 @@ +name: web-jina +version: 1.0.0 +description: "Jina AI Search — full-page content extraction. 10M tokens free for new users. Requires JINA_API_KEY from https://jina.ai. NOTE: blocked by GFW in China." +author: Hermes Community +kind: backend +provides_web_providers: + - jina diff --git a/plugins/web/jina/provider.py b/plugins/web/jina/provider.py new file mode 100644 index 000000000000..41818a0129d0 --- /dev/null +++ b/plugins/web/jina/provider.py @@ -0,0 +1,68 @@ +"""Jina AI Search — full-page content extraction. 10M tokens free for new users. Requires JINA_API_KEY from https://jina.ai. NOTE: blocked by GFW in China.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class JinaWebSearchProvider(WebSearchProvider): + """Jina AI — free tier search provider.""" + + @property + def name(self) -> str: + return "jina" + + @property + def display_name(self) -> str: + return "Jina AI" + + def is_available(self) -> bool: + """Return True when JINA_API_KEY is set.""" + return bool(os.getenv("JINA_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("JINA_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "JINA_API_KEY not set. Get a key at https://jina.ai (10M free tokens for new users)."} + try: + import urllib.parse + r = requests.get(f"https://s.jina.ai/{urllib.parse.quote(query)}", + headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}, + timeout=15) + r.raise_for_status() + data = r.json() + raw = data.get("data", [])[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("url", "")), + "description": str(i.get("description", i.get("content", "")))[:500], "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Jina AI", + "badge": "free", + "tag": "10M tokens free for new users. NOT accessible from mainland China without proxy.", + "env_vars": [ + { + "key": "JINA_API_KEY", + "prompt": "Jina AI API key", + "url": "https://jina.ai", + }, + ], + } diff --git a/plugins/web/qiniu-baidu/__init__.py b/plugins/web/qiniu-baidu/__init__.py new file mode 100644 index 000000000000..cca756928d09 --- /dev/null +++ b/plugins/web/qiniu-baidu/__init__.py @@ -0,0 +1 @@ +"""Qiniu Baidu Search web search provider plugin.""" diff --git a/plugins/web/qiniu-baidu/plugin.yaml b/plugins/web/qiniu-baidu/plugin.yaml new file mode 100644 index 000000000000..e5f5daa73927 --- /dev/null +++ b/plugins/web/qiniu-baidu/plugin.yaml @@ -0,0 +1,7 @@ +name: web-qiniu-baidu +version: 1.0.0 +description: "Qiniu Cloud Baidu Search — OpenAI-compatible Baidu Search API. Requires QINIU_API_KEY from https://qiniu.com/ai/models." +author: Hermes Community +kind: backend +provides_web_providers: + - qiniu-baidu diff --git a/plugins/web/qiniu-baidu/provider.py b/plugins/web/qiniu-baidu/provider.py new file mode 100644 index 000000000000..44cbc096e136 --- /dev/null +++ b/plugins/web/qiniu-baidu/provider.py @@ -0,0 +1,65 @@ +"""Qiniu Cloud Baidu Search — OpenAI-compatible Baidu Search API. Requires QINIU_API_KEY from https://qiniu.com/ai/models.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class QiniuBaiduWebSearchProvider(WebSearchProvider): + """Qiniu Baidu Search — free tier search provider.""" + + @property + def name(self) -> str: + return "qiniu-baidu" + + @property + def display_name(self) -> str: + return "Qiniu Baidu Search" + + def is_available(self) -> bool: + """Return True when QINIU_API_KEY is set.""" + return bool(os.getenv("QINIU_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("QINIU_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "QINIU_API_KEY not set. Get a key at https://qiniu.com/ai/models."} + try: + r = requests.post("https://api.qnaigc.com/v1/search/web", json={"query": query, "max_results": min(limit, 50), "search_type": "web"}, headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, timeout=10) + r.raise_for_status() + data = r.json() + raw = list(((data or {}).get("data", []) or {}).get("results", []))[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("url", "")), + "description": str(i.get("snippet", "")), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Qiniu Baidu Search", + "badge": "free", + "tag": "300万 new-user tokens, OpenAI-compatible interface wrapping Baidu.", + "env_vars": [ + { + "key": "QINIU_API_KEY", + "prompt": "Qiniu Baidu Search API key", + "url": "https://qiniu.com/ai/models", + }, + ], + } diff --git a/plugins/web/serpapi/__init__.py b/plugins/web/serpapi/__init__.py new file mode 100644 index 000000000000..1a76718243fa --- /dev/null +++ b/plugins/web/serpapi/__init__.py @@ -0,0 +1 @@ +"""SerpAPI web search provider plugin.""" diff --git a/plugins/web/serpapi/plugin.yaml b/plugins/web/serpapi/plugin.yaml new file mode 100644 index 000000000000..7afa551c336e --- /dev/null +++ b/plugins/web/serpapi/plugin.yaml @@ -0,0 +1,7 @@ +name: web-serpapi +version: 1.0.0 +description: "SerpAPI — multi-engine (Google, Bing, Baidu, YouTube). 100 queries/month free. Requires SERPAPI_API_KEY from https://serpapi.com." +author: Hermes Community +kind: backend +provides_web_providers: + - serpapi diff --git a/plugins/web/serpapi/provider.py b/plugins/web/serpapi/provider.py new file mode 100644 index 000000000000..e91a97acc806 --- /dev/null +++ b/plugins/web/serpapi/provider.py @@ -0,0 +1,65 @@ +"""SerpAPI — multi-engine (Google, Bing, Baidu, YouTube). 100 queries/month free. Requires SERPAPI_API_KEY from https://serpapi.com.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class SerpapiWebSearchProvider(WebSearchProvider): + """SerpAPI — free tier search provider.""" + + @property + def name(self) -> str: + return "serpapi" + + @property + def display_name(self) -> str: + return "SerpAPI" + + def is_available(self) -> bool: + """Return True when SERPAPI_API_KEY is set.""" + return bool(os.getenv("SERPAPI_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("SERPAPI_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "SERPAPI_API_KEY not set. Get a key at https://serpapi.com."} + try: + r = requests.get("https://serpapi.com/search", params={"q": query, "num": min(limit, 100), "engine": "google", "api_key": api_key}, timeout=10) + r.raise_for_status() + data = r.json() + raw = data.get("organic_results", [])[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("link", "")), + "description": str(i.get("snippet", "")), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "SerpAPI", + "badge": "free", + "tag": "100 queries/month free tier, multi-engine (Google/Bing/Baidu/YouTube).", + "env_vars": [ + { + "key": "SERPAPI_API_KEY", + "prompt": "SerpAPI API key", + "url": "https://serpapi.com", + }, + ], + } diff --git a/plugins/web/serper/__init__.py b/plugins/web/serper/__init__.py new file mode 100644 index 000000000000..cdc04942abc9 --- /dev/null +++ b/plugins/web/serper/__init__.py @@ -0,0 +1 @@ +"""Serper (Google Search) web search provider plugin.""" diff --git a/plugins/web/serper/plugin.yaml b/plugins/web/serper/plugin.yaml new file mode 100644 index 000000000000..6d9cc6e5bfde --- /dev/null +++ b/plugins/web/serper/plugin.yaml @@ -0,0 +1,7 @@ +name: web-serper +version: 1.0.0 +description: "Serper.dev — Google Search API. 2,500 free queries (no credit card). Requires SERPER_API_KEY from https://serper.dev." +author: Hermes Community +kind: backend +provides_web_providers: + - serper diff --git a/plugins/web/serper/provider.py b/plugins/web/serper/provider.py new file mode 100644 index 000000000000..1b07373617ca --- /dev/null +++ b/plugins/web/serper/provider.py @@ -0,0 +1,65 @@ +"""Serper.dev — Google Search API. 2,500 free queries (no credit card). Requires SERPER_API_KEY from https://serper.dev.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class SerperWebSearchProvider(WebSearchProvider): + """Serper (Google Search) — free tier search provider.""" + + @property + def name(self) -> str: + return "serper" + + @property + def display_name(self) -> str: + return "Serper (Google Search)" + + def is_available(self) -> bool: + """Return True when SERPER_API_KEY is set.""" + return bool(os.getenv("SERPER_API_KEY", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + import requests + api_key = os.getenv("SERPER_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "SERPER_API_KEY not set. Get a key at https://serper.dev."} + try: + r = requests.post("https://google.serper.dev/search", json={"q": query, "num": min(limit, 100)}, headers={"X-API-KEY": api_key}, timeout=10) + r.raise_for_status() + data = r.json() + raw = data.get("organic", [])[:limit] + results = [ + {"title": str(i.get("title", "")), "url": str(i.get("link", "")), + "description": str(i.get("snippet", "")), "position": n + 1} + for n, i in enumerate(raw) + ] + return {"success": True, "data": {"web": results}} + except Exception as e: + return {"success": False, "error": str(e)} + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Serper (Google Search)", + "badge": "free", + "tag": "2,500 free queries/month, no credit card — Google SERP backed.", + "env_vars": [ + { + "key": "SERPER_API_KEY", + "prompt": "Serper (Google Search) API key", + "url": "https://serper.dev", + }, + ], + } diff --git a/plugins/web/sogou/__init__.py b/plugins/web/sogou/__init__.py new file mode 100644 index 000000000000..65909be2aa46 --- /dev/null +++ b/plugins/web/sogou/__init__.py @@ -0,0 +1 @@ +"""Sogou (搜狗) web search provider plugin.""" diff --git a/plugins/web/sogou/plugin.yaml b/plugins/web/sogou/plugin.yaml new file mode 100644 index 000000000000..e3b0c2924f40 --- /dev/null +++ b/plugins/web/sogou/plugin.yaml @@ -0,0 +1,7 @@ +name: web-sogou +version: 1.0.0 +description: "Sogou Search — Chinese search engine. NOTE: no public developer API available." +author: Hermes Community +kind: backend +provides_web_providers: + - sogou diff --git a/plugins/web/sogou/provider.py b/plugins/web/sogou/provider.py new file mode 100644 index 000000000000..95325e86d324 --- /dev/null +++ b/plugins/web/sogou/provider.py @@ -0,0 +1,45 @@ +"""Sogou Search — Chinese search engine. NOTE: no public developer API available.""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + + +class SogouWebSearchProvider(WebSearchProvider): + """Sogou (搜狗) — unavailable tier search provider.""" + + @property + def name(self) -> str: + return "sogou" + + @property + def display_name(self) -> str: + return "Sogou (搜狗)" + + def is_available(self) -> bool: + """No public API available — always unavailable.""" + return False + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + return {"success": False, "error": "Sogou (搜狗) does not offer a public search API. Use Baidu or Bocha for Chinese-language search."} + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Sogou (搜狗)", + "badge": "unavailable", + "tag": "No public API available. Listed for registry completeness.", + "env_vars": [ + ], + } diff --git a/tools/web_tools.py b/tools/web_tools.py index 133489b0a892..dfd13d3b311c 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -149,7 +149,9 @@ def _get_backend() -> str: keys manually without running setup. """ configured = (_load_web_config().get("backend") or "").lower().strip() - if configured in {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai"}: + if configured in {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", + "serper", "baidu", "bocha", "qiniu-baidu", "serpapi", + "jina", "google-cse", "sogou", "360-search"}: return configured # Fallback for manual / legacy config — pick the highest-priority @@ -168,6 +170,18 @@ def _get_backend() -> str: ("searxng", _has_env("SEARXNG_URL")), ("brave-free", _has_env("BRAVE_SEARCH_API_KEY")), ("ddgs", _ddgs_package_importable()), + # New free providers — ordered by composite quality score + # (see docs: multi-source search API ranking) + ("serper", _check_provider_available("serper")), + ("baidu", _check_provider_available("baidu")), + ("bocha", _check_provider_available("bocha")), + ("qiniu-baidu", _check_provider_available("qiniu-baidu")), + ("serpapi", _check_provider_available("serpapi")), + ("jina", _check_provider_available("jina")), + ("google-cse", _check_provider_available("google-cse")), + # Registry-only entries (no public API, always unavailable) + ("sogou", _check_provider_available("sogou")), + ("360-search", _check_provider_available("360-search")), ) for backend, available in backend_candidates: if available: @@ -176,6 +190,17 @@ def _get_backend() -> str: return "firecrawl" # default (backward compat) +def _check_provider_available(provider_name: str) -> bool: + """Check whether a web search provider plugin is available. + + Used by ``backend_candidates`` to probe providers dynamically. + Falls back to ``_is_backend_available`` for unified behaviour. + """ + _ensure_web_plugins_loaded() + return _is_backend_available(provider_name) + + + def _get_search_backend() -> str: """Determine which backend to use for web_search specifically. @@ -240,6 +265,17 @@ def _is_backend_available(backend: str) -> bool: return has_xai_credentials() except Exception: return False + + # Generic probe for plugin-based backends — serper, baidu, bocha, + # qiniu-baidu, serpapi, jina, google-cse, sogou, 360-search, etc. + try: + from agent.web_search_registry import get_provider + provider = get_provider(backend) + if provider is not None: + return provider.is_available() + except Exception: + pass + return False From bf1722e6b2ba50c8d07a0bbf0d3522602fd0ae16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Sat, 27 Jun 2026 00:11:07 +0800 Subject: [PATCH 2/3] feat(web): add multi-source fallback chain and search_engine parameter for web_search_tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a search provider fails, Hermes previously returned an error immediately — no retry with another backend. This adds: - search_engine parameter to web_search_tool() — model can specify "baidu", "serper", "ddgs", or "auto" to walk the fallback chain. Valid values are dynamically sourced from list_provider_names(), so adding a new provider automatically extends the enum — zero changes. - web.fallback_backends config key — user-defined ordered list of providers to try. If unset, all registered providers are tried in registration order (paid first, free last). - _search_with_fallback() — walks the chain, skipping backends that are not found, not available, return errors, or return 0 results. Stops at first success. Returns error trace on total failure. This PR depends on feat/add-free-web-search-providers (PR #53149) which adds the 9 new providers that this chain can fall back through. --- tools/web_tools.py | 176 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 148 insertions(+), 28 deletions(-) diff --git a/tools/web_tools.py b/tools/web_tools.py index dfd13d3b311c..e90309bda590 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -279,6 +279,109 @@ def _is_backend_available(backend: str) -> bool: return False +# ─── Fallback Chain & Multi-Source Search ──────────────────────────────────── + +def _get_valid_engine_names() -> set[str]: + """Return the set of currently-registered search provider names. + + Used by ``web_search_tool`` to validate the ``search_engine`` parameter. + Dynamically sourced from the provider registry so adding a new provider + automatically extends the valid choices — no hardcoded list. + """ + try: + from agent.web_search_registry import list_provider_names + return set(list_provider_names()) + except Exception: + return set() + + +def _get_fallback_chain() -> list[str]: + """Return an ordered list of backends to try when ``search_engine == "auto"``. + + Priority: + 1. ``web.backend`` — explicit choice from config (always first) + 2. ``web.fallback_backends`` — user-configured ordered list + 3. All other registered providers not already in the chain + """ + cfg = _load_web_config() + + # Build unique-ordered chain. Explicit backend goes first, then + # user-configured fallback list, then auto-discovered remainder. + chain: list[str] = [] + + primary = (cfg.get("backend") or "").lower().strip() + if primary: + chain.append(primary) + + user_fbs = cfg.get("fallback_backends", []) + if isinstance(user_fbs, str): + user_fbs = [b.strip() for b in user_fbs.split(",") if b.strip()] + for b in user_fbs: + if b not in chain: + chain.append(b) + + # Append remaining registered providers (those not already listed). + # The registry returns providers in registration order, which for + # bundled plugins is the backend_candidates tuple order. + try: + from agent.web_search_registry import list_provider_names + for name in list_provider_names(): + if name not in chain: + chain.append(name) + except Exception: + pass + + return chain + + +def _search_with_fallback( + query: str, + limit: int, + chain: list[str], +) -> tuple[dict | None, list[str]]: + """Try each backend in *chain* until one succeeds. + + Returns ``(success_result, error_trace)``. A backend is skipped + when it is not found, not available, returns an error, or returns + zero results. ``success_result`` is ``None`` when all backends fail. + """ + from agent.web_search_registry import get_provider + + errors: list[str] = [] + for name in chain: + provider = get_provider(name) + if provider is None or not provider.supports_search(): + errors.append(f"{name}: not found or search unsupported") + continue + if not provider.is_available(): + errors.append(f"{name}: not available (missing key or package)") + continue + + try: + result = provider.search(query, limit) + except Exception as exc: + errors.append(f"{name}: {exc}") + continue + + if not isinstance(result, dict): + errors.append(f"{name}: unexpected response type {type(result).__name__}") + continue + if not result.get("success"): + errors.append(f"{name}: {result.get('error', 'unknown error')}") + continue + if not len(result.get("data", {}).get("web", [])): + errors.append(f"{name}: returned 0 results") + continue + + # Success — return immediately + return result, errors + + return None, errors + + +# ─── Tool Functions ────────────────────────────────────────────────────────── + + def _ddgs_package_importable() -> bool: """Return True when the ``ddgs`` Python package can be imported. @@ -820,12 +923,14 @@ def _ensure_web_plugins_loaded() -> None: logger.warning("Web plugin discovery failed (non-fatal): %s", exc) -def web_search_tool(query: str, limit: int = 5) -> str: +def web_search_tool(query: str, limit: int = 5, search_engine: str = "auto") -> str: """ - Search the web for information using available search API backend. + Search the web for information using available search API backends. This function provides a generic interface for web search that can work - with multiple backends (Parallel or Firecrawl). + with multiple backends. When ``search_engine`` is ``"auto"`` (default), + backends from ``web.fallback_backends`` are tried in order until one + succeeds. Explicit engine names run a single backend with no fallback. Note: This function returns search result metadata only (URLs, titles, descriptions). Use web_extract_tool to get full content from specific URLs. @@ -833,6 +938,10 @@ def web_search_tool(query: str, limit: int = 5) -> str: Args: query (str): The search query to look up limit (int): Maximum number of results to return (default: 5) + search_engine (str): Which provider to use. ``"auto"`` walks the + fallback chain. Any registered provider name (e.g. ``"baidu"``, + ``"serper"``, ``"ddgs"``) runs that single backend. Defaults to + ``"auto"``. Returns: str: JSON string containing search results with the following structure: @@ -850,9 +959,6 @@ def web_search_tool(query: str, limit: int = 5) -> str: ] } } - - Raises: - Exception: If search fails or API key is not set """ try: limit = int(limit) @@ -876,38 +982,52 @@ def web_search_tool(query: str, limit: int = 5) -> str: if is_interrupted(): return tool_error("Interrupted", success=False) - # Dispatch through the web search registry. All 7 providers - # (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl) - # now live as plugins; the dispatcher is just a registry lookup + - # delegation. Sync only — every provider's search() is sync. _ensure_web_plugins_loaded() from agent.web_search_registry import ( - get_active_search_provider, get_provider as _wsp_get_provider, ) - backend = _get_search_backend() - provider = _wsp_get_provider(backend) if backend else None - if provider is None or not provider.supports_search(): - # Fall back to availability-walked active provider when the - # configured backend isn't a registered search provider (typo, - # uninstalled plugin, or capability mismatch). - provider = get_active_search_provider() - - if provider is None: + # --- Explicit engine requested --- + if search_engine and search_engine != "auto": + provider = _wsp_get_provider(search_engine) + if provider is None: + valid = _get_valid_engine_names() + return tool_error( + f"Unknown search engine: '{search_engine}'. " + f"Valid values: {', '.join(repr(e) for e in sorted(valid))}" + ) + if not provider.supports_search(): + return tool_error( + f"Provider '{search_engine}' does not support search." + ) + logger.info( + "Web search via %s (explicit): '%s' (limit: %d)", + provider.name, query, limit, + ) + response_data = provider.search(query, limit) + debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", [])) + result_json = json.dumps(response_data, indent=2, ensure_ascii=False) + debug_call_data["final_response_size"] = len(result_json) + _debug.log_call("web_search_tool", debug_call_data) + _debug.save() + return result_json + + # --- Auto mode: fallback chain --- + chain = _get_fallback_chain() + response_data, errors = _search_with_fallback(query, limit, chain) + + if response_data is None: + summary = "; ".join(errors[-3:]) if errors else "All backends exhausted" response_data = { "success": False, "error": ( - "No web search provider configured. " - "Run `hermes tools` to set one up." + f"No web search backend available. Tried: " + f"{', '.join(chain)}. Last errors: {summary}" ), } - else: - logger.info( - "Web search via %s: '%s' (limit: %d)", - provider.name, query, limit, - ) - response_data = provider.search(query, limit) + elif errors: + # Attach fallback trace for observability + response_data.setdefault("_fallback_trace", errors) debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", [])) result_json = json.dumps(response_data, indent=2, ensure_ascii=False) From 798df3daa544e7c93250379c996f5973925027aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Sat, 27 Jun 2026 00:13:18 +0800 Subject: [PATCH 3/3] feat(tools): add hermes tools web reorder for interactive search provider priority New subcommand: hermes tools web reorder Shows the current web.fallback_backends list with numbered indices. User enters a space-separated reorder (e.g. '3 1 2 4 5'). Validated for completeness (must include each number exactly once). Writes result back to config.yaml as web.fallback_backends. This gives users a simple way to prioritize search engines after adding new providers via feat/add-free-web-search-providers (PR #53149). --- hermes_cli/subcommands/tools.py | 18 ++++++ hermes_cli/tools_config.py | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/hermes_cli/subcommands/tools.py b/hermes_cli/subcommands/tools.py index 19b85db5f179..c489842529cb 100644 --- a/hermes_cli/subcommands/tools.py +++ b/hermes_cli/subcommands/tools.py @@ -92,4 +92,22 @@ def build_tools_parser(subparsers, *, cmd_tools: Callable) -> None: metavar="KEY", help="Post-setup hook key (e.g. agent_browser, camofox, kittentts)", ) + + # hermes tools web reorder + tools_web_p = tools_sub.add_parser( + "web", + help="Manage web search backend configuration", + ) + tools_web_sub = tools_web_p.add_subparsers(dest="web_action") + + tools_web_reorder_p = tools_web_sub.add_parser( + "reorder", + help="Interactively reorder web search provider priority", + description=( + "Show current fallback order and let you rearrange it.\n" + "Providers are tried left-to-right on search failure.\n" + "Writes the new order to config.yaml as web.fallback_backends." + ), + ) + tools_parser.set_defaults(func=cmd_tools) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index f76c56e667f3..64c03aa1f087 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -3373,6 +3373,111 @@ def _reconfigure_simple_requirements(ts_key: str): # ─── Main Entry Point ───────────────────────────────────────────────────────── +def _tools_web_reorder(config: dict) -> None: + """``hermes tools web reorder`` — interactively adjust web search backend priority. + + Reads the current ``web.fallback_backends`` list, shows it with + numbered indices, lets the user enter a space-separated reorder + (e.g. ``3 1 2``), validates, and writes the result back to config. + """ + web_cfg = config.setdefault("web", {}) + current = web_cfg.get("fallback_backends", []) + if isinstance(current, str): + current = [b.strip() for b in current.split(",") if b.strip()] + + if not current: + # Populate from auto-discovered providers + try: + from tools.web_tools import _get_fallback_chain + current = _get_fallback_chain() + # Strip the explicit backend if it was added by the chain helper + explicit = (web_cfg.get("backend") or "").lower().strip() + if explicit and current and current[0] == explicit: + current = current[1:] + except Exception: + pass + if not current: + print("No web search providers registered. Run `hermes tools` to set up web search first.") + return + + print() + print(color("⚕ Web Search Provider Order", Colors.CYAN, Colors.BOLD)) + print(color(" Current fallback order (tried left to right on failure):", Colors.DIM)) + print() + for i, name in enumerate(current, 1): + label = _provider_display_name(name) + print(f" {color(str(i), Colors.BOLD)} {label} {color(f'({name})', Colors.DIM)}") + print() + print(color(" Enter a new order as space-separated numbers, e.g.: 3 1 2 4 5", Colors.DIM)) + print(color(" Press Enter to keep current order.", Colors.DIM)) + print() + + try: + user_input = input(" New order: ").strip() + except (EOFError, KeyboardInterrupt): + print() + return + + if not user_input: + print(color(" ✓ Order unchanged.", Colors.GREEN)) + return + + try: + indices = [int(x) for x in user_input.split()] + except ValueError: + print(color(" ✗ Invalid input — enter space-separated numbers only.", Colors.RED)) + return + + if set(indices) != set(range(1, len(current) + 1)): + print(color( + f" ✗ Must include each number 1–{len(current)} exactly once.", + Colors.RED, + )) + return + + reordered = [current[i - 1] for i in indices] + web_cfg["fallback_backends"] = reordered + + # Save config + try: + from hermes_cli.config import save_config + save_config(config) + except Exception: + print(color(" ✗ Could not save config.", Colors.RED)) + return + + print() + print(color(" ✓ Updated fallback order:", Colors.GREEN)) + for i, name in enumerate(reordered, 1): + label = _provider_display_name(name) + print(f" {i}. {label}") + print() + + +def _provider_display_name(name: str) -> str: + """Return a human-readable label for a web provider.""" + labels = { + "brave-free": "Brave Search (Free)", + "ddgs": "DuckDuckGo", + "serper": "Serper (Google)", + "baidu": "Baidu", + "bocha": "Bocha (博查)", + "qiniu-baidu": "Qiniu Baidu", + "serpapi": "SerpAPI", + "jina": "Jina AI", + "google-cse": "Google CSE", + "sogou": "Sogou (搜狗)", + "360-search": "360 Search", + "firecrawl": "Firecrawl", + "tavily": "Tavily", + "exa": "Exa", + "parallel": "Parallel", + "searxng": "SearXNG", + "xai": "X (xAI)", + } + return labels.get(name, name.replace("-", " ").title()) + + def tools_command(args=None, first_install: bool = False, config: dict = None): """Entry point for `hermes tools` and `hermes setup tools`. @@ -3386,6 +3491,12 @@ def tools_command(args=None, first_install: bool = False, config: dict = None): """ if config is None: config = load_config() + + # ── Subcommand: hermes tools web reorder ── + if args and getattr(args, "tools_action", None) == "web": + if getattr(args, "web_action", None) == "reorder": + _tools_web_reorder(config) + return enabled_platforms = _get_enabled_platforms() print()