diff --git a/tools/web_tools.py b/tools/web_tools.py index 352b4a55b1302..5509af4b5b4be 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -126,7 +126,7 @@ def _get_backend() -> str: keys manually without running setup. """ configured = (_load_web_config().get("backend") or "").lower().strip() - if configured in ("parallel", "firecrawl", "tavily", "exa"): + if configured in ("parallel", "firecrawl", "tavily", "exa", "searxng"): return configured # Fallback for manual / legacy config — pick the highest-priority @@ -155,6 +155,8 @@ def _is_backend_available(backend: str) -> bool: return check_firecrawl_api_key() if backend == "tavily": return _has_env("TAVILY_API_KEY") + if backend == "searxng": + return _has_env("SEARXNG_URL") return False # ─── Firecrawl Client ──────────────────────────────────────────────────────── @@ -1071,6 +1073,112 @@ async def _parallel_extract(urls: List[str]) -> List[Dict[str, Any]]: return results +# ─── SearXNG Search & Extract Helpers ───────────────────────────────────────── + +def _searxng_base_url() -> str: + """Return the configured SearXNG base URL, or empty string if not set.""" + return os.getenv("SEARXNG_URL", "").strip().rstrip("/") + + +def _searxng_search(query: str, limit: int = 10) -> dict: + """Search using a self-hosted SearXNG instance and return results as a dict.""" + from tools.interrupt import is_interrupted + if is_interrupted(): + return {"error": "Interrupted", "success": False} + + base = _searxng_base_url() + if not base: + return {"error": "SEARXNG_URL environment variable is not set. " + "Configure a self-hosted SearXNG instance to use this backend.", "success": False} + + params = { + "q": query, + "format": "json", + "limit": min(limit, 20), + } + logger.info("SearXNG search: '%s' (base=%s, limit=%d)", query, base, limit) + + response = httpx.get( + f"{base}/search", + params=params, + timeout=30, + follow_redirects=True, + ) + response.raise_for_status() + data = response.json() + + web_results = [] + for i, result in enumerate(data.get("results", [])[:limit]): + web_results.append({ + "title": result.get("title", ""), + "url": result.get("url", ""), + "description": result.get("content", "") or result.get("description", ""), + "position": i + 1, + }) + + return {"success": True, "data": {"web": web_results}} + + +async def _searxng_extract(urls: List[str]) -> List[Dict[str, Any]]: + """Extract content from URLs using direct HTTP fetch via httpx. + + Returns a list of result dicts matching the structure expected by the + LLM post-processing pipeline (url, title, content, metadata). + """ + from tools.interrupt import is_interrupted + if is_interrupted(): + return [{"url": u, "error": "Interrupted", "title": ""} for u in urls] + + results: List[Dict[str, Any]] = [] + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client: + for url in urls: + if is_interrupted(): + results.append({"url": url, "error": "Interrupted", "title": ""}) + continue + + try: + logger.info("SearXNG extract: %s", url) + resp = await client.get(url, headers={ + "User-Agent": "Mozilla/5.0 (compatible; Hermes/1.0; +http://hermes-agent.ai)", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }) + resp.raise_for_status() + html = resp.text + + # Basic title extraction + title = "" + title_match = re.search(r"