From e027e041b3ea57bc7751f009237f51c084f94989 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@srv1486023.hstgr.cloud>
Date: Fri, 17 Apr 2026 02:31:35 +0000
Subject: [PATCH] fix(web): add runtime backend failover for search and extract

---
 tests/tools/test_web_tools_failover.py   | 126 ++++++
 tools/web_tools.py                       | 500 +++++++++++++++--------
 website/docs/integrations/index.md       |   2 +-
 website/docs/user-guide/configuration.md |   2 +
 4 files changed, 449 insertions(+), 181 deletions(-)
 create mode 100644 tests/tools/test_web_tools_failover.py

diff --git a/tests/tools/test_web_tools_failover.py b/tests/tools/test_web_tools_failover.py
new file mode 100644
index 0000000000000..7890c05a689f9
--- /dev/null
+++ b/tests/tools/test_web_tools_failover.py
@@ -0,0 +1,126 @@
+"""Regression tests for runtime web backend failover.
+
+Covers:
+- retryable search failures fail over to the next backend
+- non-retryable search failures do not silently hop providers
+- retryable extract failures fail over to the next backend
+- backend error classification
+"""
+
+import json
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+
+class TestRetryableErrorClassification:
+    def test_retryable_credit_error(self):
+        from tools.web_tools import _is_retryable_backend_error
+
+        assert _is_retryable_backend_error(RuntimeError("402 Insufficient credits")) is True
+
+    def test_non_retryable_bad_request_error(self):
+        from tools.web_tools import _is_retryable_backend_error
+
+        assert _is_retryable_backend_error(RuntimeError("400 bad request: invalid query")) is False
+
+
+class TestWebSearchFailover:
+    def test_explicit_backend_disables_cross_provider_failover(self):
+        with patch("tools.web_tools._load_web_config", return_value={"backend": "firecrawl"}), \
+             patch("tools.web_tools._is_backend_available", return_value=True):
+            from tools.web_tools import _get_backend_fallback_chain
+
+            assert _get_backend_fallback_chain("firecrawl") == ["firecrawl"]
+
+    def test_search_fails_over_on_retryable_backend_error(self):
+        with patch("tools.web_tools._get_backend", return_value="firecrawl"), \
+             patch("tools.web_tools._get_backend_fallback_chain", return_value=["firecrawl", "exa"]), \
+             patch(
+                 "tools.web_tools._dispatch_web_search_backend",
+                 side_effect=[
+                     RuntimeError("402 Insufficient credits"),
+                     {
+                         "success": True,
+                         "data": {"web": [{"title": "Recovered", "url": "https://example.com", "description": "ok"}]},
+                     },
+                 ],
+             ) as mock_dispatch, \
+             patch("tools.interrupt.is_interrupted", return_value=False):
+            from tools.web_tools import web_search_tool
+
+            result = json.loads(web_search_tool("fallback query", limit=3))
+
+        assert result["success"] is True
+        assert result["data"]["web"][0]["title"] == "Recovered"
+        assert mock_dispatch.call_count == 2
+        assert mock_dispatch.call_args_list[0].args[0] == "firecrawl"
+        assert mock_dispatch.call_args_list[1].args[0] == "exa"
+
+    def test_search_does_not_failover_on_non_retryable_error(self):
+        with patch("tools.web_tools._get_backend", return_value="firecrawl"), \
+             patch("tools.web_tools._get_backend_fallback_chain", return_value=["firecrawl", "exa"]), \
+             patch(
+                 "tools.web_tools._dispatch_web_search_backend",
+                 side_effect=RuntimeError("400 bad request: invalid query"),
+             ) as mock_dispatch, \
+             patch("tools.interrupt.is_interrupted", return_value=False):
+            from tools.web_tools import web_search_tool
+
+            result = json.loads(web_search_tool("bad query", limit=3))
+
+        assert "error" in result
+        assert "invalid query" in result["error"].lower()
+        assert mock_dispatch.call_count == 1
+
+
+class TestWebExtractFailover:
+    def test_all_error_extract_results_trigger_failover(self):
+        from tools.web_tools import _extract_results_should_failover
+
+        assert _extract_results_should_failover([
+            {"url": "https://example.com", "error": "402 Insufficient credits", "content": ""},
+            {"url": "https://example.org", "error": "429 Too Many Requests", "content": ""},
+        ]) is True
+
+    def test_partial_extract_results_do_not_trigger_failover(self):
+        from tools.web_tools import _extract_results_should_failover
+
+        assert _extract_results_should_failover([
+            {"url": "https://example.com", "error": "402 Insufficient credits", "content": ""},
+            {"url": "https://example.org", "error": None, "content": "Recovered body"},
+        ]) is False
+
+    @pytest.mark.asyncio
+    async def test_policy_blocked_url_never_reaches_backend_dispatch(self):
+        with patch("tools.web_tools.is_safe_url", return_value=True), \
+             patch("tools.web_tools.check_website_access", return_value={"host": "blocked.example", "rule": "deny", "source": "test", "message": "Blocked by policy"}), \
+             patch("tools.web_tools._dispatch_web_extract_backend", new=AsyncMock()) as mock_dispatch:
+            from tools.web_tools import web_extract_tool
+
+            result = json.loads(await web_extract_tool(["https://blocked.example"], use_llm_processing=False))
+
+        assert result["results"][0]["error"] == "Blocked by policy"
+        mock_dispatch.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_extract_fails_over_on_retryable_backend_error(self):
+        with patch("tools.web_tools._get_backend", return_value="firecrawl"), \
+             patch("tools.web_tools._get_backend_fallback_chain", return_value=["firecrawl", "exa"]), \
+             patch("tools.web_tools.is_safe_url", return_value=True), \
+             patch(
+                 "tools.web_tools._dispatch_web_extract_backend",
+                 new=AsyncMock(side_effect=[
+                     RuntimeError("402 Insufficient credits"),
+                     [{"url": "https://example.com", "title": "Recovered", "content": "Extracted body"}],
+                 ]),
+             ) as mock_dispatch:
+            from tools.web_tools import web_extract_tool
+
+            result = json.loads(await web_extract_tool(["https://example.com"], use_llm_processing=False))
+
+        assert result["results"][0]["title"] == "Recovered"
+        assert result["results"][0]["content"] == "Extracted body"
+        assert mock_dispatch.await_count == 2
+        assert mock_dispatch.await_args_list[0].args[0] == "firecrawl"
+        assert mock_dispatch.await_args_list[1].args[0] == "exa"
diff --git a/tools/web_tools.py b/tools/web_tools.py
index c24f1fc38aa16..d74f5116c6497 100644
--- a/tools/web_tools.py
+++ b/tools/web_tools.py
@@ -119,6 +119,93 @@ def _is_backend_available(backend: str) -> bool:
         return _has_env("TAVILY_API_KEY")
     return False
 
+
+def _get_backend_fallback_chain(initial_backend: Optional[str] = None) -> List[str]:
+    """Build ordered backend candidates for runtime failover.
+
+    When the user explicitly configures ``web.backend``, honor that choice and do
+    not silently hop to other providers. Cross-provider failover is only allowed
+    for auto-detected backend selection.
+    """
+    primary = (initial_backend or _get_backend()).strip().lower()
+    configured = (_load_web_config().get("backend") or "").strip().lower()
+    if configured in ("firecrawl", "parallel", "tavily", "exa"):
+        return [primary] if primary else []
+
+    ordered: List[str] = []
+    for backend in (primary, "firecrawl", "parallel", "tavily", "exa"):
+        if backend and backend not in ordered and _is_backend_available(backend):
+            ordered.append(backend)
+    if primary and primary not in ordered:
+        ordered.insert(0, primary)
+    return ordered
+
+
+def _is_retryable_backend_error(exc: Exception) -> bool:
+    """Return True when a backend error merits cross-provider failover."""
+    message = str(exc).lower()
+    retryable_markers = (
+        "402",
+        "payment required",
+        "insufficient credits",
+        "quota",
+        "rate limit",
+        "rate-limit",
+        "too many requests",
+        "429",
+        "service unavailable",
+        "503",
+        "bad gateway",
+        "502",
+        "gateway timeout",
+        "504",
+        "temporarily unavailable",
+        "upstream",
+        "timed out",
+        "timeout",
+        "connection reset",
+        "connection aborted",
+        "connection refused",
+        "network",
+        "billing",
+        "credit balance",
+    )
+    non_retryable_markers = (
+        "unsupported parameter",
+        "invalid parameter",
+        "invalid query",
+        "malformed",
+        "bad request",
+        "400",
+        "401",
+        "403",
+        "forbidden",
+        "unauthorized",
+        "not found",
+        "404",
+        "blocked: url contains",
+        "private or internal network address",
+    )
+    if any(marker in message for marker in non_retryable_markers):
+        return False
+    return any(marker in message for marker in retryable_markers)
+
+
+def _log_backend_failover(operation: str, from_backend: str, to_backend: str, exc: Exception) -> None:
+    """Emit a concise failover log for observability."""
+    logger.warning(
+        "%s backend %s failed (%s); retrying with %s",
+        operation,
+        from_backend,
+        str(exc),
+        to_backend,
+    )
+
+
+class RetryableWebBackendError(RuntimeError):
+    """Provider/runtime backend failure that should trigger cross-provider retry."""
+
+
 # ─── Firecrawl Client ────────────────────────────────────────────────────────
 
 _firecrawl_client = None
@@ -398,6 +485,170 @@ def _normalize_result_list(values: Any) -> List[Dict[str, Any]]:
     return normalized
 
 
+
+def _dispatch_web_search_backend(backend: str, query: str, limit: int) -> Dict[str, Any]:
+    """Execute search against a specific backend."""
+    if backend == "parallel":
+        return _parallel_search(query, limit)
+    if backend == "exa":
+        return _exa_search(query, limit)
+    if backend == "tavily":
+        logger.info("Tavily search: '%s' (limit: %d)", query, limit)
+        raw = _tavily_request("search", {
+            "query": query,
+            "max_results": min(limit, 20),
+            "include_raw_content": False,
+            "include_images": False,
+        })
+        return _normalize_tavily_search_results(raw)
+
+    logger.info("Searching the web via Firecrawl for: '%s' (limit: %d)", query, limit)
+    response = _get_firecrawl_client().search(query=query, limit=limit)
+    return {
+        "success": True,
+        "data": {
+            "web": _extract_web_search_results(response)
+        }
+    }
+
+
+async def _firecrawl_extract_urls(
+    safe_urls: List[str],
+    format: Optional[str],
+) -> List[Dict[str, Any]]:
+    """Extract URLs with Firecrawl and raise on provider-wide retryable failures."""
+    if format == "markdown":
+        formats: List[str] = ["markdown"]
+    elif format == "html":
+        formats = ["html"]
+    else:
+        formats = ["markdown", "html"]
+
+    results: List[Dict[str, Any]] = []
+    retryable_failures: List[str] = []
+
+    from tools.interrupt import is_interrupted as _is_interrupted
+    for url in safe_urls:
+        if _is_interrupted():
+            results.append({"url": url, "error": "Interrupted", "title": ""})
+            continue
+
+        blocked = check_website_access(url)
+        if blocked:
+            logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"])
+            results.append({
+                "url": url, "title": "", "content": "",
+                "error": blocked["message"],
+                "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]},
+            })
+            continue
+
+        try:
+            logger.info("Scraping: %s", url)
+            try:
+                scrape_result = await asyncio.wait_for(
+                    asyncio.to_thread(
+                        _get_firecrawl_client().scrape,
+                        url=url,
+                        formats=formats,
+                    ),
+                    timeout=60,
+                )
+            except asyncio.TimeoutError as timeout_exc:
+                raise RetryableWebBackendError(
+                    "Scrape timed out after 60s — page may be too large or unresponsive"
+                ) from timeout_exc
+
+            scrape_payload = _extract_scrape_payload(scrape_result)
+            metadata = scrape_payload.get("metadata", {})
+            content_markdown = scrape_payload.get("markdown")
+            content_html = scrape_payload.get("html")
+
+            if not isinstance(metadata, dict):
+                if hasattr(metadata, 'model_dump'):
+                    metadata = metadata.model_dump()
+                elif hasattr(metadata, '__dict__'):
+                    metadata = metadata.__dict__
+                else:
+                    metadata = {}
+
+            title = metadata.get("title", "")
+            final_url = metadata.get("sourceURL", url)
+            final_blocked = check_website_access(final_url)
+            if final_blocked:
+                logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"])
+                results.append({
+                    "url": final_url, "title": title, "content": "", "raw_content": "",
+                    "error": final_blocked["message"],
+                    "blocked_by_policy": {"host": final_blocked["host"], "rule": final_blocked["rule"], "source": final_blocked["source"]},
+                })
+                continue
+
+            chosen_content = content_markdown if (format == "markdown" or (format is None and content_markdown)) else content_html or content_markdown or ""
+            results.append({
+                "url": final_url,
+                "title": title,
+                "content": chosen_content,
+                "raw_content": chosen_content,
+                "metadata": metadata,
+            })
+
+        except Exception as scrape_err:
+            logger.debug("Scrape failed for %s: %s", url, scrape_err)
+            if _is_retryable_backend_error(scrape_err):
+                retryable_failures.append(str(scrape_err))
+            results.append({
+                "url": url,
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": str(scrape_err)
+            })
+
+    successful_results = [r for r in results if r.get("content")]
+    if retryable_failures and not successful_results:
+        raise RetryableWebBackendError("; ".join(retryable_failures))
+    return results
+
+
+def _extract_results_should_failover(results: List[Dict[str, Any]]) -> bool:
+    """Return True when an extract backend produced only retryable failures."""
+    if not results:
+        return False
+    if any(result.get("content") for result in results):
+        return False
+    errors = [str(result.get("error", "")) for result in results if result.get("error")]
+    if not errors:
+        return False
+    return all(_is_retryable_backend_error(RuntimeError(err)) for err in errors)
+
+
+async def _dispatch_web_extract_backend(
+    backend: str,
+    safe_urls: List[str],
+    format: Optional[str],
+) -> List[Dict[str, Any]]:
+    """Execute extract against a specific backend."""
+    if backend == "parallel":
+        results = await _parallel_extract(safe_urls)
+    elif backend == "exa":
+        results = _exa_extract(safe_urls)
+    elif backend == "tavily":
+        logger.info("Tavily extract: %d URL(s)", len(safe_urls))
+        raw = _tavily_request("extract", {
+            "urls": safe_urls,
+            "include_images": False,
+        })
+        results = _normalize_tavily_documents(raw, fallback_url=safe_urls[0] if safe_urls else "")
+    else:
+        results = await _firecrawl_extract_urls(safe_urls, format)
+
+    if _extract_results_should_failover(results):
+        errors = [str(result.get("error", "")) for result in results if result.get("error")]
+        raise RetryableWebBackendError("; ".join(errors))
+    return results
+
+
 def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
     """Extract Firecrawl search results across SDK/direct/gateway response shapes."""
     response_plain = _to_plain_object(response)
@@ -1082,65 +1333,33 @@ def web_search_tool(query: str, limit: int = 5) -> str:
         if is_interrupted():
             return tool_error("Interrupted", success=False)
 
-        # Dispatch to the configured backend
         backend = _get_backend()
-        if backend == "parallel":
-            response_data = _parallel_search(query, limit)
-            debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", []))
-            result_json = json.dumps(response_data, indent=2, ensure_ascii=False)
-            debug_call_data["final_response_size"] = len(result_json)
-            _debug.log_call("web_search_tool", debug_call_data)
-            _debug.save()
-            return result_json
-
-        if backend == "exa":
-            response_data = _exa_search(query, limit)
-            debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", []))
-            result_json = json.dumps(response_data, indent=2, ensure_ascii=False)
-            debug_call_data["final_response_size"] = len(result_json)
-            _debug.log_call("web_search_tool", debug_call_data)
-            _debug.save()
-            return result_json
-
-        if backend == "tavily":
-            logger.info("Tavily search: '%s' (limit: %d)", query, limit)
-            raw = _tavily_request("search", {
-                "query": query,
-                "max_results": min(limit, 20),
-                "include_raw_content": False,
-                "include_images": False,
-            })
-            response_data = _normalize_tavily_search_results(raw)
-            debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", []))
-            result_json = json.dumps(response_data, indent=2, ensure_ascii=False)
-            debug_call_data["final_response_size"] = len(result_json)
-            _debug.log_call("web_search_tool", debug_call_data)
-            _debug.save()
-            return result_json
-
-        logger.info("Searching the web for: '%s' (limit: %d)", query, limit)
-
-        response = _get_firecrawl_client().search(
-            query=query,
-            limit=limit
-        )
-
-        web_results = _extract_web_search_results(response)
-        results_count = len(web_results)
-        logger.info("Found %d search results", results_count)
-        
-        # Build response with just search metadata (URLs, titles, descriptions)
-        response_data = {
-            "success": True,
-            "data": {
-                "web": web_results
-            }
-        }
-        
-        # Capture debug information
-        debug_call_data["results_count"] = results_count
-        
-        # Convert to JSON
+        backend_chain = _get_backend_fallback_chain(backend)
+        backend_attempts: List[Dict[str, Any]] = []
+        response_data = None
+
+        for idx, candidate in enumerate(backend_chain):
+            try:
+                response_data = _dispatch_web_search_backend(candidate, query, limit)
+                backend_attempts.append({"backend": candidate, "status": "success"})
+                if candidate != backend:
+                    logger.info("web_search failover succeeded via %s (primary=%s)", candidate, backend)
+                break
+            except Exception as exc:
+                backend_attempts.append({
+                    "backend": candidate,
+                    "status": "error",
+                    "error": str(exc),
+                    "retryable": _is_retryable_backend_error(exc),
+                })
+                has_more = idx < len(backend_chain) - 1
+                if has_more and _is_retryable_backend_error(exc):
+                    _log_backend_failover("web_search", candidate, backend_chain[idx + 1], exc)
+                    continue
+                raise
+
+        debug_call_data["backend_attempts"] = backend_attempts
+        debug_call_data["results_count"] = len((response_data or {}).get("data", {}).get("web", []))
         result_json = json.dumps(response_data, indent=2, ensure_ascii=False)
         
         debug_call_data["final_response_size"] = len(result_json)
@@ -1223,143 +1442,64 @@ async def web_extract_tool(
     try:
         logger.info("Extracting content from %d URL(s)", len(urls))
 
-        # ── SSRF protection — filter out private/internal URLs before any backend ──
+        # ── SSRF + website policy protection before any backend ──
         safe_urls = []
-        ssrf_blocked: List[Dict[str, Any]] = []
+        blocked_results: List[Dict[str, Any]] = []
         for url in urls:
             if not is_safe_url(url):
-                ssrf_blocked.append({
+                blocked_results.append({
                     "url": url, "title": "", "content": "",
                     "error": "Blocked: URL targets a private or internal network address",
                 })
-            else:
-                safe_urls.append(url)
+                continue
+
+            blocked = check_website_access(url)
+            if blocked:
+                logger.info("Blocked web_extract for %s by rule %s before backend dispatch", blocked["host"], blocked["rule"])
+                blocked_results.append({
+                    "url": url,
+                    "title": "",
+                    "content": "",
+                    "error": blocked["message"],
+                    "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]},
+                })
+                continue
+
+            safe_urls.append(url)
 
         # Dispatch only safe URLs to the configured backend
+        backend_attempts: List[Dict[str, Any]] = []
         if not safe_urls:
             results = []
         else:
             backend = _get_backend()
-
-            if backend == "parallel":
-                results = await _parallel_extract(safe_urls)
-            elif backend == "exa":
-                results = _exa_extract(safe_urls)
-            elif backend == "tavily":
-                logger.info("Tavily extract: %d URL(s)", len(safe_urls))
-                raw = _tavily_request("extract", {
-                    "urls": safe_urls,
-                    "include_images": False,
-                })
-                results = _normalize_tavily_documents(raw, fallback_url=safe_urls[0] if safe_urls else "")
-            else:
-                # ── Firecrawl extraction ──
-                # Determine requested formats for Firecrawl v2
-                formats: List[str] = []
-                if format == "markdown":
-                    formats = ["markdown"]
-                elif format == "html":
-                    formats = ["html"]
-                else:
-                    # Default: request markdown for LLM-readiness and include html as backup
-                    formats = ["markdown", "html"]
-
-                # Always use individual scraping for simplicity and reliability
-                # Batch scraping adds complexity without much benefit for small numbers of URLs
-                results: List[Dict[str, Any]] = []
-
-                from tools.interrupt import is_interrupted as _is_interrupted
-                for url in safe_urls:
-                    if _is_interrupted():
-                        results.append({"url": url, "error": "Interrupted", "title": ""})
+            backend_chain = _get_backend_fallback_chain(backend)
+            results = []
+            for idx, candidate in enumerate(backend_chain):
+                try:
+                    results = await _dispatch_web_extract_backend(candidate, safe_urls, format)
+                    backend_attempts.append({"backend": candidate, "status": "success"})
+                    if candidate != backend:
+                        logger.info("web_extract failover succeeded via %s (primary=%s)", candidate, backend)
+                    break
+                except Exception as exc:
+                    backend_attempts.append({
+                        "backend": candidate,
+                        "status": "error",
+                        "error": str(exc),
+                        "retryable": _is_retryable_backend_error(exc),
+                    })
+                    has_more = idx < len(backend_chain) - 1
+                    if has_more and _is_retryable_backend_error(exc):
+                        _log_backend_failover("web_extract", candidate, backend_chain[idx + 1], exc)
                         continue
+                    raise
 
-                    # Website policy check — block before fetching
-                    blocked = check_website_access(url)
-                    if blocked:
-                        logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"])
-                        results.append({
-                            "url": url, "title": "", "content": "",
-                            "error": blocked["message"],
-                            "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]},
-                        })
-                        continue
+        debug_call_data["backend_attempts"] = backend_attempts
 
-                    try:
-                        logger.info("Scraping: %s", url)
-                        # Run synchronous Firecrawl scrape in a thread with a
-                        # 60s timeout so a hung fetch doesn't block the session.
-                        try:
-                            scrape_result = await asyncio.wait_for(
-                                asyncio.to_thread(
-                                    _get_firecrawl_client().scrape,
-                                    url=url,
-                                    formats=formats,
-                                ),
-                                timeout=60,
-                            )
-                        except asyncio.TimeoutError:
-                            logger.warning("Firecrawl scrape timed out for %s", url)
-                            results.append({
-                                "url": url, "title": "", "content": "",
-                                "error": "Scrape timed out after 60s — page may be too large or unresponsive. Try browser_navigate instead.",
-                            })
-                            continue
-
-                        scrape_payload = _extract_scrape_payload(scrape_result)
-                        metadata = scrape_payload.get("metadata", {})
-                        title = ""
-                        content_markdown = scrape_payload.get("markdown")
-                        content_html = scrape_payload.get("html")
-
-                        # Ensure metadata is a dict (not an object)
-                        if not isinstance(metadata, dict):
-                            if hasattr(metadata, 'model_dump'):
-                                metadata = metadata.model_dump()
-                            elif hasattr(metadata, '__dict__'):
-                                metadata = metadata.__dict__
-                            else:
-                                metadata = {}
-
-                        # Get title from metadata
-                        title = metadata.get("title", "")
-
-                        # Re-check final URL after redirect
-                        final_url = metadata.get("sourceURL", url)
-                        final_blocked = check_website_access(final_url)
-                        if final_blocked:
-                            logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"])
-                            results.append({
-                                "url": final_url, "title": title, "content": "", "raw_content": "",
-                                "error": final_blocked["message"],
-                                "blocked_by_policy": {"host": final_blocked["host"], "rule": final_blocked["rule"], "source": final_blocked["source"]},
-                            })
-                            continue
-
-                        # Choose content based on requested format
-                        chosen_content = content_markdown if (format == "markdown" or (format is None and content_markdown)) else content_html or content_markdown or ""
-
-                        results.append({
-                            "url": final_url,
-                            "title": title,
-                            "content": chosen_content,
-                            "raw_content": chosen_content,
-                            "metadata": metadata  # Now guaranteed to be a dict
-                        })
-
-                    except Exception as scrape_err:
-                        logger.debug("Scrape failed for %s: %s", url, scrape_err)
-                        results.append({
-                            "url": url,
-                            "title": "",
-                            "content": "",
-                            "raw_content": "",
-                            "error": str(scrape_err)
-                        })
-
-        # Merge any SSRF-blocked results back in
-        if ssrf_blocked:
-            results = ssrf_blocked + results
+        # Merge any blocked results back in
+        if blocked_results:
+            results = blocked_results + results
 
         response = {"results": results}
         
diff --git a/website/docs/integrations/index.md b/website/docs/integrations/index.md
index ccb7853702367..d939fe67f4298 100644
--- a/website/docs/integrations/index.md
+++ b/website/docs/integrations/index.md
@@ -38,7 +38,7 @@ web:
   backend: firecrawl    # firecrawl | parallel | tavily | exa
 ```
 
-If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`.
+If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`. In auto-detect mode, Hermes can retry other available web backends when the primary one fails with retryable provider errors such as credits/quota/rate-limit exhaustion or transient upstream outages. If you explicitly set `web.backend`, Hermes honors that provider pin and does not silently switch providers.
 
 ## Browser Automation
 
diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md
index bef9b5cfd55a9..4abedafb137ab 100644
--- a/website/docs/user-guide/configuration.md
+++ b/website/docs/user-guide/configuration.md
@@ -1118,6 +1118,8 @@ web:
 
 **Backend selection:** If `web.backend` is not set, the backend is auto-detected from available API keys. If only `EXA_API_KEY` is set, Exa is used. If only `TAVILY_API_KEY` is set, Tavily is used. If only `PARALLEL_API_KEY` is set, Parallel is used. Otherwise Firecrawl is the default.
 
+**Runtime failover:** When `web.backend` is left unset and Hermes auto-detects the provider, it can retry other available web backends on retryable provider errors such as insufficient credits (`402`), quota/rate-limit issues, and transient upstream outages. If you explicitly set `web.backend`, Hermes honors that pin and does not silently switch providers. It also does not fail over for user/query errors like invalid parameters or malformed requests.
+
 **Self-hosted Firecrawl:** Set `FIRECRAWL_API_URL` to point at your own instance. When a custom URL is set, the API key becomes optional (set `USE_DB_AUTHENTICATION=false` on the server to disable auth).
 
 **Parallel search modes:** Set `PARALLEL_SEARCH_MODE` to control search behavior — `fast`, `one-shot`, or `agentic` (default: `agentic`).