feat: keyless web tier is now a 5-vendor free rotation (Exa/Parallel/Tavily/Firecrawl/Keenable) with ring failover + honest doctor readiness

agent/web_search_registry.py

@@ -168,36 +168,40 @@ def _read_config_key(*path: str) -> Optional[str]:
 
 # Keyless free-tier walk — strictly LAST-resort, tried only after the
 # availability-filtered legacy walk finds nothing (i.e. the user has zero
-# web credentials and no importable ddgs). These providers expose public
-# anonymous MCP endpoints (see plugins/web/keyless_mcp.py). Like opencode,
-# unpinned keyless traffic is split 50/50 between Exa and Parallel per
-# process (see _keyless_preference()); an explicit `hermes tools` pick
-# (web.backend / web.<capability>_backend) bypasses this walk entirely.
+# web credentials and no importable ddgs). All five vendors expose public
+# anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless
+# traffic round-robins across the ring per request (the ring cursor lives
+# in keyless_mcp; an explicit `hermes tools` pick bypasses this walk
+# entirely, and rate-limited requests fail over to the next ring vendor).
 # Disable the tier with ``web.keyless_fallback: false``.
 _KEYLESS_PREFERENCE = (
     "exa",
     "parallel",
+    "tavily",
+    "firecrawl",
+    "keenable",
 )
 
 
 def _keyless_preference() -> tuple:
-    """Return the keyless walk order, split 50/50 per process.
-
-    Mirrors opencode's session-checksum A/B split between Exa and
-    Parallel: the per-process random session id (also used as Parallel's
-    free-tier rate-limit token) picks which vendor goes first, so keyless
-    load spreads evenly across both free tiers fleet-wide while staying
-    stable within one process. The runner-up stays in the walk as a
-    fallback if the first isn't registered. Explicit user selection never
-    reaches this function — configured names resolve in step 1.
+    """Return the keyless walk order for resolution.
+
+    Delegates the entry-vendor choice to the ring cursor in
+    :mod:`plugins.web.keyless_mcp` (round-robin per request, seeded by the
+    per-process random session id) so resolution and dispatch agree on
+    which vendor a fresh install starts at. The remaining vendors follow
+    in ring order as fallbacks for registration gaps.
     """
     try:
-        from plugins.web.keyless_mcp import _SESSION_ID
+        from plugins.web.keyless_mcp import _KEYLESS_RING, _ring_cursor
 
-        if int(_SESSION_ID, 16) % 2:
-            return ("parallel", "exa")
-    except Exception as exc:  # noqa: BLE001 — split is best-effort
-        logger.debug("keyless 50/50 split unavailable: %s", exc)
+        start = _ring_cursor % len(_KEYLESS_RING)
+        return tuple(
+            _KEYLESS_RING[(start + i) % len(_KEYLESS_RING)]
+            for i in range(len(_KEYLESS_RING))
+        )
+    except Exception as exc:  # noqa: BLE001 — ring optional in stripped envs
+        logger.debug("keyless ring order unavailable: %s", exc)
     return _KEYLESS_PREFERENCE
 
 

contributors/emails/lakshya.agarwal@tavily.com

@@ -0,0 +1 @@
+lakshyaag-tavily

hermes_cli/config_defaults.py

@@ -492,17 +492,20 @@
         "search_backend": "",    # per-capability override for web_search (e.g. "searxng")
         "extract_backend": "",   # per-capability override for web_extract (e.g. "native")
         "extract_char_limit": 15000,  # per-page char budget for web_extract; larger pages truncate + store full text in cache/web
-        # Keyless free-tier fallback: with NO web backend configured or keyed,
-        # web_search/web_extract fall back to Parallel's / Exa's public
-        # anonymous MCP endpoints (rate-limited free tiers). Never pre-empts
-        # a configured or keyed backend. Set false to disable entirely.
+        # Keyless free-tier ring: with NO web backend configured or keyed,
+        # web_search/web_extract rotate round-robin across five vendors'
+        # public free tiers (exa, parallel, tavily, firecrawl, keenable),
+        # failing over to the next ring vendor on rate limits. Never
+        # pre-empts a configured or keyed backend. Set false to disable.
         "keyless_fallback": True,
-        # Per-provider tier selection for providers with both a keyless free
-        # endpoint and a keyed paid SDK path (exa, parallel). Set by the
-        # `hermes tools` picker's "Free (keyless)" / "Paid (API key)" rows.
+        # Per-provider tier selection for ring vendors with both a keyless
+        # free endpoint and a keyed paid path (exa, parallel, tavily,
+        # firecrawl, keenable). Set by the `hermes tools` picker's
+        # "Free (keyless)" / "Paid (API key)" rows.
         #   free  — always use the anonymous free endpoint (even with a key)
-        #   paid  — always use the keyed SDK path (missing key = error)
-        #   unset — auto: keyed when the API key is present, else keyless
+        #   paid  — always use the keyed path (missing key = error; vendor
+        #           is also excluded from the keyless ring)
+        #   unset — auto: keyed when the API key is present, else the ring
         "provider_tier": {},
     },
 
@@ -4117,13 +4120,21 @@
         "advanced": True,
     },
     "TAVILY_API_KEY": {
-        "description": "Tavily API key for AI-native web search and extract",
+        "description": "Tavily API key for AI-native web search and extract (optional — keyless works without it)",
         "prompt": "Tavily API key",
         "url": "https://app.tavily.com/home",
         "tools": ["web_search", "web_extract"],
         "password": True,
         "category": "tool",
     },
+    "KEENABLE_API_KEY": {
+        "description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)",
+        "prompt": "Keenable API key",
+        "url": "https://keenable.ai",
+        "tools": ["web_search", "web_extract"],
+        "password": True,
+        "category": "tool",
+    },
     "SEARXNG_URL": {
         "description": "URL of your SearXNG instance for free self-hosted web search",
         "prompt": "SearXNG URL (e.g. http://localhost:8080)",

hermes_cli/doctor.py

@@ -295,6 +295,60 @@ def _doctor_tool_availability_detail(toolset: str) -> str:
     return ""
 
 
+def _doctor_web_capability_rows() -> list[tuple[str, str, str]]:
+    """Return doctor rows for web search/extract provider readiness (#78412).
+
+    Each row is ``(status, label, detail)`` where *status* is ``ok`` or ``warn``.
+    Uses the same active-provider resolvers as the tools, but reports readiness
+    from ``is_available()`` so an explicitly selected but unconfigured backend
+    does not look healthy.
+    """
+    rows: list[tuple[str, str, str]] = []
+    try:
+        from agent.web_search_registry import (
+            get_active_extract_provider,
+            get_active_search_provider,
+        )
+        from tools.web_tools import _ensure_web_plugins_loaded, _provider_is_ready
+
+        # Doctor runs in a fresh process — bundled web providers register
+        # during plugin discovery, which nothing has triggered yet here.
+        # Without this the registry is empty and every row reads
+        # "no provider selected or registered" (idempotent, cheap on rerun).
+        _ensure_web_plugins_loaded()
+    except Exception:
+        return rows
+
+    for capability, getter in (
+        ("web search", get_active_search_provider),
+        ("web extract", get_active_extract_provider),
+    ):
+        try:
+            provider = getter()
+        except Exception:
+            provider = None
+        if provider is None:
+            rows.append(
+                (
+                    "warn",
+                    capability,
+                    "(no provider selected or registered)",
+                )
+            )
+            continue
+        name = getattr(provider, "name", None) or type(provider).__name__
+        if _provider_is_ready(provider):
+            rows.append(("ok", capability, f"({name})"))
+        else:
+            rows.append(
+                (
+                    "warn",
+                    capability,
+                    f"({name} selected; provider not configured)",
+                )
+            )
+    return rows
+
 def _apply_doctor_tool_availability_overrides(available: list[str], unavailable: list[dict]) -> tuple[list[str], list[dict]]:
     """Adjust runtime-gated tool availability for doctor diagnostics."""
     updated_available = list(available)
@@ -2835,11 +2889,26 @@ def _probe_azure_entra() -> _ConnectivityResult:
 
         available, unavailable = check_tool_availability()
         available, unavailable = _apply_doctor_tool_availability_overrides(available, unavailable)
-
+
+        # Web is split into search/extract readiness rows so an explicitly
+        # selected but unconfigured backend cannot look healthy (#78412).
+        web_rows = []
+        if "web" in available or any(item.get("name") == "web" for item in unavailable):
+            web_rows = _doctor_web_capability_rows()
+            if web_rows:
+                available = [tid for tid in available if tid != "web"]
+                unavailable = [item for item in unavailable if item.get("name") != "web"]
+
         for tid in available:
             info = TOOLSET_REQUIREMENTS.get(tid, {})
             check_ok(info.get("name", tid), _doctor_tool_availability_detail(tid))
-
+
+        for status, label, detail in web_rows:
+            if status == "ok":
+                check_ok(label, detail)
+            else:
+                check_warn(label, detail)
+
         for item in unavailable:
             env_vars = item.get("missing_vars") or item.get("env_vars") or []
             if env_vars:
@@ -2852,7 +2921,8 @@ def _probe_azure_entra() -> _ConnectivityResult:
         # current CLI platform. Default-off or explicitly disabled toolsets may
         # still show warnings above, but should not pollute the final summary.
         api_disabled = _missing_api_key_toolsets_for_summary(unavailable)
-        if api_disabled:
+        web_not_ready = any(status != "ok" for status, _, _ in web_rows)
+        if api_disabled or web_not_ready:
             issues.append("Run 'hermes setup' to configure missing API keys for full tool access")
     except Exception as e:
         check_warn("Could not check tool availability", f"({e})")

hermes_cli/nous_subscription.py

@@ -446,6 +446,7 @@ def _entitled_for(category: str) -> bool:
     # Per-capability overrides: if set, they determine which backend is active for
     # search/extract independently of web.backend.
     web_search_backend = str(web_cfg.get("search_backend") or "").strip().lower()
+    web_extract_backend = str(web_cfg.get("extract_backend") or "").strip().lower()
     tts_provider = str(tts_cfg.get("provider") or "edge").strip().lower()
     # STT default is "local" (faster-whisper) per DEFAULT_CONFIG, which
     # requires `pip install faster-whisper`. For Nous subscribers we'd
@@ -505,6 +506,9 @@ def _entitled_for(category: str) -> bool:
     direct_firecrawl = bool(get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL"))
     direct_parallel = bool(get_env_value("PARALLEL_API_KEY"))
     direct_tavily = bool(get_env_value("TAVILY_API_KEY"))
+    # Keyless Tavily is opt-in: selecting it in `hermes tools` / setup writes
+    # web.backend (or a per-capability override) without requiring a key.
+    tavily_selected = "tavily" in {web_backend, web_search_backend, web_extract_backend}
     direct_searxng = bool(get_env_value("SEARXNG_URL"))
     direct_fal = fal_key_is_configured()
     direct_fal_video = direct_fal  # same FAL_KEY; separate var so use_gateway is independent
@@ -537,6 +541,7 @@ def _entitled_for(category: str) -> bool:
         direct_exa = False
         direct_parallel = False
         direct_tavily = False
+        tavily_selected = False
     if image_use_gateway:
         direct_fal = False
     if video_use_gateway:
@@ -624,6 +629,8 @@ def _entitled_for(category: str) -> bool:
         # different browser choice wins over the env var.
         direct_camofox = False
 
+
+    tavily_ready = direct_tavily or tavily_selected
     web_managed = web_backend == "firecrawl" and managed_web_available and not direct_firecrawl
     web_active = bool(
         web_tool_enabled
@@ -632,19 +639,25 @@ def _entitled_for(category: str) -> bool:
             or (web_backend == "exa" and direct_exa)
             or (web_backend == "firecrawl" and direct_firecrawl)
             or (web_backend == "parallel" and direct_parallel)
-            or (web_backend == "tavily" and direct_tavily)
+            or (web_backend == "tavily" and tavily_ready)
             or (web_backend == "searxng" and direct_searxng)
             # Per-capability overrides: search_backend or extract_backend may be set
             # without web.backend (using the new split config from #20061)
             or (web_search_backend == "searxng" and direct_searxng)
             or (web_search_backend == "exa" and direct_exa)
             or (web_search_backend == "firecrawl" and direct_firecrawl)
             or (web_search_backend == "parallel" and direct_parallel)
-            or (web_search_backend == "tavily" and direct_tavily)
+            or (web_search_backend == "tavily" and tavily_ready)
+            or (web_extract_backend == "tavily" and tavily_ready)
         )
     )
     web_available = bool(
-        managed_web_available or direct_exa or direct_firecrawl or direct_parallel or direct_tavily or direct_searxng
+        managed_web_available
+        or direct_exa
+        or direct_firecrawl
+        or direct_parallel
+        or tavily_ready
+        or direct_searxng
     )
 
     image_managed = image_tool_enabled and managed_image_available and not direct_fal
@@ -754,8 +767,8 @@ def _entitled_for(category: str) -> bool:
             managed_by_nous=web_managed,
             direct_override=web_active and not web_managed,
             toolset_enabled=web_tool_enabled,
-            current_provider=web_backend or web_search_backend or "",
-            explicit_configured=bool(web_backend or web_search_backend),
+            current_provider=web_backend or web_search_backend or web_extract_backend or "",
+            explicit_configured=bool(web_backend or web_search_backend or web_extract_backend),
         ),
         "image_gen": NousFeatureState(
             key="image_gen",

plugins/web/exa/provider.py

@@ -143,14 +143,14 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
 
             from agent.web_search_provider import get_provider_env
 
-            from plugins.web.keyless_mcp import exa_search_keyless, use_keyless
+            from plugins.web.keyless_mcp import search_with_failover, use_keyless
 
             if use_keyless("exa", get_provider_env("EXA_API_KEY")):
                 # Keyless free tier — public MCP endpoint, no SDK needed.
                 logger.info(
                     "Exa keyless search: '%s' (limit=%d)", query, limit
                 )
-                return exa_search_keyless(query, limit)
+                return search_with_failover("exa", query, limit)
 
             logger.info("Exa search: '%s' (limit=%d)", query, limit)
             response = _get_exa_client().search(
@@ -198,12 +198,12 @@ def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
 
             from agent.web_search_provider import get_provider_env
 
-            from plugins.web.keyless_mcp import exa_extract_keyless, use_keyless
+            from plugins.web.keyless_mcp import extract_with_failover, use_keyless
 
             if use_keyless("exa", get_provider_env("EXA_API_KEY")):
                 # Keyless free tier — public MCP endpoint, no SDK needed.
                 logger.info("Exa keyless extract: %d URL(s)", len(urls))
-                return exa_extract_keyless(list(urls))
+                return extract_with_failover("exa", list(urls))
 
             logger.info("Exa extract: %d URL(s)", len(urls))
             response = _get_exa_client().get_contents(urls, text=True)

plugins/web/firecrawl/plugin.yaml

@@ -1,6 +1,6 @@
 name: web-firecrawl
 version: 1.0.0
-description: "Firecrawl web search + content extraction. Supports direct API and Nous-hosted tool-gateway routing for subscribers. Requires FIRECRAWL_API_KEY (or FIRECRAWL_API_URL for self-hosted), or an active Nous subscription with FIRECRAWL_GATEWAY_URL."
+description: "Firecrawl web search + content extraction. Supports keyless cloud, direct API, and Nous-hosted tool-gateway routing for subscribers."
 author: NousResearch
 kind: backend
 provides_web_providers: