fix(web): a benched lazy-install must not disable an importable SDK

plugins/web/_common.py

@@ -7,6 +7,9 @@
 
 from __future__ import annotations
 
+import importlib.util
+import sys
+
 import logging
 from typing import Any, Awaitable, Callable, Dict, List, Optional
 
@@ -167,16 +170,71 @@ def titled_rows(raw_results: List[Dict[str, Any]], description_key: str) -> List
     ]
 
 
+def _sdk_importable(module_name: str) -> bool:
+    """True when ``import <module_name>`` would succeed.
+
+    ``sys.modules`` is checked first: an already-imported module is importable
+    by definition, and ``find_spec`` raises ``ValueError`` on entries with no
+    ``__spec__`` (which is how tests inject a stub SDK).
+    """
+    if module_name in sys.modules:
+        return True
+    try:
+        return importlib.util.find_spec(module_name) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+# Feature -> top-level import the factory actually needs (usually the feature id
+# with its namespace stripped, per the "search.<vendor>" convention).
+_FEATURE_MODULE_PREFIXES = ("search.", "extract.", "export.")
+# Features whose package name is NOT the vendor slug (``exa-py`` ships ``exa_py``).
+_IMPORT_NAME_OVERRIDES = {"search.exa": "exa_py"}
+
+
+def _feature_module(feature: str) -> str:
+    """The importable package a feature's factory needs, from the feature id.
+
+    The override table is consulted first (``search.exa`` -> ``exa_py``) so a
+    vendor whose package name is not its slug is never probed as the wrong
+    module; everything else strips the namespace prefix (``search.parallel`` ->
+    ``parallel``, ``search.firecrawl`` -> ``firecrawl``).
+    """
+    override = _IMPORT_NAME_OVERRIDES.get(feature)
+    if override is not None:
+        return override
+    return feature.split(".", 1)[-1] if feature.startswith(_FEATURE_MODULE_PREFIXES) else feature
+
+
 def lazy_ensure(feature: str) -> None:
-    """Best-effort ``tools.lazy_deps.ensure``: its own ImportError is benign and swallowed;
-    an install hint (any other error) is re-raised as ImportError."""
+    """Try to install ``feature``'s packages; refuse only when the import would fail.
+
+    The import at the call site is the real gate: whether the SDK can be
+    installed is only interesting when it is not already importable.
+    ``tools.lazy_deps.ensure`` reports an unusable feature with
+    ``FeatureUnavailable`` (a ``RuntimeError``, not an ``ImportError``), so the
+    old ``except ImportError`` never caught the case it was written for, and
+    the broad ``except Exception`` turned "cannot install" into a hard failure
+    even when the SDK would have imported. On a host with
+    ``security.allow_lazy_installs=false`` that made the provider unusable
+    whether or not the package was present.
+
+    So: try to install, and if that is impossible, check whether we needed it.
+    Only a genuinely missing package raises, and it carries the install hint.
+    Unrelated faults still surface as ``ImportError`` for the caller.
+    """
     try:
-        from tools.lazy_deps import ensure as _lazy_ensure
-        _lazy_ensure(feature, prompt=False)
+        from tools.lazy_deps import FeatureUnavailable, ensure as _lazy_ensure
     except ImportError:
-        pass
-    except Exception as exc:  # noqa: BLE001
-        raise ImportError(str(exc))
+        return  # lazy-deps helper itself unavailable — let the import decide
+    try:
+        _lazy_ensure(feature, prompt=False)
+    except FeatureUnavailable as exc:
+        if not _sdk_importable(_feature_module(feature)):
+            raise ImportError(str(exc)) from exc
+        # Already importable; nothing needed installing.
+    except Exception as exc:  # noqa: BLE001 — surface real faults as ImportError
+        raise ImportError(str(exc)) from exc
 
 
 def cached_sdk_client(slot: str, env_var: str, missing_key_error: str, feature: str, factory: Callable[[str], Any]) -> Any:

tests/plugins/web/test_lazy_ensure_importable_sdk.py

@@ -0,0 +1,127 @@
+"""``lazy_ensure`` must not fail when a feature's SDK is already importable.
+
+The helper's contract (docstring and callers) says the import at the call site
+is the real gate: an install that cannot happen is only an error when the
+package is genuinely absent. ``tools.lazy_deps.ensure`` signals an unusable
+feature with ``FeatureUnavailable`` — a ``RuntimeError``, not an
+``ImportError`` — so the pre-split handler never caught it and the broad
+``except Exception`` re-raised it as ``ImportError`` even when the package was
+present. On a host with ``security.allow_lazy_installs=false`` that made the
+provider unusable regardless of whether the SDK was installed.
+
+Ported from the per-provider fix (PR for "a benched lazy-install must not
+disable an importable SDK") to the shared ``plugins/web/_common.lazy_ensure``
+that succeeded it — same contract, every web vendor covered.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+from plugins.web import _common
+from tools.lazy_deps import FeatureUnavailable
+
+
+@pytest.fixture
+def fake_parallel_sdk(monkeypatch):
+    """Make ``import parallel`` succeed without installing anything."""
+    module = types.ModuleType("parallel")
+
+    class Parallel:
+        def __init__(self, api_key):
+            self.api_key = api_key
+
+    class AsyncParallel:
+        def __init__(self, api_key):
+            self.api_key = api_key
+
+    module.Parallel = Parallel
+    module.AsyncParallel = AsyncParallel
+    monkeypatch.setitem(sys.modules, "parallel", module)
+    return module
+
+
+@pytest.fixture
+def fake_exa_sdk(monkeypatch):
+    """Make ``import exa_py`` succeed without installing anything."""
+    module = types.ModuleType("exa_py")
+
+    class Exa:
+        def __init__(self, api_key):
+            self.api_key = api_key
+
+    module.Exa = Exa
+    monkeypatch.setitem(sys.modules, "exa_py", module)
+    return module
+
+
+def _deny_lazy_install(*_args, **_kwargs):
+    raise FeatureUnavailable(
+        "search.parallel",
+        ("parallel-web==0.4.2",),
+        "lazy installs disabled (security.allow_lazy_installs=false)",
+    )
+
+
+def _deny_exa_lazy_install(*_args, **_kwargs):
+    raise FeatureUnavailable(
+        "search.exa",
+        ("exa-py==2.10.2",),
+        "lazy installs disabled (security.allow_lazy_installs=false)",
+    )
+
+
+def test_importable_sdk_survives_disabled_lazy_installs(monkeypatch, fake_parallel_sdk):
+    """Feature reported unavailable, but the package imports: not an error."""
+    monkeypatch.setattr("tools.lazy_deps.ensure", _deny_lazy_install)
+
+    _common.lazy_ensure("search.parallel")  # must not raise
+
+    from parallel import Parallel
+
+    assert Parallel(api_key="k").api_key == "k"
+
+
+def test_importable_exa_sdk_survives_disabled_lazy_installs(monkeypatch, fake_exa_sdk):
+    """Same for a vendor whose package name is not its slug: ``exa-py`` imports as ``exa_py``."""
+    monkeypatch.setattr("tools.lazy_deps.ensure", _deny_exa_lazy_install)
+
+    _common.lazy_ensure("search.exa")  # must not raise
+
+    from exa_py import Exa
+
+    assert Exa(api_key="k").api_key == "k"
+
+
+def test_missing_sdk_still_reports_the_install_hint(monkeypatch):
+    """Genuinely absent package keeps the actionable ImportError."""
+    monkeypatch.delitem(sys.modules, "parallel", raising=False)
+    monkeypatch.setattr("tools.lazy_deps.ensure", _deny_lazy_install)
+
+    with pytest.raises(ImportError) as excinfo:
+        _common.lazy_ensure("search.parallel")
+
+    assert "parallel-web" in str(excinfo.value)
+
+
+def test_unrelated_failure_is_still_surfaced(monkeypatch, fake_parallel_sdk):
+    """A non-availability error is a real fault and must not be swallowed."""
+
+    def boom(*_args, **_kwargs):
+        raise OSError("disk exploded")
+
+    monkeypatch.setattr("tools.lazy_deps.ensure", boom)
+
+    with pytest.raises(ImportError, match="disk exploded"):
+        _common.lazy_ensure("search.parallel")
+
+
+def test_absent_helper_degrades_to_the_import(monkeypatch):
+    """Without lazy_deps at all, the call-site import remains the only gate."""
+    monkeypatch.setitem(sys.modules, "tools", types.ModuleType("tools"))
+    monkeypatch.setitem(sys.modules, "tools.lazy_deps", None)  # import raises
+
+    _common.lazy_ensure("search.parallel")  # must not raise, must not install