From da579238e80832eccd9b47c84ad6ec13356d439c Mon Sep 17 00:00:00 2001 From: Beto de Paola Date: Fri, 7 Aug 2026 13:31:27 -0700 Subject: [PATCH 1/2] feat(providers): discover pip-installed model providers via entry points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model-provider discovery was filesystem-only (bundled dir, $HERMES_HOME, legacy providers/*.py). The general PluginManager scans the hermes_agent.plugins entry-point group but deliberately does NOT import kind=model-provider manifests (providers/ owns their lifecycle), so a pip-installed provider was recorded yet never called register_provider() — it never appeared in the picker, contradicting the 'Distribute via pip' docs. Add a _discover_entry_point_providers() step that scans the hermes_agent.plugins group and imports each entry, supporting both a module:func callable target and a bare self-registering module target. - Runs BEFORE filesystem plugins (lowest precedence): last-writer-wins means bundled/$HERMES_HOME profiles always override a pip provider of the same name, so a third-party package cannot hijack a first-party provider id. - Per-entry failures are isolated (logged + skipped), so one broken package can't break discovery. - Docs updated to describe the real mechanism; tests cover callable + module targets, failure isolation, and first-party precedence. --- providers/__init__.py | 83 +++++++++- tests/providers/test_entry_point_discovery.py | 156 ++++++++++++++++++ .../developer-guide/model-provider-plugin.md | 22 ++- 3 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 tests/providers/test_entry_point_discovery.py diff --git a/providers/__init__.py b/providers/__init__.py index 4d828c561d34..b34bc4f61cc1 100644 --- a/providers/__init__.py +++ b/providers/__init__.py @@ -1,9 +1,11 @@ """Provider module registry. -Provider profiles can live in two places: +Provider profiles can live in three places: 1. Bundled plugins: ``plugins/model-providers//`` (shipped with hermes-agent) 2. User plugins: ``$HERMES_HOME/plugins/model-providers//`` +3. Pip-installed plugins: distributions exposing a ``hermes_agent.plugins`` + entry point (``module:func`` callable or a self-registering ``module``) Each plugin directory contains: - ``__init__.py`` — calls ``register_provider(profile)`` at import @@ -144,6 +146,65 @@ def _import_plugin_dir(plugin_dir: Path, source: str) -> None: sys.modules.pop(module_name, None) +def _discover_entry_point_providers() -> None: + """Import pip-installed provider plugins via the ``hermes_agent.plugins`` + entry-point group so they self-register. + + A distribution ships:: + + [project.entry-points."hermes_agent.plugins"] + acme-inference = "acme_hermes_plugin:register" + + The target may be either a **callable** (``module:func`` — invoked with no + args; typically calls ``register_provider(profile)``) or a **module** + (``module`` — imported for its module-level ``register_provider`` side + effect, mirroring the directory-plugin ``__init__.py`` contract). + + Failures are swallowed per-entry (a broken third-party package must not + break provider discovery) and logged at warning level. This scan runs + first, so filesystem plugins (bundled + ``$HERMES_HOME``) keep their + documented override precedence via last-writer-wins in + ``register_provider()`` — a pip package cannot hijack a first-party + provider name. + """ + try: + import importlib.metadata as _md + except Exception: # pragma: no cover — importlib.metadata always present ≥3.8 + return + + group = "hermes_agent.plugins" + try: + eps = _md.entry_points() + # Python 3.10+ exposes .select(); older returns a dict-like mapping. + if hasattr(eps, "select"): + group_eps = list(eps.select(group=group)) + else: # pragma: no cover — legacy interpreters + group_eps = list(eps.get(group, [])) # type: ignore[attr-defined] + except Exception as exc: + logger.debug("entry-point provider scan skipped: %s", exc) + return + + for ep in group_eps: + try: + loaded = ep.load() + except Exception as exc: + logger.warning( + "Failed to load entry-point provider plugin %r: %s", ep.name, exc + ) + continue + # ``module:func`` → callable we invoke; bare ``module`` → import side + # effect already happened during load(). Only call when it's callable. + if callable(loaded): + try: + loaded() + except Exception as exc: + logger.warning( + "Entry-point provider plugin %r raised on invocation: %s", + ep.name, + exc, + ) + + def _discover_providers() -> None: """Populate the registry by importing every provider plugin. @@ -160,6 +221,22 @@ def _discover_providers() -> None: return _discovered = True + # 0. Pip-installed plugins — entry points in the ``hermes_agent.plugins`` + # group (the same group the general PluginManager uses). The manager + # records model-provider manifests for introspection but deliberately + # does NOT import them — provider lifecycle is owned here — so without + # this step a ``pip install``ed provider never calls + # ``register_provider()`` and is never selectable. + # + # Discovered FIRST, i.e. lowest precedence: because + # ``register_provider()`` is last-writer-wins, running this before the + # filesystem steps means a bundled or ``$HERMES_HOME`` profile of the + # same name always overrides a pip-installed one. That prevents a + # third-party package from silently hijacking a first-party provider + # name (e.g. ``openrouter``) while still letting pip packages add + # genuinely new providers. + _discover_entry_point_providers() + # 1. Bundled plugins — shipped with hermes-agent. if _BUNDLED_PLUGINS_DIR.is_dir(): for child in sorted(_BUNDLED_PLUGINS_DIR.iterdir()): @@ -196,3 +273,7 @@ def _discover_providers() -> None: ) except Exception: pass + + # (Pip entry-point providers are discovered in step 0, before the + # filesystem plugins, so first-party profiles always win on name + # collision — see _discover_entry_point_providers.) diff --git a/tests/providers/test_entry_point_discovery.py b/tests/providers/test_entry_point_discovery.py new file mode 100644 index 000000000000..b82e748cfacb --- /dev/null +++ b/tests/providers/test_entry_point_discovery.py @@ -0,0 +1,156 @@ +"""Tests for pip entry-point provider discovery (hermes_agent.plugins group). + +Verifies that ``providers/__init__.py`` imports provider plugins exposed via a +distribution's ``hermes_agent.plugins`` entry point, supporting both a +``module:func`` callable target and a bare self-registering ``module`` target. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +import providers + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _clear_provider_caches(): + providers._REGISTRY.clear() + providers._ALIASES.clear() + providers._PROVIDER_LIST_CACHE = None + providers._discovered = False + for mod in list(sys.modules.keys()): + if mod.startswith("plugins.model_providers") or mod.startswith( + "_hermes_user_provider" + ): + del sys.modules[mod] + + +@pytest.fixture(autouse=True) +def _restore_real_discovery(): + """Snapshot registry state; on teardown re-run REAL discovery. + + These tests monkeypatch ``importlib.metadata.entry_points`` and evict the + ``plugins.model_providers`` submodules to force re-discovery. Without an + explicit restore, the emptied registry / ``sys.modules`` would leak into + later tests (e.g. ``from plugins.model_providers.custom import ...``). + + This fixture is autouse and declared before ``monkeypatch`` is requested, + so it tears down LAST — after ``entry_points`` is restored to the real + implementation — letting the final ``_discover_providers()`` repopulate + both the registry and ``sys.modules`` from the real filesystem plugins. + """ + yield + _clear_provider_caches() + providers._discover_providers() + + + +class _FakeEP: + def __init__(self, name, loader): + self.name = name + self.group = "hermes_agent.plugins" + self._loader = loader + + def load(self): + return self._loader() + + +class _FakeEntryPoints: + def __init__(self, eps): + self._eps = eps + + def select(self, group): + return [e for e in self._eps if e.group == group] + + +def _register_via_callable(): + from providers.base import ProviderProfile + + def register(): + providers.register_provider( + ProviderProfile(name="ep-callable", aliases=("epc",), base_url="https://a.test/v1") + ) + + return register # ep.load() returns the callable; discovery invokes it + + +def _register_via_module(): + # ep.load() returns a non-callable object; the import side effect already + # registered the profile (mirrors a bare ``module`` target). + from providers.base import ProviderProfile + + providers.register_provider( + ProviderProfile(name="ep-module", base_url="https://b.test/v1") + ) + return object() # non-callable → discovery must NOT try to call it + + +def test_entry_point_callable_and_module_targets(monkeypatch): + fake_eps = _FakeEntryPoints( + [ + _FakeEP("ep-callable", _register_via_callable), + _FakeEP("ep-module", _register_via_module), + ] + ) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _clear_provider_caches() + try: + assert providers.get_provider_profile("ep-callable") is not None + assert providers.get_provider_profile("epc") is not None # alias + assert providers.get_provider_profile("ep-module") is not None + finally: + _clear_provider_caches() + + +def test_entry_point_failure_is_isolated(monkeypatch): + def _boom(): + raise RuntimeError("broken plugin") + + fake_eps = _FakeEntryPoints( + [ + _FakeEP("broken", _boom), + _FakeEP("ep-callable", _register_via_callable), + ] + ) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _clear_provider_caches() + try: + # A broken entry point must not prevent the good one from registering. + assert providers.get_provider_profile("ep-callable") is not None + finally: + _clear_provider_caches() + + +def test_filesystem_plugins_win_over_entry_points(monkeypatch): + """Entry points scan last, so a bundled/user profile of the same name wins.""" + from providers.base import ProviderProfile + + def _register_ep_openrouter(): + def register(): + providers.register_provider( + ProviderProfile(name="openrouter", base_url="https://impostor.test/v1") + ) + + return register + + fake_eps = _FakeEntryPoints([_FakeEP("openrouter", _register_ep_openrouter)]) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _clear_provider_caches() + try: + p = providers.get_provider_profile("openrouter") + assert p is not None + # The bundled OpenRouter profile (real base_url) must win, not the impostor. + assert "impostor.test" not in (p.base_url or "") + finally: + _clear_provider_caches() diff --git a/website/docs/developer-guide/model-provider-plugin.md b/website/docs/developer-guide/model-provider-plugin.md index dd1be9f291b7..ea48f80b66db 100644 --- a/website/docs/developer-guide/model-provider-plugin.md +++ b/website/docs/developer-guide/model-provider-plugin.md @@ -248,14 +248,32 @@ The general `PluginManager` (the thing `hermes plugins` operates on) **sees** mo ## Distribute via pip -Like any Hermes plugin, model providers can ship as a pip package. Add an entry point to your `pyproject.toml`: +Model providers can ship as a pip package. Expose an entry point in the +`hermes_agent.plugins` group in your `pyproject.toml`: ```toml [project.entry-points."hermes_agent.plugins"] acme-inference = "acme_hermes_plugin:register" ``` -…where `acme_hermes_plugin:register` is a function that calls `register_provider(profile)`. The general PluginManager picks up entry-point plugins during `discover_and_load()`. For `kind: model-provider` pip plugins, you still need to declare the kind in your manifest (or rely on the source-text heuristic). +The target may be either: + +- a **callable** (`module:func`) — invoked with no arguments; it should call + `register_provider(profile)`, or +- a **bare module** (`module`) — imported for its module-level + `register_provider(...)` side effect, mirroring the directory-plugin + `__init__.py` contract. + +`providers/__init__.py` discovers these entry points itself (the general +`PluginManager` records model-provider manifests but never imports them, so it +cannot register the profile). Entry-point plugins are discovered **before** +filesystem plugins, giving them the lowest precedence: because +`register_provider()` is last-writer-wins, a bundled or `$HERMES_HOME` profile +of the same name always overrides a pip-installed one. A pip package can add a +genuinely new provider, but cannot silently hijack a first-party provider name. + +A broken entry point is isolated — it is logged at warning level and skipped, +and never blocks discovery of the other providers. See [Building a Hermes Plugin](/developer-guide/plugins#distribute-via-pip) for the full entry-points setup. From c7f3e2bd2d5fb630b98bd95597b695469dac3125 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:51:17 -0700 Subject: [PATCH 2/2] fix: gate entry-point provider scan on plugins.enabled and skip register(ctx) targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups on salvaged #81419: - Honor the plugins.enabled allow-list / plugins.disabled deny-list (same opt-in contract as the general PluginManager) — installed != loaded. - Skip callables that require arguments: general plugins share the hermes_agent.plugins group with register(ctx) targets; invoking them zero-arg would TypeError-spam every startup. - Fix test docstring (entry points are discovered FIRST, lowest precedence) and docs mechanism wording; document the config gate. - New tests: opt-in gate, deny-list, register(ctx) never invoked. E2E-verified with a real pip-built package against a temp HERMES_HOME. --- providers/__init__.py | 65 ++++++++++++++++- tests/providers/test_entry_point_discovery.py | 71 ++++++++++++++++++- .../developer-guide/model-provider-plugin.md | 37 +++++++--- 3 files changed, 161 insertions(+), 12 deletions(-) diff --git a/providers/__init__.py b/providers/__init__.py index b34bc4f61cc1..011e84afa723 100644 --- a/providers/__init__.py +++ b/providers/__init__.py @@ -160,6 +160,18 @@ def _discover_entry_point_providers() -> None: (``module`` — imported for its module-level ``register_provider`` side effect, mirroring the directory-plugin ``__init__.py`` contract). + Gating and safety: + + * **Opt-in.** Entry-point plugins are subject to the same + ``plugins.enabled`` allow-list (and ``plugins.disabled`` deny-list) the + general PluginManager enforces — a pip package is never imported just + because it is installed. An entry point whose name is not enabled is + skipped without loading. + * **Provider targets only.** The ``hermes_agent.plugins`` group is shared + with general plugins whose target is ``register(ctx)``. Callables that + require arguments are skipped here (the PluginManager owns them); + provider registration hooks take no arguments by contract. + Failures are swallowed per-entry (a broken third-party package must not break provider discovery) and logged at warning level. This scan runs first, so filesystem plugins (bundled + ``$HERMES_HOME``) keep their @@ -172,6 +184,18 @@ def _discover_entry_point_providers() -> None: except Exception: # pragma: no cover — importlib.metadata always present ≥3.8 return + # Same opt-in gate as the general PluginManager: only entry points named + # in ``plugins.enabled`` load, and ``plugins.disabled`` always wins. + try: + from hermes_cli.plugins import _get_disabled_plugins, _get_enabled_plugins + + enabled = _get_enabled_plugins() # None = nothing enabled yet (opt-in default) + disabled = _get_disabled_plugins() + except Exception: # pragma: no cover — config layer unavailable + enabled, disabled = None, set() + if not enabled: + return + group = "hermes_agent.plugins" try: eps = _md.entry_points() @@ -185,6 +209,11 @@ def _discover_entry_point_providers() -> None: return for ep in group_eps: + if ep.name not in enabled or ep.name in disabled: + logger.debug( + "entry-point provider %r skipped: not enabled in config", ep.name + ) + continue try: loaded = ep.load() except Exception as exc: @@ -193,8 +222,18 @@ def _discover_entry_point_providers() -> None: ) continue # ``module:func`` → callable we invoke; bare ``module`` → import side - # effect already happened during load(). Only call when it's callable. + # effect already happened during load(). Only call when it's callable + # AND zero-arg: general plugins in this shared group expose + # ``register(ctx)`` (requires an argument) and belong to the + # PluginManager, not the provider registry. if callable(loaded): + if _requires_arguments(loaded): + logger.debug( + "entry-point %r skipped by provider scan: target requires " + "arguments (general plugin owned by PluginManager)", + ep.name, + ) + continue try: loaded() except Exception as exc: @@ -205,6 +244,30 @@ def _discover_entry_point_providers() -> None: ) +def _requires_arguments(fn) -> bool: + """True when ``fn`` cannot be called with zero arguments. + + Used to distinguish provider registration hooks (zero-arg by contract) + from general plugin hooks (``register(ctx)``) sharing the same entry-point + group. Unintrospectable callables (C extensions) are treated as zero-arg + and left to the per-entry exception guard. + """ + import inspect + + try: + sig = inspect.signature(fn) + except (TypeError, ValueError): # pragma: no cover — builtins/C callables + return False + for param in sig.parameters.values(): + if param.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) and param.default is inspect.Parameter.empty: + return True + return False + + def _discover_providers() -> None: """Populate the registry by importing every provider plugin. diff --git a/tests/providers/test_entry_point_discovery.py b/tests/providers/test_entry_point_discovery.py index b82e748cfacb..86965f47fb34 100644 --- a/tests/providers/test_entry_point_discovery.py +++ b/tests/providers/test_entry_point_discovery.py @@ -60,6 +60,19 @@ def load(self): return self._loader() +def _enable(monkeypatch, *names, disabled=()): + """Gate helper: mark entry-point names enabled/disabled in config. + + ``_discover_entry_point_providers`` enforces the PluginManager's + ``plugins.enabled`` opt-in allow-list, so tests must enable their fake + entry points explicitly. + """ + import hermes_cli.plugins as hp + + monkeypatch.setattr(hp, "_get_enabled_plugins", lambda: set(names)) + monkeypatch.setattr(hp, "_get_disabled_plugins", lambda: set(disabled)) + + class _FakeEntryPoints: def __init__(self, eps): self._eps = eps @@ -100,6 +113,7 @@ def test_entry_point_callable_and_module_targets(monkeypatch): import importlib.metadata as md monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "ep-callable", "ep-module") _clear_provider_caches() try: assert providers.get_provider_profile("ep-callable") is not None @@ -109,6 +123,57 @@ def test_entry_point_callable_and_module_targets(monkeypatch): _clear_provider_caches() +def test_entry_point_not_enabled_is_skipped(monkeypatch): + """Entry points honor the plugins.enabled opt-in gate — installed ≠ loaded.""" + fake_eps = _FakeEntryPoints([_FakeEP("ep-callable", _register_via_callable)]) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "some-other-plugin") # ep-callable NOT enabled + _clear_provider_caches() + try: + assert providers.get_provider_profile("ep-callable") is None + finally: + _clear_provider_caches() + + +def test_entry_point_disabled_wins_over_enabled(monkeypatch): + """plugins.disabled is a deny-list that beats plugins.enabled.""" + fake_eps = _FakeEntryPoints([_FakeEP("ep-callable", _register_via_callable)]) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "ep-callable", disabled=("ep-callable",)) + _clear_provider_caches() + try: + assert providers.get_provider_profile("ep-callable") is None + finally: + _clear_provider_caches() + + +def test_general_plugin_register_ctx_not_invoked(monkeypatch): + """A register(ctx)-style general plugin sharing the group is never called.""" + calls = [] + + def _general_plugin_target(): + def register(ctx): # requires an argument — PluginManager contract + calls.append(ctx) + + return register + + fake_eps = _FakeEntryPoints([_FakeEP("general-plugin", _general_plugin_target)]) + import importlib.metadata as md + + monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "general-plugin") + _clear_provider_caches() + try: + providers._discover_providers() + assert calls == [] # never invoked (would have been a TypeError anyway) + finally: + _clear_provider_caches() + + def test_entry_point_failure_is_isolated(monkeypatch): def _boom(): raise RuntimeError("broken plugin") @@ -122,6 +187,7 @@ def _boom(): import importlib.metadata as md monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "broken", "ep-callable") _clear_provider_caches() try: # A broken entry point must not prevent the good one from registering. @@ -131,7 +197,9 @@ def _boom(): def test_filesystem_plugins_win_over_entry_points(monkeypatch): - """Entry points scan last, so a bundled/user profile of the same name wins.""" + """Entry points are discovered FIRST (lowest precedence): last-writer-wins + in register_provider() means a bundled/user profile of the same name + overrides a pip impostor.""" from providers.base import ProviderProfile def _register_ep_openrouter(): @@ -146,6 +214,7 @@ def register(): import importlib.metadata as md monkeypatch.setattr(md, "entry_points", lambda: fake_eps) + _enable(monkeypatch, "openrouter") # enabled, so precedence is what's tested _clear_provider_caches() try: p = providers.get_provider_profile("openrouter") diff --git a/website/docs/developer-guide/model-provider-plugin.md b/website/docs/developer-guide/model-provider-plugin.md index ea48f80b66db..5127107fa312 100644 --- a/website/docs/developer-guide/model-provider-plugin.md +++ b/website/docs/developer-guide/model-provider-plugin.md @@ -264,16 +264,33 @@ The target may be either: `register_provider(...)` side effect, mirroring the directory-plugin `__init__.py` contract. -`providers/__init__.py` discovers these entry points itself (the general -`PluginManager` records model-provider manifests but never imports them, so it -cannot register the profile). Entry-point plugins are discovered **before** -filesystem plugins, giving them the lowest precedence: because -`register_provider()` is last-writer-wins, a bundled or `$HERMES_HOME` profile -of the same name always overrides a pip-installed one. A pip package can add a -genuinely new provider, but cannot silently hijack a first-party provider name. - -A broken entry point is isolated — it is logged at warning level and skipped, -and never blocks discovery of the other providers. +`providers/__init__.py` discovers these entry points itself — the general +`PluginManager` never invokes provider registration for pip packages (its +entry-point path targets `register(ctx)`-style general plugins, gated by +`plugins.enabled`), so the provider registry does its own scan. Two rules +apply: + +- **Opt-in required.** The same `plugins.enabled` allow-list (and + `plugins.disabled` deny-list) from `config.yaml` governs this scan. A pip + package is never imported just because it is installed — users must add the + entry-point name to `plugins.enabled`: + + ```yaml + plugins: + enabled: + - acme-inference + ``` + +- **Lowest precedence.** Entry-point plugins are discovered **before** + filesystem plugins: because `register_provider()` is last-writer-wins, a + bundled or `$HERMES_HOME` profile of the same name always overrides a + pip-installed one. A pip package can add a genuinely new provider, but + cannot silently hijack a first-party provider name. + +Targets that require arguments (a general plugin's `register(ctx)`) are +skipped by the provider scan — they belong to the `PluginManager`. A broken +entry point is isolated — it is logged at warning level and skipped, and never +blocks discovery of the other providers. See [Building a Hermes Plugin](/developer-guide/plugins#distribute-via-pip) for the full entry-points setup.