perf(runtime): read-only config loads + per-message provider-resolution memo

gateway/run.py

@@ -2531,6 +2531,140 @@ def _own_policy_open_startup_violation(config) -> Optional[str]:
 _UNSET = object()
 
 
+# ---------------------------------------------------------------------------
+# Per-message runtime-resolution memo.
+#
+# resolve_runtime_provider() walks config.yaml (a full load_config() deepcopy
+# inside the resolve tree) plus the credential pool / auth stores on EVERY
+# call, measured ~2.35 ms warm on this host. The gateway resolver
+# (_resolve_session_agent_runtime) calls it 5-8x per inbound message flow, so
+# per-message provider resolution costs ~12-19 ms of pure CPU before the LLM
+# call. The MoA path already caches resolve_runtime_provider behind a 300s
+# TTL (#66793, agent/moa_loop.py _runtime_cache); the gateway path never got
+# the same treatment.
+#
+# Keyed on the files the resolution actually reads (config.yaml + profile and
+# global auth.json), so a config edit or `hermes auth add` invalidates
+# immediately, with a TTL backstop for env-var-only changes (the resolver also
+# reads ~25 env vars; the merged MoA cache accepts the same 300s staleness for
+# those). Never cache: the vertex OAuth path (token minted per call, 5-min
+# refresh margin) and AuthError/fallback results (a transient failure must not
+# be pinned for the whole TTL — same rule moa_loop documents).
+# ---------------------------------------------------------------------------
+_runtime_resolve_memo_lock = threading.Lock()
+_runtime_resolve_memo: dict[tuple, tuple[float, dict]] = {}
+_RUNTIME_RESOLVE_MEMO_TTL_SECONDS = 300.0  # mirrors agent/moa_loop
+
+_VERTEX_PROVIDER_ALIASES = frozenset(
+    {"vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"}
+)
+
+
+def _runtime_resolve_memo_signature() -> tuple:
+    """(hermes_home, config sig, profile auth sig, global auth sig).
+
+    ``hermes_home`` is part of the key because a multiplex gateway resolves
+    multiple profiles' agents in the same OS process (the desktop tui_gateway
+    switches profiles per request via ``set_hermes_home_override``). Without
+    it, two profiles whose config.yaml/auth.json happen to share the same
+    (mtime_ns, size) — e.g. ``hermes profile create --clone-all`` copies the
+    tree with mtime-preserving ``shutil.copy2`` — would resolve to the same
+    memo slot and one profile would receive the other's cached api_key /
+    base_url for up to the TTL. Same profile-boundary fix as #78185 applies
+    to agent/moa_loop.py's sibling cache.
+
+    A missing file contributes None so an auth.json that appears later
+    (first `hermes auth add`) invalidates the memo.
+    """
+    from hermes_cli.config import get_config_path
+
+    def _sig(path) -> tuple | None:
+        try:
+            st = path.stat()
+            return (st.st_mtime_ns, st.st_size)
+        except OSError:
+            return None
+
+    try:
+        cfg_path = get_config_path()
+    except Exception:
+        cfg_path = None
+    try:
+        from hermes_cli.auth import _auth_file_path, _global_auth_file_path
+
+        auth_path = _auth_file_path()
+        global_path = _global_auth_file_path()
+    except Exception:
+        auth_path = global_path = None
+    return (
+        str(get_hermes_home()),
+        _sig(cfg_path) if cfg_path is not None else None,
+        _sig(auth_path) if auth_path is not None else None,
+        _sig(global_path) if global_path is not None else None,
+    )
+
+
+def _memoized_resolve_runtime(
+    *,
+    requested: Optional[str] = None,
+    target_model: Optional[str] = None,
+    explicit_api_key: Optional[str] = None,
+    explicit_base_url: Optional[str] = None,
+) -> dict:
+    """resolve_runtime_provider() with the per-message memo applied.
+
+    Returns a FRESH top-level dict each call (callers mutate it, e.g.
+    ``runtime_kwargs.pop("model", None)``), so a cached entry can never be
+    corrupted by a caller. The credential_pool object is shared across hits
+    within the TTL — safe because any store change bumps the auth.json mtime
+    and invalidates the memo.
+    """
+    from hermes_cli.runtime_provider import resolve_runtime_provider
+
+    # Preserve the resolver's original call shape: only pass kwargs that are
+    # actually set, so a patched/alternate resolver that accepts the legacy
+    # signature (requested / explicit_*) keeps working unchanged.
+    kwargs: dict = {}
+    if requested is not None:
+        kwargs["requested"] = requested
+    if target_model is not None:
+        kwargs["target_model"] = target_model
+    if explicit_api_key is not None:
+        kwargs["explicit_api_key"] = explicit_api_key
+    if explicit_base_url is not None:
+        kwargs["explicit_base_url"] = explicit_base_url
+
+    if requested in _VERTEX_PROVIDER_ALIASES:
+        # Vertex mints a fresh OAuth token per call (5-min refresh margin);
+        # caching it would serve an expired token.
+        return resolve_runtime_provider(**kwargs)
+
+    key = (
+        requested,
+        target_model,
+        explicit_api_key,
+        explicit_base_url,
+        _runtime_resolve_memo_signature(),
+    )
+    now = time.monotonic()
+    with _runtime_resolve_memo_lock:
+        entry = _runtime_resolve_memo.get(key)
+    if entry is not None:
+        stamped_at, cached = entry
+        if now - stamped_at < _RUNTIME_RESOLVE_MEMO_TTL_SECONDS:
+            return dict(cached)
+
+    runtime = resolve_runtime_provider(**kwargs)
+    # Never cache the vertex-shaped result (resolved through the default
+    # config) — same reason as the requested-alias bypass above.
+    if str(runtime.get("provider") or "").strip().lower() not in _VERTEX_PROVIDER_ALIASES:
+        with _runtime_resolve_memo_lock:
+            # Store a COPY: the caller gets `runtime` and may mutate it (e.g.
+            # pop("model")), and the memo must never share its own reference.
+            _runtime_resolve_memo[key] = (now, dict(runtime))
+    return runtime
+
+
 def _resolve_runtime_agent_kwargs() -> dict:
     """Resolve provider credentials for gateway-created AIAgent instances.
 
@@ -2545,14 +2679,13 @@ def _resolve_runtime_agent_kwargs() -> dict:
     before giving up.
     """
     from hermes_cli.runtime_provider import (
-        resolve_runtime_provider,
         format_runtime_provider_error,
         _get_model_config,
     )
     from hermes_cli.auth import AuthError, is_rate_limited_auth_error
 
     try:
-        runtime = resolve_runtime_provider()
+        runtime = _memoized_resolve_runtime()
     except AuthError as auth_exc:
         # Distinguish a transient rate-limit/quota cap (credentials are fine,
         # re-auth cannot help) from a genuine auth failure (expired/revoked
@@ -2604,12 +2737,9 @@ def _resolve_runtime_agent_kwargs() -> dict:
 
 def _resolve_runtime_agent_kwargs_for_provider(provider: str) -> dict:
     """Resolve runtime credentials for a specific provider (e.g. from channel override)."""
-    from hermes_cli.runtime_provider import (
-        resolve_runtime_provider,
-        format_runtime_provider_error,
-    )
+    from hermes_cli.runtime_provider import format_runtime_provider_error
     try:
-        runtime = resolve_runtime_provider(requested=provider)
+        runtime = _memoized_resolve_runtime(requested=provider)
     except Exception as exc:
         raise RuntimeError(format_runtime_provider_error(exc)) from exc
     return {

hermes_cli/runtime_provider.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import copy
 import logging
 import os
 import re
@@ -42,7 +43,7 @@
 )
 from hermes_cli.config import (
     get_compatible_custom_providers,
-    load_config,
+    load_config_readonly,
     normalize_extra_headers,
 )
 from hermes_cli.providers import custom_provider_aliases, custom_provider_slug
@@ -315,10 +316,16 @@ def _auto_detect_local_model(base_url: str) -> str:
 
 
 def _get_model_config() -> Dict[str, Any]:
-    config = load_config()
+    config = load_config_readonly()
     model_cfg = config.get("model")
     if isinstance(model_cfg, dict):
-        cfg = dict(model_cfg)
+        # Shallow dict() of the model section would leave nested values
+        # (fallback_providers, overrides, …) shared with the cached config;
+        # a caller mutating one would corrupt the read-only cache for every
+        # other caller. The model section is a handful of keys, so deepcopy
+        # the section (µs) instead of the whole config (the ~265µs deepcopy
+        # load_config() applies).
+        cfg = copy.deepcopy(model_cfg)
         # Accept "model" as alias for "default" (users intuitively write model.model)
         if not cfg.get("default") and cfg.get("model"):
             cfg["default"] = cfg["model"]
@@ -705,7 +712,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
             if (canonical or "").strip().lower() == requested_norm:
                 return None
 
-    config = load_config()
+    config = load_config_readonly()
 
     # First check providers: dict (new-style user-defined providers)
     providers = config.get("providers")
@@ -842,7 +849,7 @@ def find_custom_provider_identity(base_url: str) -> Optional[str]:
     if not target:
         return None
     try:
-        config = load_config()
+        config = load_config_readonly()
     except Exception:
         return None
 
@@ -895,7 +902,7 @@ def find_custom_provider_identity_by_model(model: str) -> Optional[str]:
     if not target:
         return None
     try:
-        config = load_config()
+        config = load_config_readonly()
     except Exception:
         return None
 
@@ -1691,8 +1698,8 @@ def resolve_runtime_provider(
     #
     # Fail fast with a typed error so the fallback chain can advance to
     # the next provider instead of using a disabled one.
-    from hermes_cli.config import is_provider_enabled, load_config
-    _full_cfg = load_config()
+    from hermes_cli.config import is_provider_enabled
+    _full_cfg = load_config_readonly()
     _provs_cfg = _full_cfg.get("providers") if isinstance(_full_cfg, dict) else None
     if isinstance(_provs_cfg, dict):
         _block = _provs_cfg.get(requested_provider)
@@ -2130,7 +2137,7 @@ def resolve_runtime_provider(
                 code="no_aws_credentials",
             )
         # Read bedrock-specific config from config.yaml
-        _bedrock_cfg = load_config().get("bedrock", {})
+        _bedrock_cfg = load_config_readonly().get("bedrock", {})
         # Region priority: config.yaml bedrock.region → env var → us-east-1
         region = (_bedrock_cfg.get("region") or "").strip() or resolve_bedrock_region()
         auth_source = resolve_aws_auth_env_var() or "aws-sdk-default-chain"

tests/agent/test_bedrock_adapter.py

@@ -1143,7 +1143,7 @@ def _resolve(self, monkeypatch, *, bearer: bool):
                 "provider": "bedrock",
             },
         )
-        monkeypatch.setattr(rp, "load_config", lambda: {"bedrock": {}})
+        monkeypatch.setattr(rp, "load_config_readonly", lambda: {"bedrock": {}})
         return rp.resolve_runtime_provider(requested="bedrock")
 
     def test_bearer_token_forces_converse_for_claude(self, monkeypatch):

tests/agent/test_nous_portal_anthropic_wire.py

@@ -80,7 +80,7 @@ class TestRuntimeResolution:
 
     @pytest.fixture(autouse=True)
     def _stub_portal_credentials(self, monkeypatch):
-        monkeypatch.setattr(rp, "load_config", lambda: {})
+        monkeypatch.setattr(rp, "load_config_readonly", lambda: {})
         monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "nous")
         monkeypatch.setattr(rp, "load_pool", lambda p: SimpleNamespace(
             has_credentials=lambda: False,