diff --git a/gateway/run.py b/gateway/run.py index bd2937bee9f77..8cd69350abade 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2531,6 +2531,140 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: _UNSET = object() +# --------------------------------------------------------------------------- +# Per-message runtime-resolution memo. +# +# resolve_runtime_provider() walks config.yaml (a full load_config() deepcopy +# inside the resolve tree) plus the credential pool / auth stores on EVERY +# call, measured ~2.35 ms warm on this host. The gateway resolver +# (_resolve_session_agent_runtime) calls it 5-8x per inbound message flow, so +# per-message provider resolution costs ~12-19 ms of pure CPU before the LLM +# call. The MoA path already caches resolve_runtime_provider behind a 300s +# TTL (#66793, agent/moa_loop.py _runtime_cache); the gateway path never got +# the same treatment. +# +# Keyed on the files the resolution actually reads (config.yaml + profile and +# global auth.json), so a config edit or `hermes auth add` invalidates +# immediately, with a TTL backstop for env-var-only changes (the resolver also +# reads ~25 env vars; the merged MoA cache accepts the same 300s staleness for +# those). Never cache: the vertex OAuth path (token minted per call, 5-min +# refresh margin) and AuthError/fallback results (a transient failure must not +# be pinned for the whole TTL — same rule moa_loop documents). +# --------------------------------------------------------------------------- +_runtime_resolve_memo_lock = threading.Lock() +_runtime_resolve_memo: dict[tuple, tuple[float, dict]] = {} +_RUNTIME_RESOLVE_MEMO_TTL_SECONDS = 300.0 # mirrors agent/moa_loop + +_VERTEX_PROVIDER_ALIASES = frozenset( + {"vertex", "google-vertex", "vertex-ai", "gcp-vertex", "vertexai"} +) + + +def _runtime_resolve_memo_signature() -> tuple: + """(hermes_home, config sig, profile auth sig, global auth sig). + + ``hermes_home`` is part of the key because a multiplex gateway resolves + multiple profiles' agents in the same OS process (the desktop tui_gateway + switches profiles per request via ``set_hermes_home_override``). Without + it, two profiles whose config.yaml/auth.json happen to share the same + (mtime_ns, size) — e.g. ``hermes profile create --clone-all`` copies the + tree with mtime-preserving ``shutil.copy2`` — would resolve to the same + memo slot and one profile would receive the other's cached api_key / + base_url for up to the TTL. Same profile-boundary fix as #78185 applies + to agent/moa_loop.py's sibling cache. + + A missing file contributes None so an auth.json that appears later + (first `hermes auth add`) invalidates the memo. + """ + from hermes_cli.config import get_config_path + + def _sig(path) -> tuple | None: + try: + st = path.stat() + return (st.st_mtime_ns, st.st_size) + except OSError: + return None + + try: + cfg_path = get_config_path() + except Exception: + cfg_path = None + try: + from hermes_cli.auth import _auth_file_path, _global_auth_file_path + + auth_path = _auth_file_path() + global_path = _global_auth_file_path() + except Exception: + auth_path = global_path = None + return ( + str(get_hermes_home()), + _sig(cfg_path) if cfg_path is not None else None, + _sig(auth_path) if auth_path is not None else None, + _sig(global_path) if global_path is not None else None, + ) + + +def _memoized_resolve_runtime( + *, + requested: Optional[str] = None, + target_model: Optional[str] = None, + explicit_api_key: Optional[str] = None, + explicit_base_url: Optional[str] = None, +) -> dict: + """resolve_runtime_provider() with the per-message memo applied. + + Returns a FRESH top-level dict each call (callers mutate it, e.g. + ``runtime_kwargs.pop("model", None)``), so a cached entry can never be + corrupted by a caller. The credential_pool object is shared across hits + within the TTL — safe because any store change bumps the auth.json mtime + and invalidates the memo. + """ + from hermes_cli.runtime_provider import resolve_runtime_provider + + # Preserve the resolver's original call shape: only pass kwargs that are + # actually set, so a patched/alternate resolver that accepts the legacy + # signature (requested / explicit_*) keeps working unchanged. + kwargs: dict = {} + if requested is not None: + kwargs["requested"] = requested + if target_model is not None: + kwargs["target_model"] = target_model + if explicit_api_key is not None: + kwargs["explicit_api_key"] = explicit_api_key + if explicit_base_url is not None: + kwargs["explicit_base_url"] = explicit_base_url + + if requested in _VERTEX_PROVIDER_ALIASES: + # Vertex mints a fresh OAuth token per call (5-min refresh margin); + # caching it would serve an expired token. + return resolve_runtime_provider(**kwargs) + + key = ( + requested, + target_model, + explicit_api_key, + explicit_base_url, + _runtime_resolve_memo_signature(), + ) + now = time.monotonic() + with _runtime_resolve_memo_lock: + entry = _runtime_resolve_memo.get(key) + if entry is not None: + stamped_at, cached = entry + if now - stamped_at < _RUNTIME_RESOLVE_MEMO_TTL_SECONDS: + return dict(cached) + + runtime = resolve_runtime_provider(**kwargs) + # Never cache the vertex-shaped result (resolved through the default + # config) — same reason as the requested-alias bypass above. + if str(runtime.get("provider") or "").strip().lower() not in _VERTEX_PROVIDER_ALIASES: + with _runtime_resolve_memo_lock: + # Store a COPY: the caller gets `runtime` and may mutate it (e.g. + # pop("model")), and the memo must never share its own reference. + _runtime_resolve_memo[key] = (now, dict(runtime)) + return runtime + + def _resolve_runtime_agent_kwargs() -> dict: """Resolve provider credentials for gateway-created AIAgent instances. @@ -2545,14 +2679,13 @@ def _resolve_runtime_agent_kwargs() -> dict: before giving up. """ from hermes_cli.runtime_provider import ( - resolve_runtime_provider, format_runtime_provider_error, _get_model_config, ) from hermes_cli.auth import AuthError, is_rate_limited_auth_error try: - runtime = resolve_runtime_provider() + runtime = _memoized_resolve_runtime() except AuthError as auth_exc: # Distinguish a transient rate-limit/quota cap (credentials are fine, # re-auth cannot help) from a genuine auth failure (expired/revoked @@ -2604,12 +2737,9 @@ def _resolve_runtime_agent_kwargs() -> dict: def _resolve_runtime_agent_kwargs_for_provider(provider: str) -> dict: """Resolve runtime credentials for a specific provider (e.g. from channel override).""" - from hermes_cli.runtime_provider import ( - resolve_runtime_provider, - format_runtime_provider_error, - ) + from hermes_cli.runtime_provider import format_runtime_provider_error try: - runtime = resolve_runtime_provider(requested=provider) + runtime = _memoized_resolve_runtime(requested=provider) except Exception as exc: raise RuntimeError(format_runtime_provider_error(exc)) from exc return { diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index bc8b25924b7ca..a6a6d6e0bd5ba 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import logging import os import re @@ -42,7 +43,7 @@ ) from hermes_cli.config import ( get_compatible_custom_providers, - load_config, + load_config_readonly, normalize_extra_headers, ) from hermes_cli.providers import custom_provider_aliases, custom_provider_slug @@ -315,10 +316,16 @@ def _auto_detect_local_model(base_url: str) -> str: def _get_model_config() -> Dict[str, Any]: - config = load_config() + config = load_config_readonly() model_cfg = config.get("model") if isinstance(model_cfg, dict): - cfg = dict(model_cfg) + # Shallow dict() of the model section would leave nested values + # (fallback_providers, overrides, …) shared with the cached config; + # a caller mutating one would corrupt the read-only cache for every + # other caller. The model section is a handful of keys, so deepcopy + # the section (µs) instead of the whole config (the ~265µs deepcopy + # load_config() applies). + cfg = copy.deepcopy(model_cfg) # Accept "model" as alias for "default" (users intuitively write model.model) if not cfg.get("default") and cfg.get("model"): cfg["default"] = cfg["model"] @@ -705,7 +712,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An if (canonical or "").strip().lower() == requested_norm: return None - config = load_config() + config = load_config_readonly() # First check providers: dict (new-style user-defined providers) providers = config.get("providers") @@ -842,7 +849,7 @@ def find_custom_provider_identity(base_url: str) -> Optional[str]: if not target: return None try: - config = load_config() + config = load_config_readonly() except Exception: return None @@ -895,7 +902,7 @@ def find_custom_provider_identity_by_model(model: str) -> Optional[str]: if not target: return None try: - config = load_config() + config = load_config_readonly() except Exception: return None @@ -1691,8 +1698,8 @@ def resolve_runtime_provider( # # Fail fast with a typed error so the fallback chain can advance to # the next provider instead of using a disabled one. - from hermes_cli.config import is_provider_enabled, load_config - _full_cfg = load_config() + from hermes_cli.config import is_provider_enabled + _full_cfg = load_config_readonly() _provs_cfg = _full_cfg.get("providers") if isinstance(_full_cfg, dict) else None if isinstance(_provs_cfg, dict): _block = _provs_cfg.get(requested_provider) @@ -2130,7 +2137,7 @@ def resolve_runtime_provider( code="no_aws_credentials", ) # Read bedrock-specific config from config.yaml - _bedrock_cfg = load_config().get("bedrock", {}) + _bedrock_cfg = load_config_readonly().get("bedrock", {}) # Region priority: config.yaml bedrock.region → env var → us-east-1 region = (_bedrock_cfg.get("region") or "").strip() or resolve_bedrock_region() auth_source = resolve_aws_auth_env_var() or "aws-sdk-default-chain" diff --git a/tests/agent/test_bedrock_adapter.py b/tests/agent/test_bedrock_adapter.py index 8994688e0f211..80c483c5a4211 100644 --- a/tests/agent/test_bedrock_adapter.py +++ b/tests/agent/test_bedrock_adapter.py @@ -1143,7 +1143,7 @@ def _resolve(self, monkeypatch, *, bearer: bool): "provider": "bedrock", }, ) - monkeypatch.setattr(rp, "load_config", lambda: {"bedrock": {}}) + monkeypatch.setattr(rp, "load_config_readonly", lambda: {"bedrock": {}}) return rp.resolve_runtime_provider(requested="bedrock") def test_bearer_token_forces_converse_for_claude(self, monkeypatch): diff --git a/tests/agent/test_nous_portal_anthropic_wire.py b/tests/agent/test_nous_portal_anthropic_wire.py index 2b4521c87e921..3a2b01d41075b 100644 --- a/tests/agent/test_nous_portal_anthropic_wire.py +++ b/tests/agent/test_nous_portal_anthropic_wire.py @@ -80,7 +80,7 @@ class TestRuntimeResolution: @pytest.fixture(autouse=True) def _stub_portal_credentials(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: {}) + monkeypatch.setattr(rp, "load_config_readonly", lambda: {}) monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "nous") monkeypatch.setattr(rp, "load_pool", lambda p: SimpleNamespace( has_credentials=lambda: False, diff --git a/tests/gateway/test_runtime_resolve_fastpath.py b/tests/gateway/test_runtime_resolve_fastpath.py new file mode 100644 index 0000000000000..f24485ae264fb --- /dev/null +++ b/tests/gateway/test_runtime_resolve_fastpath.py @@ -0,0 +1,300 @@ +"""Regression tests for the per-message runtime-resolution fast path. + +Two layers are pinned here: + +Layer A — ``hermes_cli/runtime_provider.py``: the resolve tree previously +called ``load_config()`` (a defensive deepcopy of the whole config, ~265us on +a warm cache) 5-7 times per ``resolve_runtime_provider()`` call — measured +74% of the ~2.35 ms/call cost. All sites are read-only against config, so +they now use ``load_config_readonly()`` (mtime-cached, no deepcopy), and +``_get_model_config()`` deep-copies only the small model section to preserve +its mutation-safety contract. + +Layer B — ``gateway/run.py``: the gateway resolver calls +``resolve_runtime_provider()`` 5-8x per inbound message. The MoA path already +caches it behind a 300s TTL (#66793); the gateway path now gets the same +treatment via ``_memoized_resolve_runtime``, keyed on the files the +resolution reads (config.yaml + profile/global auth.json) with a TTL +backstop. Vertex (per-call OAuth token) and fallback/AuthError results are +never cached. + +The measured-work pins assert the MECHANISM (work actually skipped), which is +what a regression would break; the behavior-parity pins assert the RESULT is +unchanged. +""" + +import time +from unittest.mock import MagicMock + +import pytest + +import gateway.run as gateway_run +import hermes_cli.config as config_mod +import hermes_cli.runtime_provider as rp_mod + + +@pytest.fixture(autouse=True) +def _clear_memo(): + """Each test starts with an empty gateway memo. + + Tolerates the attribute being absent (the sabotage gate runs this test + file against PRE-FIX base code, where the memo does not exist yet) so + the base leg fails on the measured-work assertions — a genuine pin + failure — instead of erroring at setup. + """ + lock = getattr(gateway_run, "_runtime_resolve_memo_lock", None) + memo = getattr(gateway_run, "_runtime_resolve_memo", None) + if lock is not None and memo is not None: + with lock: + memo.clear() + yield + if lock is not None and memo is not None: + with lock: + memo.clear() + + +# --------------------------------------------------------------------------- +# Layer A — the resolve tree must not deepcopy the whole config +# --------------------------------------------------------------------------- + +class TestResolveTreeReadOnlyConfig: + def test_get_model_config_does_not_call_deepcopy_load_config(self, monkeypatch): + """The model-config read must use the read-only loader, not the + deepcopy variant. load_config() is what a regression would reintroduce.""" + calls = {"deepcopy": 0} + + def counting_load_config(*a, **k): + calls["deepcopy"] += 1 + return {} + + monkeypatch.setattr(config_mod, "load_config", counting_load_config) + rp_mod._get_model_config() + assert calls["deepcopy"] == 0 + + def test_resolve_tree_uses_readonly_config_loader(self, monkeypatch): + """No load_config() (deepcopy) call may originate from the + runtime_provider resolve tree — every site there was converted to the + read-only loader. (auth.py's error-hint builder legitimately calls + load_config on the failure path; that is not the resolve tree.)""" + import traceback + + calls = {"from_rp": 0, "elsewhere": 0} + orig = config_mod.load_config + + def counting(*a, **k): + frames = traceback.extract_stack(limit=4) + # The DIRECT caller is frames[-3] (counting <- caller <- ...). + caller = frames[-3] if len(frames) >= 3 else None + from_rp = bool( + caller and "hermes_cli/runtime_provider.py" in caller.filename + ) + if from_rp: + calls["from_rp"] += 1 + else: + calls["elsewhere"] += 1 + return orig(*a, **k) + + monkeypatch.setattr(config_mod, "load_config", counting) + try: + rp_mod.resolve_runtime_provider(requested="openai") + except Exception: + pass # no credentials on CI — we only assert the loader used + assert calls["from_rp"] == 0 + assert calls["elsewhere"] >= 1 # error-hint path still works + + def test_get_model_config_mutation_is_isolated(self): + """The returned model config is a private copy: mutating it (even + nested) must not corrupt the shared read-only config cache.""" + cfg_a = rp_mod._get_model_config() + cfg_b = rp_mod._get_model_config() + cfg_a["default"] = "HACKED" + assert cfg_b.get("default") != "HACKED" + # Nested values must also be isolated (the deepcopy of the section). + if isinstance(cfg_a, dict) and isinstance(cfg_b, dict): + assert cfg_a is not cfg_b + + +# --------------------------------------------------------------------------- +# Layer B — gateway memo: mechanism pins +# --------------------------------------------------------------------------- + +class TestGatewayRuntimeMemo: + def _counting_resolve(self, monkeypatch, *, real_delegate=False): + calls = {"n": 0, "last": None} + orig = rp_mod.resolve_runtime_provider + + def fake(*a, **k): + calls["n"] += 1 + calls["last"] = (a, k) + if real_delegate: + # Behavior-parity mode: run the REAL resolver. + return orig(*a, **k) + # Deterministic success result (CI has no provider configured, so + # the real resolver raises AuthError and nothing would be cached). + return { + "api_key": "test-key", + "base_url": "https://example.com/v1", + "provider": k.get("requested") or "openai", + "requested_provider": k.get("requested"), + "api_mode": "chat_completions", + "command": None, + "args": [], + "credential_pool": None, + "max_output_tokens": None, + } + + monkeypatch.setattr(rp_mod, "resolve_runtime_provider", fake) + return calls + + def test_second_resolve_hits_memo(self, monkeypatch): + """Two identical calls within the TTL must resolve ONCE — the + per-message win (5-8 resolver calls per message → 1).""" + calls = self._counting_resolve(monkeypatch) + r1 = gateway_run._memoized_resolve_runtime() + r2 = gateway_run._memoized_resolve_runtime() + assert calls["n"] == 1 + assert r1 == r2 + + def test_different_requested_provider_is_separate_key(self, monkeypatch): + calls = self._counting_resolve(monkeypatch) + try: + gateway_run._memoized_resolve_runtime() + except Exception: + pass + try: + gateway_run._memoized_resolve_runtime(requested="openai") + except Exception: + pass + # Two different keys → two resolves (whatever they resolve to). + assert calls["n"] == 2 + + def test_config_change_invalidates_memo(self, monkeypatch, tmp_path): + """A config.yaml mtime bump must force a fresh resolve immediately + (a `/model` switch or `hermes config` edit must not wait for the TTL).""" + calls = self._counting_resolve(monkeypatch) + gateway_run._memoized_resolve_runtime() + assert calls["n"] == 1 + + # Rewrite config.yaml so its mtime/size change. + from hermes_cli.config import get_config_path + cfg_path = get_config_path() + cfg_path.parent.mkdir(parents=True, exist_ok=True) + orig = cfg_path.read_text() if cfg_path.exists() else "" + try: + cfg_path.write_text(orig + "\n# memo-invalidation probe\n") + gateway_run._memoized_resolve_runtime() + assert calls["n"] == 2 + finally: + if orig: + cfg_path.write_text(orig) + + def test_ttl_expiry_forces_re_resolve(self, monkeypatch): + calls = self._counting_resolve(monkeypatch) + gateway_run._memoized_resolve_runtime() + assert calls["n"] == 1 + # Rewind the cached stamp past the TTL. + with gateway_run._runtime_resolve_memo_lock: + for key, (stamp, val) in list(gateway_run._runtime_resolve_memo.items()): + gateway_run._runtime_resolve_memo[key] = ( + stamp - gateway_run._RUNTIME_RESOLVE_MEMO_TTL_SECONDS - 1, + val, + ) + gateway_run._memoized_resolve_runtime() + assert calls["n"] == 2 + + def test_vertex_requested_never_cached(self, monkeypatch): + """Vertex mints a per-call OAuth token — every call must resolve.""" + calls = self._counting_resolve(monkeypatch) + try: + gateway_run._memoized_resolve_runtime(requested="vertex") + except Exception: + pass + try: + gateway_run._memoized_resolve_runtime(requested="vertex") + except Exception: + pass + assert calls["n"] == 2 + + def test_returned_dict_is_fresh_copy(self, monkeypatch): + """Callers mutate the returned dict (e.g. pop('model')) — the cached + entry must never be corrupted by that.""" + calls = self._counting_resolve(monkeypatch) + r1 = gateway_run._memoized_resolve_runtime() + r1["provider"] = "HACKED" + r2 = gateway_run._memoized_resolve_runtime() + assert calls["n"] == 1 + assert r2.get("provider") != "HACKED" + + def test_gateway_kwargs_wrapper_uses_memo(self, monkeypatch): + """The actual per-message entry (_resolve_runtime_agent_kwargs) must + resolve once across two calls (success path).""" + calls = {"n": 0} + orig = rp_mod.resolve_runtime_provider + + def counting(*a, **k): + calls["n"] += 1 + return { + "api_key": "k", + "base_url": "https://example.com/v1", + "provider": k.get("requested") or "openai", + "requested_provider": k.get("requested"), + "api_mode": "chat_completions", + "command": None, + "args": [], + "credential_pool": None, + "max_output_tokens": None, + } + + monkeypatch.setattr(rp_mod, "resolve_runtime_provider", counting) + gateway_run._resolve_runtime_agent_kwargs() + gateway_run._resolve_runtime_agent_kwargs() + assert calls["n"] == 1 + + +# --------------------------------------------------------------------------- +# Layer B — the _resolve_runtime_agent_kwargs wrapper itself still works +# --------------------------------------------------------------------------- + +class TestRuntimeKwargsWrapper: + def test_wrapper_returns_expected_shape(self, monkeypatch): + """The gateway wrapper must keep returning the documented dict shape + (max_tokens computed from env/config per call, outside the memo).""" + fake_runtime = { + "api_key": "k", + "base_url": "https://example.com/v1", + "provider": "openai", + "requested_provider": "openai", + "api_mode": "chat_completions", + "command": None, + "args": ["a"], + "credential_pool": None, + } + monkeypatch.setattr( + rp_mod, "resolve_runtime_provider", lambda *a, **k: dict(fake_runtime) + ) + out = gateway_run._resolve_runtime_agent_kwargs() + assert out["provider"] == "openai" + assert out["base_url"] == "https://example.com/v1" + assert out["args"] == ["a"] + assert "max_tokens" in out + + def test_for_provider_wrapper_uses_memo(self, monkeypatch): + calls = {"n": 0} + + def counting(*a, **k): + calls["n"] += 1 + return { + "api_key": "k", + "base_url": "https://example.com/v1", + "provider": k.get("requested") or "openai", + "requested_provider": k.get("requested"), + "api_mode": "chat_completions", + "command": None, + "args": [], + "credential_pool": None, + } + + monkeypatch.setattr(rp_mod, "resolve_runtime_provider", counting) + gateway_run._resolve_runtime_agent_kwargs_for_provider("openai") + gateway_run._resolve_runtime_agent_kwargs_for_provider("openai") + assert calls["n"] == 1 diff --git a/tests/gateway/test_runtime_resolve_fastpath_profile_isolation.py b/tests/gateway/test_runtime_resolve_fastpath_profile_isolation.py new file mode 100644 index 0000000000000..34599945cc36a --- /dev/null +++ b/tests/gateway/test_runtime_resolve_fastpath_profile_isolation.py @@ -0,0 +1,92 @@ +"""Profile-isolation regression tests for the gateway runtime-resolution memo. + +The per-message memo in gateway/run.py (_memoized_resolve_runtime) caches +resolve_runtime_provider() results keyed on config/auth file signatures plus +a TTL. A multiplex gateway resolves multiple profiles' agents in the same OS +process (the desktop tui_gateway switches profiles per request via +set_hermes_home_override), so the key must carry the profile identity: +without hermes_home, two profiles whose config.yaml/auth.json share the same +(mtime_ns, size) — exactly what `hermes profile create --clone-all` produces +via mtime-preserving shutil.copy2 — would resolve to the same memo slot and +one profile would receive the other's cached api_key/base_url for up to the +300s TTL. Same profile-boundary fix as #78185 applies to agent/moa_loop.py's +sibling cache. +""" + +from pathlib import Path + +import pytest + +from hermes_constants import get_hermes_home + +# The two_profiles fixture lives in the sibling profile-isolation suite. +from tests.test_profile_isolation_runtime import two_profiles # noqa: F401 + + +def _under_override(home: Path, fn): + """Run ``fn`` with the profile override set to ``home`` and reset after.""" + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + token = set_hermes_home_override(str(home)) + try: + return fn() + finally: + reset_hermes_home_override(token) + + +class TestGatewayRuntimeMemoProfileIsolation: + """gateway/run.py's runtime memo must not leak credentials across profiles.""" + + def test_memo_does_not_leak_credentials_across_profiles( + self, two_profiles, monkeypatch + ): + prof_a, prof_b = two_profiles + import gateway.run as gateway_run + import hermes_cli.runtime_provider as rp_mod + + with gateway_run._runtime_resolve_memo_lock: + gateway_run._runtime_resolve_memo.clear() + + def fake_resolve(*, requested=None, target_model=None, **_kw): + # A realistic resolver reads the active profile's own config for + # credentials — simulated here by keying off the live override. + home = str(get_hermes_home()) + return { + "api_key": f"secret-for-{Path(home).name}", + "base_url": f"https://{Path(home).name}.example.com/v1", + "provider": requested or "openai", + "requested_provider": requested, + "api_mode": "chat_completions", + "command": None, + "args": [], + "credential_pool": None, + } + + monkeypatch.setattr(rp_mod, "resolve_runtime_provider", fake_resolve) + + resolved_a = _under_override( + prof_a, lambda: gateway_run._memoized_resolve_runtime() + ) + resolved_b = _under_override( + prof_b, lambda: gateway_run._memoized_resolve_runtime() + ) + + assert resolved_a["api_key"] == f"secret-for-{prof_a.name}" + assert resolved_b["api_key"] == f"secret-for-{prof_b.name}", ( + "profile B's runtime resolution must not receive profile A's " + "cached credentials for the same memo key" + ) + assert resolved_a["base_url"] != resolved_b["base_url"] + + def test_signature_includes_hermes_home(self, two_profiles): + """The memo signature must differ between profiles even when the + config/auth files are byte-identical (cloned profiles share + mtime_ns/size via mtime-preserving copy2).""" + import gateway.run as gateway_run + + prof_a, prof_b = two_profiles + sig_a = _under_override(prof_a, gateway_run._runtime_resolve_memo_signature) + sig_b = _under_override(prof_b, gateway_run._runtime_resolve_memo_signature) + assert sig_a != sig_b + assert sig_a[0] == str(prof_a) + assert sig_b[0] == str(prof_b) diff --git a/tests/hermes_cli/test_canonical_custom_identity.py b/tests/hermes_cli/test_canonical_custom_identity.py index 80ef6c362406a..9eb975572996e 100644 --- a/tests/hermes_cli/test_canonical_custom_identity.py +++ b/tests/hermes_cli/test_canonical_custom_identity.py @@ -41,7 +41,7 @@ def keyed_provider_config(monkeypatch): } } } - monkeypatch.setattr(rp, "load_config", lambda *a, **k: config) + monkeypatch.setattr(rp, "load_config_readonly", lambda *a, **k: config) monkeypatch.setattr("hermes_cli.config.load_config", lambda *a, **k: config) monkeypatch.setattr(rp, "_get_model_config", lambda: {}) return config @@ -93,7 +93,7 @@ def test_legacy_unkeyed_entry_keeps_its_name_identity(monkeypatch): } ] } - monkeypatch.setattr(rp, "load_config", lambda *a, **k: config) + monkeypatch.setattr(rp, "load_config_readonly", lambda *a, **k: config) monkeypatch.setattr("hermes_cli.config.load_config", lambda *a, **k: config) monkeypatch.setattr(rp, "_get_model_config", lambda: {}) diff --git a/tests/hermes_cli/test_custom_provider_identity.py b/tests/hermes_cli/test_custom_provider_identity.py index c4f36ea37f199..134d486e9ab9e 100644 --- a/tests/hermes_cli/test_custom_provider_identity.py +++ b/tests/hermes_cli/test_custom_provider_identity.py @@ -14,7 +14,7 @@ def test_matches_legacy_custom_providers_list(monkeypatch): monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ {"name": "MiMo v2.5 Pro", "base_url": "https://api.mimo.example/v1"} @@ -30,7 +30,7 @@ def test_matches_legacy_custom_providers_list(monkeypatch): def test_matches_providers_dict_by_key(monkeypatch): monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: {"providers": {"local": {"api": "http://127.0.0.1:8000/v1"}}}, ) assert ( @@ -50,7 +50,7 @@ def test_matches_providers_dict_by_stable_key_not_display_name(monkeypatch): } monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: config, ) slug = rp.find_custom_provider_identity("http://127.0.0.1:8000/v1") @@ -64,7 +64,7 @@ def test_matches_providers_dict_by_stable_key_not_display_name(monkeypatch): def test_match_ignores_trailing_slash_and_case(monkeypatch): monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ {"name": "local", "base_url": "http://Localhost:8000/v1/"} @@ -80,7 +80,7 @@ def test_match_ignores_trailing_slash_and_case(monkeypatch): def test_no_match_returns_none(monkeypatch): monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ {"name": "other", "base_url": "https://elsewhere.example/v1"} @@ -92,7 +92,7 @@ def test_no_match_returns_none(monkeypatch): def test_empty_base_url_returns_none(monkeypatch): monkeypatch.setattr( - rp, "load_config", lambda: {"custom_providers": [{"name": "x"}]} + rp, "load_config_readonly", lambda: {"custom_providers": [{"name": "x"}]} ) assert rp.find_custom_provider_identity("") is None assert rp.find_custom_provider_identity(None) is None @@ -110,7 +110,7 @@ def test_identity_resolves_back_through_named_lookup(monkeypatch): } ] } - monkeypatch.setattr(rp, "load_config", lambda: config) + monkeypatch.setattr(rp, "load_config_readonly", lambda: config) slug = rp.find_custom_provider_identity("https://api.mimo.example/v1") assert slug == "custom:mimo-v2.5-pro" diff --git a/tests/hermes_cli/test_runtime_provider_resolution.py b/tests/hermes_cli/test_runtime_provider_resolution.py index 1f90e3edb9c61..a7a151aaf6569 100644 --- a/tests/hermes_cli/test_runtime_provider_resolution.py +++ b/tests/hermes_cli/test_runtime_provider_resolution.py @@ -556,7 +556,7 @@ def test_named_custom_provider_uses_saved_credentials(monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ { @@ -597,7 +597,7 @@ def test_bare_custom_resolves_providers_dict_entry_named_custom(monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "providers": { "custom": { @@ -639,7 +639,7 @@ def test_named_custom_provider_same_url_uses_matching_key_env_and_api_mode(monke monkeypatch.setenv("CLAUDE_KEY", "claude-secret") monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ { @@ -684,7 +684,7 @@ def test_named_custom_provider_falls_back_to_openai_api_key(monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ { @@ -725,7 +725,7 @@ def test_named_custom_provider_wins_over_builtin_alias(monkeypatch): """ monkeypatch.setattr( rp, - "load_config", + "load_config_readonly", lambda: { "custom_providers": [ { @@ -837,7 +837,7 @@ def test_auto_detected_nous_auth_failure_falls_through_to_openrouter(monkeypatch monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.delenv("OPENAI_BASE_URL", raising=False) monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) - monkeypatch.setattr(rp, "load_config", lambda: {}) + monkeypatch.setattr(rp, "load_config_readonly", lambda: {}) # resolve_provider returns "nous" (stale active_provider in auth.json) monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "nous") @@ -1379,7 +1379,7 @@ def _patch_bedrock(monkeypatch, config_default=""): monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "bedrock") monkeypatch.setattr(rp, "_get_model_config", lambda: {"default": config_default}) - monkeypatch.setattr(rp, "load_config", lambda: {"bedrock": {}}) + monkeypatch.setattr(rp, "load_config_readonly", lambda: {"bedrock": {}}) monkeypatch.setattr(ba, "has_aws_credentials", lambda: True) monkeypatch.setattr(ba, "resolve_aws_auth_env_var", lambda: "AWS_PROFILE") monkeypatch.setattr(ba, "resolve_bedrock_region", lambda: "eu-north-1") diff --git a/tests/tui_gateway/test_custom_provider_session_persistence.py b/tests/tui_gateway/test_custom_provider_session_persistence.py index 2f0e677f2cba4..9f087fcd51c93 100644 --- a/tests/tui_gateway/test_custom_provider_session_persistence.py +++ b/tests/tui_gateway/test_custom_provider_session_persistence.py @@ -66,7 +66,7 @@ def _custom_agent(base_url=MIMO_URL): class TestRuntimeModelConfigPersistsEntryIdentity: def test_persists_menu_key_instead_of_resolved_custom(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: LEGACY_LIST_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: LEGACY_LIST_CONFIG) from tui_gateway.server import _runtime_model_config @@ -80,7 +80,7 @@ def test_persists_menu_key_instead_of_resolved_custom(self, monkeypatch): def test_keeps_bare_custom_when_no_entry_matches(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: {}) + monkeypatch.setattr(rp, "load_config_readonly", lambda: {}) from tui_gateway.server import _runtime_model_config @@ -92,7 +92,7 @@ def test_non_custom_provider_untouched(self, monkeypatch): def _boom(): raise AssertionError("identity lookup must not run for built-ins") - monkeypatch.setattr(rp, "load_config", _boom) + monkeypatch.setattr(rp, "load_config_readonly", _boom) from tui_gateway.server import _runtime_model_config @@ -106,7 +106,7 @@ def _boom(): def _make_agent_with_override(override, monkeypatch, config, model_cfg=None): """Run _make_agent through the REAL resolve_runtime_provider against a patched config, returning the kwargs AIAgent was constructed with.""" - monkeypatch.setattr(rp, "load_config", lambda: config) + monkeypatch.setattr(rp, "load_config_readonly", lambda: config) monkeypatch.setattr(rp, "_get_model_config", lambda: model_cfg or {}) # Keep credential-pool resolution off the developer's real HERMES home. monkeypatch.setattr(rp, "_try_resolve_from_custom_pool", lambda *a, **k: None) @@ -132,7 +132,7 @@ def test_round_trip_restores_entry_credentials(self, monkeypatch): """persist → stored-overrides → _make_agent resolves the entry's api_key again (the exact path that raised "No LLM provider configured" before the fix).""" - monkeypatch.setattr(rp, "load_config", lambda: LEGACY_LIST_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: LEGACY_LIST_CONFIG) from tui_gateway.server import ( _runtime_model_config, @@ -202,7 +202,7 @@ class TestBareCustomNoBaseUrlHealsFromConfig: def test_canonical_identity_recovers_from_config_when_no_base_url( self, monkeypatch ): - monkeypatch.setattr(rp, "load_config", lambda: NAMED_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: NAMED_CONFIG) monkeypatch.setattr(rp, "_get_model_config", lambda: NAMED_CONFIG["model"]) # No base_url to reverse-lookup → must fall back to config.model.provider. @@ -213,7 +213,7 @@ def test_canonical_identity_recovers_from_config_when_no_base_url( def test_persist_recovers_entry_when_agent_has_no_base_url(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: NAMED_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: NAMED_CONFIG) monkeypatch.setattr(rp, "_get_model_config", lambda: NAMED_CONFIG["model"]) from tui_gateway.server import _runtime_model_config @@ -225,7 +225,7 @@ def test_persist_recovers_entry_when_agent_has_no_base_url(self, monkeypatch): assert config["provider"] == "custom:mimo-v2.5-pro" def test_restore_heals_bare_custom_row_without_base_url(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: NAMED_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: NAMED_CONFIG) monkeypatch.setattr(rp, "_get_model_config", lambda: NAMED_CONFIG["model"]) from tui_gateway.server import _stored_session_runtime_overrides @@ -269,7 +269,7 @@ def test_first_db_row_persists_entry_identity_not_bare_custom(self, monkeypatch) composer override's RESOLVED provider. A named custom provider's resolved value is bare "custom" — persisting that verbatim seeds the unresumable row. It must be healed to ``custom:`` here.""" - monkeypatch.setattr(rp, "load_config", lambda: NAMED_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: NAMED_CONFIG) monkeypatch.setattr(rp, "_get_model_config", lambda: NAMED_CONFIG["model"]) captured = {} @@ -335,7 +335,7 @@ def create_session(self, key, **kwargs): class TestModelNameRecoversEntryIdentity: def test_identity_by_model_from_providers_dict_models_list(self, monkeypatch): - monkeypatch.setattr(rp, "load_config", lambda: ULTRA_CONFIG) + monkeypatch.setattr(rp, "load_config_readonly", lambda: ULTRA_CONFIG) assert ( rp.find_custom_provider_identity_by_model("hermes-ultra-sft")