From 83c6e59fd367a290eccbfcb26816f82364d8771f Mon Sep 17 00:00:00 2001 From: Chen Jin Date: Sat, 8 Aug 2026 12:56:55 +0800 Subject: [PATCH 1/3] fix(oneshot): consult fallback_providers at resolution time (#81209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway already had `_try_resolve_fallback_provider` that runs at provider-resolution time, before `AIAgent` is constructed — so a gateway turn survives a primary-provider quota window with a healthy `fallback_providers` chain. The CLI/oneshot lane was bare: a `hermes -z` invocation during a 429 window raised `resolve_runtime_provider(...)` outright, never reaching the `fallback_model=_fb` wiring in `AIAgent(...)` that handles mid-session failures. The docs claim "Where Fallback Works: CLI sessions ✔" but in practice the CLI was only covered for failures after the session started. Fix: extract the fallback loop into `hermes_cli.oneshot._resolve_runtime_with_fallback` and call it in place of the bare `resolve_runtime_provider` call in `_run_agent`. On total failure (primary + every fallback), the primary error is re-raised — it names the operator's configured primary provider, which is what the operator needs to fix first. Adds 5 regression tests covering: primary success short-circuit, primary 429 → fallback success, primary failure with no chain, primary failure with every entry also failing, and first-fallback failure → second-fallback success. --- hermes_cli/oneshot.py | 111 ++++++++++++- tests/hermes_cli/test_oneshot_fallback.py | 180 ++++++++++++++++++++++ 2 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 tests/hermes_cli/test_oneshot_fallback.py diff --git a/hermes_cli/oneshot.py b/hermes_cli/oneshot.py index f13fe64029d5..bcc088c2aa88 100644 --- a/hermes_cli/oneshot.py +++ b/hermes_cli/oneshot.py @@ -310,6 +310,94 @@ def _create_session_db_for_oneshot(): return None +def _resolve_runtime_with_fallback( + effective_provider: Optional[str], + effective_model: str, + explicit_base_url: Optional[str], + cfg: dict, +) -> tuple[Optional[dict], Optional[Exception]]: + """Resolve a runtime provider, falling back to ``fallback_providers`` on quota/429. + + The gateway has a dedicated resolution-time fallback path + (``_try_resolve_fallback_provider`` in ``gateway/run.py``) that runs + *before* ``AIAgent`` is constructed — exactly when a oneshot invocation + in the same situation was failing outright (#81209). The docs claim + "Where Fallback Works: CLI sessions ✔", but in practice the CLI was + only covered for failures *after* the session started, not at resolution + time. This helper ports the gateway's loop into the oneshot path so + headless invocations (``hermes -z`` from cron, queue workers, ops + scripts) survive a primary provider quota window. + + Returns ``(runtime_dict, None)`` on success. On primary failure with + fallback success, returns ``(runtime_dict, None)`` and logs the original + primary error at INFO. On total failure, returns + ``(None, primary_error)`` so the caller can raise the right diagnostic; + if every fallback entry also failed, the primary error is the + operator-facing message because it names the configured primary + provider (a fallback entry's failure is by definition not what the + operator configured first). + """ + from hermes_cli.fallback_config import resolve_entry_api_key + from hermes_cli.runtime_provider import resolve_runtime_provider + + primary_exc: Optional[Exception] = None + try: + runtime = resolve_runtime_provider( + requested=effective_provider, + target_model=effective_model or None, + explicit_base_url=explicit_base_url, + ) + return runtime, None + except Exception as exc: + primary_exc = exc + + # Primary failed. Walk the configured fallback chain in order, applying + # the same managed-overlay / ${VAR}-expansion semantics as the gateway's + # _try_resolve_fallback_provider (get_fallback_chain is the single source + # of truth for both). + try: + fb_list = get_fallback_chain(cfg) + except Exception: + fb_list = [] + + if not fb_list: + return None, primary_exc + + logging.info( + "Primary provider resolution failed (%s); attempting %d configured fallback(s).", + primary_exc, + len(fb_list), + ) + for entry in fb_list: + try: + runtime = resolve_runtime_provider( + requested=entry.get("provider"), + explicit_base_url=entry.get("base_url"), + explicit_api_key=resolve_entry_api_key(entry), + ) + logging.info( + "Oneshot fallback provider resolved: %s model=%s", + entry.get("provider") or runtime.get("provider"), + entry.get("model"), + ) + # Annotate with the fallback entry's model so AIAgent constructs + # against the fallback's model, not the originally-requested + # one — same as the gateway does at run.py:2540. + return runtime, None + except Exception as fb_exc: + logging.debug( + "Oneshot fallback entry %s failed: %s", + entry.get("provider"), + fb_exc, + ) + continue + + # Every fallback failed too. Surface the primary error to the caller — + # it names the operator's configured primary provider, which is what + # the operator needs to fix first. + return None, primary_exc + + def _run_agent( prompt: str, model: Optional[str] = None, @@ -382,12 +470,29 @@ def _run_agent( if detected: effective_provider, effective_model = detected - runtime = resolve_runtime_provider( - requested=effective_provider, - target_model=effective_model or None, + runtime, fallback_resolution_error = _resolve_runtime_with_fallback( + effective_provider=effective_provider, + effective_model=effective_model, explicit_base_url=explicit_base_url_from_alias, + cfg=cfg, ) + # If the primary failed AND every fallback entry also failed, surface a + # unified error so the operator sees one message instead of a noisy + # DEBUG log of every fallback entry's individual failure. When the + # primary error was the genuine root cause (e.g. config syntax), bubble + # it up directly so the operator is not pointed at a fallback that was + # not actually the problem (#81209). + if runtime is None: + if fallback_resolution_error is not None: + raise fallback_resolution_error + raise RuntimeError( + "No usable provider resolved for oneshot invocation: primary " + "provider failed and fallback_providers chain returned no " + "runtime. Check `hermes fallback list` and provider " + "credentials." + ) + # Pull in explicit toolsets when provided; otherwise use whatever the user # has enabled for "cli". sorted() gives stable ordering for config-derived # sets; explicit values preserve user order. diff --git a/tests/hermes_cli/test_oneshot_fallback.py b/tests/hermes_cli/test_oneshot_fallback.py new file mode 100644 index 000000000000..f29a57710145 --- /dev/null +++ b/tests/hermes_cli/test_oneshot_fallback.py @@ -0,0 +1,180 @@ +"""Regression tests for #81209: CLI/oneshot must consult fallback_providers +at resolution time, not only after the session starts. + +Before the fix, ``_run_agent`` called ``resolve_runtime_provider`` bare, +so a quota-exhausted primary (429) raised *before* ``AIAgent`` was +constructed and the ``fallback_model=_fb`` wiring that handles +mid-session failures was never reached. The gateway already had this +behaviour via ``_try_resolve_fallback_provider``; the helper introduced +here brings the oneshot path to parity. +""" + +from unittest.mock import patch + +import pytest + +from hermes_cli import oneshot + + +@pytest.fixture +def cfg_with_fallback(): + return { + "model": {"default": "primary-model"}, + "fallback_providers": [ + {"provider": "anthropic", "model": "haiku"}, + {"provider": "openai", "model": "gpt-4o-mini"}, + ], + } + + +@pytest.fixture +def cfg_no_fallback(): + return { + "model": {"default": "primary-model"}, + } + + +class TestResolveRuntimeWithFallback: + def test_primary_success_short_circuits(self, cfg_with_fallback): + primary_runtime = {"provider": "openai", "api_key": "k1"} + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value=primary_runtime, + ) as resolve: + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_with_fallback, + ) + + assert runtime is primary_runtime + assert err is None + # Primary success must not touch fallback entries. + assert resolve.call_count == 1 + + def test_primary_quota_failure_invokes_fallback(self, cfg_with_fallback): + primary_err = RuntimeError("Codex provider quota exhausted (429)") + fallback_runtime = {"provider": "anthropic", "api_key": "k2"} + + # First call raises, second call (fallback entry) succeeds. + call_log = [] + + def fake_resolve(*, requested, target_model=None, **kwargs): + call_log.append(requested) + if requested == "openai": + raise primary_err + return fallback_runtime + + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=fake_resolve, + ), patch( + "hermes_cli.fallback_config.resolve_entry_api_key", + return_value="resolved-key", + ): + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_with_fallback, + ) + + assert err is None + assert runtime is fallback_runtime + # First call: primary; second call: first fallback entry. + assert call_log == ["openai", "anthropic"] + + def test_primary_failure_no_fallback_chain_returns_primary_error( + self, cfg_no_fallback + ): + primary_err = RuntimeError("primary down") + + def fake_resolve(**kwargs): + raise primary_err + + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=fake_resolve, + ): + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_no_fallback, + ) + + assert runtime is None + # Operator gets the primary's error message (not a fallback that + # was never configured in the first place). + assert err is primary_err + + def test_all_fallbacks_exhausted_returns_primary_error( + self, cfg_with_fallback + ): + primary_err = RuntimeError("primary quota") + + def fake_resolve(*, requested, **kwargs): + # Primary (first call) raises the original primary error; + # subsequent fallback entries raise their own distinct errors. + if requested == "openai": + raise primary_err + raise RuntimeError(f"{requested} also down") + + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=fake_resolve, + ): + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_with_fallback, + ) + + # Three calls: primary + 2 fallback entries, all fail. + assert runtime is None + # Primary error wins (operator-facing) — not the second fallback's. + assert err is primary_err + + def test_second_fallback_succeeds_when_first_also_fails( + self, cfg_with_fallback + ): + primary_err = RuntimeError("primary quota") + openai_runtime = {"provider": "openai", "api_key": "openai-key"} + + # Distinguish the primary call (first) from the fallback entry call + # (third) — they both target openai, so the side_effect needs a + # per-call gate. + state = {"calls": 0} + + def fake_resolve(*, requested, **kwargs): + state["calls"] += 1 + if state["calls"] == 1: + # First call: primary's configured provider. + assert requested == "openai" + raise primary_err + if requested == "anthropic": + # Second call: first fallback entry, also fails. + raise RuntimeError("anthropic auth invalid") + # Third call: openai fallback entry — succeeds. + assert state["calls"] == 3 + return openai_runtime + + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=fake_resolve, + ), patch( + "hermes_cli.fallback_config.resolve_entry_api_key", + return_value="key", + ): + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_with_fallback, + ) + + assert err is None + assert runtime is openai_runtime + assert state["calls"] == 3 \ No newline at end of file From 3931ecab94d487403b9982eac38acfc35bded60e Mon Sep 17 00:00:00 2001 From: Chen Jin Date: Sat, 8 Aug 2026 14:52:34 +0800 Subject: [PATCH 2/3] fix(oneshot): stamp fallback entry's model onto runtime and honor it in AIAgent (#81209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback resolver noted in its docstring that the fallback entry's model would be carried into the runtime dict (same as the gateway does at run.py:2540), but never actually did so. The returned runtime was the raw provider dict, so AIAgent still received the *primary* model — meaning a oneshot invocation that fell back to e.g. Anthropic credentials would still be asked for the primary provider's model (likely failing or mis-routing). This follow-up: - Passes `target_model=entry.get('model')` to `resolve_runtime_provider` for each fallback entry so the resolver's api_mode is derived correctly. - Stamps `runtime['model'] = entry['model']` on the returned dict so AIAgent sees the fallback's model. - Honors `runtime.get('model')` in `_run_agent` (falling back to `effective_model` on primary success). - Updates the two existing assertions to compare by content rather than identity (the helper now returns a fresh dict with the model field). - Adds a regression test asserting the fallback model is injected. --- hermes_cli/oneshot.py | 11 +++++- tests/hermes_cli/test_oneshot_fallback.py | 42 +++++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/hermes_cli/oneshot.py b/hermes_cli/oneshot.py index bcc088c2aa88..07413949387d 100644 --- a/hermes_cli/oneshot.py +++ b/hermes_cli/oneshot.py @@ -372,6 +372,7 @@ def _resolve_runtime_with_fallback( try: runtime = resolve_runtime_provider( requested=entry.get("provider"), + target_model=entry.get("model") or None, explicit_base_url=entry.get("base_url"), explicit_api_key=resolve_entry_api_key(entry), ) @@ -383,6 +384,9 @@ def _resolve_runtime_with_fallback( # Annotate with the fallback entry's model so AIAgent constructs # against the fallback's model, not the originally-requested # one — same as the gateway does at run.py:2540. + fallback_model = entry.get("model") + if fallback_model: + runtime = {**runtime, "model": fallback_model} return runtime, None except Exception as fb_exc: logging.debug( @@ -526,13 +530,18 @@ def _run_agent( # gateway sessions. _fb = get_fallback_chain(cfg) + # ``runtime.get("model")`` is set by ``_resolve_runtime_with_fallback`` + # when the fallback chain supplied a model that differs from the + # primary one; honour it so AIAgent constructs against the + # fallback's model instead of the originally-requested one. + agent_model = runtime.get("model") or effective_model agent = AIAgent( api_key=runtime.get("api_key"), base_url=runtime.get("base_url"), provider=runtime.get("provider"), requested_provider=runtime.get("requested_provider"), api_mode=runtime.get("api_mode"), - model=effective_model, + model=agent_model, enabled_toolsets=toolsets_list, quiet_mode=True, platform="cli", diff --git a/tests/hermes_cli/test_oneshot_fallback.py b/tests/hermes_cli/test_oneshot_fallback.py index f29a57710145..7af965c674cd 100644 --- a/tests/hermes_cli/test_oneshot_fallback.py +++ b/tests/hermes_cli/test_oneshot_fallback.py @@ -81,7 +81,10 @@ def fake_resolve(*, requested, target_model=None, **kwargs): ) assert err is None - assert runtime is fallback_runtime + # The fallback entry's ``model`` is now stamped onto a copy of + # ``fallback_runtime`` so the helper always returns a fresh dict; + # compare by content rather than identity. + assert runtime == {**fallback_runtime, "model": "haiku"} # First call: primary; second call: first fallback entry. assert call_log == ["openai", "anthropic"] @@ -176,5 +179,38 @@ def fake_resolve(*, requested, **kwargs): ) assert err is None - assert runtime is openai_runtime - assert state["calls"] == 3 \ No newline at end of file + assert runtime == {**openai_runtime, "model": "gpt-4o-mini"} + assert state["calls"] == 3 + + def test_fallback_model_is_injected_into_runtime(self, cfg_with_fallback): + """The fallback entry's ``model`` must be carried into the returned + runtime dict so AIAgent constructs against the fallback's model + rather than the originally-requested primary model (#81209).""" + primary_err = RuntimeError("primary quota") + fallback_runtime = {"provider": "anthropic", "api_key": "anth-key"} + + def fake_resolve(*, requested, **kwargs): + if requested == "openai": + raise primary_err + return fallback_runtime + + with patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + side_effect=fake_resolve, + ), patch( + "hermes_cli.fallback_config.resolve_entry_api_key", + return_value="key", + ): + runtime, err = oneshot._resolve_runtime_with_fallback( + effective_provider="openai", + effective_model="gpt-4o", + explicit_base_url=None, + cfg=cfg_with_fallback, + ) + + assert err is None + # ``cfg_with_fallback`` declares the first fallback entry with + # model ``haiku``; that key must appear on the returned runtime so + # AIAgent constructs against the fallback's model instead of the + # originally-requested primary model (#81209). + assert runtime.get("model") == "haiku" \ No newline at end of file From 72270a722a31e0cc8aeb3c9ad65856099d2decaa Mon Sep 17 00:00:00 2001 From: Chen Jin Date: Thu, 13 Aug 2026 10:37:56 +0800 Subject: [PATCH 3/3] style(oneshot): add missing trailing newline to fallback test (#81209) Co-Authored-By: Claude Fable 5 --- tests/hermes_cli/test_oneshot_fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hermes_cli/test_oneshot_fallback.py b/tests/hermes_cli/test_oneshot_fallback.py index 7af965c674cd..e44b46be5f5e 100644 --- a/tests/hermes_cli/test_oneshot_fallback.py +++ b/tests/hermes_cli/test_oneshot_fallback.py @@ -213,4 +213,4 @@ def fake_resolve(*, requested, **kwargs): # model ``haiku``; that key must appear on the returned runtime so # AIAgent constructs against the fallback's model instead of the # originally-requested primary model (#81209). - assert runtime.get("model") == "haiku" \ No newline at end of file + assert runtime.get("model") == "haiku"