fix(oneshot): consult fallback_providers at resolution time

hermes_cli/oneshot.py

@@ -310,6 +310,98 @@ def _create_session_db_for_oneshot():
         return None
 
 
+def _resolve_runtime_with_fallback(
+    effective_provider: Optional[str],
+    effective_model: str,
+    explicit_base_url: Optional[str],
+    cfg: dict,
+) -> tuple[Optional[dict], Optional[Exception]]:
+    """Resolve a runtime provider, falling back to ``fallback_providers`` on quota/429.
+
+    The gateway has a dedicated resolution-time fallback path
+    (``_try_resolve_fallback_provider`` in ``gateway/run.py``) that runs
+    *before* ``AIAgent`` is constructed — exactly when a oneshot invocation
+    in the same situation was failing outright (#81209). The docs claim
+    "Where Fallback Works: CLI sessions ✔", but in practice the CLI was
+    only covered for failures *after* the session started, not at resolution
+    time. This helper ports the gateway's loop into the oneshot path so
+    headless invocations (``hermes -z`` from cron, queue workers, ops
+    scripts) survive a primary provider quota window.
+
+    Returns ``(runtime_dict, None)`` on success. On primary failure with
+    fallback success, returns ``(runtime_dict, None)`` and logs the original
+    primary error at INFO. On total failure, returns
+    ``(None, primary_error)`` so the caller can raise the right diagnostic;
+    if every fallback entry also failed, the primary error is the
+    operator-facing message because it names the configured primary
+    provider (a fallback entry's failure is by definition not what the
+    operator configured first).
+    """
+    from hermes_cli.fallback_config import resolve_entry_api_key
+    from hermes_cli.runtime_provider import resolve_runtime_provider
+
+    primary_exc: Optional[Exception] = None
+    try:
+        runtime = resolve_runtime_provider(
+            requested=effective_provider,
+            target_model=effective_model or None,
+            explicit_base_url=explicit_base_url,
+        )
+        return runtime, None
+    except Exception as exc:
+        primary_exc = exc
+
+    # Primary failed. Walk the configured fallback chain in order, applying
+    # the same managed-overlay / ${VAR}-expansion semantics as the gateway's
+    # _try_resolve_fallback_provider (get_fallback_chain is the single source
+    # of truth for both).
+    try:
+        fb_list = get_fallback_chain(cfg)
+    except Exception:
+        fb_list = []
+
+    if not fb_list:
+        return None, primary_exc
+
+    logging.info(
+        "Primary provider resolution failed (%s); attempting %d configured fallback(s).",
+        primary_exc,
+        len(fb_list),
+    )
+    for entry in fb_list:
+        try:
+            runtime = resolve_runtime_provider(
+                requested=entry.get("provider"),
+                target_model=entry.get("model") or None,
+                explicit_base_url=entry.get("base_url"),
+                explicit_api_key=resolve_entry_api_key(entry),
+            )
+            logging.info(
+                "Oneshot fallback provider resolved: %s model=%s",
+                entry.get("provider") or runtime.get("provider"),
+                entry.get("model"),
+            )
+            # Annotate with the fallback entry's model so AIAgent constructs
+            # against the fallback's model, not the originally-requested
+            # one — same as the gateway does at run.py:2540.
+            fallback_model = entry.get("model")
+            if fallback_model:
+                runtime = {**runtime, "model": fallback_model}
+            return runtime, None
+        except Exception as fb_exc:
+            logging.debug(
+                "Oneshot fallback entry %s failed: %s",
+                entry.get("provider"),
+                fb_exc,
+            )
+            continue
+
+    # Every fallback failed too. Surface the primary error to the caller —
+    # it names the operator's configured primary provider, which is what
+    # the operator needs to fix first.
+    return None, primary_exc
+
+
 def _run_agent(
     prompt: str,
     model: Optional[str] = None,
@@ -382,12 +474,29 @@ def _run_agent(
                 if detected:
                     effective_provider, effective_model = detected
 
-    runtime = resolve_runtime_provider(
-        requested=effective_provider,
-        target_model=effective_model or None,
+    runtime, fallback_resolution_error = _resolve_runtime_with_fallback(
+        effective_provider=effective_provider,
+        effective_model=effective_model,
         explicit_base_url=explicit_base_url_from_alias,
+        cfg=cfg,
     )
 
+    # If the primary failed AND every fallback entry also failed, surface a
+    # unified error so the operator sees one message instead of a noisy
+    # DEBUG log of every fallback entry's individual failure.  When the
+    # primary error was the genuine root cause (e.g. config syntax), bubble
+    # it up directly so the operator is not pointed at a fallback that was
+    # not actually the problem (#81209).
+    if runtime is None:
+        if fallback_resolution_error is not None:
+            raise fallback_resolution_error
+        raise RuntimeError(
+            "No usable provider resolved for oneshot invocation: primary "
+            "provider failed and fallback_providers chain returned no "
+            "runtime.  Check `hermes fallback list` and provider "
+            "credentials."
+        )
+
     # Pull in explicit toolsets when provided; otherwise use whatever the user
     # has enabled for "cli". sorted() gives stable ordering for config-derived
     # sets; explicit values preserve user order.
@@ -421,13 +530,18 @@ def _run_agent(
         # gateway sessions.
         _fb = get_fallback_chain(cfg)
 
+        # ``runtime.get("model")`` is set by ``_resolve_runtime_with_fallback``
+        # when the fallback chain supplied a model that differs from the
+        # primary one; honour it so AIAgent constructs against the
+        # fallback's model instead of the originally-requested one.
+        agent_model = runtime.get("model") or effective_model
         agent = AIAgent(
             api_key=runtime.get("api_key"),
             base_url=runtime.get("base_url"),
             provider=runtime.get("provider"),
             requested_provider=runtime.get("requested_provider"),
             api_mode=runtime.get("api_mode"),
-            model=effective_model,
+            model=agent_model,
             enabled_toolsets=toolsets_list,
             quiet_mode=True,
             platform="cli",

tests/hermes_cli/test_oneshot_fallback.py

@@ -0,0 +1,216 @@
+"""Regression tests for #81209: CLI/oneshot must consult fallback_providers
+at resolution time, not only after the session starts.
+
+Before the fix, ``_run_agent`` called ``resolve_runtime_provider`` bare,
+so a quota-exhausted primary (429) raised *before* ``AIAgent`` was
+constructed and the ``fallback_model=_fb`` wiring that handles
+mid-session failures was never reached.  The gateway already had this
+behaviour via ``_try_resolve_fallback_provider``; the helper introduced
+here brings the oneshot path to parity.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from hermes_cli import oneshot
+
+
+@pytest.fixture
+def cfg_with_fallback():
+    return {
+        "model": {"default": "primary-model"},
+        "fallback_providers": [
+            {"provider": "anthropic", "model": "haiku"},
+            {"provider": "openai", "model": "gpt-4o-mini"},
+        ],
+    }
+
+
+@pytest.fixture
+def cfg_no_fallback():
+    return {
+        "model": {"default": "primary-model"},
+    }
+
+
+class TestResolveRuntimeWithFallback:
+    def test_primary_success_short_circuits(self, cfg_with_fallback):
+        primary_runtime = {"provider": "openai", "api_key": "k1"}
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            return_value=primary_runtime,
+        ) as resolve:
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_with_fallback,
+            )
+
+        assert runtime is primary_runtime
+        assert err is None
+        # Primary success must not touch fallback entries.
+        assert resolve.call_count == 1
+
+    def test_primary_quota_failure_invokes_fallback(self, cfg_with_fallback):
+        primary_err = RuntimeError("Codex provider quota exhausted (429)")
+        fallback_runtime = {"provider": "anthropic", "api_key": "k2"}
+
+        # First call raises, second call (fallback entry) succeeds.
+        call_log = []
+
+        def fake_resolve(*, requested, target_model=None, **kwargs):
+            call_log.append(requested)
+            if requested == "openai":
+                raise primary_err
+            return fallback_runtime
+
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            side_effect=fake_resolve,
+        ), patch(
+            "hermes_cli.fallback_config.resolve_entry_api_key",
+            return_value="resolved-key",
+        ):
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_with_fallback,
+            )
+
+        assert err is None
+        # The fallback entry's ``model`` is now stamped onto a copy of
+        # ``fallback_runtime`` so the helper always returns a fresh dict;
+        # compare by content rather than identity.
+        assert runtime == {**fallback_runtime, "model": "haiku"}
+        # First call: primary; second call: first fallback entry.
+        assert call_log == ["openai", "anthropic"]
+
+    def test_primary_failure_no_fallback_chain_returns_primary_error(
+        self, cfg_no_fallback
+    ):
+        primary_err = RuntimeError("primary down")
+
+        def fake_resolve(**kwargs):
+            raise primary_err
+
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            side_effect=fake_resolve,
+        ):
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_no_fallback,
+            )
+
+        assert runtime is None
+        # Operator gets the primary's error message (not a fallback that
+        # was never configured in the first place).
+        assert err is primary_err
+
+    def test_all_fallbacks_exhausted_returns_primary_error(
+        self, cfg_with_fallback
+    ):
+        primary_err = RuntimeError("primary quota")
+
+        def fake_resolve(*, requested, **kwargs):
+            # Primary (first call) raises the original primary error;
+            # subsequent fallback entries raise their own distinct errors.
+            if requested == "openai":
+                raise primary_err
+            raise RuntimeError(f"{requested} also down")
+
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            side_effect=fake_resolve,
+        ):
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_with_fallback,
+            )
+
+        # Three calls: primary + 2 fallback entries, all fail.
+        assert runtime is None
+        # Primary error wins (operator-facing) — not the second fallback's.
+        assert err is primary_err
+
+    def test_second_fallback_succeeds_when_first_also_fails(
+        self, cfg_with_fallback
+    ):
+        primary_err = RuntimeError("primary quota")
+        openai_runtime = {"provider": "openai", "api_key": "openai-key"}
+
+        # Distinguish the primary call (first) from the fallback entry call
+        # (third) — they both target openai, so the side_effect needs a
+        # per-call gate.
+        state = {"calls": 0}
+
+        def fake_resolve(*, requested, **kwargs):
+            state["calls"] += 1
+            if state["calls"] == 1:
+                # First call: primary's configured provider.
+                assert requested == "openai"
+                raise primary_err
+            if requested == "anthropic":
+                # Second call: first fallback entry, also fails.
+                raise RuntimeError("anthropic auth invalid")
+            # Third call: openai fallback entry — succeeds.
+            assert state["calls"] == 3
+            return openai_runtime
+
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            side_effect=fake_resolve,
+        ), patch(
+            "hermes_cli.fallback_config.resolve_entry_api_key",
+            return_value="key",
+        ):
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_with_fallback,
+            )
+
+        assert err is None
+        assert runtime == {**openai_runtime, "model": "gpt-4o-mini"}
+        assert state["calls"] == 3
+
+    def test_fallback_model_is_injected_into_runtime(self, cfg_with_fallback):
+        """The fallback entry's ``model`` must be carried into the returned
+        runtime dict so AIAgent constructs against the fallback's model
+        rather than the originally-requested primary model (#81209)."""
+        primary_err = RuntimeError("primary quota")
+        fallback_runtime = {"provider": "anthropic", "api_key": "anth-key"}
+
+        def fake_resolve(*, requested, **kwargs):
+            if requested == "openai":
+                raise primary_err
+            return fallback_runtime
+
+        with patch(
+            "hermes_cli.runtime_provider.resolve_runtime_provider",
+            side_effect=fake_resolve,
+        ), patch(
+            "hermes_cli.fallback_config.resolve_entry_api_key",
+            return_value="key",
+        ):
+            runtime, err = oneshot._resolve_runtime_with_fallback(
+                effective_provider="openai",
+                effective_model="gpt-4o",
+                explicit_base_url=None,
+                cfg=cfg_with_fallback,
+            )
+
+        assert err is None
+        # ``cfg_with_fallback`` declares the first fallback entry with
+        # model ``haiku``; that key must appear on the returned runtime so
+        # AIAgent constructs against the fallback's model instead of the
+        # originally-requested primary model (#81209).
+        assert runtime.get("model") == "haiku"