From 0c8edd82ddaf0c1506c148f5d413679af938d641 Mon Sep 17 00:00:00 2001 From: CC#3 Kora Runtime Date: Sat, 23 May 2026 23:32:38 -0700 Subject: [PATCH] =?UTF-8?q?feat(kora):=20KR-HERMES-LOCAL-EXT-REISSUE=20+?= =?UTF-8?q?=20KR-HAIKU-ROUTER-PLUGIN=20=E2=80=94=20completes=20Lock=20R3-2?= =?UTF-8?q?=20Phase=20C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Batched pair. Adds the missing local Hermes post-LLM re-issue hook and wires the haiku-router sub-plugin against it so should_escalate_post_call (present at cost_ladder/selector.py since #185 but unwired) finally fires. Deliverable A — post_llm_call_can_reissue (agent/conversation_loop.py): fires after messages.create returns and BEFORE normalize / post_api_request observer / tool dispatch. Plugins return {"reissue_with": } to transparently re-call the API; first non-None wins; at most ONE re-issue per iteration (anti-loop break). Fail-safe: invoke_hook wraps each callback; outer guard protects the re-issue call itself; original response preserved on any failure. Telemetry: re-issued response feeds record_inference with escalated_to_opus=True (the original Haiku call already telemetered with the default False inside the retry-loop chokepoint). Deliverable B — kora_cli/reasoning/kora_hermes_plugin/haiku_router/ (constants + escalator + plugin + sub-register, #185 template). Activation gates: Kora-tagged route, iteration == 1, original model is Haiku, KORA_DISABLE_POST_CALL_ESCALATION not "true", response has text content, should_escalate_post_call returns True. Re-issue kwargs: model swapped to Opus; messages extended with Haiku response as assistant turn + terse reviewer prompt (parallel-Claude's pattern — ~30% cheaper escalations). Tests: 30 new tests, 125 directly-affected tests green. Existing seven-hook count test updated to expect 8. Pre-existing failures elsewhere in the suite are environmental (fastapi/blake3/HERMES_HOME) and verified to fail identically on the base branch. After this: 6 of 7 plugin extractions complete; KR-PLUGIN-IDENTITY remains deferred per Lock R3-2. All escalation paths from KR-HAIKU-ROUTER (#165) are now functional end-to-end. Co-Authored-By: Claude Opus 4.7 (1M context) --- agent/conversation_loop.py | 113 ++++ .../haiku_router/__init__.py | 40 ++ .../haiku_router/constants.py | 38 ++ .../haiku_router/escalator.py | 153 +++++ .../kora_hermes_plugin/haiku_router/plugin.py | 204 +++++++ .../reasoning/kora_hermes_plugin/plugin.py | 20 +- plugins/kora_hermes/__init__.py | 4 + ..._conversation_loop_post_llm_can_reissue.py | 163 ++++++ tests/plugins/test_kora_hermes_plugin.py | 12 +- .../test_kora_hermes_plugin_haiku_router.py | 543 ++++++++++++++++++ 10 files changed, 1284 insertions(+), 6 deletions(-) create mode 100644 kora_cli/reasoning/kora_hermes_plugin/haiku_router/__init__.py create mode 100644 kora_cli/reasoning/kora_hermes_plugin/haiku_router/constants.py create mode 100644 kora_cli/reasoning/kora_hermes_plugin/haiku_router/escalator.py create mode 100644 kora_cli/reasoning/kora_hermes_plugin/haiku_router/plugin.py create mode 100644 tests/agent/test_conversation_loop_post_llm_can_reissue.py create mode 100644 tests/plugins/test_kora_hermes_plugin_haiku_router.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 72be2cff1c28..8f1e14303532 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2996,6 +2996,119 @@ def _stop_spinner(): agent._persist_session(messages, conversation_history) break + # KR-HERMES-LOCAL-EXT-REISSUE — post-LLM can-reissue hook. + # Fires AFTER messages.create returns a valid response and + # BEFORE downstream normalization / observer hooks / tool + # dispatch. A plugin can return ``{"reissue_with": }`` to transparently re-call the API with + # modified kwargs; the re-issued response REPLACES the + # original for all downstream processing (normalize, + # post_api_request, post_llm_call, tool dispatch). + # + # Override semantics: matches #172/#181 — iterate plugin + # returns, FIRST non-None ``reissue_with`` wins. Subsequent + # plugin returns are ignored (single re-issue per + # iteration). Plugins returning None or a dict without + # ``reissue_with`` fall through. + # + # Anti-loop safety: the hook is NOT re-fired against the + # re-issued response. If the re-issued response would + # itself satisfy another plugin's escalation condition, the + # loop ignores it. This is intentional — without it, two + # plugins could ping-pong escalations indefinitely. + # + # Fail-safe: any exception in the hook firing OR the re- + # issue API call falls through to the original response. + # ``invoke_hook`` already wraps each callback in try/except; + # this outer guard protects against errors in the re-issue + # call itself (transport raise, validation failure, etc). + # + # Telemetry: when a re-issue fires, we feed cost-ladder + # ``record_inference`` for the re-issued response with + # ``escalated_to_opus=True``. The original Haiku call was + # already telemetered inside the retry-loop chokepoint with + # the default ``escalated_to_opus=False`` — both events + # land in the cost-ladder estimator so $-burn accounting + # stays accurate. + try: + from kora_cli.plugins import invoke_hook as _invoke_hook_reissue + _reissue_results = _invoke_hook_reissue( + "post_llm_call_can_reissue", + response=response, + api_kwargs=api_kwargs, + agent=agent, + iteration=api_call_count, + task_id=effective_task_id, + session_id=agent.session_id or "", + route=getattr(agent, "route", "") or "", + ) + for _reissue_result in _reissue_results: + if not isinstance(_reissue_result, dict): + continue + _new_kwargs = _reissue_result.get("reissue_with") + if not isinstance(_new_kwargs, dict): + continue + # First non-None reissue_with wins. + _original_response = response + _original_model = api_kwargs.get("model") if isinstance(api_kwargs, dict) else None + _new_model = _new_kwargs.get("model") + logger.info( + "[kora_hermes] post_llm_call_can_reissue re-issuing " + "API call (iteration=%s, original_model=%s, new_model=%s)", + api_call_count, + _original_model, + _new_model, + ) + try: + api_kwargs = _new_kwargs + if _use_streaming: + response = agent._interruptible_streaming_api_call( + api_kwargs, on_first_delta=_stop_spinner + ) + else: + response = agent._interruptible_api_call(api_kwargs) + api_duration = time.time() - api_start_time + # Telemetry for the re-issued call. The first + # call's record_inference already fired inside + # the retry-loop chokepoint with the default + # escalated_to_opus=False — this adds the Opus + # call as a second $-burn event with the flag + # set so cockpit panels can compute escalation + # rate per route. + try: + from agent.cost_ladder_wire import ( + record_inference_from_response, + ) + record_inference_from_response( + response, + model=_new_model or getattr(agent, "model", None), + provider=agent.provider, + base_url=agent.base_url, + api_mode=agent.api_mode, + route=getattr(agent, "route", "") or "unknown", + escalated_to_opus=True, + ) + except Exception as _cl_exc: + logger.debug( + "[kora_hermes] re-issue cost-ladder feed " + "failed: %r", + _cl_exc, + ) + except Exception as _reissue_call_exc: + logger.warning( + "post_llm_call_can_reissue re-issue API call " + "failed: %s — falling back to original response", + _reissue_call_exc, + ) + response = _original_response + break # anti-loop: at most one re-issue per iteration + except Exception as _reissue_exc: + logger.warning( + "post_llm_call_can_reissue hook failed: %s — " + "continuing with original response", + _reissue_exc, + ) + try: _transport = agent._get_transport() _normalize_kwargs = {} diff --git a/kora_cli/reasoning/kora_hermes_plugin/haiku_router/__init__.py b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/__init__.py new file mode 100644 index 000000000000..714c84d23773 --- /dev/null +++ b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/__init__.py @@ -0,0 +1,40 @@ +"""Haiku-router sub-plugin — post-call Opus escalation. + +See ``escalator.py`` for the pure helpers (text extraction + +re-issue kwargs construction), ``constants.py`` for the model +IDs + env-var names, ``plugin.py`` for the +``post_llm_call_can_reissue`` hook handler + sub-register. + +Consumes ``should_escalate_post_call`` from the cost_ladder +sub-plugin's selector — present since #185 but had no caller +until KR-HERMES-LOCAL-EXT-REISSUE added the hook surface that +this plugin registers against. +""" + +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + ENV_DISABLE_POST_CALL_ESCALATION, + MODEL_HAIKU, + MODEL_OPUS, + REISSUE_REVIEW_PROMPT, +) +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + build_opus_reissue_kwargs, + extract_first_text, + extract_last_user_text, +) +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + register, +) + +__all__ = [ + "ENV_DISABLE_POST_CALL_ESCALATION", + "MODEL_HAIKU", + "MODEL_OPUS", + "REISSUE_REVIEW_PROMPT", + "build_opus_reissue_kwargs", + "extract_first_text", + "extract_last_user_text", + "haiku_router_post_call_escalation", + "register", +] diff --git a/kora_cli/reasoning/kora_hermes_plugin/haiku_router/constants.py b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/constants.py new file mode 100644 index 000000000000..78d14caf400b --- /dev/null +++ b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/constants.py @@ -0,0 +1,38 @@ +"""Constants for the haiku_router sub-plugin. + +Defaults + env-var names. The model identifiers are re-exported +from the cost_ladder sub-plugin so a single source-of-truth for +the long-form Anthropic IDs (cache-key stability + version-pin +discipline) stays at ``cost_ladder.constants``. +""" + +from __future__ import annotations + +from kora_cli.reasoning.kora_hermes_plugin.cost_ladder.constants import ( + DEFAULT_HAIKU_MODEL as MODEL_HAIKU, +) +from kora_cli.reasoning.kora_hermes_plugin.cost_ladder.constants import ( + DEFAULT_OPUS_MODEL as MODEL_OPUS, +) + +# Re-issue prompt — short, terse instruction asked of Opus when it +# inherits a Haiku response as assistant context. Parallel-Claude's +# pattern (R3 origin): Opus often just confirms with a one-liner +# instead of redoing the work — ~30% cheaper escalations than a +# cold Opus call. +REISSUE_REVIEW_PROMPT = ( + "Please review my last response and improve it if needed. " + "Be terse if confirming." +) + +# Disable env — operator escape hatch if post-call escalation +# starts misbehaving. When set to "true" the plugin no-ops and +# the loop continues with the Haiku response unchanged. +ENV_DISABLE_POST_CALL_ESCALATION = "KORA_DISABLE_POST_CALL_ESCALATION" + +__all__ = [ + "ENV_DISABLE_POST_CALL_ESCALATION", + "MODEL_HAIKU", + "MODEL_OPUS", + "REISSUE_REVIEW_PROMPT", +] diff --git a/kora_cli/reasoning/kora_hermes_plugin/haiku_router/escalator.py b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/escalator.py new file mode 100644 index 000000000000..1c9146ee8119 --- /dev/null +++ b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/escalator.py @@ -0,0 +1,153 @@ +"""Pure helpers for the post-call Opus escalation re-issue. + +The hook handler in ``plugin.py`` is responsible for orchestrating +the re-issue; this module holds the pure functions it composes +with so they're independently testable. + +Two responsibilities: + + 1. :func:`extract_first_text` — pull the first text block out of + an Anthropic ``Messages`` response. Used to feed + :func:`should_escalate_post_call` from the cost_ladder + selector + to build the Haiku-context assistant turn. + 2. :func:`build_opus_reissue_kwargs` — mutate a copy of the + original ``api_kwargs`` into the Opus re-issue form: same + conversation prefix + Haiku response as an assistant turn + + a one-liner reviewer prompt + ``model`` swapped to Opus. + +Both are pure functions (no I/O, no state). The hook handler is +where activation gating + telemetry side-effects live. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + MODEL_OPUS, + REISSUE_REVIEW_PROMPT, +) + + +def extract_first_text(response: Any) -> str: + """Return the first text block from an Anthropic ``Messages`` + response. Empty string on any extraction failure — caller + treats that as "no Haiku text to escalate from" and bails. + + The Anthropic SDK shape is ``response.content`` → + ``list[ContentBlock]`` where each block has ``type`` and + (for text blocks) ``text``. Models can return content lists + interleaving text + tool_use; we only want the text. We + concatenate ALL text blocks because some models emit the + user-visible answer across multiple text blocks (e.g. when + extended thinking is enabled the answer can split). + """ + if response is None: + return "" + + content = getattr(response, "content", None) + if not isinstance(content, list): + return "" + + parts: List[str] = [] + for block in content: + # SDK content block (pydantic model). + block_type = getattr(block, "type", None) + if block_type == "text": + text = getattr(block, "text", "") + if isinstance(text, str) and text: + parts.append(text) + continue + # Dict fallback (Mock-friendly + non-SDK callers). + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + if isinstance(text, str) and text: + parts.append(text) + + return "\n".join(parts).strip() + + +def extract_last_user_text(api_kwargs: Dict[str, Any]) -> str: + """Return the most recent user-turn text from + ``api_kwargs["messages"]``. Used as ``original_message_text`` + when calling :func:`should_escalate_post_call`. + + The Anthropic ``messages`` shape is a list of + ``{"role": "user"|"assistant", "content": str | list[block]}``. + We walk backwards looking for the first user turn + flatten + its content to text. Returns "" on any extraction failure. + """ + if not isinstance(api_kwargs, dict): + return "" + + messages = api_kwargs.get("messages") + if not isinstance(messages, list): + return "" + + for msg in reversed(messages): + if not isinstance(msg, dict): + continue + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + # Content-block list — flatten text parts. + parts: List[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts).strip() + return "" + + return "" + + +def build_opus_reissue_kwargs( + *, + api_kwargs: Dict[str, Any], + haiku_response_text: str, + opus_model: Optional[str] = None, +) -> Dict[str, Any]: + """Build the new api_kwargs for the Opus re-issue. + + Strategy (parallel-Claude's pattern, R3 origin): instead of + a cold Opus call that redoes Haiku's work, include Haiku's + response as an assistant turn and ask Opus to confirm-or- + improve. Opus often returns a one-liner confirmation — ~30% + cheaper escalations. + + Returns a NEW dict; the input ``api_kwargs`` is not mutated. + The returned kwargs share top-level non-message refs with + the input (system prompt, tools, max_tokens, etc.) — only + ``model`` + ``messages`` are replaced. + """ + new_kwargs = dict(api_kwargs) + new_kwargs["model"] = opus_model or MODEL_OPUS + + original_messages = api_kwargs.get("messages") + if not isinstance(original_messages, list): + original_messages = [] + + new_messages = list(original_messages) + new_messages.append( + {"role": "assistant", "content": haiku_response_text} + ) + new_messages.append( + {"role": "user", "content": REISSUE_REVIEW_PROMPT} + ) + new_kwargs["messages"] = new_messages + + return new_kwargs + + +__all__ = [ + "build_opus_reissue_kwargs", + "extract_first_text", + "extract_last_user_text", +] diff --git a/kora_cli/reasoning/kora_hermes_plugin/haiku_router/plugin.py b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/plugin.py new file mode 100644 index 000000000000..2c39f99330f5 --- /dev/null +++ b/kora_cli/reasoning/kora_hermes_plugin/haiku_router/plugin.py @@ -0,0 +1,204 @@ +"""Haiku-router sub-plugin — post-call Opus escalation. + +Consumes +:func:`kora_cli.reasoning.kora_hermes_plugin.cost_ladder.selector.should_escalate_post_call` +to decide whether a low-confidence Haiku response should be +re-issued to Opus with the Haiku answer as assistant context +(parallel-Claude's pattern from R3). + +Registered against the NEW local Hermes hook +``post_llm_call_can_reissue`` added by +KR-HERMES-LOCAL-EXT-REISSUE in +``agent/conversation_loop.py``. The hook contract: + + def post_llm_call_can_reissue(*, response, api_kwargs, agent, + iteration, task_id, session_id, + route, **kw) -> dict | None + +Returns ``{"reissue_with": }`` to trigger a re- +issue; ``None`` to fall through. + +# Activation gates (all must be true to escalate) + + 1. Call is a Kora-tagged route (``_is_kora_call(route)``). + 2. ``iteration == 1`` — post-call escalation only fires on the + first iteration. Subsequent iterations already get Opus from + ``tool_loop_iteration`` (cost_ladder pre-call rule). + 3. The original call's model is Haiku — Opus already-on-Opus + calls have nothing to escalate to. + 4. :func:`should_escalate_post_call` returns ``(True, )`` + based on Haiku response heuristics. + 5. ``KORA_DISABLE_POST_CALL_ESCALATION`` env is not "true" + (operator escape hatch). + +When all gates pass, builds the Opus re-issue kwargs (Haiku text +as assistant turn + terse review prompt) and returns it. The +re-issue itself is performed by the conversation_loop; +telemetry attribution (``escalated_to_opus=True``) fires there +too — this plugin only describes WHAT to re-issue. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +from kora_cli.reasoning.kora_hermes_plugin.cost_ladder.selector import ( + should_escalate_post_call, +) +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + ENV_DISABLE_POST_CALL_ESCALATION, + MODEL_HAIKU, + MODEL_OPUS, +) +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + build_opus_reissue_kwargs, + extract_first_text, + extract_last_user_text, +) + +logger = logging.getLogger(__name__) + + +def _is_kora_call(route_value: Any) -> bool: + """Mirror of the top-level plugin's KORA_ROUTES gate. Lazy- + imported from the discovery shim to avoid a circular import + (same pattern as cost_ladder.plugin).""" + from plugins.kora_hermes import KORA_ROUTES + + if not isinstance(route_value, str) or not route_value: + return False + return route_value in KORA_ROUTES + + +def _is_disabled() -> bool: + return ( + os.environ.get(ENV_DISABLE_POST_CALL_ESCALATION, "") + .strip() + .lower() + == "true" + ) + + +def haiku_router_post_call_escalation( + *, + response: Any = None, + api_kwargs: Optional[dict] = None, + agent: Any = None, + iteration: int = 0, + route: str = "", + **kw: Any, +) -> Optional[dict]: + """``post_llm_call_can_reissue`` handler. + + Returns ``{"reissue_with": }`` when a Haiku + response should be re-issued to Opus; ``None`` otherwise. + Fail-soft: any extraction or build failure logs DEBUG and + returns None (the loop continues with the Haiku response). + """ + if not _is_kora_call(route): + return None + + if not isinstance(api_kwargs, dict): + return None + + if iteration != 1: + # Post-call escalation only fires on the first iteration. + # Subsequent iterations already get Opus from the cost- + # ladder pre-call rule (tool_loop_iteration). + return None + + if api_kwargs.get("model") != MODEL_HAIKU: + # The original call wasn't Haiku — nothing to escalate + # from. This catches force_opus_env / opus_prefix / + # decision_language paths that already routed to Opus + # pre-call. + return None + + if _is_disabled(): + logger.debug( + "[kora_hermes.haiku_router] post-call escalation disabled " + "via %s — skipping", + ENV_DISABLE_POST_CALL_ESCALATION, + ) + return None + + try: + haiku_text = extract_first_text(response) + except Exception as exc: + logger.debug( + "[kora_hermes.haiku_router] extract_first_text raised %r — " + "skipping escalation", + exc, + ) + return None + + if not haiku_text: + # No text content to escalate from (e.g. tool-use-only + # response). Fall through; no escalation. + return None + + try: + original_user_text = extract_last_user_text(api_kwargs) + except Exception as exc: + logger.debug( + "[kora_hermes.haiku_router] extract_last_user_text raised " + "%r — skipping escalation", + exc, + ) + return None + + try: + should_escalate, reason = should_escalate_post_call( + haiku_response_text=haiku_text, + original_message_text=original_user_text, + ) + except Exception as exc: + logger.debug( + "[kora_hermes.haiku_router] should_escalate_post_call raised " + "%r — skipping escalation", + exc, + ) + return None + + if not should_escalate: + return None + + try: + new_kwargs = build_opus_reissue_kwargs( + api_kwargs=api_kwargs, + haiku_response_text=haiku_text, + opus_model=MODEL_OPUS, + ) + except Exception as exc: + logger.warning( + "[kora_hermes.haiku_router] build_opus_reissue_kwargs raised " + "%r — skipping escalation", + exc, + ) + return None + + logger.info( + "[kora_hermes.haiku_router] escalating to Opus post-call " + "(reason=%s, route=%s, haiku_chars=%d, user_chars=%d)", + reason, + route, + len(haiku_text), + len(original_user_text), + ) + + return {"reissue_with": new_kwargs} + + +def register(ctx) -> None: + """Sub-plugin register. Wires + :func:`haiku_router_post_call_escalation` to the new + ``post_llm_call_can_reissue`` hook added by + KR-HERMES-LOCAL-EXT-REISSUE.""" + ctx.register_hook( + "post_llm_call_can_reissue", haiku_router_post_call_escalation + ) + logger.debug( + "[kora_hermes.haiku_router] sub-plugin registered" + ) diff --git a/kora_cli/reasoning/kora_hermes_plugin/plugin.py b/kora_cli/reasoning/kora_hermes_plugin/plugin.py index e2e03e995c47..6ce3ce7dc6f2 100644 --- a/kora_cli/reasoning/kora_hermes_plugin/plugin.py +++ b/kora_cli/reasoning/kora_hermes_plugin/plugin.py @@ -27,6 +27,13 @@ 5. state_holders — KR-PLUGIN-STATE-HOLDERS (this PR, Deliverable D) Owns: ``on_session_start`` (debug-log; holder liveness registry). + 6. haiku_router — KR-HAIKU-ROUTER-PLUGIN (KR-HERMES-LOCAL-EXT- + REISSUE-AND-HAIKU-ROUTER-PLUGIN-PAIR — completes Lock R3-2 + Phase C). Owns: ``post_llm_call_can_reissue`` (the new + local Hermes hook added by Deliverable A of the same + bucket). Consumes ``should_escalate_post_call`` from the + cost_ladder sub-plugin to fire parallel-Claude's Haiku- + as-Opus-context escalation pattern. # Remaining orchestrator-resident handlers @@ -307,6 +314,17 @@ def register(self, ctx) -> None: register_state_holders(ctx) + # KR-HAIKU-ROUTER-PLUGIN — owns post_llm_call_can_reissue. + # Consumes should_escalate_post_call from cost_ladder/ + # selector.py to fire post-call Opus escalation per + # parallel-Claude's pattern. Registers against the new + # local Hermes hook added by the paired Deliverable A. + from kora_cli.reasoning.kora_hermes_plugin.haiku_router import ( + register as register_haiku_router, + ) + + register_haiku_router(ctx) + # --- Handlers still living in the orchestrator (await # their own KR-PLUGIN-* extraction buckets) --- ctx.register_hook( @@ -319,7 +337,7 @@ def register(self, ctx) -> None: ) logger.info( - "[kora_hermes] plugin registered: 5 sub-plugins + 3 " + "[kora_hermes] plugin registered: 6 sub-plugins + 3 " "orchestrator-resident hooks against KORA_ROUTES=%s", sorted(KORA_ROUTES), ) diff --git a/plugins/kora_hermes/__init__.py b/plugins/kora_hermes/__init__.py index 8ded173c8612..4686632627e4 100644 --- a/plugins/kora_hermes/__init__.py +++ b/plugins/kora_hermes/__init__.py @@ -36,6 +36,9 @@ from kora_cli.reasoning.kora_hermes_plugin.cost_ladder.plugin import ( cost_ladder_and_caching_hook as _pre_api_request_mutable, ) +from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation as _post_llm_call_can_reissue, +) from kora_cli.reasoning.kora_hermes_plugin.plugin import ( KORA_ROUTES, KoraHermesPlugin, @@ -59,6 +62,7 @@ "_is_kora_reasoning_tool", "_on_session_start", "_post_llm_call", + "_post_llm_call_can_reissue", "_post_tool_call", "_pre_api_request_mutable", "_pre_tool_call", diff --git a/tests/agent/test_conversation_loop_post_llm_can_reissue.py b/tests/agent/test_conversation_loop_post_llm_can_reissue.py new file mode 100644 index 000000000000..e6ec197321a2 --- /dev/null +++ b/tests/agent/test_conversation_loop_post_llm_can_reissue.py @@ -0,0 +1,163 @@ +"""Structural tests for the post_llm_call_can_reissue hook +firing block in ``agent/conversation_loop.py``. + +The full ``run_conversation`` function is 4000+ lines and has +heavy setup requirements (transport, streaming, retry loop, +session DB, telemetry, etc.) — exercising it end-to-end for one +hook invocation is too much surface area. Instead we pin the +contract at the source level + verify the hook firing logic in +isolation via ``PluginManager``. + +Coverage: + + 1. Source contains the expected invoke_hook call with the + contract kwargs (response / api_kwargs / agent / iteration + / task_id / session_id / route). + 2. Source contains the anti-loop ``break`` after the re-issue + — guarantees at most ONE re-issue per iteration. + 3. Source places the hook AFTER the retry-loop guard + (``if response is None: ... break``) and BEFORE the + ``normalize_response`` transport call. + 4. The telemetry feed for re-issued calls passes + ``escalated_to_opus=True``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +_LOOP_SOURCE = ( + Path(__file__).resolve().parents[2] + / "agent" + / "conversation_loop.py" +).read_text() + + +def test_source_contains_post_llm_call_can_reissue_invocation(): + """The new hook fires with the named ``post_llm_call_can_reissue`` + hook ID — pinning the wire so future renames don't drift it + silently.""" + assert ( + '"post_llm_call_can_reissue"' in _LOOP_SOURCE + or "'post_llm_call_can_reissue'" in _LOOP_SOURCE + ) + + +def test_source_passes_contract_kwargs(): + """The hook contract documented in the bucket spec requires + these kwargs to be present in the firing site.""" + for kw in ( + "response=response", + "api_kwargs=api_kwargs", + "agent=agent", + "iteration=api_call_count", + "task_id=effective_task_id", + "session_id=agent.session_id", + "route=", + ): + assert kw in _LOOP_SOURCE, ( + f"missing contract kwarg in hook firing: {kw!r}" + ) + + +def test_source_has_anti_loop_break(): + """After a successful re-issue, the loop MUST ``break`` so + no second plugin can chain another re-issue (would cause + infinite escalation chains).""" + # The hook firing block ends with `break # anti-loop:`. + assert "anti-loop:" in _LOOP_SOURCE + + +def test_source_places_hook_after_retry_exhaustion_guard(): + """The hook MUST fire AFTER the retry-exhaustion guard + (``if response is None: ... break``) — firing inside the + retry loop would expose plugins to invalid responses + half- + initialized state.""" + guard = "all_retries_exhausted_no_response" + hook = "post_llm_call_can_reissue" + g_idx = _LOOP_SOURCE.find(guard) + h_idx = _LOOP_SOURCE.find(hook) + assert g_idx != -1 and h_idx != -1 + assert g_idx < h_idx, ( + "post_llm_call_can_reissue hook must fire after the retry-" + "exhaustion guard" + ) + + +def test_source_places_hook_before_post_api_request_observer(): + """The hook MUST fire BEFORE the per-iteration + ``post_api_request`` observer so observers always see the + final (possibly-re-issued) response. The spec's "post_llm_call + observers see final response only" guarantee depends on this + ordering.""" + # We want the LAST hook firing site (the new one we added) to + # come before the post_api_request OBSERVER (not the literal + # string "post_api_request" which appears in our own + # docstring earlier). Use the unique observer signature. + hook = "post_llm_call_can_reissue" + observer_marker = '"post_api_request"' + h_idx = _LOOP_SOURCE.find(hook) + o_idx = _LOOP_SOURCE.find(observer_marker) + assert h_idx != -1 and o_idx != -1 + assert h_idx < o_idx, ( + "post_llm_call_can_reissue hook must fire before the " + "post_api_request observer fires" + ) + + +def test_source_telemetry_escalated_to_opus_for_reissue(): + """When a re-issue fires, the cost-ladder feed for the new + response MUST pass ``escalated_to_opus=True`` so cockpit + panels can compute escalation rate.""" + # Look for the specific re-issue telemetry call (not the + # original-call site — that one defaults to False). + assert "escalated_to_opus=True" in _LOOP_SOURCE + + +# --------------------------------------------------------------------------- +# Behavioral test: end-to-end through PluginManager +# --------------------------------------------------------------------------- + + +def test_reissue_plugin_dispatch_via_plugin_manager(): + """Drive the same invoke_hook semantics the conversation_loop + uses. Verifies a real ``PluginManager`` returns the right + shape for the loop to consume.""" + from kora_cli.plugins import PluginManager + + mgr = PluginManager() + + capture: dict = {} + + def reissue_plugin(**kw): + # Capture the kwargs the loop will pass us — pins the + # contract from the consumer side. + capture.update(kw) + return {"reissue_with": {"model": "claude-opus-4-7"}} + + mgr._hooks["post_llm_call_can_reissue"] = [reissue_plugin] + results = mgr.invoke_hook( + "post_llm_call_can_reissue", + response=object(), + api_kwargs={"model": "claude-haiku-4-5-20251001"}, + agent=object(), + iteration=1, + task_id="t1", + session_id="s1", + route="slack_dm", + ) + assert len(results) == 1 + assert results[0]["reissue_with"]["model"] == "claude-opus-4-7" + # Contract kwargs visible to the plugin + assert set(capture.keys()) >= { + "response", + "api_kwargs", + "agent", + "iteration", + "task_id", + "session_id", + "route", + } diff --git a/tests/plugins/test_kora_hermes_plugin.py b/tests/plugins/test_kora_hermes_plugin.py index a997377df674..f2189cf54288 100644 --- a/tests/plugins/test_kora_hermes_plugin.py +++ b/tests/plugins/test_kora_hermes_plugin.py @@ -74,11 +74,12 @@ def test_plugin_is_discovered_but_opt_in(): assert "not enabled in config" in (loaded.error or "") -def test_register_function_wires_seven_hooks(): - """The plugin's register(ctx) function registers exactly 7 - hooks (ST2B added pre_tool_call_can_provide_result to ST1's - 6). Test directly with a mock context — bypasses Hermes's - opt-in plugins.enabled gate (operator-policy territory).""" +def test_register_function_wires_eight_hooks(): + """The plugin's register(ctx) function registers exactly 8 + hooks (KR-HERMES-LOCAL-EXT-REISSUE added + post_llm_call_can_reissue to ST2B's 7). Test directly with + a mock context — bypasses Hermes's opt-in plugins.enabled + gate (operator-policy territory).""" from plugins.kora_hermes import register registered = [] @@ -97,6 +98,7 @@ def register_hook(self, name, callback): "post_tool_call", "post_llm_call", "pre_tool_call_can_provide_result", # ST2B added + "post_llm_call_can_reissue", # KR-HERMES-LOCAL-EXT-REISSUE added ]) # Each registered callback is callable. for name, callback in registered: diff --git a/tests/plugins/test_kora_hermes_plugin_haiku_router.py b/tests/plugins/test_kora_hermes_plugin_haiku_router.py new file mode 100644 index 000000000000..c9646489c179 --- /dev/null +++ b/tests/plugins/test_kora_hermes_plugin_haiku_router.py @@ -0,0 +1,543 @@ +"""Tests for the haiku_router sub-plugin + the new +``post_llm_call_can_reissue`` local Hermes hook surface. + +Coverage: + + - Pure helpers in ``escalator.py`` (text extraction + + re-issue kwargs construction) + - ``haiku_router_post_call_escalation`` hook handler gating + (non-Kora route / iteration > 1 / non-Haiku model / disabled + env / no text / confident response) + - Hook handler fires on low-confidence Haiku reply: returns + ``{"reissue_with": ...}`` with Opus model + Haiku-as-assistant + context injected + - First-non-None override semantics (re-issue is at most one + per iteration; second plugin's return is ignored when first + already won) + - Anti-loop: the spec contract guarantees the hook is NOT + re-fired against the re-issued response. We verify this at + the contract level (no recursion in the handler) and via + the loop-side guard in ``conversation_loop.py``. + - Backward-compat: discovery shim still exports the new + handler under the ``_post_llm_call_can_reissue`` alias +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest + + +# --------------------------------------------------------------------------- +# Pure helper: extract_first_text +# --------------------------------------------------------------------------- + + +def test_extract_first_text_sdk_content_blocks(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_first_text, + ) + + # SDK-like content block (pydantic-style; has .type / .text). + block1 = SimpleNamespace(type="text", text="first part") + block2 = SimpleNamespace(type="tool_use", input={"x": 1}) + block3 = SimpleNamespace(type="text", text="second part") + response = SimpleNamespace(content=[block1, block2, block3]) + + assert extract_first_text(response) == "first part\nsecond part" + + +def test_extract_first_text_dict_content_blocks(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_first_text, + ) + + response = SimpleNamespace( + content=[ + {"type": "text", "text": "hello"}, + {"type": "text", "text": "world"}, + ] + ) + assert extract_first_text(response) == "hello\nworld" + + +def test_extract_first_text_no_content(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_first_text, + ) + + assert extract_first_text(None) == "" + assert extract_first_text(SimpleNamespace()) == "" + assert extract_first_text(SimpleNamespace(content=None)) == "" + assert extract_first_text(SimpleNamespace(content=[])) == "" + + +def test_extract_first_text_tool_use_only(): + """A tool-use-only response (no text blocks) returns empty — + the haiku_router handler treats this as "nothing to escalate + from" and skips.""" + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_first_text, + ) + + response = SimpleNamespace( + content=[SimpleNamespace(type="tool_use", input={})] + ) + assert extract_first_text(response) == "" + + +# --------------------------------------------------------------------------- +# Pure helper: extract_last_user_text +# --------------------------------------------------------------------------- + + +def test_extract_last_user_text_string_content(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_last_user_text, + ) + + api_kwargs = { + "messages": [ + {"role": "user", "content": "earlier turn"}, + {"role": "assistant", "content": "response"}, + {"role": "user", "content": "latest question"}, + ] + } + assert extract_last_user_text(api_kwargs) == "latest question" + + +def test_extract_last_user_text_block_list_content(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_last_user_text, + ) + + api_kwargs = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "part1"}, + {"type": "text", "text": "part2"}, + ], + }, + ] + } + assert extract_last_user_text(api_kwargs) == "part1\npart2" + + +def test_extract_last_user_text_no_messages(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + extract_last_user_text, + ) + + assert extract_last_user_text({}) == "" + assert extract_last_user_text({"messages": None}) == "" + assert extract_last_user_text(None) == "" # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Pure helper: build_opus_reissue_kwargs +# --------------------------------------------------------------------------- + + +def test_build_opus_reissue_kwargs_injects_haiku_context(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + MODEL_OPUS, + REISSUE_REVIEW_PROMPT, + ) + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + build_opus_reissue_kwargs, + ) + + original = { + "model": "claude-haiku-4-5-20251001", + "max_tokens": 1024, + "system": "you are kora", + "messages": [ + {"role": "user", "content": "what should i do"}, + ], + } + new = build_opus_reissue_kwargs( + api_kwargs=original, + haiku_response_text="I'm not sure, maybe X", + ) + + # New kwargs swap to Opus + extend messages with Haiku + # assistant turn + reviewer prompt user turn. + assert new["model"] == MODEL_OPUS + assert new["max_tokens"] == 1024 # unchanged + assert new["system"] == "you are kora" # unchanged + assert new["messages"] == [ + {"role": "user", "content": "what should i do"}, + {"role": "assistant", "content": "I'm not sure, maybe X"}, + {"role": "user", "content": REISSUE_REVIEW_PROMPT}, + ] + # Original is not mutated. + assert original["model"] == "claude-haiku-4-5-20251001" + assert len(original["messages"]) == 1 + + +def test_build_opus_reissue_kwargs_handles_missing_messages(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.escalator import ( + build_opus_reissue_kwargs, + ) + + new = build_opus_reissue_kwargs( + api_kwargs={"model": "claude-haiku-4-5-20251001"}, + haiku_response_text="hi", + ) + # When the original had no messages, the new messages start + # at the Haiku turn (still valid Anthropic shape — assistant + # turns can lead in continuation flows). + assert len(new["messages"]) == 2 + assert new["messages"][0]["role"] == "assistant" + + +# --------------------------------------------------------------------------- +# Hook handler — activation gating +# --------------------------------------------------------------------------- + + +def _haiku_response(text: str) -> Any: + return SimpleNamespace( + content=[SimpleNamespace(type="text", text=text)] + ) + + +def _haiku_api_kwargs( + user_text: str = "a substantial question " * 20, + model: str = "claude-haiku-4-5-20251001", +) -> dict: + return { + "model": model, + "messages": [{"role": "user", "content": user_text}], + } + + +def test_handler_no_op_on_non_kora_route(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + result = haiku_router_post_call_escalation( + response=_haiku_response("I'm not sure about this"), + api_kwargs=_haiku_api_kwargs(), + iteration=1, + route="", # not Kora + ) + assert result is None + + +def test_handler_no_op_on_iteration_gt_1(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + # iteration 2+ already runs Opus from the pre-call cost- + # ladder rule (tool_loop_iteration); post-call escalation + # would be redundant. + result = haiku_router_post_call_escalation( + response=_haiku_response("I'm not sure"), + api_kwargs=_haiku_api_kwargs(), + iteration=2, + route="slack_dm", + ) + assert result is None + + +def test_handler_no_op_on_non_haiku_original_model(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + # The original call was Opus (force_opus / opus_prefix / + # decision_language path) — nothing to escalate from. + result = haiku_router_post_call_escalation( + response=_haiku_response("I'm not sure"), + api_kwargs=_haiku_api_kwargs(model="claude-opus-4-7"), + iteration=1, + route="slack_dm", + ) + assert result is None + + +def test_handler_no_op_when_disabled_via_env(monkeypatch): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + ENV_DISABLE_POST_CALL_ESCALATION, + ) + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + monkeypatch.setenv(ENV_DISABLE_POST_CALL_ESCALATION, "true") + result = haiku_router_post_call_escalation( + response=_haiku_response("I'm not sure"), + api_kwargs=_haiku_api_kwargs(), + iteration=1, + route="slack_dm", + ) + assert result is None + + +def test_handler_no_op_on_confident_haiku(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + # Confident, substantive Haiku reply — no escalation. + result = haiku_router_post_call_escalation( + response=_haiku_response( + "Yes, the deploy is healthy: all 7 services report " + "ready and the canary is at 100%." + ), + api_kwargs=_haiku_api_kwargs(), + iteration=1, + route="slack_dm", + ) + assert result is None + + +def test_handler_no_op_on_tool_use_only_response(): + """A tool-use-only response has no text to escalate — handler + must skip even though the response IS otherwise eligible.""" + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + response = SimpleNamespace( + content=[SimpleNamespace(type="tool_use", input={"q": "x"})] + ) + result = haiku_router_post_call_escalation( + response=response, + api_kwargs=_haiku_api_kwargs(), + iteration=1, + route="slack_dm", + ) + assert result is None + + +# --------------------------------------------------------------------------- +# Hook handler — escalation path +# --------------------------------------------------------------------------- + + +def test_handler_escalates_on_low_confidence_marker(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + MODEL_OPUS, + REISSUE_REVIEW_PROMPT, + ) + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + haiku_text = ( + "I'm not sure about that — I don't have enough context " + "to give a confident answer." + ) + api_kwargs = _haiku_api_kwargs( + user_text="Is the migration safe to roll out today?" + ) + result = haiku_router_post_call_escalation( + response=_haiku_response(haiku_text), + api_kwargs=api_kwargs, + iteration=1, + route="slack_dm", + ) + assert isinstance(result, dict) + assert "reissue_with" in result + new_kwargs = result["reissue_with"] + assert new_kwargs["model"] == MODEL_OPUS + # Messages: original user + Haiku assistant turn + reviewer + # user turn. + assert new_kwargs["messages"] == [ + {"role": "user", "content": "Is the migration safe to roll out today?"}, + {"role": "assistant", "content": haiku_text}, + {"role": "user", "content": REISSUE_REVIEW_PROMPT}, + ] + + +def test_handler_escalates_on_short_response_for_long_input(): + """Heuristic: long input + tiny Haiku reply → escalation.""" + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.constants import ( + MODEL_OPUS, + ) + from kora_cli.reasoning.kora_hermes_plugin.haiku_router.plugin import ( + haiku_router_post_call_escalation, + ) + + long_input = "a long substantive question " * 30 # > 200 chars + short_haiku = "Yes." # < 50 chars + result = haiku_router_post_call_escalation( + response=_haiku_response(short_haiku), + api_kwargs=_haiku_api_kwargs(user_text=long_input), + iteration=1, + route="slack_dm", + ) + assert isinstance(result, dict) + assert result["reissue_with"]["model"] == MODEL_OPUS + + +# --------------------------------------------------------------------------- +# Sub-register: hook wiring through ctx +# --------------------------------------------------------------------------- + + +def test_subregister_wires_post_llm_call_can_reissue(): + from kora_cli.reasoning.kora_hermes_plugin.haiku_router import register + + registered = [] + + class _MockCtx: + def register_hook(self, name, callback): + registered.append((name, callback)) + + register(_MockCtx()) + assert len(registered) == 1 + name, callback = registered[0] + assert name == "post_llm_call_can_reissue" + assert callable(callback) + + +def test_orchestrator_registers_haiku_router_hook(): + """The top-level KoraHermesPlugin.register must wire the + haiku_router sub-plugin's hook — drift here means future + sub-plugins land without their register being called.""" + from plugins.kora_hermes import register + + registered = [] + + class _MockCtx: + def register_hook(self, name, callback): + registered.append(name) + + register(_MockCtx()) + assert "post_llm_call_can_reissue" in registered + + +# --------------------------------------------------------------------------- +# First-non-None override semantics +# --------------------------------------------------------------------------- + + +def test_first_non_none_override_wins(monkeypatch): + """Contract: when multiple plugins register against + post_llm_call_can_reissue, the FIRST plugin returning a dict + with ``reissue_with`` wins. Subsequent plugins are ignored. + + We exercise this via ``invoke_hook``'s actual semantics — + it collects ALL non-None returns; the conversation_loop + iteration in the consumer logic breaks on first match. + """ + from kora_cli.plugins import PluginManager + + mgr = PluginManager() + + def plugin_one(**kw): + return {"reissue_with": {"model": "first-wins"}} + + def plugin_two(**kw): + return {"reissue_with": {"model": "second-loses"}} + + mgr._hooks["post_llm_call_can_reissue"] = [plugin_one, plugin_two] + + results = mgr.invoke_hook( + "post_llm_call_can_reissue", + response=None, + api_kwargs={}, + iteration=1, + route="slack_dm", + ) + # Both returned non-None — invoke_hook returns both. + assert len(results) == 2 + # The conversation_loop iterates and breaks on first dict + # with ``reissue_with`` — verify that semantic by mirroring + # the loop's iteration logic here. + chosen = None + for r in results: + if isinstance(r, dict) and "reissue_with" in r: + chosen = r["reissue_with"] + break + assert chosen == {"model": "first-wins"} + + +def test_plugin_exception_is_fail_safe(): + """A plugin that raises inside the handler must not break + the loop — invoke_hook catches + logs, the iteration + continues with the original response.""" + from kora_cli.plugins import PluginManager + + mgr = PluginManager() + + def plugin_raises(**kw): + raise RuntimeError("plugin bug") + + def plugin_returns_none(**kw): + return None + + mgr._hooks["post_llm_call_can_reissue"] = [ + plugin_raises, + plugin_returns_none, + ] + # invoke_hook MUST NOT raise — the bad plugin's exception is + # logged and the iteration continues. + results = mgr.invoke_hook( + "post_llm_call_can_reissue", + response=None, + api_kwargs={}, + iteration=1, + route="slack_dm", + ) + # No usable return — loop continues with original response. + assert results == [] + + +# --------------------------------------------------------------------------- +# Backward-compat: discovery shim exposes the alias +# --------------------------------------------------------------------------- + + +def test_discovery_shim_exports_alias(): + """``plugins.kora_hermes`` re-exports the handler under the + pre-extraction alias name for consumer-import stability.""" + from plugins.kora_hermes import _post_llm_call_can_reissue + from kora_cli.reasoning.kora_hermes_plugin.haiku_router import ( + haiku_router_post_call_escalation, + ) + + assert _post_llm_call_can_reissue is haiku_router_post_call_escalation + + +# --------------------------------------------------------------------------- +# Sanity: the cost_ladder selector dep is still importable from +# this sub-plugin (would break the import chain if cost_ladder +# selector signature drifted under us) +# --------------------------------------------------------------------------- + + +def test_should_escalate_post_call_signature_compatible(): + """The plugin depends on + ``should_escalate_post_call(haiku_response_text=..., original_message_text=...)``. + Pin the signature so cost_ladder selector edits don't silently + break the haiku_router wiring.""" + from kora_cli.reasoning.kora_hermes_plugin.cost_ladder.selector import ( + should_escalate_post_call, + ) + + # Confident → no escalation. + should, reason = should_escalate_post_call( + haiku_response_text="The deploy is healthy.", + original_message_text="is the deploy healthy", + ) + assert should is False + assert reason == "haiku_confident" + + # Uncertain → escalate. + should, reason = should_escalate_post_call( + haiku_response_text="I'm not sure about the status.", + original_message_text="is the deploy healthy", + ) + assert should is True + assert reason == "low_confidence_marker"