/model: probe effort combo on switch, stop passing reasoning_effort to litellm

agent/config.py

@@ -33,14 +33,15 @@ class Config(BaseModel):
     confirm_cpu_jobs: bool = True
     auto_file_upload: bool = False
 
-    # Reasoning effort for models that support it (GPT-5 / o-series, Claude
-    # extended thinking, HF reasoning models like MiniMax M2 / Kimi K2).
-    # Defaults to "high" — we'd rather spend tokens thinking than ship a
-    # wrong ML recipe. Users can dial down with `/effort low|medium|off`.
-    # "minimal" is an OpenAI-only level and is normalized to "low" for HF
-    # router models (MiniMax requires ≥low). Ignored for non-reasoning models.
-    # Valid values: None | "minimal" | "low" | "medium" | "high"
-    reasoning_effort: str | None = "high"
+    # Reasoning effort *preference* — the ceiling the user wants. The probe
+    # on `/model` walks a cascade down from here (``max`` → ``xhigh`` → ``high``
+    # → …) and caches per-model what the provider actually accepted in
+    # ``Session.model_effective_effort``. Default ``max`` because we'd rather
+    # burn tokens thinking than ship a wrong ML recipe; the cascade lands on
+    # whichever level the model supports (``high`` for GPT-5 / HF router,
+    # ``xhigh`` or ``max`` for Anthropic 4.6 / 4.7). ``None`` = thinking off.
+    # Valid values: None | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"
+    reasoning_effort: str | None = "max"
 
 
 def substitute_env_vars(obj: Any) -> Any:

agent/core/agent_loop.py

@@ -136,6 +136,58 @@ def _is_transient_error(error: Exception) -> bool:
     return any(pattern in err_str for pattern in transient_patterns)
 
 
+def _is_effort_config_error(error: Exception) -> bool:
+    """Catch the two 400s the effort probe also handles — thinking
+    unsupported for this model, or the specific effort level invalid.
+
+    This is our safety net for the case where ``/effort`` was changed
+    mid-conversation (which clears the probe cache) and the new level
+    doesn't work for the current model. We heal the cache and retry once.
+    """
+    from agent.core.effort_probe import _is_invalid_effort, _is_thinking_unsupported
+    return _is_thinking_unsupported(error) or _is_invalid_effort(error)
+
+
+async def _heal_effort_and_rebuild_params(
+    session: Session, error: Exception, llm_params: dict,
+) -> dict:
+    """Update the session's effort cache based on ``error`` and return new
+    llm_params. Called only when ``_is_effort_config_error(error)`` is True.
+
+    Two branches:
+      • thinking-unsupported → cache ``None`` for this model, next call
+        strips thinking entirely
+      • invalid-effort → re-run the full cascade probe; the result lands
+        in the cache
+    """
+    from agent.core.effort_probe import ProbeInconclusive, _is_thinking_unsupported, probe_effort
+
+    model = session.config.model_name
+    if _is_thinking_unsupported(error):
+        session.model_effective_effort[model] = None
+        logger.info("healed: %s doesn't support thinking — stripped", model)
+    else:
+        try:
+            outcome = await probe_effort(
+                model, session.config.reasoning_effort, session.hf_token,
+            )
+            session.model_effective_effort[model] = outcome.effective_effort
+            logger.info(
+                "healed: %s effort cascade → %s", model, outcome.effective_effort,
+            )
+        except ProbeInconclusive:
+            # Transient during healing — strip thinking for safety, next
+            # call will either succeed or surface the real error.
+            session.model_effective_effort[model] = None
+            logger.info("healed: %s probe inconclusive — stripped", model)
+
+    return _resolve_llm_params(
+        model,
+        session.hf_token,
+        reasoning_effort=session.effective_effort_for(model),
+    )
+
+
 def _friendly_error_message(error: Exception) -> str | None:
     """Return a user-friendly message for known error types, or None to fall back to traceback."""
     err_str = str(error).lower()
@@ -243,6 +295,7 @@ class LLMResult:
 async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> LLMResult:
     """Call the LLM with streaming, emitting assistant_chunk events."""
     response = None
+    _healed_effort = False  # one-shot safety net per call
     for _llm_attempt in range(_MAX_LLM_RETRIES):
         try:
             response = await acompletion(
@@ -258,6 +311,14 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) ->
         except ContextWindowExceededError:
             raise
         except Exception as e:
+            if not _healed_effort and _is_effort_config_error(e):
+                _healed_effort = True
+                llm_params = await _heal_effort_and_rebuild_params(session, e, llm_params)
+                await session.send_event(Event(
+                    event_type="tool_log",
+                    data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."},
+                ))
+                continue
             if _llm_attempt < _MAX_LLM_RETRIES - 1 and _is_transient_error(e):
                 _delay = _LLM_RETRY_DELAYS[_llm_attempt]
                 logger.warning(
@@ -328,6 +389,7 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) ->
 async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) -> LLMResult:
     """Call the LLM without streaming, emit assistant_message at the end."""
     response = None
+    _healed_effort = False
     for _llm_attempt in range(_MAX_LLM_RETRIES):
         try:
             response = await acompletion(
@@ -342,6 +404,14 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params)
         except ContextWindowExceededError:
             raise
         except Exception as e:
+            if not _healed_effort and _is_effort_config_error(e):
+                _healed_effort = True
+                llm_params = await _heal_effort_and_rebuild_params(session, e, llm_params)
+                await session.send_event(Event(
+                    event_type="tool_log",
+                    data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."},
+                ))
+                continue
             if _llm_attempt < _MAX_LLM_RETRIES - 1 and _is_transient_error(e):
                 _delay = _LLM_RETRY_DELAYS[_llm_attempt]
                 logger.warning(
@@ -490,10 +560,13 @@ async def run_agent(
             tools = session.tool_router.get_tool_specs_for_llm()
             try:
                 # ── Call the LLM (streaming or non-streaming) ──
+                # Pull the per-model probed effort from the session cache when
+                # available; fall back to the raw preference for models we
+                # haven't probed yet (e.g. research sub-model).
                 llm_params = _resolve_llm_params(
                     session.config.model_name,
                     session.hf_token,
-                    reasoning_effort=session.config.reasoning_effort,
+                    reasoning_effort=session.effective_effort_for(session.config.model_name),
                 )
                 if session.stream:
                     llm_result = await _call_llm_streaming(session, messages, tools, llm_params)

agent/core/effort_probe.py

@@ -0,0 +1,229 @@
+"""Probe-and-cascade for reasoning effort on /model switch.
+
+We don't maintain a per-model capability table. Instead, the first time a
+user picks a model we fire a 1-token ping with the same params we'd use
+for real and walk down a cascade (``max`` → ``xhigh`` → ``high`` → …)
+until the provider stops rejecting us. The result is cached per-model on
+the session, so real messages don't pay the probe cost again.
+
+Three outcomes, classified from the 400 error text:
+
+* success → cache the effort that worked
+* ``"thinking ... not supported"`` → model doesn't do thinking at all;
+  cache ``None`` so we stop sending thinking params
+* ``"effort ... invalid"`` / synonyms → cascade walks down and retries
+
+Transient errors (5xx, timeout, connection reset) bubble out as
+``ProbeInconclusive`` so the caller can complete the switch with a
+warning instead of blocking on a flaky provider.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+
+from litellm import acompletion
+
+from agent.core.llm_params import UnsupportedEffortError, _resolve_llm_params
+
+logger = logging.getLogger(__name__)
+
+
+# Cascade: for each user-stated preference, the ordered list of levels to
+# try. First success wins. ``max`` / ``xhigh`` are Anthropic-only; providers
+# that don't accept them raise ``UnsupportedEffortError`` synchronously (no
+# wasted network round-trip) and we advance to the next level.
+_EFFORT_CASCADE: dict[str, list[str]] = {
+    "max":     ["max", "xhigh", "high", "medium", "low"],
+    "xhigh":   ["xhigh", "high", "medium", "low"],
+    "high":    ["high", "medium", "low"],
+    "medium":  ["medium", "low"],
+    "minimal": ["minimal", "low"],
+    "low":     ["low"],
+}
+
+_PROBE_TIMEOUT = 15.0
+_PROBE_MAX_TOKENS = 16
+
+
+class ProbeInconclusive(Exception):
+    """The probe couldn't reach a verdict (transient network / provider error).
+
+    Caller should complete the switch with a warning — the next real call
+    will re-surface the error if it's persistent.
+    """
+
+
+@dataclass
+class ProbeOutcome:
+    """What the probe learned. ``effective_effort`` semantics match the cache:
+
+    * str → send this level
+    * None → model doesn't support thinking; strip it
+    """
+    effective_effort: str | None
+    attempts: int
+    elapsed_ms: int
+    note: str | None = None  # e.g. "max not supported, falling back"
+
+
+def _is_thinking_unsupported(e: Exception) -> bool:
+    """Model rejected any thinking config.
+
+    Matches Anthropic's 'thinking.type.enabled is not supported for this
+    model' as well as the adaptive variant. Substring-match because the
+    exact wording shifts across API versions.
+    """
+    s = str(e).lower()
+    return "thinking" in s and "not supported" in s
+
+
+def _is_invalid_effort(e: Exception) -> bool:
+    """The requested effort level isn't accepted for this model.
+
+    Covers both API responses (Anthropic/OpenAI 400 with "invalid", "must
+    be one of", etc.) and LiteLLM's local validation that fires *before*
+    the request (e.g. "effort='max' is only supported by Claude Opus 4.6"
+    — LiteLLM knows max is Opus-4.6-only and raises synchronously). The
+    cascade walks down on either.
+
+    Explicitly returns False when the message is really about thinking
+    itself (e.g. Anthropic's 4.7 error mentions ``output_config.effort``
+    in its fix hint, but the actual failure is ``thinking.type.enabled``
+    being unsupported). That case is caught by ``_is_thinking_unsupported``.
+    """
+    if _is_thinking_unsupported(e):
+        return False
+    s = str(e).lower()
+    if "effort" not in s and "output_config" not in s:
+        return False
+    return any(
+        phrase in s
+        for phrase in (
+            "invalid", "not supported", "must be one of", "not a valid",
+            "unrecognized", "unknown",
+            # LiteLLM's own pre-flight validation phrasing.
+            "only supported by", "is only supported",
+        )
+    )
+
+
+def _is_transient(e: Exception) -> bool:
+    """Network / provider-side flake. Keep in sync with agent_loop's list.
+
+    Also matches by type for ``asyncio.TimeoutError`` — its ``str(e)`` is
+    empty, so substring matching alone misses it.
+    """
+    if isinstance(e, (asyncio.TimeoutError, TimeoutError)):
+        return True
+    s = str(e).lower()
+    return any(
+        p in s
+        for p in (
+            "timeout", "timed out", "429", "rate limit",
+            "503", "service unavailable", "502", "bad gateway",
+            "500", "internal server error", "overloaded", "capacity",
+            "connection reset", "connection refused", "connection error",
+            "eof", "broken pipe",
+        )
+    )
+
+
+async def probe_effort(
+    model_name: str,
+    preference: str | None,
+    hf_token: str | None,
+) -> ProbeOutcome:
+    """Walk the cascade for ``preference`` on ``model_name``.
+
+    Returns the first effort the provider accepts, or ``None`` if it
+    rejects thinking altogether. Raises ``ProbeInconclusive`` only for
+    transient errors (5xx, timeout) — persistent 4xx that aren't thinking/
+    effort related bubble as the original exception so callers can surface
+    them (auth, model-not-found, quota, etc.).
+    """
+    loop = asyncio.get_event_loop()
+    start = loop.time()
+    attempts = 0
+
+    if not preference:
+        # User explicitly turned effort off — nothing to probe. A bare
+        # ping with no thinking params is pointless; just report "off".
+        return ProbeOutcome(effective_effort=None, attempts=0, elapsed_ms=0)
+
+    cascade = _EFFORT_CASCADE.get(preference, [preference])
+    skipped: list[str] = []  # levels the provider rejected synchronously
+
+    last_error: Exception | None = None
+    for effort in cascade:
+        try:
+            params = _resolve_llm_params(
+                model_name, hf_token, reasoning_effort=effort, strict=True,
+            )
+        except UnsupportedEffortError:
+            # Provider can't even accept this effort name (e.g. "max" on
+            # HF router). Skip without a network call.
+            skipped.append(effort)
+            continue
+
+        attempts += 1
+        try:
+            await asyncio.wait_for(
+                acompletion(
+                    messages=[{"role": "user", "content": "ping"}],
+                    max_tokens=_PROBE_MAX_TOKENS,
+                    stream=False,
+                    **params,
+                ),
+                timeout=_PROBE_TIMEOUT,
+            )
+        except Exception as e:
+            last_error = e
+            if _is_thinking_unsupported(e):
+                elapsed = int((loop.time() - start) * 1000)
+                return ProbeOutcome(
+                    effective_effort=None,
+                    attempts=attempts,
+                    elapsed_ms=elapsed,
+                    note="model doesn't support reasoning, dropped",
+                )
+            if _is_invalid_effort(e):
+                logger.debug("probe: %s rejected effort=%s, trying next", model_name, effort)
+                continue
+            if _is_transient(e):
+                raise ProbeInconclusive(str(e)) from e
+            # Persistent non-thinking 4xx (auth, quota, model-not-found) —
+            # let the caller classify & surface.
+            raise
+        else:
+            elapsed = int((loop.time() - start) * 1000)
+            note = None
+            if effort != preference:
+                note = f"{preference} not supported, using {effort}"
+            return ProbeOutcome(
+                effective_effort=effort,
+                attempts=attempts,
+                elapsed_ms=elapsed,
+                note=note,
+            )
+
+    # Cascade exhausted without a success. This only happens when every
+    # level was either rejected synchronously (``UnsupportedEffortError``,
+    # e.g. preference=max on HF and we also somehow filtered all others)
+    # or the provider 400'd ``invalid effort`` on every level.
+    elapsed = int((loop.time() - start) * 1000)
+    if last_error is not None and not _is_invalid_effort(last_error):
+        raise last_error
+    note = (
+        "no effort level accepted — proceeding without thinking"
+        if not skipped
+        else f"provider rejected all efforts ({', '.join(skipped)})"
+    )
+    return ProbeOutcome(
+        effective_effort=None,
+        attempts=attempts,
+        elapsed_ms=elapsed,
+        note=note,
+    )