diff --git a/api/config.py b/api/config.py index 5c2d5a489e..8bf52034da 100644 --- a/api/config.py +++ b/api/config.py @@ -1252,6 +1252,14 @@ def _resolve_cli_toolsets(cfg=None): "nemotron": "nvidia", "mimo": "xiaomi", "xiaomi-mimo": "xiaomi", + # Vercel AI Gateway is a registered production provider in the installed + # Agent (hermes_cli PROVIDER_REGISTRY canonical ``ai-gateway``); mirror + # its alias family so standalone WebUI installs resolve every documented + # spelling to the same recognized reasoning lane. (#6018 gate 2026-08-13) + "vercel": "ai-gateway", + "vercel-ai-gateway": "ai-gateway", + "ai_gateway": "ai-gateway", + "aigateway": "ai-gateway", # Legacy alias — earlier WebUI builds wrote ``provider: local`` for unknown # loopback endpoints, but ``local`` is not registered in # ``hermes_cli.auth.PROVIDER_REGISTRY``. Routing it through ``custom`` @@ -1311,6 +1319,10 @@ def _resolve_provider_alias(name: str) -> str: return _agent_aliases[raw] except Exception: pass + # Keep Agent-owned Azure Foundry aliases available in standalone WebUI + # installs where hermes_cli is intentionally absent (including CI). + if raw in {"azure", "azure-ai-foundry", "azure-ai"}: + return "azure-foundry" return _PROVIDER_ALIASES.get(raw, name) @@ -3204,7 +3216,9 @@ def get_effective_default_model(config_data: dict | None = None) -> str: # importing from the agent tree (which may not be installed). Any drift here # will show up in the shared test suite since both sides accept the same set. # Keep this WebUI-visible set aligned with hermes-agent#29248. -VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh", "max") +VALID_REASONING_EFFORTS = ( + "minimal", "low", "medium", "high", "xhigh", "max", "ultra", +) def parse_reasoning_effort(effort): @@ -3515,24 +3529,6 @@ def _zai_glm_thinking_toggle_supported(model_id: str, provider_id: str) -> bool return cls in {"effort", "thinking"} -_OPENAI_FAMILY_REASONING_PROVIDERS = frozenset({ - "openai-codex", "openai", "openai-api", - "azure-foundry", "azure-openai", "azure", -}) - -_GPT_5_6_REASONING_MODELS = frozenset({ - "gpt-5.6", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", -}) - - -def _is_gpt_5_6_reasoning_model(bare_model: str) -> bool: - """Return whether an OpenAI-family model uses GPT-5.6's max ladder.""" - return str(bare_model or "").strip().lower() in _GPT_5_6_REASONING_MODELS - - def _filter_reasoning_efforts_for_provider( efforts: list[str], model_id: str, @@ -3546,41 +3542,74 @@ def _filter_reasoning_efforts_for_provider( ] normalized = list(dict.fromkeys(normalized)) provider = _resolve_provider_alias(str(provider_id or "").strip().lower()) - bare = _strip_provider_hint_for_reasoning(model_id).lower().rsplit("/", 1)[-1] - # OpenAI-family lanes cap pre-GPT-5.6 GPT-5 models at xhigh and o-series at - # high. GPT-5.6's alias and Sol/Terra/Luna variants natively accept max. - if provider in _OPENAI_FAMILY_REASONING_PROVIDERS: - if bare.startswith(("o1", "o3", "o4")): - return [eff for eff in normalized if eff in {"low", "medium", "high"}] - if bare.startswith("gpt-5") and not _is_gpt_5_6_reasoning_model(bare): - return [eff for eff in normalized if eff != "max"] - # Providers whose native ladder tops out below 'max' must NOT advertise it, - # otherwise a stored/CLI 'max' degrades WORSE than the - # prior max->xhigh coercion (Gemini's adapter treats unknown 'max' as medium; - # pre-adaptive Anthropic manual-thinking lacks a 'max' budget and falls to 8k). - # Dropping 'max' here lets the existing downgrade ladder land on xhigh/high. + bare = _strip_provider_hint_for_reasoning(model_id, provider).lower().rsplit("/", 1)[-1] + # Model-scoped hard ceilings follow the MODEL across every serving lane — + # direct OpenAI-family providers, aggregators (OpenRouter/Nous), Copilot, + # and custom gateways alike. Older GPT-5 generations reject the generic + # max/ultra tiers and o-series models reject anything above ``high`` no + # matter which route delivers them; GPT-5.6 is the only GPT-5 generation + # exposing max/ultra. Hermes Agent normalizes Codex's product-level + # ``ultra`` to the Responses API wire value ``max``. (#6018) + if bare.startswith(("o1", "o3", "o4")): + return [eff for eff in normalized if eff in {"low", "medium", "high"}] + if bare.startswith("gpt-5") and not _is_gpt_5_6_family(bare): + return [eff for eff in normalized if eff not in {"max", "ultra"}] + # First-party GPT-5.6 lookalikes must not regain a top tier merely because + # their authoritative capability result is empty. Keep genuinely unknown + # custom/future models on the historical preserve-verbatim path. + if ( + provider in { + "openai-codex", "openai", "openai-api", + "azure-openai", "azure-foundry", + } + and "gpt-5.6" in bare + and not _is_gpt_5_6_family(bare) + ): + return [eff for eff in normalized if eff not in {"max", "ultra"}] + + # Generic top tiers must not be advertised to providers whose native ladder + # tops out lower; the downgrade ladder then lands on xhigh/high safely. if provider in {"gemini", "google", "google-gemini", "google-vertex", "vertex"}: - return [eff for eff in normalized if eff != "max"] - # Legacy Claude is pre-adaptive whether served natively OR via Azure Foundry / - # Bedrock / Vertex — the ceiling follows the MODEL, not just the provider name. - _anthropic_lanes = { - "anthropic", "claude", "anthropic-claude", - "azure-foundry", "azure-openai", "azure", "bedrock", "aws-bedrock", - "vertex", "google-vertex", - } - if provider in _anthropic_lanes and "claude" in bare and _is_pre_adaptive_anthropic(bare): - return [eff for eff in normalized if eff != "max"] + return [eff for eff in normalized if eff not in {"max", "ultra"}] + # Legacy Claude is pre-adaptive whether served natively, via cloud hosts + # (Azure Foundry / Bedrock / Vertex), OR through aggregator/routed lanes + # (OpenRouter, Nous, AI Gateway, custom gateways). The ceiling follows the + # MODEL across every serving lane — no provider gate, exactly like the + # older-GPT-5 and o-series ceilings above. (#6018 gate 2026-08-13) + if "claude" in bare and _is_pre_adaptive_anthropic(bare): + return [eff for eff in normalized if eff not in {"max", "ultra"}] + # Vercel AI Gateway forwards the reasoning config to the routed model, so + # adaptive Claude keeps ``max``; the Codex product-only ``ultra`` tier does + # not exist on that wire and must map down to ``max`` instead of leaking + # through. (#6018 gate 2026-08-13) + if provider == "ai-gateway": + return [eff for eff in normalized if eff != "ultra"] # Z.AI / GLM native-endpoint gate: see _zai_glm_reasoning_efforts_supported. # True → keep the full ladder (GLM-5.2+); False → strip it entirely (pre-5.2 # GLM and forced-thinking GLM-4.7); None → not a zai GLM case, defer. zai_supports = _zai_glm_reasoning_efforts_supported(model_id, provider_id) if zai_supports is True: - return normalized + # Z.AI documents ``max`` as the top GLM-5.2 reasoning_effort value; the + # Codex product-only ``ultra`` tier must not leak into its native ladder. + return [eff for eff in normalized if eff != "ultra"] if zai_supports is False: return [] + # DEFAULT-DENY for custom/unrecognized providers: their native effort + # ladders are unknown, so the supra-ceiling max/ultra tiers must not leak + # through heuristic or metadata fallbacks — unless the operator explicitly + # authorized them via a provider ``reasoning_efforts`` allowlist. An empty + # provider id skips this gate: the caller simply didn't name a provider, + # which is not the same as naming one we don't recognize. (#6018) + if provider and not _provider_known_reasoning_capable(provider): + # The model-scoped allowlist is the most specific operator authority. + allow = set(_configured_model_reasoning_efforts(provider, model_id)) + allow.update(_provider_configured_reasoning_efforts(provider)) + normalized = [ + eff for eff in normalized + if eff not in {"max", "ultra"} or eff in allow + ] return normalized - _KNOWN_REASONING_PROVIDERS = frozenset({ "anthropic", "claude", "anthropic-claude", "openai", "openai-api", "openai-codex", @@ -3588,21 +3617,68 @@ def _filter_reasoning_efforts_for_provider( "bedrock", "aws-bedrock", "vertex", "google-vertex", "gemini", "google", "google-gemini", "deepseek", "x-ai", "xai", "grok", - "copilot", "github-copilot", "openrouter", + # OpenRouter and the Nous Portal are recognized aggregators: model-scoped + # ceilings above still cap what each routed model can accept. (#6018) + "copilot", "github-copilot", "openrouter", "nous", + # Vercel AI Gateway is a registered production provider that forwards the + # reasoning config to the routed model (canonical slug ``ai-gateway``; + # aliases vercel / vercel-ai-gateway / ai_gateway / aigateway all resolve + # to it). Recognizing it keeps adaptive Claude ``max`` intact while the + # filter above maps the Codex-only ``ultra`` down to ``max``. (#6018) + "ai-gateway", + # First-class WebUI catalog providers are RECOGNIZED lanes, not custom / + # unknown gateways — the max/ultra default-deny must not strip them. + # (Canonical post-alias slugs; e.g. xai-oauth is the xAI Grok OAuth lane.) + "xai-oauth", "zai", "kimi-coding", "minimax", "minimax-cn", + "opencode-zen", "opencode-go", "mistralai", "alibaba", + "nvidia", "xiaomi", "actual", }) def _provider_known_reasoning_capable(provider_id) -> bool: """True if the provider is one we recognize as reasoning-capable. - Used to gate the 'max' default-deny: for a RECOGNIZED provider whose specific - model we couldn't resolve (empty capability list), preserve 'max' since those - providers genuinely support it; for a truly unknown/custom provider, degrade - 'max' -> 'xhigh' so we never send a supra-ceiling level that would 400. + Used to gate the top-tier default-deny: for a RECOGNIZED provider whose + specific model we couldn't resolve (empty capability list), preserve + ``max``/``ultra``; for a truly unknown/custom provider, degrade either to + ``xhigh`` so we never send a supra-ceiling level that would 400. """ prov = _resolve_provider_alias(str(provider_id or "").strip().lower()) return prov in _KNOWN_REASONING_PROVIDERS +def _provider_configured_reasoning_efforts(provider_id: str) -> list[str]: + """Return the explicitly configured reasoning_efforts allowlist for *provider_id*. + + Reads ``providers..reasoning_efforts`` or the matching named + ``custom_providers[]`` entry. Only valid levels (plus the ``none`` sentinel) + survive, lowercased, in configured order; [] when nothing valid is set. + This is the operator's explicit authorization consulted by the + default-deny top-tier gate in _filter_reasoning_efforts_for_provider. + """ + provider = str(provider_id or "").strip().lower() + if not provider: + return [] + entries = None + try: + if provider.startswith("custom:"): + for entry in _custom_provider_entries(): + if _custom_provider_slug_from_name(entry.get("name")) == provider: + entries = entry.get("reasoning_efforts") + break + else: + prov_entry = (cfg.get("providers") or {}).get(provider, {}) + if isinstance(prov_entry, dict): + entries = prov_entry.get("reasoning_efforts") + except Exception: + return [] + if not isinstance(entries, list): + return [] + return [ + str(x).strip().lower() + for x in entries + if str(x).strip().lower() in {*VALID_REASONING_EFFORTS, "none"} + ] + def _is_pre_adaptive_anthropic(bare_model: str) -> bool: """True for Claude models that predate the adaptive-thinking (4.6+) generation. @@ -3657,7 +3733,8 @@ def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]: if bare.startswith(("gpt-5", "o1", "o3", "o4")): if bare.startswith(("o1", "o3", "o4")): return ["low", "medium", "high"] - return list(VALID_REASONING_EFFORTS) + # Mirror Hermes Agent's static no-catalog Copilot GPT-5 ceiling. + return ["minimal", "low", "medium", "high"] prefixes = ( "deepseek/", "anthropic/", @@ -3669,16 +3746,27 @@ def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]: "tencent/hy3-preview", "xiaomi/", ) + # The fallback branches here return the expanded global effort list when + # capability metadata is unavailable. They MUST route through + # _filter_reasoning_efforts_for_provider so the GPT-5.6 model check, the + # provider ceilings, and the unknown-provider max/ultra default-deny still + # apply — otherwise unsupported models get offered ultra. (#6018) if any(model.startswith(prefix) for prefix in prefixes): - return list(VALID_REASONING_EFFORTS) + return _filter_reasoning_efforts_for_provider( + list(VALID_REASONING_EFFORTS), model, provider + ) if _nested_gateway_route_reasoning(model): - return list(VALID_REASONING_EFFORTS) + return _filter_reasoning_efforts_for_provider( + list(VALID_REASONING_EFFORTS), model, provider + ) # Named custom providers often rewrite model ids with dots, underscores, or # extra vendor namespaces. Normalize those shapes before applying family-level # reasoning heuristics so "deepseek.v3.2", "deepseek_v4_flash", and # "vendor.deepseek.v3.2" are treated consistently. if any(_candidate_supports_reasoning(candidate) for candidate in _reasoning_name_candidates(bare)): - return list(VALID_REASONING_EFFORTS) + return _filter_reasoning_efforts_for_provider( + list(VALID_REASONING_EFFORTS), model, provider + ) return [] @@ -3894,6 +3982,35 @@ def _lmstudio_model_reasoning_options( ) +def _resolve_reasoning_context( + model_id: str | None, + provider_id: str | None, + base_url: str | None, +) -> tuple[str, str, str | None]: + """Canonicalize the model routing tuple used by reasoning capability gates.""" + model = str(model_id or "").strip() + provider = str(provider_id or "").strip().lower() + resolved_base_url = str(base_url or "").strip() or None + if model and not provider: + try: + resolved_model, resolved_provider, inferred_base_url = resolve_model_provider(model) + model = str(resolved_model or model).strip() + provider = str(resolved_provider or "").strip().lower() + if resolved_base_url is None: + resolved_base_url = str(inferred_base_url or "").strip() or None + except Exception: + model_cfg = cfg.get("model") + if isinstance(model_cfg, dict): + provider = str(model_cfg.get("provider") or "").strip().lower() + return model, _resolve_provider_alias(provider), resolved_base_url + + +def _is_gpt_5_6_family(model_id: str | None) -> bool: + """Match GPT-5.6 and its variants without accepting lookalike versions.""" + bare = str(model_id or "").strip().lower().rsplit("/", 1)[-1] + return re.match(r"^gpt-5\.6(?:$|[-_:])", bare) is not None + + def resolve_model_reasoning_efforts( model_id: str | None = None, provider_id: str | None = None, @@ -3902,26 +4019,28 @@ def resolve_model_reasoning_efforts( """Return supported reasoning-effort levels for *model_id*, or [] if none. Always passes the sourced list through _filter_reasoning_efforts_for_provider - so the hard provider ceilings (OpenAI-family GPT-5 before 5.6 at xhigh and - o-series at high; Gemini + pre-adaptive/cloud-hosted Claude at xhigh) are - applied uniformly. The UI dropdown and coercion therefore agree: ``max`` is - retained for GPT-5.6 and other models whose native ladder includes it, and - stripped where it would be rejected or mishandled. + so hard provider/model ceilings are applied uniformly. Older OpenAI-family + GPT-5 models cap at xhigh, GPT-5.6 exposes max/ultra, and Gemini plus + pre-adaptive/cloud-hosted Claude cap below the generic top tiers. The UI + dropdown and streaming coercion therefore agree on every offered level. """ - raw = _resolve_model_reasoning_efforts_impl(model_id, provider_id, base_url) + model, provider, resolved_base_url = _resolve_reasoning_context( + model_id, provider_id, base_url + ) + raw = _resolve_model_reasoning_efforts_impl(model, provider, resolved_base_url) if not raw: return raw # Forced-thinking models (GLM-4.7 on native zai) cannot have reasoning # disabled, so the 'none' sentinel must NOT appear in their supported list — # otherwise the UI offers an "off" option that has no effect and contradicts # the forced-tier contract. (#6219 round-3) - if _zai_glm_classification(model_id, provider_id) == "forced": + if _zai_glm_classification(model, provider) == "forced": return [] # Preserve any explicit 'none' sentinel (valid UI option = "no reasoning"); # the ceiling filter only knows the reasoning LEVELS. had_none = "none" in raw filtered = _filter_reasoning_efforts_for_provider( - [e for e in raw if e != "none"], str(model_id or ""), str(provider_id or "") + [e for e in raw if e != "none"], model, provider ) if had_none: # Keep 'none' in its original leading position if it was there. @@ -3956,6 +4075,35 @@ def _configured_reasoning_effort_lists(provider_entry, model_id: str) -> list: return configured_lists +def _configured_model_reasoning_efforts(provider_id: str, model_id: str) -> list[str]: + """Return the explicit model-level effort allowlist for a provider route.""" + provider = str(provider_id or "").strip().lower() + model = _strip_provider_hint_for_reasoning(model_id, provider) + provider_entry = None + try: + if provider.startswith("custom:"): + provider_entry = next( + ( + entry for entry in _custom_provider_entries() + if _custom_provider_slug_from_name(entry.get("name")) == provider + ), + None, + ) + else: + candidate = (cfg.get("providers") or {}).get(provider) + provider_entry = candidate if isinstance(candidate, dict) else None + except Exception: + return [] + configured = _configured_reasoning_effort_lists(provider_entry, model) + if not configured or not isinstance(configured[0], list): + return [] + return [ + str(level).strip().lower() + for level in configured[0] + if str(level).strip().lower() in {*VALID_REASONING_EFFORTS, "none"} + ] + + def _resolve_model_reasoning_efforts_impl( model_id: str | None = None, provider_id: str | None = None, @@ -3972,7 +4120,9 @@ def _resolve_model_reasoning_efforts_impl( try: _, provider, resolved_base_url = resolve_model_provider(model) except Exception: - provider = str((cfg.get("model") or {}).get("provider") or "").strip().lower() + model_cfg = cfg.get("model") + if isinstance(model_cfg, dict): + provider = str(model_cfg.get("provider") or "").strip().lower() provider = _resolve_provider_alias(provider) @@ -4017,6 +4167,17 @@ def _resolve_model_reasoning_efforts_impl( return _filtered except Exception: pass + # 0. Provider config: providers..reasoning_efforts or named + # custom_providers[].reasoning_efforts. When the user has explicitly listed + # valid efforts for a provider, return that list directly — no heuristics, + # no models.dev lookup. + # Only short-circuits when the filtered list is non-empty; an all-invalid + # list (e.g. typos) falls through to heuristics instead of hiding reasoning. + _re_list = _provider_configured_reasoning_efforts(provider) + if _re_list: + _filtered = list(dict.fromkeys(_re_list)) + if _filtered: + return _filtered if provider in {"copilot", "github-copilot"}: try: @@ -4063,6 +4224,18 @@ def _resolve_model_reasoning_efforts_impl( # _models_dev_reasoning_efforts already applies the provider/model filter # internally, so it is returned as-is here (filtering again would be # redundant — the filter is idempotent but the double pass obscures flow). + # GPT-5.6 top-tier support is a first-party transport contract. A stale or + # negative registry answer must not erase it after explicit config has had + # its higher precedence above. + if provider in { + "openai-codex", "openai", "openai-api", "azure", "azure-openai", + "azure-foundry", + }: + if _is_gpt_5_6_family(hinted_model): + return _filter_reasoning_efforts_for_provider( + list(VALID_REASONING_EFFORTS), hinted_model, provider + ) + metadata_efforts = _models_dev_reasoning_efforts(hinted_model, provider) if metadata_efforts is not None: return metadata_efforts @@ -4080,41 +4253,55 @@ def coerce_reasoning_effort_for_model( raw = str(effort or "").strip().lower() if not raw: return "" + model, provider, resolved_base_url = _resolve_reasoning_context( + model_id, provider_id, base_url + ) # Forced-thinking models (GLM-4.7 on native zai) cannot have reasoning # disabled at all — a stored 'none' must coerce to '' (provider default = # thinking on) so streaming does not build disabled reasoning for a model # that forces thinking on regardless. Checked BEFORE the generic 'none' # early-return below so the forced-tier contract wins. (#6219 round-3) - if raw == "none" and _zai_glm_classification(model_id, provider_id) == "forced": + if raw == "none" and _zai_glm_classification(model, provider) == "forced": return "" if raw == "none": return "none" if raw not in VALID_REASONING_EFFORTS: return "" supported = resolve_model_reasoning_efforts( - model_id, - provider_id=provider_id, - base_url=base_url, + model, + provider_id=provider, + base_url=resolved_base_url, ) # Hard provider ceilings must win regardless of what the sourced capability # list says. resolve_model_reasoning_efforts() draws from hermes_cli / - # models.dev / heuristics, and those can (a) return [] for an unrecognized - # model or (b) wrongly advertise 'max' for a provider - # whose native ladder tops out lower. _filter_reasoning_efforts_for_provider - # encodes the known ceilings (OpenAI-family GPT-5 before 5.6, Gemini, and - # pre-adaptive Anthropic all cap below 'max'); if it actively EXCLUDES the - # requested level, honor that ceiling and degrade down the ladder even when - # the sourced list is empty or (mistakenly) includes the level. This keeps a - # stored/CLI 'max' from reaching an adapter that would silently downgrade it - # worse than xhigh/high (Gemini→medium, legacy Claude manual-thinking→8k). - # GPT-5.6 is intentionally not capped. For providers with NO ceiling rule the - # filter returns the full list unchanged, so genuinely unknown models still - # preserve the configured effort (#3505 behavior). + # models.dev / heuristics, and those can return [] for an unrecognized model + # or advertise generic top tiers for a provider whose native ladder is lower. + # The filter encodes known ceilings (older OpenAI-family GPT-5, Gemini, and + # pre-adaptive Anthropic), while preserving max/ultra for GPT-5.6. If it + # excludes the requested level, degrade down the ladder even when the sourced + # list is empty or overly broad. ceiling = _filter_reasoning_efforts_for_provider( - list(VALID_REASONING_EFFORTS), str(model_id or ""), str(provider_id or "") + list(VALID_REASONING_EFFORTS), model, provider ) + # For a NAMED unknown/custom provider whose top-tier request was denied by + # the default-deny gate (no explicit provider/model allowlist — otherwise + # ``raw`` would survive in ``ceiling``) and whose capability set is empty, + # the only universally proven landing level is ``high``: nothing proves an + # unknown OpenAI-compatible endpoint accepts ``xhigh``. Recognized model + # families keep their proven ladders: a non-empty capability set degrades + # through the supported ladder below, and model-scoped family ceilings + # (older GPT-5 → xhigh, o-series → high) stay encoded in ``ceiling`` + # itself. (#6018 gate 2026-08-13) + if ( + raw in {"max", "ultra"} + and not supported + and provider + and not _provider_known_reasoning_capable(provider) + and raw not in ceiling + ): + return "high" if ceiling and raw not in ceiling: - ladder = list(VALID_REASONING_EFFORTS) # ascending: minimal..xhigh..max + ladder = list(VALID_REASONING_EFFORTS) # ascending: minimal..xhigh..max..ultra try: raw_idx = ladder.index(raw) except ValueError: @@ -4127,12 +4314,13 @@ def coerce_reasoning_effort_for_model( # both for models KNOWN not to support reasoning AND for models we simply # don't recognize (custom providers, aggregator-rewritten ids, brand-new # releases). Coercion exists to avoid sending a level a KNOWN-incompatible - # model rejects (e.g. pre-5.6 GPT-5 'max', o1/o3/o4 above 'high') - - # those paths return a NON-empty clamped set, so the degrade ladder below + # model rejects (e.g. older openai-codex GPT-5 top tiers, or o1/o3/o4 + # above ``high``) — those paths return a NON-empty clamped set, so the # still applies. When the set is empty we can't tell "unsupported" from # "unknown", so preserve the user's configured effort verbatim where it is # still valid. (#3505 review) # + # EXCEPTION for 'max' (the #3505 default-deny refinement, maintainer call # 2026-07-11): 'max' is ABOVE the universally-safe ceiling 'xhigh'. A # genuinely unknown/custom provider will 400 on it. So when the @@ -4150,17 +4338,27 @@ def coerce_reasoning_effort_for_model( # verbatim, which Z.AI would silently ignore. This keeps the value actually # sent in agreement with the UI (which offers no options for these models). if not supported: - if _zai_glm_reasoning_efforts_supported(model_id, provider_id) is False: + if _zai_glm_reasoning_efforts_supported(model, provider) is False: return "" - if raw == "max" and not _provider_known_reasoning_capable(provider_id): - return "xhigh" + if raw in {"max", "ultra"} and not _provider_known_reasoning_capable(provider): + # An explicit provider/model allowlist is authoritative — the + # ceiling filter preserves an authorized top tier through the + # default-deny, so honor it verbatim. + if provider and raw in ceiling: + return raw + # A named unknown/custom lane without authorization lands on the + # universally proven ``high`` ceiling (#6018 gate 2026-08-13); an + # UNNAMED (empty) provider keeps the historical conservative + # ``xhigh`` landing — the caller simply didn't name a provider, + # which is not the same as naming one we don't recognize. + return "high" if provider else "xhigh" return raw if raw in supported: return raw # Degrade to the closest *lower* supported level instead of silently - # disabling reasoning. e.g. max -> xhigh -> high, or xhigh -> high when the + # disabling reasoning. For example, ultra/max -> xhigh -> high when the # target model caps below the configured effort. Never escalate. - ladder = list(VALID_REASONING_EFFORTS) # ascending: minimal..xhigh..max + ladder = list(VALID_REASONING_EFFORTS) # ascending: minimal..xhigh..max..ultra try: raw_idx = ladder.index(raw) except ValueError: @@ -4205,6 +4403,9 @@ def get_reasoning_status( if not resolve_base_url and model_cfg.get("base_url"): resolve_base_url = str(model_cfg["base_url"]).strip() + resolve_model, resolve_provider, resolve_base_url = _resolve_reasoning_context( + resolve_model, resolve_provider, resolve_base_url + ) supported_efforts = resolve_model_reasoning_efforts( resolve_model, provider_id=resolve_provider, diff --git a/static/commands.js b/static/commands.js index 8bef376d1b..aa1c27fc0d 100644 --- a/static/commands.js +++ b/static/commands.js @@ -30,7 +30,7 @@ const COMMANDS=[ {name:'background',desc:t('cmd_background'),fn:cmdBackground,arg:'prompt', noEcho:true}, {name:'status', desc:t('cmd_status'), fn:cmdStatus}, {name:'voice', desc:t('cmd_voice'), fn:cmdVoice, noEcho:true}, - {name:'reasoning', desc:t('cmd_reasoning'), fn:cmdReasoning, arg:'show|hide|none|minimal|low|medium|high|xhigh|max', subArgs:['show','hide','none','minimal','low','medium','high','xhigh','max'], noEcho:true}, + {name:'reasoning', desc:t('cmd_reasoning'), fn:cmdReasoning, arg:'show|hide|none|minimal|low|medium|high|xhigh|max|ultra', subArgs:['show','hide','none','minimal','low','medium','high','xhigh','max','ultra'], noEcho:true}, {name:'yolo', desc:t('cmd_yolo'), fn:cmdYolo, noEcho:true}, {name:'branch', desc:t('cmd_branch'), fn:cmdBranch, arg:'[name]', noEcho:true}, ]; @@ -1832,13 +1832,13 @@ function cmdReasoning(args){ const BRAIN='\uD83E\uDDE0'; // Matches hermes_constants.VALID_REASONING_EFFORTS + 'none' (CLI parity). // Keep this WebUI effort list in sync with hermes-agent#29248. - const EFFORTS=['none','minimal','low','medium','high','xhigh','max']; + const EFFORTS=['none','minimal','low','medium','high','xhigh','max','ultra']; // Shared status renderer used by the no-args branch and as a fallback. function _fmtStatus(st){ const vis=(st && st.show_reasoning===false)?'off':'on'; const eff=(st && st.reasoning_effort)||'default'; return BRAIN+' Reasoning effort: '+eff+' \u00B7 display: '+vis - +' | /reasoning show|hide|none|minimal|low|medium|high|xhigh|max'; + +' | /reasoning show|hide|none|minimal|low|medium|high|xhigh|max|ultra'; } if(!arg){ // Status — read from the same config.yaml keys the CLI uses. @@ -1867,7 +1867,7 @@ function cmdReasoning(args){ // Takes effect on the NEXT session/turn (agent re-reads config at // construction time), matching CLI semantics where `/reasoning high` // also forces an agent re-init. - api('/api/reasoning',{method:'POST',body:JSON.stringify({effort:arg})}) + api('/api/reasoning',{method:'POST',body:JSON.stringify({effort:arg,..._reasoningEffortContext()})}) .then(function(st){ const eff=(st && st.reasoning_effort)||arg; showToast(BRAIN+' Reasoning effort: '+eff+' (saved; applies to next turn)'); diff --git a/static/index.html b/static/index.html index 1f55169ac2..8ac0f1f4bf 100644 --- a/static/index.html +++ b/static/index.html @@ -762,6 +762,7 @@

What can I help with?

High
Extra High
Max
+
Ultra
diff --git a/static/style.css b/static/style.css index 86dbcfd2be..6713737efe 100644 --- a/static/style.css +++ b/static/style.css @@ -2639,7 +2639,8 @@ .composer-reasoning-icon,.composer-reasoning-chevron{display:inline-flex;align-items:center;justify-content:center;flex-shrink:0;line-height:1;} .composer-reasoning-chevron{display:none;} .composer-reasoning-label{display:inline-flex;align-items:center;justify-content:center;min-width:1.6ch;font-size:11px;font-weight:700;letter-spacing:0;text-transform:none;} - .composer-reasoning-dropdown{display:none;position:absolute;bottom:calc(100% + 4px);left:0;min-width:140px;background:var(--surface);border:1px solid var(--border2);border-radius:10px;box-shadow:0 -4px 24px rgba(0,0,0,.4);z-index:200;padding:4px;overflow:hidden;} + .composer-reasoning-dropdown{display:none;position:absolute;bottom:calc(100% + 4px);left:0;min-width:140px;background:var(--surface);border:1px solid var(--border2);border-radius:10px;box-shadow:0 -4px 24px rgba(0,0,0,.4);z-index:200;padding:4px;overflow:hidden;max-height:min(60vh,calc(100vh - 96px),480px);overflow-y:auto;overscroll-behavior:contain;} + @supports (height:100dvh){.composer-reasoning-dropdown{max-height:min(60dvh,calc(100dvh - 96px),480px);}} .composer-reasoning-dropdown.open{display:block;} .reasoning-option{padding:8px 14px;border-radius:6px;cursor:pointer;font-size:13px;color:var(--text);white-space:nowrap;transition:background-color .12s;} .reasoning-option:hover{background:rgba(255,255,255,.07);} diff --git a/static/ui.js b/static/ui.js index bcd41c7b67..414ff8260f 100644 --- a/static/ui.js +++ b/static/ui.js @@ -5057,6 +5057,7 @@ function _formatReasoningEffortLabel(effort){ if(effort==='high') return 'High'; if(effort==='xhigh') return 'XHigh'; if(effort==='max') return 'Max'; + if(effort==='ultra') return 'Ultra'; return effort.charAt(0).toUpperCase()+effort.slice(1); } diff --git a/tests/test_issue1103_reasoning_chip_visibility.py b/tests/test_issue1103_reasoning_chip_visibility.py index 71e96a669b..0954f517fc 100644 --- a/tests/test_issue1103_reasoning_chip_visibility.py +++ b/tests/test_issue1103_reasoning_chip_visibility.py @@ -45,6 +45,9 @@ def test_reasoning_chip_html_starts_hidden(): assert 'data-effort="max"' in src, ( "composer reasoning dropdown must include Max option" ) + assert 'data-effort="ultra"' in src, ( + "composer reasoning dropdown must include Ultra option" + ) def test_ui_js_passes_model_context_to_reasoning_api(): diff --git a/tests/test_reasoning_chip_js_behaviour.py b/tests/test_reasoning_chip_js_behaviour.py index d783f70b15..6ba7849997 100644 --- a/tests/test_reasoning_chip_js_behaviour.py +++ b/tests/test_reasoning_chip_js_behaviour.py @@ -375,8 +375,8 @@ def test_default_toggle_undefined_keeps_prior_behavior(self, driver_meta_path): # (off). The earlier round-2 fix kept the chip visible but the only rendered # option was "None" — so a user could turn thinking OFF but never back ON. # These tests drive the actual _applyReasoningOptions against a simulated -# dropdown containing all 8 options from static/index.html (Default/none/ -# minimal/low/medium/high/xhigh/max) and pin which are visible per tier. +# dropdown containing all 9 options from static/index.html (Default/none/ +# minimal/low/medium/high/xhigh/max/ultra) and pin which are visible per tier. # ───────────────────────────────────────────────────────────────────────────── @@ -419,7 +419,7 @@ def test_default_toggle_undefined_keeps_prior_behavior(self, driver_meta_path): } // Options mirror static/index.html's composerReasoningDropdown exactly. -const options = ['', 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'].map(makeOption); +const options = ['', 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra'].map(makeOption); const optionEls = options; const els = { diff --git a/tests/test_reasoning_dropdown_viewport.py b/tests/test_reasoning_dropdown_viewport.py new file mode 100644 index 0000000000..a4f174b5e9 --- /dev/null +++ b/tests/test_reasoning_dropdown_viewport.py @@ -0,0 +1,161 @@ +"""#6018 gate blocker 4 — the 9-row reasoning dropdown must keep every row +reachable on short viewports. + +The composer reasoning dropdown opens UPWARD (``bottom:calc(100% + 4px)``) +above its chip. With nine rows (Default..Ultra) and no height cap, a short +viewport (e.g. 390×300 landscape keyboard-overlap, or a small phone) pushes +the top rows — including the override-clearing ``Default`` row — above the +viewport with ``overflow:hidden`` and no scroll path to reach them. + +Real-browser measurement (Playwright) of the production markup + styles: +the ``#composerReasoningDropdown`` block from ``static/index.html`` is mounted +inside a composer-footer-shaped anchor at the bottom of the page, with the real +``static/style.css`` applied. For desktop (1280×800), mobile portrait +(390×844), and the reviewer's short-landscape case (390×300) we assert: + + * the dropdown fits inside the viewport (no row above y=0); + * every row — first (``Default``) and last (``Ultra``) — is reachable: + either directly visible or scrollable-to via ``overflow-y:auto``; + * after scrolling to top/bottom, both boundary rows are inside the dropdown's + visible box. + +Skips cleanly where Playwright or a chromium binary is unavailable (matching +the repo's other browser tests, e.g. test_wakeup_card_responsive.py). +""" + +import re +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +INDEX_HTML = (ROOT / "static" / "index.html").read_text(encoding="utf-8") +STYLE_CSS = (ROOT / "static" / "style.css").read_text(encoding="utf-8") + + +def _dropdown_markup() -> str: + """Extract the real #composerReasoningDropdown block from index.html.""" + m = re.search( + r'
]*id="composerReasoningDropdown"[^>]*>' + r".*?
\s*
", + INDEX_HTML, + re.S, + ) + assert m, "composerReasoningDropdown markup not found in index.html" + # Trim the trailing sibling-closing captured for a complete block. + markup = m.group(0) + assert 'data-effort="ultra"' in markup, "Ultra row missing from dropdown markup" + assert 'data-effort=""' in markup, "Default row missing from dropdown markup" + return markup[: markup.rindex("")] + + +def _measure(width: int, height: int): + try: + from playwright.sync_api import sync_playwright + except Exception: # pragma: no cover - dependency missing path + pytest.skip("playwright is unavailable; run the reasoning dropdown viewport test") + + playwright = sync_playwright().start() + try: + browser = playwright.chromium.launch( + headless=True, + args=["--no-sandbox", "--disable-dev-shm-usage"], + ) + except Exception as exc: # pragma: no cover - no browser binary in sandbox + playwright.stop() + pytest.skip(f"chromium unavailable for browser measurement: {exc}") + + try: + page = browser.new_page(viewport={"width": width, "height": height}) + # Composer-footer-shaped anchor pinned to the bottom of the viewport, + # mirroring the production placement of the upward-opening dropdown. + page.set_content( + "" + '
' + + _dropdown_markup() + + "
" + ) + page.add_style_tag(content=STYLE_CSS) + result = page.evaluate( + """ + () => { + const dd = document.getElementById('composerReasoningDropdown'); + dd.classList.add('open'); + const rows = Array.from(dd.querySelectorAll('.reasoning-option')); + const first = rows[0]; + const last = rows[rows.length - 1]; + const style = getComputedStyle(dd); + const box = () => dd.getBoundingClientRect(); + const inBox = (el) => { + const b = box(); + const r = el.getBoundingClientRect(); + return r.top >= b.top - 1 && r.bottom <= b.bottom + 1; + }; + const out = { + rowCount: rows.length, + firstLabel: first.textContent.trim(), + lastLabel: last.textContent.trim(), + overflowY: style.overflowY, + menuTop: box().top, + menuBottom: box().bottom, + viewportH: window.innerHeight, + scrollable: dd.scrollHeight > dd.clientHeight + 1, + }; + dd.scrollTop = 0; + out.firstReachableAtTop = inBox(first); + out.firstTopAtTop = first.getBoundingClientRect().top; + dd.scrollTop = dd.scrollHeight; + out.lastReachableAtBottom = inBox(last); + out.lastBottomAtBottom = last.getBoundingClientRect().bottom; + return out; + } + """ + ) + finally: + browser.close() + playwright.stop() + return result + + +def _assert_all_rows_reachable(m): + # The dropdown itself must sit fully inside the viewport. + assert m["menuTop"] >= -1, f"dropdown top is above the viewport: {m}" + assert m["menuBottom"] <= m["viewportH"] + 1, m + # Nine rows: Default..Ultra. + assert m["rowCount"] == 9, m + assert m["firstLabel"] == "Default", m + assert m["lastLabel"] == "Ultra", m + # Both boundary rows are reachable: at scrollTop=0 the Default row is + # inside the dropdown's box AND on-screen; after scrolling to the bottom + # the Ultra row is inside the box. + assert m["firstReachableAtTop"], f"Default row unreachable: {m}" + assert m["firstTopAtTop"] >= -1, f"Default row rendered above the viewport: {m}" + assert m["lastReachableAtBottom"], f"Ultra row unreachable: {m}" + assert m["lastBottomAtBottom"] <= m["viewportH"] + 1, m + # When content is taller than the capped menu, scrolling must be enabled — + # overflow-y:auto like the sibling model/session dropdowns. + if m["scrollable"]: + assert m["overflowY"] in ("auto", "scroll"), ( + f"scrollable dropdown must not clip with overflow:{m['overflowY']}: {m}" + ) + + +def test_desktop_1280x800_every_row_reachable(): + _assert_all_rows_reachable(_measure(1280, 800)) + + +def test_mobile_portrait_390x844_every_row_reachable(): + _assert_all_rows_reachable(_measure(390, 844)) + + +def test_short_landscape_390x300_default_row_reachable(): + # The reviewer's reproduction: 390×300 previously measured menuTop=-50 with + # overflow hidden — the Default row sat off-viewport with no scroll path. + m = _measure(390, 300) + _assert_all_rows_reachable(m) + # At this height the 9-row list cannot fit uncapped; the viewport-bounded + # max-height must engage and hand the overflow to the scroll container. + assert m["scrollable"], f"expected the height cap to engage at 390x300: {m}" + assert m["overflowY"] in ("auto", "scroll"), m diff --git a/tests/test_reasoning_effort_model_capabilities.py b/tests/test_reasoning_effort_model_capabilities.py index 910c1197f8..84fdd3e2c4 100644 --- a/tests/test_reasoning_effort_model_capabilities.py +++ b/tests/test_reasoning_effort_model_capabilities.py @@ -36,6 +36,27 @@ def test_openai_codex_gpt5_supports_reasoning_effort_levels(): assert "high" in efforts assert "xhigh" in efforts assert "max" not in efforts + assert "ultra" not in efforts + + +def test_openai_codex_gpt56_supports_max_and_ultra_effort_levels(): + efforts = cfg.resolve_model_reasoning_efforts( + "gpt-5.6-sol", + provider_id="openai-codex", + ) + assert "xhigh" in efforts + assert "max" in efforts + assert "ultra" in efforts + assert cfg.coerce_reasoning_effort_for_model( + "max", + "gpt-5.6-sol", + provider_id="openai-codex", + ) == "max" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", + "gpt-5.6-sol", + provider_id="openai-codex", + ) == "ultra" def test_openai_codex_prefixed_gpt5_supports_reasoning_effort_levels(): @@ -47,6 +68,7 @@ def test_openai_codex_prefixed_gpt5_supports_reasoning_effort_levels(): assert "high" in efforts assert "xhigh" in efforts assert "max" not in efforts + assert "ultra" not in efforts def test_openai_codex_max_effort_is_clamped_before_streaming(): @@ -111,15 +133,21 @@ def test_coerce_preserves_effort_for_unrecognized_model(): "some-unknown-model-xyz", provider_id="some-custom-provider", ) == "high" - # #3505 default-deny refinement (maintainer 2026-07-11): 'max' is above the - # universally safe ceiling, so on an UNRECOGNIZED provider it degrades to - # xhigh (a truly-unknown provider would 400 on max). All OTHER levels still - # preserve verbatim below. + # #3505 default-deny refinement, tightened by the 2026-08-13 gate: 'max' / + # 'ultra' are supra-ceiling levels, so on an UNRECOGNIZED provider they + # degrade to the universally proven 'high' ceiling — nothing proves an + # unknown OpenAI-compatible endpoint accepts xhigh either. All OTHER + # levels still preserve verbatim below. assert cfg.coerce_reasoning_effort_for_model( "max", "brand-new-model-2099", provider_id="some-custom-provider", - ) == "xhigh" + ) == "high" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", + "brand-new-model-2099", + provider_id="some-custom-provider", + ) == "high" # 'none' / unset still pass through unchanged for unknown models. assert cfg.coerce_reasoning_effort_for_model( "none", "some-unknown-model-xyz", provider_id="custom" @@ -257,6 +285,51 @@ def test_named_custom_provider_model_reasoning_efforts_take_precedence(monkeypat monkeypatch.setitem(cfg.cfg, "custom_providers", original) +def test_model_only_top_tiers_remain_authoritative_in_public_filter(monkeypatch): + monkeypatch.setitem(cfg.cfg, "custom_providers", [{ + "name": "llm-proxy", + "models": {"inkling": {"reasoning_efforts": ["high", "max", "ultra"]}}, + }]) + assert cfg.resolve_model_reasoning_efforts( + "inkling", provider_id="custom:llm-proxy" + ) == ["high", "max", "ultra"] + for effort in ("max", "ultra"): + assert cfg.coerce_reasoning_effort_for_model( + effort, "inkling", provider_id="custom:llm-proxy" + ) == effort + + +def test_actual_metadata_ladder_retains_max(monkeypatch): + monkeypatch.setattr( + cfg, "_models_dev_reasoning_efforts", + lambda *args, **kwargs: ["minimal", "low", "medium", "high", "xhigh", "max"], + ) + efforts = cfg.resolve_model_reasoning_efforts("glm-5.2-nvfp4", provider_id="actual") + assert "max" in efforts + assert cfg.coerce_reasoning_effort_for_model( + "max", "glm-5.2-nvfp4", provider_id="actual" + ) == "max" + + +def test_copilot_standalone_fallback_caps_gpt56(monkeypatch): + import builtins + + real_import = builtins.__import__ + + def fail_cli_models(name, *args, **kwargs): + if name == "hermes_cli.models": + raise ImportError("forced standalone fallback") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fail_cli_models) + efforts = cfg.resolve_model_reasoning_efforts("gpt-5.6-sol", provider_id="copilot") + assert efforts == ["minimal", "low", "medium", "high"] + for effort in ("xhigh", "max", "ultra"): + assert cfg.coerce_reasoning_effort_for_model( + effort, "gpt-5.6-sol", provider_id="copilot" + ) == "high" + + def test_model_reasoning_efforts_fall_back_to_provider_when_invalid(monkeypatch): original = cfg.cfg.get("providers") monkeypatch.setitem( @@ -494,13 +567,18 @@ def test_max_degrades_for_azure_bedrock_hosted_legacy_claude(): def test_max_degrades_on_unknown_provider_but_other_levels_preserved(): - # #3505 default-deny refinement (maintainer call 2026-07-11): 'max' is above - # the universally safe ceiling, so an unknown/custom provider (empty capability list) - # must degrade 'max'->'xhigh' rather than send an unsupported level — while all - # other levels keep the conservative preserve-verbatim behavior. + # #3505 default-deny refinement, tightened by the 2026-08-13 gate: max and + # ultra sit above the universal ceiling, so an unknown/custom provider + # (empty capability list, no explicit allowlist) must degrade both to the + # universally proven 'high' — an unknown OpenAI-compatible endpoint has no + # authority proving xhigh support. All other levels keep the conservative + # preserve-verbatim behavior. assert cfg.coerce_reasoning_effort_for_model( "max", model_id="some-unknown-model", provider_id="customprovider" - ) == "xhigh" + ) == "high" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model_id="some-unknown-model", provider_id="customprovider" + ) == "high" # other levels still preserved verbatim for an unknown provider for eff in ("minimal", "low", "medium", "high", "xhigh"): assert cfg.coerce_reasoning_effort_for_model( @@ -594,3 +672,376 @@ def test_qwen_prefixed_alias_reasoning_detection(): f"{model} must remain reasoning-capable (DeepSeek-R1 hybrid, " f"Qwen 2.x must not shadow the DeepSeek detector)" ) + +# ── PR #6018: max/ultra coercion leak closures ─────────────────────────────── + +def test_named_custom_provider_hints_keep_model_scoped_effort_ceilings(monkeypatch): + original = cfg.cfg.get("custom_providers") + monkeypatch.setitem( + cfg.cfg, + "custom_providers", + [{"name": "frontier-gw", "reasoning_efforts": ["high", "xhigh", "max", "ultra"]}], + ) + try: + cases = ( + ("@custom:frontier-gw:gpt-5.5", "xhigh", False), + ("@custom:frontier-gw:o3", "high", False), + ("@custom:frontier-gw:gpt-5.6-sol", "ultra", True), + ("@custom:frontier-gw:unknown-frontier-model", "ultra", True), + ) + for model, expected, keeps_top_tiers in cases: + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id="custom:frontier-gw") + assert ("max" in efforts and "ultra" in efforts) is keeps_top_tiers, model + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model, provider_id="custom:frontier-gw" + ) == expected, model + inferred_efforts = cfg.resolve_model_reasoning_efforts(model) + assert ("max" in inferred_efforts and "ultra" in inferred_efforts) is keeps_top_tiers + assert cfg.coerce_reasoning_effort_for_model("ultra", model) == expected + finally: + if original is None: + cfg.cfg.pop("custom_providers", None) + else: + monkeypatch.setitem(cfg.cfg, "custom_providers", original) + + +def test_custom_provider_default_denies_max_ultra_for_recognized_models(): + # Finding 1: a recognized reasoning-capable model family behind a custom / + # unknown provider must NOT inherit the generic max/ultra tiers by default — + # the provider's native ladder is unknown, so the top tiers are denied. + for model in ( + "kimi-k2.5", "moonshotai.kimi-k2.5", "deepseek-v4-flash", + "zai-org/GLM-5.2", "minimax-m3-pro", + ): + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id="custom:unlisted") + assert efforts, f"{model} should still expose the standard ladder" + assert "xhigh" in efforts + assert "max" not in efforts and "ultra" not in efforts, ( + f"{model} via unrecognized custom provider must default-deny max/ultra" + ) + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model, provider_id="custom:unlisted" + ) == "xhigh" + assert cfg.coerce_reasoning_effort_for_model( + "max", model, provider_id="custom:unlisted" + ) == "xhigh" + + +def test_gpt_5_6_first_party_fallback_is_exact_and_includes_azure_foundry(monkeypatch): + monkeypatch.setattr(cfg, "_models_dev_reasoning_efforts", lambda *_args: []) + for provider in ( + "openai-codex", "openai", "azure-foundry", + "azure", "azure-ai-foundry", "azure-ai", + ): + assert cfg._resolve_provider_alias(provider) in { + "openai-codex", "openai", "azure-foundry" + } + for model in ("gpt-5.6", "gpt-5.6-sol"): + for candidate in (model, f"@{provider}:{model}"): + efforts = cfg.resolve_model_reasoning_efforts(candidate, provider_id=provider) + assert "max" in efforts and "ultra" in efforts + assert cfg.coerce_reasoning_effort_for_model( + "ultra", candidate, provider_id=provider + ) == "ultra" + for lookalike in ("not-gpt-5.6", "gpt-5.60"): + for candidate in (lookalike, f"@{provider}:{lookalike}"): + efforts = cfg.resolve_model_reasoning_efforts(candidate, provider_id=provider) + assert "max" not in efforts and "ultra" not in efforts + assert cfg.coerce_reasoning_effort_for_model( + "ultra", candidate, provider_id=provider + ) not in {"max", "ultra"} + + +def test_gpt_5_6_status_matches_resolver_and_coercer_for_azure_aliases(monkeypatch): + monkeypatch.setattr(cfg, "_models_dev_reasoning_efforts", lambda *_args: []) + monkeypatch.setattr(cfg, "_load_yaml_config_file", lambda *_args: { + "agent": {"reasoning_effort": "ultra"}, + }) + for provider in ("azure-foundry", "azure", "azure-ai-foundry", "azure-ai"): + positive = cfg.get_reasoning_status( + model_id="gpt-5.6-sol", provider_id=provider + ) + assert "ultra" in positive["supported_efforts"] + assert positive["reasoning_effort"] == "ultra" + + for lookalike in ("not-gpt-5.6", "gpt-5.60"): + rejected = cfg.get_reasoning_status( + model_id=lookalike, provider_id=provider + ) + assert "max" not in rejected["supported_efforts"] + assert "ultra" not in rejected["supported_efforts"] + assert rejected["reasoning_effort"] not in {"max", "ultra"} + + +def test_azure_aliases_keep_lower_model_ceilings(monkeypatch): + monkeypatch.setattr( + cfg, "_models_dev_reasoning_efforts", + lambda *_args: list(cfg.VALID_REASONING_EFFORTS), + ) + for provider in ("azure-foundry", "azure", "azure-ai-foundry", "azure-ai"): + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "gpt-5.5", provider_id=provider + ) == "xhigh" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "o3", provider_id=provider + ) == "high" + + +def test_reasoning_context_fallback_accepts_legacy_string_model_config(monkeypatch): + monkeypatch.setitem(cfg.cfg, "model", "gpt-5.6-sol") + monkeypatch.setattr( + cfg, "resolve_model_provider", lambda *_args: (_ for _ in ()).throw(RuntimeError("probe")) + ) + assert cfg._resolve_reasoning_context("gpt-5.6-sol", None, None) == ( + "gpt-5.6-sol", "", None + ) + + +def test_custom_provider_explicit_allowlist_authorizes_max_ultra(monkeypatch): + # Finding 1 (exception): an explicit provider reasoning_efforts allowlist + # is the operator's authorization — max/ultra survive when listed. + original = cfg.cfg.get("custom_providers") + monkeypatch.setitem( + cfg.cfg, + "custom_providers", + [{"name": "frontier-gw", "reasoning_efforts": ["high", "xhigh", "max", "ultra"]}], + ) + try: + assert cfg.resolve_model_reasoning_efforts( + "some-model", provider_id="custom:frontier-gw" + ) == ["high", "xhigh", "max", "ultra"] + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "some-model", provider_id="custom:frontier-gw" + ) == "ultra" + assert cfg.coerce_reasoning_effort_for_model( + "max", "some-model", provider_id="custom:frontier-gw" + ) == "max" + finally: + if original is None: + cfg.cfg.pop("custom_providers", None) + else: + monkeypatch.setitem(cfg.cfg, "custom_providers", original) + + +def test_aggregator_routes_cap_older_gpt5_and_o_series(): + # Finding 2: model-scoped ceilings follow the MODEL across aggregator lanes. + # Older GPT-5 + o-series via OpenRouter/Nous must never see max/ultra. + for prov in ("openrouter", "nous"): + for model in ("openai/gpt-5.5", "openai/gpt-5.1"): + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model, provider_id=prov + ) == "xhigh", f"{model} via {prov}: ultra must downgrade to xhigh" + assert cfg.coerce_reasoning_effort_for_model( + "max", model, provider_id=prov + ) == "xhigh" + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id=prov) + assert "max" not in efforts and "ultra" not in efforts + for model in ("openai/o3", "openai/o4-mini"): + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model, provider_id=prov + ) == "high", f"{model} via {prov}: ultra must downgrade to high" + assert cfg.coerce_reasoning_effort_for_model( + "xhigh", model, provider_id=prov + ) == "high" + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id=prov) + assert set(efforts) <= {"low", "medium", "high"} + # GPT-5.6 via the same aggregator keeps the generic top tiers — the + # ceiling is model-scoped, not aggregator-scoped. + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "openai/gpt-5.6-sol", provider_id=prov + ) == "ultra" + + +def test_copilot_fallback_caps_max_ultra(): + # The standalone fallback mirrors Agent's static high ceiling for every + # Copilot GPT-5 model, including GPT-5.6. + expected = ["minimal", "low", "medium", "high"] + assert cfg._heuristic_reasoning_efforts("gpt-5.5", "copilot") == expected + assert cfg._heuristic_reasoning_efforts("gpt-5.6-sol", "copilot") == expected + assert cfg._heuristic_reasoning_efforts("o3", "github-copilot") == ["low", "medium", "high"] + + +def test_metadata_unavailable_fallback_applies_gpt56_check(monkeypatch): + # Finding 4: when capability metadata has no answer, the fallback branches + # returning the expanded global effort list must still apply the GPT-5.6 + # model check and the unknown-provider max/ultra default-deny. + monkeypatch.setattr(cfg, "_models_dev_reasoning_efforts", lambda *a, **k: None) + efforts = cfg.resolve_model_reasoning_efforts("openai/gpt-5.5", provider_id="openrouter") + assert "xhigh" in efforts + assert "max" not in efforts and "ultra" not in efforts + efforts56 = cfg.resolve_model_reasoning_efforts("openai/gpt-5.6-sol", provider_id="openrouter") + assert "max" in efforts56 and "ultra" in efforts56 + # Same gap closed for heuristic-recognized families on unknown providers. + custom = cfg.resolve_model_reasoning_efforts("kimi-k2.5", provider_id="custom:unlisted") + assert "xhigh" in custom + assert "max" not in custom and "ultra" not in custom + + +def test_negative_metadata_does_not_hide_first_party_gpt56_contract(monkeypatch): + monkeypatch.setattr(cfg, "_models_dev_reasoning_efforts", lambda *a, **k: []) + efforts = cfg.resolve_model_reasoning_efforts( + "gpt-5.6-sol", provider_id="openai-codex" + ) + assert "max" in efforts and "ultra" in efforts + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "gpt-5.6-sol", provider_id="openai-codex" + ) == "ultra" + + + +# --- 2026-08-13 gate regressions (#6018) ----------------------------------- + + +def test_unknown_provider_top_tiers_land_on_universal_high_ceiling(tmp_path, monkeypatch): + # Gate blocker 1: for unknown/custom providers WITHOUT an explicit + # provider/model allowlist, max AND ultra must land on the universally + # proven 'high' ceiling — an unknown OpenAI-compatible endpoint has no + # authority proving xhigh support. + # Resolver: no ladder is advertised at all for the unknown model. + assert cfg.resolve_model_reasoning_efforts( + "frontier-model-x", provider_id="unknown-gw" + ) == [] + # Coercion: both supra-ceiling tiers degrade to high, not xhigh. + for eff in ("max", "ultra"): + assert cfg.coerce_reasoning_effort_for_model( + eff, "frontier-model-x", provider_id="unknown-gw" + ) == "high", f"{eff} must land on the universal high ceiling" + # Wire: the /api/reasoning persistence path reports the coerced high, so + # what boot/status/chip read agrees with what streaming would send. + cfgfile = tmp_path / "config.yaml" + cfgfile.write_text("agent: {}\n", encoding="utf-8") + monkeypatch.setattr(cfg, "_get_config_path", lambda: cfgfile) + monkeypatch.setattr(cfg, "reload_config", lambda: None) + status = cfg.set_reasoning_effort( + "ultra", model_id="frontier-model-x", provider_id="unknown-gw" + ) + assert status["reasoning_effort"] == "high" + status = cfg.get_reasoning_status( + model_id="frontier-model-x", provider_id="unknown-gw" + ) + assert status["reasoning_effort"] == "high" + + +def test_unknown_provider_allowlists_stay_authoritative_over_high_ceiling(monkeypatch): + # Gate blocker 1 (control): explicit provider- and model-level allowlists + # remain authoritative — an authorized top tier survives verbatim instead + # of landing on the universal high ceiling. + original = cfg.cfg.get("custom_providers") + monkeypatch.setitem(cfg.cfg, "custom_providers", [ + {"name": "authorized-gw", "reasoning_efforts": ["high", "max", "ultra"]}, + {"name": "model-gw", "models": {"inkling": {"reasoning_efforts": ["high", "max"]}}}, + ]) + try: + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "some-model", provider_id="custom:authorized-gw" + ) == "ultra" + assert cfg.coerce_reasoning_effort_for_model( + "max", "inkling", provider_id="custom:model-gw" + ) == "max" + # The model-level allowlist does not leak to sibling models: an + # unlisted model on the same unknown gateway still lands on high. + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "other-model", provider_id="custom:model-gw" + ) == "high" + finally: + if original is None: + cfg.cfg.pop("custom_providers", None) + else: + monkeypatch.setitem(cfg.cfg, "custom_providers", original) + + +def test_pre_adaptive_claude_ceiling_follows_model_across_aggregators(): + # Gate blocker 2: the pre-adaptive Claude ceiling is model-scoped and must + # apply on OpenRouter/Nous and every routed lane, for prefixed and bare + # legacy IDs alike — not only on the named Anthropic/cloud-host lanes. + legacy_ids = ( + "anthropic/claude-sonnet-4.5", + "claude-sonnet-4-5", + "anthropic/claude-3-7-sonnet", + "claude-3-5-sonnet-20241022", + ) + for prov in ("openrouter", "nous"): + for model in legacy_ids: + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id=prov) + assert "max" not in efforts and "ultra" not in efforts, ( + f"{model} via {prov} must cap below max/ultra, got {efforts}" + ) + if efforts: # date-stamped Claude 3.x is heuristic-denied entirely + assert "xhigh" in efforts, (model, prov, efforts) + for eff in ("max", "ultra"): + assert cfg.coerce_reasoning_effort_for_model( + eff, model, provider_id=prov + ) == "xhigh", f"{eff} for {model} via {prov} must degrade to xhigh" + # Qualified @provider:model form resolves the same ceiling. + qualified = f"@{prov}:anthropic/claude-sonnet-4.5" + q_efforts = cfg.resolve_model_reasoning_efforts(qualified, provider_id=prov) + assert "max" not in q_efforts and "ultra" not in q_efforts + assert cfg.coerce_reasoning_effort_for_model( + "ultra", qualified, provider_id=prov + ) == "xhigh" + # Control: adaptive Claude via the same aggregator keeps the top tiers + # — the ceiling is model-scoped, not lane-scoped. + adaptive = cfg.resolve_model_reasoning_efforts( + "anthropic/claude-opus-4.6", provider_id=prov + ) + assert "max" in adaptive and "ultra" in adaptive + # The same model-scoped ceiling holds on unknown/custom gateway lanes. + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "claude-sonnet-4-5", provider_id="custom:some-gw" + ) == "xhigh" + + +def test_ai_gateway_alias_family_recognized_with_claude_max_contract(): + # Gate blocker 3: ai-gateway is a registered production provider that + # forwards reasoning config. Every installed Agent alias must resolve to + # the canonical slug, preserve adaptive Claude 'max', and map the + # Codex-only 'ultra' tier down to the wire 'max' instead of xhigh. + aliases = ("ai-gateway", "vercel", "vercel-ai-gateway", "ai_gateway", "aigateway") + for alias in aliases: + assert cfg._resolve_provider_alias(alias) == "ai-gateway", alias + for model in ("anthropic/claude-opus-4.6", "claude-opus-4.6"): + efforts = cfg.resolve_model_reasoning_efforts(model, provider_id=alias) + assert "max" in efforts, (alias, model, efforts) + assert "ultra" not in efforts, ( + f"ultra is Codex product-only; {alias} wire ladder tops at max" + ) + assert cfg.coerce_reasoning_effort_for_model( + "max", model, provider_id=alias + ) == "max", (alias, model) + assert cfg.coerce_reasoning_effort_for_model( + "ultra", model, provider_id=alias + ) == "max", f"ultra must map to max on {alias}, not degrade to xhigh" + # Qualified @provider:model form keeps the same contract. + qualified = f"@{alias}:anthropic/claude-opus-4.6" + q_efforts = cfg.resolve_model_reasoning_efforts(qualified, provider_id=alias) + assert "max" in q_efforts and "ultra" not in q_efforts, (alias, q_efforts) + assert cfg.coerce_reasoning_effort_for_model( + "ultra", qualified, provider_id=alias + ) == "max" + # Model-scoped ceilings still ride through the gateway: pre-adaptive + # Claude caps at xhigh, older GPT-5 at xhigh, o-series at high. + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "anthropic/claude-sonnet-4.5", provider_id=alias + ) == "xhigh" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "openai/gpt-5.5", provider_id=alias + ) == "xhigh" + assert cfg.coerce_reasoning_effort_for_model( + "ultra", "openai/o3", provider_id=alias + ) == "high" + + +def test_ai_gateway_status_reports_wire_max_for_stored_ultra(monkeypatch): + # Gate blocker 3 (status boundary): a stored 'ultra' surfaces as the wire + # 'max' for Claude 4.6 behind every ai-gateway alias, so the chip and the + # streamed value agree. + monkeypatch.setattr(cfg, "_load_yaml_config_file", lambda *_args: { + "agent": {"reasoning_effort": "ultra"}, + }) + for alias in ("ai-gateway", "vercel", "vercel-ai-gateway", "ai_gateway", "aigateway"): + status = cfg.get_reasoning_status( + model_id="anthropic/claude-opus-4.6", provider_id=alias + ) + assert "max" in status["supported_efforts"], alias + assert "ultra" not in status["supported_efforts"], alias + assert status["reasoning_effort"] == "max", alias diff --git a/tests/test_reasoning_show_hide.py b/tests/test_reasoning_show_hide.py index 26178af0f1..cf0509ac5a 100644 --- a/tests/test_reasoning_show_hide.py +++ b/tests/test_reasoning_show_hide.py @@ -12,6 +12,7 @@ import pathlib import re +import subprocess REPO = pathlib.Path(__file__).parent.parent @@ -292,6 +293,39 @@ def test_cmd_reasoning_routes_effort_through_api_reasoning(self): "(effort now round-trips through /api/reasoning → config.yaml)" ) + def test_cmd_reasoning_effort_post_drives_model_aware_downgrade(self): + src = read('static/commands.js') + fn = 'function cmdReasoning(args){' + function_body(src, 'cmdReasoning') + '}' + script = f""" +const calls=[]; +const chips=[]; +const toasts=[]; +const window={{}}; +function _reasoningEffortContext(){{return {{model:'gpt-5.5',provider:'openai-codex'}};}} +function api(path, options){{ + const body=JSON.parse(options.body); + calls.push({{path,body}}); + const downgraded=body.model==='gpt-5.5' && body.provider==='openai-codex' + ? 'xhigh' : body.effort; + return Promise.resolve({{reasoning_effort:downgraded}}); +}} +function showToast(message){{toasts.push(message);}} +function _applyReasoningChip(effort){{chips.push(effort);}} +{fn} +cmdReasoning('max'); +setImmediate(()=>{{ + if(calls.length!==1) throw new Error('slash branch did not POST exactly once'); + if(calls[0].body.model!=='gpt-5.5' || calls[0].body.provider!=='openai-codex') + throw new Error('active reasoning context missing from slash POST'); + if(chips[0]!=='xhigh' || !toasts[0].includes('xhigh')) + throw new Error('downgraded effort was not rendered'); +}}); +""" + result = subprocess.run( + ['node', '-e', script], capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, result.stderr + def test_cmd_reasoning_routes_display_through_api_reasoning(self): """show|hide|on|off must POST to /api/reasoning (config.yaml display.show_reasoning — the CLI's key) in addition to mirroring @@ -312,7 +346,7 @@ def test_cmd_reasoning_supports_all_cli_effort_levels(self): m = re.search(r'function cmdReasoning\(.*?\n\}', src, re.DOTALL) assert m fn = m.group(0) - for level in ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'): + for level in ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra'): assert f"'{level}'" in fn, ( f"cmdReasoning must accept '{level}' (CLI parity with " f"hermes_constants.parse_reasoning_effort)" @@ -325,7 +359,7 @@ def test_reasoning_subargs_match_cli_levels(self): assert m, "reasoning COMMANDS entry not found" entry = m.group(0) for suggestion in ( - 'show', 'hide', 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max' + 'show', 'hide', 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra' ): assert f"'{suggestion}'" in entry, ( f"reasoning subArgs must include '{suggestion}' for CLI parity" @@ -359,7 +393,7 @@ def test_valid_reasoning_efforts_matches_hermes_constants(self): # Snapshot-style assertion: if hermes_constants adds a level, this # test will fail fast so we know to update WebUI too. assert VALID_REASONING_EFFORTS == ( - 'minimal', 'low', 'medium', 'high', 'xhigh', 'max' + 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra' ) def test_set_reasoning_effort_persists_to_config_yaml(self, tmp_path, monkeypatch): diff --git a/tests/test_zai_reasoning_effort_gating.py b/tests/test_zai_reasoning_effort_gating.py index a20e186909..044ec4214b 100644 --- a/tests/test_zai_reasoning_effort_gating.py +++ b/tests/test_zai_reasoning_effort_gating.py @@ -158,7 +158,7 @@ def test_classification_none_for_non_glm_on_zai(): # ── supports_thinking_toggle: chip visibility contract ────────────────────────── -def _reasoning_status(model_id, provider_id="zai"): +def _reasoning_status(model_id, provider_id: str | None = "zai"): """Helper: call get_reasoning_status with a stub config so it does not depend on the active profile's config.yaml.""" import unittest.mock as mock @@ -228,6 +228,15 @@ def test_non_zai_status_toggle_defaults_to_effort_capability(): assert st2["supports_reasoning_effort"] is False +def test_qualified_zai_context_is_reused_for_coercion_and_status(monkeypatch): + monkeypatch.setattr(cfg, "_models_dev_reasoning_efforts", lambda *_args: []) + qualified = "@zai:glm-5.1" + assert cfg.coerce_reasoning_effort_for_model("high", qualified) == "" + status = _reasoning_status(qualified, provider_id=None) + assert status["supported_efforts"] == [] + assert status["supports_thinking_toggle"] is True + + # ── Aliases resolve through the same gate ──────────────────────────────────────── @pytest.mark.parametrize("alias", ["glm", "z-ai", "z.ai", "zhipu"]) @@ -268,10 +277,10 @@ def test_aggregator_providers_keep_family_reasoning(model_id, provider_id): def test_non_glm_model_on_zai_provider_unaffected(): # A non-GLM model id routed through the zai provider must not be gated # by the GLM-specific branch — the gate keys on "glm" in the bare id, so - # non-GLM models fall through unchanged. (The OpenAI-family ceiling does NOT - # fire here because that branch is keyed on provider, not model family.) + # non-GLM models bypass the GLM-specific branch. The GPT-5 ceiling is still + # model-scoped across serving lanes, so older GPT-5 cannot retain max/ultra. efforts = cfg.resolve_model_reasoning_efforts("gpt-5", provider_id="zai") - assert set(efforts) == {"minimal", "low", "medium", "high", "xhigh", "max"} + assert set(efforts) == {"minimal", "low", "medium", "high", "xhigh"} # ── Coercion agrees with advertising (UI/coercion invariant) ───────────────────── @@ -444,7 +453,7 @@ def test_set_reasoning_effort_still_rejects_invalid(): cfg.set_reasoning_effort("banana", model_id="glm-4.6", provider_id="zai") -@pytest.mark.parametrize("effort", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +@pytest.mark.parametrize("effort", ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]) def test_set_reasoning_effort_still_accepts_valid_levels(effort): """Regression guard: all valid levels + none must still save correctly.""" import unittest.mock as mock