fix(models): validate OpenRouter presets structurally on custom endpoints (#97907)

hermes_cli/models.py

@@ -3359,6 +3359,38 @@ def _openrouter_variant_base(model_id: str) -> Optional[str]:
         return base
     return None
 
+
+# OpenRouter dashboard presets are account-scoped server-side objects
+# (created in the OpenRouter dashboard), referenced as ``@preset/<slug>`` or
+# combined with a base model as ``<model>@preset/<slug>``. They never appear
+# in ``/models`` listings — OpenRouter resolves and validates the slug at
+# request time. (#97907)
+_OPENROUTER_PRESET_MARKER = "@preset/"
+_PRESET_SLUG_RE = re.compile(r"[A-Za-z0-9._~-]+")
+
+
+def _split_openrouter_preset(model_id: str) -> tuple[str, Optional[str]]:
+    """Split an OpenRouter dashboard preset reference into base and suffix.
+
+    Returns ``(base_model, suffix)`` where ``suffix`` is ``"@preset/<slug>"``
+    when exactly one marker is present and the slug is non-empty:
+
+      ``@preset/<slug>``        → ``("", "@preset/<slug>")`` — bare preset
+      ``<model>@preset/<slug>`` → ``("<model>", "@preset/<slug>")``
+      anything else             → ``(model_id, None)`` — not a preset form
+
+    Multiple markers (``a@preset/x@preset/y``) are not a defined OpenRouter
+    form and return ``(model_id, None)`` so callers keep their existing
+    behavior for ordinary identifiers. A single marker with an empty slug
+    (``@preset/``) IS returned as a preset form — the caller's slug-charset
+    check rejects it with the preset contract.
+    """
+    text = model_id or ""
+    if text.count(_OPENROUTER_PRESET_MARKER) != 1:
+        return text, None
+    base, _, slug = text.partition(_OPENROUTER_PRESET_MARKER)
+    return base, f"{_OPENROUTER_PRESET_MARKER}{slug}"
+
 # Subscription/OAuth providers whose catalogs RE-EXPOSE other vendors' models
 # would be listed here (tried only as a last resort for bare short-alias
 # resolution, after every native-vendor catalog, so they never hijack an alias
@@ -6635,6 +6667,37 @@ def validate_requested_model(
         }
 
     if normalized == "custom" or normalized.startswith("custom:"):
+        # OpenRouter dashboard presets are server-side objects that never
+        # appear in a /models listing — probing cannot confirm them, and the
+        # resulting "not found" warning embeds the raw catalog URL (which
+        # messaging platforms happily link-preview). Handle them structurally
+        # instead: OpenRouter validates the slug at request time. (#97907)
+        _preset_base, _preset_suffix = _split_openrouter_preset(requested)
+        if _preset_suffix is not None:
+            if _PRESET_SLUG_RE.fullmatch(_preset_suffix[len(_OPENROUTER_PRESET_MARKER):]) is None:
+                return {
+                    "accepted": False,
+                    "persist": False,
+                    "recognized": False,
+                    "message": (
+                        f"OpenRouter preset slugs must be non-empty URL-safe "
+                        f"identifiers using only letters, digits, '.', '_', "
+                        f"'~', or '-' (got `{_preset_suffix}`)."
+                    ),
+                }
+            if not _preset_base:
+                # Bare ``@preset/<slug>`` — nothing to validate locally.
+                return {
+                    "accepted": True,
+                    "persist": True,
+                    "recognized": False,
+                    "message": (
+                        f"Accepted OpenRouter preset reference `{requested}`. "
+                        f"Presets are resolved server-side by OpenRouter at "
+                        f"request time."
+                    ),
+                }
+
         # Try probing with correct auth for the api_mode.
         if api_mode == "anthropic_messages":
             probe = probe_api_models(
@@ -6651,7 +6714,13 @@ def validate_requested_model(
             )
         api_models = probe.get("models")
         if api_models is not None:
-            if requested_for_lookup in set(api_models):
+            # For ``<model>@preset/<slug>``, validate the BASE model against
+            # the listing and keep the preset suffix verbatim on the persisted
+            # / corrected id — the suffix is a server-side reference, not a
+            # catalog entry, and the fuzzy corrector would otherwise strip it
+            # (same rule as the ``:nitro`` routing-variant handling).
+            _lookup = _preset_base if _preset_suffix else requested_for_lookup
+            if _lookup in set(api_models):
                 return {
                     "accepted": True,
                     "persist": True,
@@ -6660,17 +6729,18 @@ def validate_requested_model(
                 }
 
             # Auto-correct if the top match is very similar (e.g. typo)
-            auto = get_close_matches(requested_for_lookup, api_models, n=1, cutoff=0.9)
+            auto = get_close_matches(_lookup, api_models, n=1, cutoff=0.9)
             if auto:
+                corrected = f"{auto[0]}{_preset_suffix}" if _preset_suffix else auto[0]
                 return {
                     "accepted": True,
                     "persist": True,
                     "recognized": True,
-                    "corrected_model": auto[0],
-                    "message": f"Auto-corrected `{requested}` → `{auto[0]}`",
+                    "corrected_model": corrected,
+                    "message": f"Auto-corrected `{requested}` → `{corrected}`",
                 }
 
-            suggestions = get_close_matches(requested, api_models, n=3, cutoff=0.5)
+            suggestions = get_close_matches(_lookup, api_models, n=3, cutoff=0.5)
             suggestion_text = ""
             if suggestions:
                 suggestion_text = "\n  Similar models: " + ", ".join(f"`{s}`" for s in suggestions)

tests/hermes_cli/test_openrouter_preset_custom_validation.py

@@ -0,0 +1,176 @@
+"""OpenRouter dashboard presets on custom endpoints must not be catalog-validated (#97907).
+
+``@preset/<slug>`` identifiers reference server-side OpenRouter dashboard
+objects. They never appear in ``/models`` listings, so the custom-provider
+validation branch used to soft-accept them with a false "not found in this
+custom endpoint's model listing" warning that embedded the raw catalog URL —
+which Telegram's link preview then fetched and rendered as a ``models.json``
+document on every preset switch.
+
+These tests pin the contract from the issue:
+
+* a bare ``@preset/<slug>`` is accepted structurally, with NO probe call and a
+  URL-free message (nothing for a link preview to fetch);
+* a combined ``<model>@preset/<slug>`` validates its BASE model against the
+  live listing and preserves the preset suffix through auto-correction
+  (mirroring the ``:nitro`` routing-variant rule);
+* a malformed slug is rejected with the slug contract, not a catalog warning;
+* non-preset identifiers keep the existing custom-branch behavior.
+"""
+
+from unittest.mock import patch
+
+from hermes_cli.models import validate_requested_model
+
+
+FAKE_CUSTOM_MODELS = [
+    "anthropic/claude-opus-4.6",
+    "anthropic/claude-sonnet-4.5",
+    "openai/gpt-5.4",
+]
+
+BASE_URL = "https://openrouter.example-proxy.dev/v1"
+
+
+def _probe_payload(models=None):
+    return {
+        "models": FAKE_CUSTOM_MODELS if models is None else models,
+        "probed_url": f"{BASE_URL}/models",
+        "resolved_base_url": BASE_URL,
+        "suggested_base_url": None,
+        "used_fallback": False,
+    }
+
+
+def _validate(model, *, models=None, provider="custom", base_url=BASE_URL, api_mode=None):
+    """Call validate_requested_model against a mocked custom-endpoint probe."""
+    with patch(
+        "hermes_cli.models.probe_api_models", return_value=_probe_payload(models)
+    ) as probe:
+        result = validate_requested_model(
+            model,
+            provider,
+            api_key="sk-test",
+            base_url=base_url,
+            api_mode=api_mode,
+        )
+    return result, probe
+
+
+class TestBarePresetCustomEndpoint:
+    """Layer 1 — bare ``@preset/<slug>`` on a custom (OpenRouter-compatible) endpoint."""
+
+    def test_bare_preset_accepted_structurally_without_probe(self):
+        result, probe = _validate("@preset/my-team-config")
+        assert result["accepted"] is True
+        assert result["persist"] is True
+        # OpenRouter validates the slug at request time; there is nothing a
+        # catalog probe could confirm for a server-side dashboard object.
+        probe.assert_not_called()
+
+    def test_bare_preset_message_carries_no_url(self):
+        result, _ = _validate("@preset/my-team-config")
+        message = result.get("message") or ""
+        # The Telegram document bug is triggered by URLs in the confirmation
+        # text; the preset path must not emit any.
+        assert "http" not in message
+        assert "/models" not in message
+        assert "not found" not in message
+
+    def test_bare_preset_on_named_custom_provider(self):
+        result, probe = _validate("@preset/fast-tier", provider="custom:my-openrouter")
+        assert result["accepted"] is True
+        probe.assert_not_called()
+
+    def test_bare_preset_reachable_and_unreachable_endpoint_agree(self):
+        # A preset reference is server-side: endpoint reachability must not
+        # change the verdict (no probe happens on this path at all).
+        reachable, _ = _validate("@preset/a")
+        with patch(
+            "hermes_cli.models.probe_api_models", return_value=_probe_payload(None)
+        ) as probe_none:
+            unreachable = validate_requested_model(
+                "@preset/a", "custom", api_key="sk-test", base_url=BASE_URL
+            )
+            probe_none.assert_not_called()
+        assert reachable["accepted"] == unreachable["accepted"] is True
+
+
+class TestCombinedPresetCustomEndpoint:
+    """Layer 2 — ``<model>@preset/<slug>`` validates the base, keeps the suffix."""
+
+    def test_combined_preset_base_listed_accepts_with_suffix(self):
+        result, probe = _validate("anthropic/claude-opus-4.6@preset/cheap")
+        assert result["accepted"] is True
+        assert result["persist"] is True
+        probe.assert_called_once()
+        # No false warning: the base was found in the listing.
+        assert result.get("message") is None
+
+    def test_combined_preset_autocorrect_preserves_suffix(self):
+        # Base is a near-typo of a listed model: the corrector must fix the
+        # base AND keep the preset suffix — not strip it.
+        result, _ = _validate("anthropic/claude-opus-4.5@preset/cheap")
+        assert result["accepted"] is True
+        assert result["corrected_model"] == "anthropic/claude-opus-4.6@preset/cheap"
+        assert "preset/cheap" in (result.get("message") or "")
+
+    def test_combined_preset_unknown_base_warns_about_base(self):
+        # The suffix alone cannot make an unknown base valid; the warning
+        # names the base model, and remains a true positive.
+        result, _ = _validate("totally-unknown-model@preset/cheap")
+        assert result["accepted"] is True  # custom endpoints soft-accept
+        message = result.get("message") or ""
+        assert "totally-unknown-model@preset/cheap" in message
+        assert "preset" in message
+
+
+class TestMalformedPresetSlug:
+    """Layer 3 — malformed preset references are rejected with the slug contract."""
+
+    def test_empty_slug_rejected(self):
+        result, probe = _validate("@preset/")
+        assert result["accepted"] is False
+        assert result["persist"] is False
+        probe.assert_not_called()
+        assert "preset" in (result.get("message") or "").lower()
+
+    def test_invalid_slug_charset_rejected(self):
+        # A charset-only violation (no whitespace) exercises this branch
+        # specifically rather than the shared spaces guard.
+        result, probe = _validate("@preset/bad!slug")
+        assert result["accepted"] is False
+        assert result["persist"] is False
+        probe.assert_not_called()
+        assert "preset" in (result.get("message") or "").lower()
+
+    def test_multiple_markers_not_a_preset_reference(self):
+        # Two markers is not a defined OpenRouter form: fall back to the
+        # existing custom-branch behavior for an ordinary (unknown) id.
+        result, probe = _validate("a@preset/x@preset/y")
+        assert result["accepted"] is True  # soft-accept, as today
+        probe.assert_called_once()
+
+
+class TestNonPresetUnchanged:
+    """Layer 4 — non-preset identifiers keep today's custom-branch behavior."""
+
+    def test_plain_unknown_model_still_soft_accepts_with_warning(self):
+        result, probe = _validate("some-hidden-model")
+        assert result["accepted"] is True
+        assert result["persist"] is True
+        assert result["recognized"] is False
+        assert "not found" in (result.get("message") or "")
+        probe.assert_called_once()
+
+    def test_plain_listed_model_still_recognized(self):
+        result, probe = _validate("openai/gpt-5.4")
+        assert result["accepted"] is True
+        assert result["recognized"] is True
+        assert result.get("message") is None
+        probe.assert_called_once()
+
+    def test_plain_typo_still_autocorrects_without_suffix_logic(self):
+        result, _ = _validate("anthropic/claude-opus-4.5")
+        assert result["accepted"] is True
+        assert result["corrected_model"] == "anthropic/claude-opus-4.6"