fix(models): OpenRouter @preset references no longer rejected by /model (salvage #89129)

hermes_cli/models.py

@@ -6608,6 +6608,47 @@ def validate_requested_model(
             "message": "Model names cannot contain spaces.",
         }
 
+    # OpenRouter presets are account-scoped configurations, so direct
+    # ``@preset/<slug>`` references never appear in the public /v1/models
+    # listing. Combined ``<model>@preset/<slug>`` references are also valid;
+    # validate their base model normally and preserve the preset suffix if a
+    # close match is auto-corrected. OpenRouter validates the preset slug when
+    # the inference request is made.
+    preset_suffix = ""
+
+    def _with_preset_suffix(model_id: str) -> str:
+        """Re-attach a preserved ``@preset/<slug>`` suffix after auto-correction."""
+        return f"{model_id}{preset_suffix}"
+
+    if normalized == "openrouter":
+        marker = "@preset/"
+        if marker in requested:
+            if requested.count(marker) != 1:
+                preset_slug = ""
+                preset_base = requested
+            else:
+                preset_base, preset_slug = requested.split(marker, 1)
+            if re.fullmatch(r"[A-Za-z0-9._~-]+", preset_slug) is None:
+                return {
+                    "accepted": False,
+                    "persist": False,
+                    "recognized": False,
+                    "message": (
+                        "OpenRouter preset slugs must be non-empty URL-safe "
+                        "identifiers using only letters, digits, '.', '_', "
+                        "'~', or '-'."
+                    ),
+                }
+            preset_suffix = f"{marker}{preset_slug}"
+            if not preset_base:
+                return {
+                    "accepted": True,
+                    "persist": True,
+                    "recognized": False,
+                    "message": None,
+                }
+            requested_for_lookup = preset_base
+
     if normalized == "lmstudio":
         from hermes_cli.auth import AuthError
         # Use probe_lmstudio_models so we can distinguish None (unreachable
@@ -7091,15 +7132,18 @@ def validate_requested_model(
             # Auto-correct if the top match is very similar (e.g. typo)
             auto = get_close_matches(requested_for_lookup, api_models, n=1, cutoff=0.9)
             if auto:
+                corrected = _with_preset_suffix(auto[0])
                 return {
                     "accepted": True,
                     "persist": True,
                     "recognized": True,
-                    "corrected_model": auto[0],
-                    "message": f"Auto-corrected `{requested}` → `{auto[0]}`",
+                    "corrected_model": corrected,
+                    "message": f"Auto-corrected `{requested}` → `{corrected}`",
                 }
 
-            suggestions = get_close_matches(requested, api_models, n=3, cutoff=0.5)
+            suggestions = get_close_matches(
+                requested_for_lookup, api_models, n=3, cutoff=0.5
+            )
             suggestion_text = ""
             if suggestions:
                 suggestion_text = "\n  Similar models: " + ", ".join(f"`{s}`" for s in suggestions)
@@ -7263,12 +7307,15 @@ def validate_requested_model(
         )
         if auto:
             corrected = catalog_lower[auto[0]]
+            corrected_with_suffix = _with_preset_suffix(corrected)
             return {
                 "accepted": True,
                 "persist": True,
                 "recognized": True,
-                "corrected_model": corrected,
-                "message": f"Auto-corrected `{requested}` → `{corrected}`",
+                "corrected_model": corrected_with_suffix,
+                "message": (
+                    f"Auto-corrected `{requested}` → `{corrected_with_suffix}`"
+                ),
             }
         suggestions = get_close_matches(
             requested_for_lookup.lower(), catalog_lower_list, n=3, cutoff=0.5

tests/hermes_cli/test_openrouter_preset_validation.py

@@ -0,0 +1,245 @@
+"""Regression coverage for OpenRouter preset references (issue #31739)."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from hermes_cli.models import validate_requested_model
+
+
+@pytest.mark.parametrize(
+    "model_name",
+    ["@preset/email-copywriter", "@preset/Foo_bar.~9"],
+)
+def test_direct_openrouter_preset_reference_skips_model_listing(model_name):
+    """An account-scoped direct preset has no public model row to probe."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        side_effect=AssertionError("direct preset references must not probe /models"),
+    ):
+        result = validate_requested_model(
+            model_name,
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result == {
+        "accepted": True,
+        "persist": True,
+        "recognized": False,
+        "message": None,
+    }
+
+
+def test_combined_openrouter_preset_reference_validates_base_model():
+    """Combined references validate the base model, not the preset-decorated ID."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        return_value=["openai/gpt-5.4"],
+    ) as mock_fetch:
+        result = validate_requested_model(
+            "openai/gpt-5.4@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    mock_fetch.assert_called_once_with("key", "https://openrouter.ai/api/v1")
+    assert result == {
+        "accepted": True,
+        "persist": True,
+        "recognized": True,
+        "message": None,
+    }
+
+
+def test_combined_openrouter_preset_reference_rejects_unknown_base_model():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
+        result = validate_requested_model(
+            "openai/gpt-5.4-preview@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+    assert result["recognized"] is False
+    assert "Similar models" in result["message"]
+    assert "openai/gpt-5.4" in result["message"]
+
+
+def test_combined_openrouter_preset_reference_preserves_suffix_on_autocorrect():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
+        result = validate_requested_model(
+            "openai/gpt-5.44@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    corrected = "openai/gpt-5.4@preset/email-copywriter"
+    assert result["accepted"] is True
+    assert result["corrected_model"] == corrected
+    assert corrected in result["message"]
+
+
+def test_combined_preset_preserves_suffix_on_catalog_autocorrect():
+    with (
+        patch("hermes_cli.models.fetch_api_models", return_value=None),
+        patch(
+            "hermes_cli.models.provider_model_ids",
+            return_value=["openai/gpt-5.4"],
+        ),
+    ):
+        result = validate_requested_model(
+            "openai/gpt-5.44@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    corrected = "openai/gpt-5.4@preset/email-copywriter"
+    assert result["accepted"] is True
+    assert result["corrected_model"] == corrected
+    assert corrected in result["message"]
+
+
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "@preset/",
+        "openai/gpt-5.4@preset/",
+        "@preset/foo/bar",
+        "@preset/foo?bar",
+        "@preset/☃",
+        "@preset/foo@preset/bar",
+        "openai/gpt-5.4@preset/foo/bar",
+    ],
+)
+def test_openrouter_preset_reference_requires_a_url_safe_slug(model_name):
+    """Malformed preset references must fail before model-list probing."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        side_effect=AssertionError("malformed presets must not probe /models"),
+    ):
+        result = validate_requested_model(
+            model_name,
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+    assert result["recognized"] is False
+    assert "URL-safe" in result["message"]
+
+
+def test_preset_reference_does_not_bypass_other_provider_validation():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["gpt-5.4"]):
+        result = validate_requested_model(
+            "@preset/email-copywriter",
+            "openai",
+            api_key="key",
+            base_url="https://api.openai.com/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+
+
+def test_preset_reference_does_not_bypass_custom_endpoint_validation():
+    probe = {
+        "models": ["local-model"],
+        "probed_url": "https://proxy.example/v1/models",
+        "resolved_base_url": "https://proxy.example/v1",
+        "suggested_base_url": None,
+        "used_fallback": False,
+    }
+    with patch("hermes_cli.models.probe_api_models", return_value=probe) as mock_probe:
+        result = validate_requested_model(
+            "@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://proxy.example/v1",
+        )
+
+    mock_probe.assert_called_once()
+    assert result["accepted"] is True
+    assert result["recognized"] is False
+    assert "custom endpoint's model listing" in result["message"]
+
+
+def test_configured_alias_switches_preset_through_real_resolution_chain(tmp_path):
+    """Exercise config loading, alias resolution, runtime resolution, and validation."""
+    (tmp_path / "config.yaml").write_text(
+        """
+model:
+  default: openai/gpt-5.4
+  provider: openrouter
+  base_url: https://openrouter.ai/api/v1
+model_aliases:
+  email-copywriter:
+    model: '@preset/email-copywriter'
+    provider: openrouter
+""".lstrip(),
+        encoding="utf-8",
+    )
+
+    script = r"""
+import json
+import hermes_cli.model_switch as model_switch
+import hermes_cli.models as models
+
+
+def fail_model_probe(*args, **kwargs):
+    raise AssertionError("preset references must not probe /models")
+
+
+models.fetch_api_models = fail_model_probe
+model_switch.get_model_capabilities = lambda *args, **kwargs: None
+model_switch.get_model_info = lambda *args, **kwargs: None
+result = model_switch.switch_model(
+    "email-copywriter",
+    current_provider="openrouter",
+    current_model="openai/gpt-5.4",
+    current_base_url="https://openrouter.ai/api/v1",
+    current_api_key="key",
+)
+print(json.dumps(vars(result)))
+"""
+    env = {
+        "HOME": str(tmp_path),
+        "HERMES_HOME": str(tmp_path),
+        "LANG": "C.UTF-8",
+        "OPENROUTER_API_KEY": "key",
+        "PATH": os.environ.get("PATH", ""),
+        "PYTHONIOENCODING": "utf-8",
+    }
+    completed = subprocess.run(
+        [sys.executable, "-c", script],
+        cwd=Path(__file__).resolve().parents[2],
+        env=env,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert completed.returncode == 0, (
+        f"subprocess failed with exit {completed.returncode}\n"
+        f"stdout:\n{completed.stdout}\n"
+        f"stderr:\n{completed.stderr}"
+    )
+    result = json.loads(completed.stdout.splitlines()[-1])
+
+    assert result["success"] is True
+    assert result["new_model"] == "@preset/email-copywriter"
+    assert result["target_provider"] == "openrouter"
+    assert result["resolved_via_alias"] == "email-copywriter"
+    assert result["warning_message"] == ""