Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion agent/agent_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,8 @@ def init_agent(
provider_sort: str = None,
provider_require_parameters: bool = False,
provider_data_collection: str = None,
provider_preferred_min_throughput: Any = None,
provider_preferred_max_latency: Any = None,
openrouter_min_coding_score: Optional[float] = None,
session_id: str = None,
tool_progress_callback: callable = None,
Expand Down Expand Up @@ -835,6 +837,8 @@ def init_agent(
agent.provider_sort = provider_sort
agent.provider_require_parameters = provider_require_parameters
agent.provider_data_collection = provider_data_collection
agent.provider_preferred_min_throughput = provider_preferred_min_throughput
agent.provider_preferred_max_latency = provider_preferred_max_latency
agent.openrouter_min_coding_score = openrouter_min_coding_score

# Store toolset filtering options
Expand Down Expand Up @@ -1693,6 +1697,7 @@ def init_agent(
# So the built-in store is created unless memory is globally disabled, while
# the external-provider block below stays gated on skip_memory.
_memory_toolset_requested = "memory" in (agent.enabled_toolsets or [])
mem_config = {}
if not skip_memory or _memory_toolset_requested:
try:
mem_config = _agent_cfg.get("memory", {})
Expand Down Expand Up @@ -1721,7 +1726,12 @@ def init_agent(
if _mem_provider_name and _mem_provider_name.strip():
from agent.memory_manager import MemoryManager as _MemoryManager
from plugins.memory import load_memory_provider as _load_mem
agent._memory_manager = _MemoryManager()
_external_prefetch_timeout = (
mem_config.get("external_prefetch_timeout") if mem_config else None
)
agent._memory_manager = _MemoryManager(
external_prefetch_timeout=_external_prefetch_timeout
)
_mp = _load_mem(_mem_provider_name)
if _mp and _mp.is_available():
agent._memory_manager.add_provider(_mp)
Expand Down
2 changes: 2 additions & 0 deletions agent/background_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,8 @@ def _bg_review_auto_deny(command, description, **kwargs):
"provider_sort",
"provider_require_parameters",
"provider_data_collection",
"provider_preferred_min_throughput",
"provider_preferred_max_latency",
):
_pref_val = getattr(agent, _pref_attr, None)
if _pref_val:
Expand Down
69 changes: 68 additions & 1 deletion agent/chat_completion_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

logger = logging.getLogger(__name__)
_OPENROUTER_PROVIDER_SORT_VALUES = {"throughput", "latency", "price"}
_OPENROUTER_PERFORMANCE_PERCENTILES = {"p50", "p75", "p90", "p99"}

# When the fallback chain is fully exhausted on a non-rate-limit failure
# (e.g. every provider returns a non-retryable client error like HTTP 400),
Expand Down Expand Up @@ -177,6 +178,59 @@ def _validated_openrouter_provider_sort(raw_sort: Any) -> Optional[str]:
return None


def _validated_openrouter_performance_preference(
raw_value: Any,
field_name: str,
) -> Optional[Any]:
"""Normalize an OpenRouter soft performance threshold.

OpenRouter accepts either one positive number or a mapping of percentile
cutoffs (p50/p75/p90/p99). Invalid entries are ignored with a warning so a
typo cannot turn a soft routing preference into an upstream 400.
"""
if isinstance(raw_value, bool):
logger.warning("Ignoring invalid OpenRouter provider.%s value %r", field_name, raw_value)
return None
if isinstance(raw_value, (int, float)):
value = float(raw_value)
if math.isfinite(value) and value > 0:
return value
logger.warning("Ignoring invalid OpenRouter provider.%s value %r", field_name, raw_value)
return None
if isinstance(raw_value, dict):
normalized: Dict[str, float] = {}
for percentile, raw_cutoff in raw_value.items():
if percentile not in _OPENROUTER_PERFORMANCE_PERCENTILES:
logger.warning(
"Ignoring unsupported OpenRouter provider.%s percentile %r",
field_name,
percentile,
)
continue
if isinstance(raw_cutoff, bool) or not isinstance(raw_cutoff, (int, float)):
logger.warning(
"Ignoring invalid OpenRouter provider.%s.%s value %r",
field_name,
percentile,
raw_cutoff,
)
continue
cutoff = float(raw_cutoff)
if math.isfinite(cutoff) and cutoff > 0:
normalized[percentile] = cutoff
else:
logger.warning(
"Ignoring invalid OpenRouter provider.%s.%s value %r",
field_name,
percentile,
raw_cutoff,
)
return normalized or None
if raw_value is not None:
logger.warning("Ignoring invalid OpenRouter provider.%s value %r", field_name, raw_value)
return None


def _provider_preferences_for_agent(agent) -> Dict[str, Any]:
"""Build the validated provider-routing object shared by request paths."""
preferences: Dict[str, Any] = {}
Expand All @@ -193,6 +247,16 @@ def _provider_preferences_for_agent(agent) -> Dict[str, Any]:
preferences["require_parameters"] = True
if agent.provider_data_collection:
preferences["data_collection"] = agent.provider_data_collection
for attribute, field_name in (
("provider_preferred_min_throughput", "preferred_min_throughput"),
("provider_preferred_max_latency", "preferred_max_latency"),
):
value = _validated_openrouter_performance_preference(
getattr(agent, attribute, None),
field_name,
)
if value is not None:
preferences[field_name] = value
return preferences


Expand Down Expand Up @@ -1540,7 +1604,10 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non
provider_profile=_profile,
ollama_num_ctx=agent._ollama_num_ctx,
# Context forwarded to profile hooks:
provider_preferences=_prefs or None,
# Provider-performance routing is valid only for aggregator endpoints.
# A provider label can coexist with an explicit custom base_url; do not
# leak OpenRouter/Nous-only request fields to that upstream.
provider_preferences=_prefs if (_is_or or _is_nous) else None,
openrouter_min_coding_score=agent.openrouter_min_coding_score,
anthropic_max_output=_ant_max,
supports_reasoning=agent._supports_reasoning_extra_body(),
Expand Down
2 changes: 2 additions & 0 deletions cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -4545,6 +4545,8 @@ def __init__(
self._providers_order = pr.get("order")
self._provider_require_params = pr.get("require_parameters", False)
self._provider_data_collection = pr.get("data_collection")
self._provider_preferred_min_throughput = pr.get("preferred_min_throughput")
self._provider_preferred_max_latency = pr.get("preferred_max_latency")

# OpenRouter Pareto Code router knob — coding-score floor (0.0-1.0).
# Only applied when model.model == "openrouter/pareto-code".
Expand Down
4 changes: 4 additions & 0 deletions cron/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -4060,6 +4060,10 @@ def run_job(
providers_ignored=pr.get("ignore"),
providers_order=pr.get("order"),
provider_sort=pr.get("sort"),
provider_require_parameters=pr.get("require_parameters", False),
provider_data_collection=pr.get("data_collection"),
provider_preferred_min_throughput=pr.get("preferred_min_throughput"),
provider_preferred_max_latency=pr.get("preferred_max_latency"),
openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
disabled_toolsets=_resolve_cron_disabled_toolsets(_cfg),
Expand Down
4 changes: 4 additions & 0 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -4859,6 +4859,8 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:
provider_sort=pr.get("sort"),
provider_require_parameters=pr.get("require_parameters", False),
provider_data_collection=pr.get("data_collection"),
provider_preferred_min_throughput=pr.get("preferred_min_throughput"),
provider_preferred_max_latency=pr.get("preferred_max_latency"),
session_id=ctx.session_id,
platform=platform_key,
user_id=ctx.source.user_id,
Expand Down Expand Up @@ -19918,6 +19920,8 @@ def run_sync():
provider_sort=pr.get("sort"),
provider_require_parameters=pr.get("require_parameters", False),
provider_data_collection=pr.get("data_collection"),
provider_preferred_min_throughput=pr.get("preferred_min_throughput"),
provider_preferred_max_latency=pr.get("preferred_max_latency"),
session_id=task_id,
platform=platform_key,
user_id=source.user_id,
Expand Down
6 changes: 6 additions & 0 deletions hermes_cli/cli_agent_setup_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,12 @@ def _init_agent(self, *, model_override: str = None, runtime_override: dict = No
provider_sort=self._provider_sort,
provider_require_parameters=self._provider_require_params,
provider_data_collection=self._provider_data_collection,
provider_preferred_min_throughput=getattr(
self, "_provider_preferred_min_throughput", None
),
provider_preferred_max_latency=getattr(
self, "_provider_preferred_max_latency", None
),
openrouter_min_coding_score=self._openrouter_min_coding_score,
session_id=self.session_id,
platform="cli",
Expand Down
6 changes: 6 additions & 0 deletions hermes_cli/cli_commands_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -2012,6 +2012,12 @@ def run_background():
provider_sort=self._provider_sort,
provider_require_parameters=self._provider_require_params,
provider_data_collection=self._provider_data_collection,
provider_preferred_min_throughput=getattr(
self, "_provider_preferred_min_throughput", None
),
provider_preferred_max_latency=getattr(
self, "_provider_preferred_max_latency", None
),
openrouter_min_coding_score=self._openrouter_min_coding_score,
fallback_model=self._fallback_model,
)
Expand Down
4 changes: 4 additions & 0 deletions run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,8 @@ def __init__(
provider_sort: str = None,
provider_require_parameters: bool = False,
provider_data_collection: str = None,
provider_preferred_min_throughput: Any = None,
provider_preferred_max_latency: Any = None,
openrouter_min_coding_score: Optional[float] = None,
session_id: str = None,
tool_progress_callback: callable = None,
Expand Down Expand Up @@ -546,6 +548,8 @@ def __init__(
provider_sort=provider_sort,
provider_require_parameters=provider_require_parameters,
provider_data_collection=provider_data_collection,
provider_preferred_min_throughput=provider_preferred_min_throughput,
provider_preferred_max_latency=provider_preferred_max_latency,
openrouter_min_coding_score=openrouter_min_coding_score,
session_id=session_id,
tool_progress_callback=tool_progress_callback,
Expand Down
41 changes: 40 additions & 1 deletion tests/agent/test_chat_completion_helpers_provider_sort.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
from agent.chat_completion_helpers import _validated_openrouter_provider_sort
from types import SimpleNamespace

from agent.chat_completion_helpers import (
_provider_preferences_for_agent,
_validated_openrouter_performance_preference,
_validated_openrouter_provider_sort,
)


def test_validated_openrouter_provider_sort_accepts_valid_values():
Expand All @@ -11,3 +17,36 @@ def test_validated_openrouter_provider_sort_rejects_invalid_values():
assert _validated_openrouter_provider_sort("intelligence") is None
assert _validated_openrouter_provider_sort("") is None
assert _validated_openrouter_provider_sort(None) is None


def test_validated_openrouter_performance_preference_accepts_number_and_percentiles():
assert _validated_openrouter_performance_preference(0.8, "preferred_max_latency") == 0.8
assert _validated_openrouter_performance_preference(
{"p50": 0.8, "p90": 3}, "preferred_max_latency"
) == {"p50": 0.8, "p90": 3.0}


def test_validated_openrouter_performance_preference_drops_invalid_entries():
assert _validated_openrouter_performance_preference(True, "preferred_max_latency") is None
assert _validated_openrouter_performance_preference(
{"p50": -1, "p100": 2, "p90": 3}, "preferred_max_latency"
) == {"p90": 3.0}


def test_provider_preferences_include_performance_thresholds():
agent = SimpleNamespace(
providers_allowed=None,
providers_ignored=None,
providers_order=None,
provider_sort="latency",
provider_require_parameters=False,
provider_data_collection="allow",
provider_preferred_min_throughput={"p50": 70},
provider_preferred_max_latency={"p50": 0.8, "p90": 3},
)
assert _provider_preferences_for_agent(agent) == {
"sort": "latency",
"data_collection": "allow",
"preferred_min_throughput": {"p50": 70.0},
"preferred_max_latency": {"p50": 0.8, "p90": 3.0},
}
24 changes: 24 additions & 0 deletions tests/run_agent/test_provider_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -830,6 +830,30 @@ def test_sort_throughput(self, monkeypatch):
kwargs = agent._build_api_kwargs([{"role": "user", "content": "hi"}])
assert kwargs["extra_body"]["provider"]["sort"] == "throughput"

def test_latency_performance_thresholds(self, monkeypatch):
agent = _make_agent(monkeypatch, "openrouter")
agent.provider_sort = "latency"
agent.provider_preferred_min_throughput = {"p50": 70}
agent.provider_preferred_max_latency = {"p50": 0.8, "p90": 3}
kwargs = agent._build_api_kwargs([{"role": "user", "content": "hi"}])
assert kwargs["extra_body"]["provider"] == {
"sort": "latency",
"preferred_min_throughput": {"p50": 70.0},
"preferred_max_latency": {"p50": 0.8, "p90": 3.0},
}

def test_preferences_do_not_leak_to_non_aggregator_base_url(self, monkeypatch):
agent = _make_agent(monkeypatch, "openrouter")
agent.base_url = "https://example.invalid/v1"
agent._base_url_lower = agent.base_url.lower()
agent.provider_sort = "latency"
agent.provider_preferred_min_throughput = {"p50": 70}
agent.provider_preferred_max_latency = {"p50": 0.8}

kwargs = agent._build_api_kwargs([{"role": "user", "content": "hi"}])

assert "provider" not in (kwargs.get("extra_body") or {})




Expand Down
Loading