diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 04ccfb6da4c4..e81be830577c 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -18,6 +18,16 @@ database: # wal_autocheckpoint: 1000 # pages between automatic checkpoints # journal_size_limit: 67108864 # cap the WAL/journal file size in bytes +# ============================================================================= +# Runtime Limits +# ============================================================================= +# Long-running Hermes server processes raise their RLIMIT_NOFILE soft limit to +# this value when the operating system permits it. The value is clamped to the +# hard limit and never lowers an already higher soft limit. Set to 0, false, or +# null to disable the adjustment. Default: 4096. +runtime: + nofile_soft_limit: 4096 + # ============================================================================= # Model Configuration # ============================================================================= diff --git a/gateway/run.py b/gateway/run.py index 8c162f7dc9a4..4e432b7c5f0c 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -26045,6 +26045,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = Useful for systemd services to avoid restart-loop deadlocks when the previous process hasn't fully exited yet. """ + from hermes_cli.resource_limits import apply_nofile_soft_limit + + apply_nofile_soft_limit() + # Snapshot the checkout revision now, while sys.modules still matches disk, # so a later `git pull` under this long-lived process can be detected (and # risky work like model switching refused) instead of crashing on a stale diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index b07081901741..fca0fa692f48 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -20,6 +20,11 @@ "wal_autocheckpoint": None, "journal_size_limit": None, }, + # Soft file-descriptor limit for long-running Hermes server processes. + # Clamped to the OS hard limit; 0/false/null disables the adjustment. + "runtime": { + "nofile_soft_limit": 4096, + }, # Global active chat session cap across CLI, TUI/dashboard, and messaging. # None/0 = unbounded. "max_concurrent_sessions": None, diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 98874824c196..6e4a619b3a95 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -10319,6 +10319,15 @@ def cmd_dashboard(args): else: os.execvpe(sys.executable, reexec_argv, env) + # Apply the final process/profile policy after dashboard routing, but before + # importing the web server or opening dashboard state. Applying it before a + # named-profile re-exec could leak that profile's higher limit into the + # machine/default dashboard, whose lower policy intentionally cannot undo it. + # This also covers Desktop SSH's isolated `serve` child, which does not route. + from hermes_cli.resource_limits import apply_nofile_soft_limit + + apply_nofile_soft_limit() + if _token_file: _ssh_session_token = _read_ssh_session_token_file(_token_file) diff --git a/hermes_cli/resource_limits.py b/hermes_cli/resource_limits.py new file mode 100644 index 000000000000..6fa0f789a98c --- /dev/null +++ b/hermes_cli/resource_limits.py @@ -0,0 +1,119 @@ +"""Best-effort process resource-limit adjustments for long-running services. + +The public helper in this module is shared by the gateway and the dashboard/ +serve entrypoints. It deliberately has no user-facing environment-variable +control: the target comes from the profile's canonical ``config.yaml`` loader. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from hermes_cli.config_defaults import DEFAULT_CONFIG + +try: # ``resource`` is POSIX-only (and unavailable on Windows). + import resource as _resource +except (ImportError, ModuleNotFoundError): # pragma: no cover - Windows only + _resource = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + +DEFAULT_NOFILE_SOFT_LIMIT = int(DEFAULT_CONFIG["runtime"]["nofile_soft_limit"]) +_MISSING = object() + + +def _configured_nofile_soft_limit( + config: Mapping[str, Any] | None, +) -> int | None: + """Resolve ``runtime.nofile_soft_limit`` from a loaded config. + + A missing key uses the default. Explicit ``0``, ``false``, and ``null`` + disable the adjustment. Other non-integer or negative values are invalid + and are ignored (the caller fails open without changing the process limit). + """ + if config is None: + try: + # Use Hermes's real, profile-aware loader rather than reading YAML + # here. This also applies managed-scope overlays and defaults. + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() + except Exception: + logger.debug("Could not load config for RLIMIT_NOFILE", exc_info=True) + return None + + if not isinstance(config, Mapping): + return None + + runtime = config.get("runtime", _MISSING) + if runtime is _MISSING: + return DEFAULT_NOFILE_SOFT_LIMIT + if not isinstance(runtime, Mapping): + return None + + raw_value = runtime.get("nofile_soft_limit", _MISSING) + if raw_value is _MISSING: + return DEFAULT_NOFILE_SOFT_LIMIT + if raw_value is None or raw_value is False: + return None + if raw_value is True or not isinstance(raw_value, int): + return None + if raw_value <= 0: + return None + return raw_value + + +def apply_nofile_soft_limit( + config: Mapping[str, Any] | None = None, +) -> bool: + """Raise this process's ``RLIMIT_NOFILE`` soft limit when possible. + + The target defaults to :data:`DEFAULT_NOFILE_SOFT_LIMIT` and can be set with + ``runtime.nofile_soft_limit``. The target is clamped to a finite hard limit, + never lowers an existing higher soft limit, and returns ``False`` for an + explicit opt-out or when the platform/sandbox refuses the operation. + + This is intentionally best-effort. Unsupported platforms, malformed + settings, and denied ``setrlimit`` calls must never prevent a server from + starting. + """ + if _resource is None: + return False + + target = _configured_nofile_soft_limit(config) + if target is None: + return False + + try: + nofile = _resource.RLIMIT_NOFILE + current_soft, current_hard = _resource.getrlimit(nofile) + # On platforms where RLIM_INFINITY is represented as -1, ordinary + # integer ordering would make an unlimited soft limit look lower than + # every positive target. Never replace infinity with a finite limit. + if current_soft == getattr(_resource, "RLIM_INFINITY", object()): + return False + if current_soft >= target: + return False + + if current_hard == getattr(_resource, "RLIM_INFINITY", object()): + new_soft = target + else: + new_soft = min(target, current_hard) + if new_soft <= current_soft: + return False + + _resource.setrlimit(nofile, (new_soft, current_hard)) + return True + except Exception: + # This helper runs before server startup and must fail open for + # unsupported/sandboxed environments and denied resource changes. + logger.debug("Could not raise RLIMIT_NOFILE soft limit", exc_info=True) + return False + + +__all__ = [ + "DEFAULT_NOFILE_SOFT_LIMIT", + "apply_nofile_soft_limit", +] diff --git a/tests/test_resource_limits.py b/tests/test_resource_limits.py new file mode 100644 index 000000000000..a55ecaccd1ad --- /dev/null +++ b/tests/test_resource_limits.py @@ -0,0 +1,342 @@ +"""Tests for configurable RLIMIT_NOFILE startup handling.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +import textwrap +from types import SimpleNamespace + +import pytest + +from hermes_cli import resource_limits + + +class _FakeResource: + RLIMIT_NOFILE = 7 + RLIM_INFINITY = 2**63 - 1 + + def __init__(self, soft: int, hard: int) -> None: + self.limits = (soft, hard) + self.set_calls: list[tuple[int, tuple[int, int]]] = [] + + def getrlimit(self, resource: int) -> tuple[int, int]: + assert resource == self.RLIMIT_NOFILE + return self.limits + + def setrlimit(self, resource: int, limits: tuple[int, int]) -> None: + assert resource == self.RLIMIT_NOFILE + self.set_calls.append((resource, limits)) + self.limits = limits + + +def test_real_config_loader_reads_runtime_nofile_setting(monkeypatch, tmp_path): + """The helper uses the canonical config loader, not a second YAML parser.""" + home = tmp_path / ".hermes" + home.mkdir() + (home / "config.yaml").write_text( + "runtime:\n nofile_soft_limit: 2048\n", + encoding="utf-8", + ) + fake_resource = _FakeResource(soft=256, hard=4096) + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit() is True + assert fake_resource.set_calls == [ + (fake_resource.RLIMIT_NOFILE, (2048, 4096)), + ] + + +def test_default_is_clamped_to_hard_limit(monkeypatch): + fake_resource = _FakeResource(soft=256, hard=1024) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit({}) is True + assert fake_resource.limits == (1024, 1024) + + +def test_finite_soft_limit_raises_when_hard_limit_is_infinite(monkeypatch): + fake_resource = _FakeResource(soft=256, hard=_FakeResource.RLIM_INFINITY) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit({}) is True + assert fake_resource.set_calls == [ + ( + fake_resource.RLIMIT_NOFILE, + (4096, fake_resource.RLIM_INFINITY), + ), + ] + + +def test_never_lowers_an_already_higher_soft_limit(monkeypatch): + fake_resource = _FakeResource(soft=8192, hard=16384) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit( + {"runtime": {"nofile_soft_limit": 4096}} + ) is False + assert fake_resource.set_calls == [] + assert fake_resource.limits == (8192, 16384) + + +@pytest.mark.parametrize("disabled", [0, False, None]) +def test_explicit_values_disable(monkeypatch, disabled): + fake_resource = _FakeResource(soft=256, hard=4096) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit( + {"runtime": {"nofile_soft_limit": disabled}} + ) is False + assert fake_resource.set_calls == [] + + +def test_unsupported_platform_is_a_safe_noop(monkeypatch): + monkeypatch.setattr(resource_limits, "_resource", None) + + assert resource_limits.apply_nofile_soft_limit({}) is False + + +def test_fresh_process_import_without_posix_resource_is_a_safe_noop(): + code = textwrap.dedent( + """ + import importlib.util + import pathlib + import sys + + sys.modules["resource"] = None + module_path = pathlib.Path(sys.argv[1]) + spec = importlib.util.spec_from_file_location( + "hermes_cli._resource_limits_without_posix_resource", + module_path, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + assert module._resource is None + assert module.apply_nofile_soft_limit({}) is False + """ + ) + + subprocess.run( + [sys.executable, "-c", code, resource_limits.__file__], + check=True, + cwd=Path(resource_limits.__file__).resolve().parents[1], + capture_output=True, + text=True, + ) + + +@pytest.mark.parametrize("invalid", [True, -1, 4096.0, "4096", object()]) +def test_invalid_values_are_safe_noops(monkeypatch, invalid): + fake_resource = _FakeResource(soft=256, hard=4096) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit( + {"runtime": {"nofile_soft_limit": invalid}} + ) is False + assert fake_resource.set_calls == [] + + +def test_setrlimit_denial_is_a_safe_noop(monkeypatch): + class _DeniedResource(_FakeResource): + def setrlimit(self, resource: int, limits: tuple[int, int]) -> None: + raise PermissionError("simulated EPERM") + + fake_resource = _DeniedResource(soft=256, hard=4096) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit({}) is False + assert fake_resource.limits == (256, 4096) + + +def test_getrlimit_failure_is_a_safe_noop(monkeypatch): + class _BrokenResource(_FakeResource): + def getrlimit(self, resource: int) -> tuple[int, int]: + raise OSError("simulated getrlimit failure") + + fake_resource = _BrokenResource(soft=256, hard=4096) + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit({}) is False + assert fake_resource.set_calls == [] + + +def test_never_lowers_an_unlimited_soft_limit(monkeypatch): + fake_resource = _FakeResource(soft=-1, hard=-1) + fake_resource.RLIM_INFINITY = -1 + monkeypatch.setattr(resource_limits, "_resource", fake_resource) + + assert resource_limits.apply_nofile_soft_limit({}) is False + assert fake_resource.set_calls == [] + assert fake_resource.limits == (-1, -1) + + +@pytest.mark.asyncio +async def test_gateway_startup_applies_limit_before_gateway_initialization(monkeypatch): + import gateway.code_skew + import gateway.run as gateway_run + + calls: list[str] = [] + + monkeypatch.setattr( + resource_limits, + "apply_nofile_soft_limit", + lambda: calls.append("limit"), + ) + + class _StopStartup(Exception): + pass + + def stop_after_limit(): + calls.append("gateway-init") + raise _StopStartup + + monkeypatch.setattr(gateway.code_skew, "record_boot_fingerprint", stop_after_limit) + + with pytest.raises(_StopStartup): + await gateway_run.start_gateway() + + assert calls == ["limit", "gateway-init"] + + +def test_serve_startup_applies_limit_before_web_server(monkeypatch): + from hermes_cli import main as cli_main + import hermes_cli.plugins + import hermes_cli.web_server + + calls: list[str] = [] + monkeypatch.setattr( + resource_limits, + "apply_nofile_soft_limit", + lambda: calls.append("limit"), + ) + monkeypatch.setattr(cli_main, "_sync_bundled_skills_quietly", lambda: None) + monkeypatch.setattr(cli_main, "_build_web_ui", lambda *args, **kwargs: True) + monkeypatch.setattr(cli_main, "_maybe_setup_dashboard_auth_interactively", lambda args: None) + monkeypatch.setattr(hermes_cli.plugins, "discover_plugins", lambda: None) + monkeypatch.setattr( + hermes_cli.web_server, + "start_server", + lambda **kwargs: calls.append("server"), + ) + + args = SimpleNamespace( + status=False, + stop=False, + headless_backend=True, + ssh_owner_nonce=None, + ssh_session_token_file=None, + host="127.0.0.1", + port=0, + no_open=True, + insecure=False, + open_profile="", + isolated=True, + skip_build=False, + ) + + cli_main.cmd_dashboard(args) + + assert calls == ["limit", "server"] + + +def test_named_profile_reroute_defers_limit_to_final_process(monkeypatch, tmp_path): + """The launcher profile must not leak its limit across machine re-exec.""" + from hermes_cli import main as cli_main + import hermes_cli.profiles + import hermes_constants + from tools.environments import local as local_environment + + calls: list[str] = [] + exec_call: dict[str, object] = {} + + monkeypatch.delenv("HERMES_DESKTOP", raising=False) + monkeypatch.setattr( + resource_limits, + "apply_nofile_soft_limit", + lambda: calls.append("limit"), + ) + monkeypatch.setattr( + hermes_cli.profiles, + "get_active_profile_name", + lambda: "worker", + ) + monkeypatch.setattr(cli_main, "_dashboard_listening", lambda *args: False) + monkeypatch.setattr( + local_environment, + "build_subprocess_env", + lambda **kwargs: {}, + ) + monkeypatch.setattr( + hermes_constants, + "get_default_hermes_root", + lambda: tmp_path, + ) + + class _ExecCalled(Exception): + pass + + def stop_at_exec(executable, argv, env): + exec_call.update(executable=executable, argv=argv, env=env) + raise _ExecCalled + + monkeypatch.setattr(cli_main.os, "execvpe", stop_at_exec) + + args = SimpleNamespace( + status=False, + stop=False, + headless_backend=True, + ssh_owner_nonce=None, + ssh_session_token_file=None, + host="127.0.0.1", + port=0, + no_open=True, + insecure=False, + open_profile="", + isolated=False, + skip_build=False, + ) + + with pytest.raises(_ExecCalled): + cli_main.cmd_dashboard(args) + + assert calls == [] + assert exec_call["argv"][1:5] == ["-m", "hermes_cli.main", "-p", "default"] + assert exec_call["env"]["HERMES_HOME"] == str(tmp_path) + + +@pytest.mark.parametrize("lifecycle_flag", ["status", "stop"]) +def test_dashboard_lifecycle_flags_skip_limit_adjustment(monkeypatch, lifecycle_flag): + """Informational/stop-only commands must not mutate process limits.""" + from hermes_cli import main as cli_main + + calls: list[str] = [] + monkeypatch.setattr( + resource_limits, + "apply_nofile_soft_limit", + lambda: calls.append("limit"), + ) + monkeypatch.setattr(cli_main, "_scan_dashboard_processes", lambda: []) + monkeypatch.setattr(cli_main, "_find_stale_dashboard_pids", lambda: []) + + args = SimpleNamespace( + status=lifecycle_flag == "status", + stop=lifecycle_flag == "stop", + headless_backend=False, + ssh_owner_nonce=None, + ssh_session_token_file=None, + host="127.0.0.1", + port=0, + no_open=True, + insecure=False, + open_profile="", + isolated=False, + skip_build=False, + ) + + with pytest.raises(SystemExit): + cli_main.cmd_dashboard(args) + + assert calls == [] diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 85d321335fad..577c51379438 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -69,6 +69,24 @@ cannot override, via a system-level managed directory. See [Managed Scope](/user-guide/managed-scope). ::: +## Runtime Limits + +Long-running Hermes server surfaces (including the gateway and +`hermes serve --isolated`) apply the configured `RLIMIT_NOFILE` soft limit +during startup when the operating system supports it: + +```yaml +runtime: + nofile_soft_limit: 4096 +``` + +The default is `4096`. Hermes clamps the target to the operating system's hard +limit and never lowers a process that already has a higher soft limit. Set the +value to `0`, `false`, or `null` to disable the adjustment. On Windows and in +sandboxes +where the limit cannot be changed, startup continues without changing the +limit. + ## Environment Variable Substitution You can reference environment variables in `config.yaml` using `${VAR_NAME}` syntax: