From 36707dd635e1b6c320b44f45f2130da3ca35d937 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:27:05 -0700 Subject: [PATCH] perf(startup): parse config + plugin manifests with libyaml CSafeLoader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The startup config/manifest reads used PyYAML's pure-Python SafeLoader, which is ~8x slower than the libyaml-backed CSafeLoader C extension. config.yaml is parsed several times during launch (cli config, raw config, early interface/redaction bridge, logging config) and every plugin manifest is parsed once — all on the slow path. Add utils.fast_safe_load (CSafeLoader-preferring, pure-Python fallback, true drop-in for safe_load) and route the hot startup parse sites through it: hermes_cli/config.py (config + manifest reads), hermes_cli/plugins.py (manifest parse), env_loader, cli.load_cli_config, hermes_logging, and the two pre-config early YAML bridges in main.py. Behavior is identical (same restricted safe tag set); only speed changes. safe_load calls on the startup path drop from ~79 to ~0, cutting the YAML parse cost from ~0.9s to ~0.15s under profiling. Adds tests/test_fast_safe_load.py asserting equivalence with safe_load across input shapes, empty-doc falsiness, C-loader preference, and that python/object tags are still rejected (safe, not full loader). --- cli.py | 4 +-- hermes_cli/config.py | 14 ++++---- hermes_cli/env_loader.py | 4 +-- hermes_cli/main.py | 8 +++-- hermes_cli/plugins.py | 4 +-- hermes_logging.py | 4 +-- tests/test_fast_safe_load.py | 62 ++++++++++++++++++++++++++++++++++++ utils.py | 28 ++++++++++++++++ 8 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 tests/test_fast_safe_load.py diff --git a/cli.py b/cli.py index b759a523615d..e209c48eecf9 100644 --- a/cli.py +++ b/cli.py @@ -175,7 +175,7 @@ def realign_markdown_tables(*args, **kwargs): try_launch_chrome_debug, ) from hermes_cli.env_loader import load_hermes_dotenv -from utils import base_url_host_matches +from utils import base_url_host_matches, fast_safe_load _hermes_home = get_hermes_home() _project_env = Path(__file__).parent / '.env' @@ -510,7 +510,7 @@ def load_cli_config() -> Dict[str, Any]: with open(config_path, "r", encoding="utf-8") as f: from hermes_cli.config import _normalize_root_model_keys - file_config = _normalize_root_model_keys(yaml.safe_load(f) or {}) + file_config = _normalize_root_model_keys(fast_safe_load(f) or {}) _file_has_terminal_config = "terminal" in file_config diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 0d62e6aec1d7..cf3cce9f89e8 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -670,7 +670,7 @@ def get_container_exec_info() -> Optional[dict]: # Re-export from hermes_constants — canonical definition lives there. from hermes_constants import get_hermes_home # noqa: F811,E402 -from utils import atomic_replace +from utils import atomic_replace, fast_safe_load def get_config_path() -> Path: """Get the main config file path.""" @@ -4592,7 +4592,7 @@ def check_config_version() -> Tuple[int, int]: try: with open(config_path, encoding="utf-8") as f: - config = yaml.safe_load(f) or {} + config = fast_safe_load(f) or {} except Exception as e: # Invalid YAML needs a parse warning, not an automatic schema rewrite # that could replace the user's broken file with defaults. @@ -5167,7 +5167,7 @@ def migrate_config(interactive: bool = True, quiet: bool = False) -> Dict[str, A continue try: with open(manifest_file, encoding="utf-8") as _mf: - manifest = yaml.safe_load(_mf) or {} + manifest = fast_safe_load(_mf) or {} except Exception: manifest = {} name = manifest.get("name") or child.name @@ -5984,7 +5984,7 @@ def read_raw_config() -> Dict[str, Any]: try: with open(config_path, encoding="utf-8") as f: - data = yaml.safe_load(f) or {} + data = fast_safe_load(f) or {} except Exception as e: _warn_config_parse_failure(config_path, e) return {} @@ -6199,7 +6199,7 @@ def _load_config_impl(*, want_deepcopy: bool) -> Dict[str, Any]: if user_sig is not None: try: with open(config_path, encoding="utf-8") as f: - user_config = yaml.safe_load(f) or {} + user_config = fast_safe_load(f) or {} if "max_turns" in user_config: agent_user_config = dict(user_config.get("agent") or {}) @@ -7273,7 +7273,7 @@ def set_config_value(key: str, value: str): if config_path.exists(): try: with open(config_path, encoding="utf-8") as f: - user_config = yaml.safe_load(f) or {} + user_config = fast_safe_load(f) or {} except Exception: user_config = {} @@ -7561,7 +7561,7 @@ def _inject_platform_plugin_env_vars() -> None: continue try: with open(manifest_path, "r", encoding="utf-8") as f: - manifest = yaml.safe_load(f) or {} + manifest = fast_safe_load(f) or {} except Exception: continue label = manifest.get("label") or manifest.get("name") or child.name diff --git a/hermes_cli/env_loader.py b/hermes_cli/env_loader.py index c7d507d8c2f3..39ff02657c66 100644 --- a/hermes_cli/env_loader.py +++ b/hermes_cli/env_loader.py @@ -7,7 +7,7 @@ from pathlib import Path from dotenv import load_dotenv -from utils import atomic_replace +from utils import atomic_replace, fast_safe_load # Env var name suffixes that indicate credential values. These are the @@ -371,7 +371,7 @@ def _load_secrets_config(home_path: Path) -> dict: return {} try: with open(config_path, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) or {} + data = fast_safe_load(f) or {} except Exception: # noqa: BLE001 return {} return data.get("secrets") or {} diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 511c435f131f..77985234f075 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -130,7 +130,9 @@ def _config_default_interface_early() -> str: import yaml as _yaml_iface with open(cfg_path, encoding="utf-8") as _f: - raw = _yaml_iface.safe_load(_f) or {} + raw = _yaml_iface.load( + _f, Loader=getattr(_yaml_iface, "CSafeLoader", None) or _yaml_iface.SafeLoader + ) or {} disp = raw.get("display", {}) if isinstance(disp, dict): iface = disp.get("interface") @@ -531,7 +533,9 @@ def _resolve_sudo_user_profile_env(name: str) -> str | None: _cfg_path = get_hermes_home() / "config.yaml" if _cfg_path.exists(): with open(_cfg_path, encoding="utf-8") as _f: - _early_cfg_raw = _yaml_early.safe_load(_f) or {} + _early_cfg_raw = _yaml_early.load( + _f, Loader=getattr(_yaml_early, "CSafeLoader", None) or _yaml_early.SafeLoader + ) or {} # Managed scope: overlay administrator-pinned values so a managed # security.redact_secrets / network.force_ipv4 wins here too. This early # bridge reads config.yaml directly (before load_config is usable), so diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index b10f9ae13597..d343b077a7a3 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -47,7 +47,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Union from hermes_constants import get_hermes_home -from utils import env_var_enabled +from utils import env_var_enabled, fast_safe_load from hermes_cli.config import cfg_get from hermes_cli.middleware import OBSERVER_SCHEMA_VERSION, VALID_MIDDLEWARE @@ -1469,7 +1469,7 @@ def _parse_manifest( if yaml is None: logger.warning("PyYAML not installed – cannot load %s", manifest_file) return None - data = yaml.safe_load(manifest_file.read_text(encoding="utf-8")) or {} + data = fast_safe_load(manifest_file.read_text(encoding="utf-8")) or {} name = data.get("name", plugin_dir.name) key = f"{prefix}/{plugin_dir.name}" if prefix else name diff --git a/hermes_logging.py b/hermes_logging.py index 9e34fbaafbcc..40d7fb44fda4 100644 --- a/hermes_logging.py +++ b/hermes_logging.py @@ -552,11 +552,11 @@ def _read_logging_config(): Returns ``(level, max_size_mb, backup_count)`` — any may be ``None``. """ try: - import yaml + from utils import fast_safe_load config_path = get_config_path() if config_path.exists(): with open(config_path, "r", encoding="utf-8") as f: - cfg = yaml.safe_load(f) or {} + cfg = fast_safe_load(f) or {} # Managed scope: an administrator can pin logging.* too. Overlay via # the shared helper (fail-open) since this reads config.yaml directly. try: diff --git a/tests/test_fast_safe_load.py b/tests/test_fast_safe_load.py new file mode 100644 index 000000000000..840829d3dfee --- /dev/null +++ b/tests/test_fast_safe_load.py @@ -0,0 +1,62 @@ +"""Invariants for utils.fast_safe_load. + +fast_safe_load is a drop-in for yaml.safe_load that prefers the libyaml +CSafeLoader C extension for speed. These tests assert the behavior contract +(it parses identically to safe_load across input shapes), not a snapshot of +any particular document. +""" + +import io + +import yaml + +from utils import fast_safe_load, _get_fast_yaml_loader + + +_DOCS = [ + "", # empty document -> None + "a: 1\nb: two\nc: 3.5\n", + "list: [1, 2, 3]\nnested:\n k: v\n flag: true\n empty: null\n", + "name: skill-x\nmetadata:\n hermes:\n tags: [alpha, beta]\n category: devops\n", + "- one\n- two\n- three\n", # top-level sequence + "scalar string", # bare scalar +] + + +def test_equivalent_to_safe_load_for_strings(): + for doc in _DOCS: + assert fast_safe_load(doc) == yaml.safe_load(doc), repr(doc) + + +def test_equivalent_to_safe_load_for_file_objects(): + for doc in _DOCS: + assert fast_safe_load(io.StringIO(doc)) == yaml.safe_load(io.StringIO(doc)), repr(doc) + + +def test_empty_document_returns_none(): + # Callers rely on ``fast_safe_load(...) or {}`` — empty must be falsy. + assert fast_safe_load("") is None + + +def test_prefers_c_loader_when_available(): + loader = _get_fast_yaml_loader() + # If libyaml is compiled in, we must be using the C loader; otherwise the + # pure-Python SafeLoader is an acceptable fallback. Either way it must be a + # safe loader (never the unsafe full Loader). + c_loader = getattr(yaml, "CSafeLoader", None) + if c_loader is not None: + assert loader is c_loader + else: + assert loader is yaml.SafeLoader + + +def test_rejects_arbitrary_python_objects_like_safe_load(): + # Safe loaders must not construct arbitrary Python objects. This tag is + # accepted by the unsafe Loader but rejected by Safe/CSafe loaders. + dangerous = "!!python/object/apply:os.system ['echo pwned']\n" + try: + fast_safe_load(dangerous) + raised = False + except yaml.YAMLError: + raised = True + assert raised, "fast_safe_load must reject python/object tags like safe_load" diff --git a/utils.py b/utils.py index d7696a059c11..7d4ee3810ac7 100644 --- a/utils.py +++ b/utils.py @@ -339,6 +339,34 @@ def safe_json_loads(text: str, default: Any = None) -> Any: return default +# ── Fast YAML loading ──────────────────────────────────────────────────── +# +# PyYAML's pure-Python SafeLoader is ~8x slower than the libyaml-backed +# ``CSafeLoader`` C extension. Startup parses config.yaml and every plugin +# manifest with the slow path, costing ~0.9s of cold-start time. The C loader +# is a true drop-in for ``safe_load`` (same restricted tag set), so prefer it +# and fall back to the pure-Python loader only when libyaml isn't compiled in. +_fast_yaml_loader = None + + +def _get_fast_yaml_loader(): + global _fast_yaml_loader + if _fast_yaml_loader is None: + _fast_yaml_loader = getattr(yaml, "CSafeLoader", None) or yaml.SafeLoader + return _fast_yaml_loader + + +def fast_safe_load(stream: Any) -> Any: + """``yaml.safe_load`` using the libyaml C loader when available. + + Accepts the same inputs as ``yaml.safe_load`` (a ``str``/``bytes`` document + or a readable file object) and returns the same parsed structure. Falls + back to PyYAML's pure-Python ``SafeLoader`` when ``CSafeLoader`` isn't + available, so behavior is identical everywhere — only the speed differs. + """ + return yaml.load(stream, Loader=_get_fast_yaml_loader()) + + # ─── Environment Variable Helpers ─────────────────────────────────────────────