Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2749,10 +2749,12 @@ def _termux_example_image_path(filename: str = "cat.png") -> str:
"/storage/emulated/0",
"/storage/self/primary",
]
# Termux/Android roots are POSIX paths — join with literal forward
# slashes so the hint stays correct even when this renders on Windows.
for root in candidates:
if os.path.isdir(root):
return os.path.join(root, "Pictures", filename)
return os.path.join("~/storage/shared", "Pictures", filename)
return f"{root}/Pictures/{filename}"
return f"~/storage/shared/Pictures/{filename}"


def _split_path_input(raw: str) -> tuple[str, str]:
Expand Down Expand Up @@ -2823,6 +2825,16 @@ def _resolve_attachment_path(raw_path: str) -> Path | None:
expanded = unquote(parsed.path or "")
if parsed.netloc and os.name == "nt":
expanded = f"//{parsed.netloc}{expanded}"
elif (
os.name == "nt"
and len(expanded) >= 3
and expanded[0] == "/"
and expanded[1].isalpha()
and expanded[2] == ":"
):
# file:///C:/... parses to path "/C:/..." — drop the
# leading slash so it resolves as a drive-letter path.
expanded = expanded[1:]
except Exception:
expanded = token
expanded = os.path.expandvars(os.path.expanduser(expanded))
Expand Down
7 changes: 5 additions & 2 deletions gateway/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,9 +337,12 @@ def _command_line_belongs_to_profile(command: str, profile_home: Path) -> bool:
explicit ``HERMES_HOME=<path>``) on its argv; the default/root gateway runs
bare with no profile flag.
"""
command_lc = command.lower()
# Normalize separators before the substring match: on Windows,
# str(Path) renders backslashes while a HERMES_HOME= value on the argv
# may carry forward slashes (Git Bash, JSON configs) — and vice versa.
command_lc = command.lower().replace("\\", "/")
profile_name = _profile_name_for_home(profile_home)
home_lc = str(profile_home).lower()
home_lc = str(profile_home).lower().replace("\\", "/")

if profile_name is not None and profile_name != "default":
profile_lc = profile_name.lower()
Expand Down
9 changes: 8 additions & 1 deletion hermes_cli/banner.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,14 @@ def cprint(text: str):
"""Print ANSI-colored text through prompt_toolkit's renderer."""
from prompt_toolkit import print_formatted_text as _pt_print
from prompt_toolkit.formatted_text import ANSI as _PT_ANSI
_pt_print(_PT_ANSI(text))
try:
_pt_print(_PT_ANSI(text))
except Exception:
# prompt_toolkit needs a real console. On Windows, a redirected or
# absent stdout (pythonw.exe, CI, `hermes ... > file`) raises
# NoConsoleScreenBufferError from its Win32Output — display helpers
# must never crash the caller over that, so degrade to plain print.
print(text)


# =========================================================================
Expand Down
6 changes: 5 additions & 1 deletion hermes_cli/browser_connect.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import os
import platform
import posixpath
import shlex
import shutil
import subprocess
Expand Down Expand Up @@ -90,7 +91,10 @@ def add_windows_install_paths(
for _, group in install_groups:
for base in filter(None, bases):
for parts in group:
add(os.path.join(base, *parts))
# Only called with WSL ``/mnt/c/...`` bases — those are
# POSIX paths regardless of the host OS, so join with
# posixpath (os.path.join would emit backslashes on nt).
add(posixpath.join(base, *parts))

if system == "Darwin":
for app in _DARWIN_APPS:
Expand Down
171 changes: 160 additions & 11 deletions hermes_cli/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,15 +346,17 @@ def _scan_gateway_pids(
looks_like_gateway_runtime_command_line,
)
current_home = str(get_hermes_home().resolve())
current_home_lc = current_home.lower()
# Forward slashes on both sides of the HERMES_HOME= match — see
# gateway.status._command_line_belongs_to_profile, which this mirrors.
current_home_lc = current_home.lower().replace("\\", "/")
current_profile_arg = _profile_arg(current_home)
current_profile_name = (
current_profile_arg.split()[-1] if current_profile_arg else ""
)
current_profile_name_lc = current_profile_name.lower()

def _matches_current_profile(command: str) -> bool:
command_lc = command.lower()
command_lc = command.lower().replace("\\", "/")
if current_profile_name:
return (
f"--profile {current_profile_name_lc}" in command_lc
Expand Down Expand Up @@ -2655,7 +2657,15 @@ def generate_systemd_unit(system: bool = False, run_as_user: str | None = None)
path_entries = _build_service_path_dirs()
resolved_node = shutil.which("node")
if resolved_node:
resolved_node_dir = str(Path(resolved_node).resolve().parent)
# Use the directory where ``node`` is *found on PATH*, NOT the
# symlink's resolved target. ``~/.local/bin/node`` is often a symlink
# into a specific profile's node install (e.g. profiles/jarvis/node/
# bin/node); calling .resolve() here would chase that symlink and bake
# one profile's node path into *every* profile's service unit. That
# cross-profile leak makes systemd_unit_is_current() perpetually false,
# so each gateway rewrites its unit + daemon-reload on every boot. Using
# the symlink's own parent keeps the generated unit profile-agnostic.
resolved_node_dir = str(Path(resolved_node).parent)
if resolved_node_dir not in path_entries:
path_entries.append(resolved_node_dir)

Expand Down Expand Up @@ -3588,6 +3598,86 @@ def _launchctl_bootstrap(
)


def _launchd_reload_log_path() -> Path:
"""Path the launchd reload watchdog tails for persistent-orphan detection."""
return get_hermes_home() / "logs" / "launchd-reload.log"


def _append_launchd_reload_log(message: str) -> None:
"""Append a timestamped line to the launchd reload log (best-effort)."""
path = _launchd_reload_log_path()
try:
path.parent.mkdir(parents=True, exist_ok=True)
from datetime import datetime as _dt

stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z")
with path.open("a", encoding="utf-8") as fh:
fh.write(f"[{stamp}] {message}\n")
except OSError:
pass


def _launchctl_label_registered(label: str) -> bool:
"""True when ``launchctl list <label>`` reports the job as registered."""
try:
result = subprocess.run(
["launchctl", "list", label],
check=False,
timeout=10,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return result.returncode == 0
except (subprocess.TimeoutExpired, OSError):
return False


def _retry_launchctl_bootstrap_until_registered(
domain: str, plist_path, label: str, *, deadline: float
) -> bool:
"""Bootstrap with retry until the label is registered or ``deadline`` passes.

Wraps :func:`_launchctl_bootstrap` (which already recovers the EIO
"already loaded" case) in a wall-clock retry loop for the *transient*
failure mode: under high load or a launchd race the bootstrap can fail
even after ``bootout`` already tore down the prior registration, leaving
the service orphaned from ``KeepAlive`` supervision. The reported incident
happened during a graceful drain (default ``agent.restart_drain_timeout``
= 180s), so a fixed ~10s window is too short — retry until ``deadline``.

Both ``CalledProcessError`` and ``TimeoutExpired`` are treated as
retryable: a ``bootstrap`` that times out after ``bootout`` still leaves
the service unloaded, so it must be retried, not allowed to escape. On
each failure a timestamped line is appended to the reload log; success is
confirmed with ``launchctl list`` (not merely a zero bootstrap exit).
Returns True once the label is registered, False if the deadline is hit.
"""
attempt = 0
while True:
attempt += 1
try:
_launchctl_bootstrap(domain, plist_path, label, timeout=30)
if _launchctl_label_registered(label):
return True
_append_launchd_reload_log(
f"bootstrap attempt {attempt} exited 0 but {domain}/{label} "
f"is not registered (launchctl list) — retrying"
)
except subprocess.CalledProcessError as exc:
_append_launchd_reload_log(
f"bootstrap attempt {attempt} failed (rc={exc.returncode}) "
f"for {domain}/{label} — retrying"
)
except subprocess.TimeoutExpired:
_append_launchd_reload_log(
f"bootstrap attempt {attempt} timed out for {domain}/{label} "
f"— retrying"
)
if time.monotonic() >= deadline:
return False
time.sleep(2)


# ── launchd unsupported marker ─────────────────────────────────────────────
# When launchd can't manage the domain on this host (error 5/125, macOS 26+),
# we write a persistent marker so `launchd_status()` can explain that launchd
Expand Down Expand Up @@ -3727,7 +3817,13 @@ def generate_launchd_plist() -> str:
priority_dirs = _build_service_path_dirs()
resolved_node = shutil.which("node")
if resolved_node:
resolved_node_dir = str(Path(resolved_node).resolve().parent)
# Use the directory where ``node`` is *found on PATH*, NOT the symlink's
# resolved target. ``~/.local/bin/node`` is often a symlink into a
# specific profile's node install; calling .resolve() would chase it and
# bake one profile's path into every profile's service definition,
# breaking profile isolation and causing perpetual unit rewrites. See
# the matching fix in generate_systemd_unit().
resolved_node_dir = str(Path(resolved_node).parent)
if resolved_node_dir not in priority_dirs:
priority_dirs.append(resolved_node_dir)
sane_path = ":".join(
Expand Down Expand Up @@ -3855,11 +3951,42 @@ def refresh_launchd_plist_if_needed() -> bool:
# Delegate to a new session: `start_new_session=True` detaches the
# helper from the gateway's process group, so the bootout that kills
# the gateway (and us) does not kill the helper before it bootstraps.
#
# The bootstrap is retried up to 5 times with verification: under
# high load (loadavg observed >= 9) or a launchd race, the bootout
# can succeed (removing the service from launchd) while the
# follow-up bootstrap fails silently. Without retry+verify the
# service stays unregistered — KeepAlive can't revive a service
# launchd no longer knows about, so the gateway stays dark until a
# manual `launchctl bootstrap`. Failures append a timestamped line
# to ~/.hermes/logs/launchd-reload.log, which the health watchdog
# can tail to detect a persistent orphan. See hermes-restart
# rootcause handoff (2026-06-26 incident).
reload_log_path = get_hermes_home() / "logs" / "launchd-reload.log"
try:
reload_log_path.parent.mkdir(parents=True, exist_ok=True)
except OSError:
pass
# Retry until launchctl LISTS the label (not merely a zero bootstrap
# exit) or the drain window elapses. The failure happens while the old
# gateway is still draining (default agent.restart_drain_timeout=180s),
# so a fixed ~10s window is too short — bound by that budget instead.
_reload_budget = int(max(30.0, _get_restart_drain_timeout()))
reload_script = (
f"sleep 2; "
f"launchctl bootout {shlex.quote(target)} 2>/dev/null; "
f"sleep 1; "
f"launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null"
f"_deadline=$(($(date +%s) + {_reload_budget})); "
f"while :; do "
f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; "
f" if launchctl list {shlex.quote(label)} >/dev/null 2>&1; then break; fi; "
f" echo \"[$(date '+%Y-%m-%d %H:%M:%S %z')] bootstrap not yet registered for {shlex.quote(target)} — retrying\" >> {shlex.quote(str(reload_log_path))}; "
f" if [ $(date +%s) -ge $_deadline ]; then break; fi; "
f" sleep 2; "
f"done; "
f"if ! launchctl list {shlex.quote(label)} >/dev/null 2>&1; then "
f" echo \"[$(date '+%Y-%m-%d %H:%M:%S %z')] FAILED launchd reload for {shlex.quote(target)} — service NOT registered after {_reload_budget}s of retries\" >> {shlex.quote(str(reload_log_path))}; "
f"fi"
)
try:
subprocess.Popen(
Expand All @@ -3877,17 +4004,39 @@ def refresh_launchd_plist_if_needed() -> bool:
)
return True

# Bootout/bootstrap so launchd picks up the new definition
# Bootout/bootstrap so launchd picks up the new definition. The reported
# incident (2026-06-26) happened when bootout succeeded but bootstrap
# failed silently under load (loadavg 9.48) during a graceful /restart
# drain, leaving the service unregistered — KeepAlive can't revive a job
# launchd no longer knows about. Retry the bootstrap (via the shared
# _launchctl_bootstrap EIO-recovery helper) until the label is actually
# registered or the drain window elapses, verify with `launchctl list`,
# and log exhaustion so the reload watchdog can detect a persistent orphan.
subprocess.run(
["launchctl", "bootout", target],
check=False,
timeout=90,
)
subprocess.run(
["launchctl", "bootstrap", domain, str(plist_path)],
check=False,
timeout=30,
)
# Size the retry window to the restart drain timeout (default 180s), not a
# fixed ~10s: the failure mode occurs while the old gateway is still
# draining, so a short window can exhaust before launchd settles.
_reload_budget = max(30.0, _get_restart_drain_timeout())
_deadline = time.monotonic() + _reload_budget
if not _retry_launchctl_bootstrap_until_registered(
domain, plist_path, label, deadline=_deadline
):
_append_launchd_reload_log(
f"FAILED launchd reload of {target} — service NOT registered after "
f"retrying for {int(_reload_budget)}s (refresh ran outside gateway "
f"process tree)"
)
logger.error(
"launchd reload of %s failed — service not registered after %ds of "
"retries; see %s",
target,
int(_reload_budget),
_launchd_reload_log_path(),
)
print(
"↻ Updated gateway launchd service definition to match the current Hermes install"
)
Expand Down
30 changes: 30 additions & 0 deletions tests/cli/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""Shared fixtures for tests/cli.

Several test modules here build a HermesCLI via ``importlib.reload(cli)``
with prompt_toolkit stubbed out in ``sys.modules`` (the ``_make_cli``
pattern from test_cli_init.py). ``importlib.reload()`` re-executes cli.py
into the SAME module dict, so those MagicMock bindings (``_pt_print``,
``_PT_ANSI``, ...) survive the ``patch.dict`` context and silently break
any later test that needs cli's real prompt_toolkit machinery — e.g.
``cli._cprint`` output vanishes into a MagicMock and capsys sees nothing.

The autouse fixture below restores the real bindings at each module
boundary by re-reloading cli (with the real prompt_toolkit back in
sys.modules) whenever the pollution is detected.
"""

import importlib
import sys
from unittest.mock import MagicMock

import pytest


@pytest.fixture(autouse=True, scope="module")
def _unpollute_cli_module():
yield
cli_mod = sys.modules.get("cli")
if cli_mod is not None and isinstance(
getattr(cli_mod, "_pt_print", None), MagicMock
):
importlib.reload(cli_mod)
18 changes: 18 additions & 0 deletions tests/cli/test_cli_file_drop.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,21 @@
from cli import _detect_file_drop


def _can_symlink():
"""Check if we can create symlinks (needs admin/dev-mode on Windows)."""
import tempfile
from pathlib import Path
try:
with tempfile.TemporaryDirectory() as d:
src = Path(d) / "src"
src.write_text("x")
lnk = Path(d) / "lnk"
lnk.symlink_to(src)
return True
except OSError:
return False


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -212,6 +227,8 @@ def test_tilde_prefixed_path(self, tmp_path, monkeypatch):
img.parent.mkdir(parents=True, exist_ok=True)
img.write_bytes(b"\x89PNG\r\n\x1a\n")
monkeypatch.setenv("HOME", str(home))
# ntpath.expanduser ignores HOME (Python 3.8+) — it wants USERPROFILE.
monkeypatch.setenv("USERPROFILE", str(home))

result = _detect_file_drop("~/storage/shared/Pictures/cat.png what is this?")

Expand Down Expand Up @@ -241,6 +258,7 @@ def test_path_that_looks_like_command_but_is_file(self, tmp_path):
assert result is not None
assert result["is_image"] is False

@pytest.mark.skipif(not _can_symlink(), reason="Symlinks need elevated privileges")
def test_symlink_to_file(self, tmp_image, tmp_path):
link = tmp_path / "link.png"
link.symlink_to(tmp_image)
Expand Down
2 changes: 2 additions & 0 deletions tests/cli/test_cli_image_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ def test_collect_query_images_supports_tilde_paths(self, tmp_path, monkeypatch):
home = tmp_path / "home"
img = _make_image(home / "storage" / "shared" / "Pictures" / "cat.png")
monkeypatch.setenv("HOME", str(home))
# ntpath.expanduser ignores HOME (Python 3.8+) — it wants USERPROFILE.
monkeypatch.setenv("USERPROFILE", str(home))

message, images = _collect_query_images("describe this", "~/storage/shared/Pictures/cat.png")

Expand Down
Loading
Loading