Skip to content

fix(macos): clearly distinguish launchd supervision from detached fal… #42567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Dr1985 wants to merge 1 commit into from
+311 −14
Closed

fix(macos): clearly distinguish launchd supervision from detached fal… #42567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 143 additions & 14 deletions hermes_cli/gateway.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1064,7 +1064,36 @@ def _recover_pending_systemd_restart(
return False


def _parse_launchd_pid_from_list_output(output: str) -> int | None:
"""Extract the PID from ``launchctl list <label>`` output.

When launchd is actively supervising a process, the output includes a
``"PID" = <number>;`` line. When the service definition is only *registered*
but not running (macOS 26+ with an unmanageable domain, fallback active),
the output lacks a PID field entirely. Returns ``None`` when no PID is
found.
"""
for line in output.splitlines():
stripped = line.strip()
if stripped.startswith('"PID"') or stripped.startswith("PID"):
parts = stripped.split("=", 1)
if len(parts) == 2:
val = parts[1].strip().rstrip(";").strip('"')
try:
return int(val)
except ValueError:
return None
return None


def _probe_launchd_service_running() -> bool:
"""Return True when launchd is actively supervising the gateway process.

``launchctl list <label>`` returns exit 0 whenever the service definition is
registered with launchd — even when ``state = not running`` (macOS 26+).
We additionally require a PID in the output to confirm launchd is actually
managing a live process, not just holding a static definition.
"""
if not get_launchd_plist_path().exists():
return False
try:
Expand All @@ -1076,7 +1105,9 @@ def _probe_launchd_service_running() -> bool:
)
except subprocess.TimeoutExpired:
return False
return result.returncode == 0
if result.returncode != 0:
return False
return _parse_launchd_pid_from_list_output(result.stdout) is not None


def get_gateway_runtime_snapshot(system: bool = False) -> GatewayRuntimeSnapshot:
Expand Down Expand Up @@ -1151,12 +1182,23 @@ def _print_gateway_process_mismatch(snapshot: GatewayRuntimeSnapshot) -> None:
if not snapshot.has_process_service_mismatch:
return
print()
print(
"⚠ Gateway process is running for this profile, but the service is not active"
)
print(f" PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
print(" This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
print(" can refuse to start another copy until this process stops.")
# Distinguish the managed detached fallback (macOS launchd exit-5 path)
# from a genuinely manual foreground/tmux/nohup run.
if _launchd_unsupported_marker_exists():
print(
"⚠ Gateway is running as a detached fallback process — "
"launchd cannot supervise it"
)
print(f" PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
print(" Auto-start at login and auto-restart on crash are NOT available.")
print(" Stop it with: hermes gateway stop")
else:
print(
"⚠ Gateway process is running for this profile, but the service is not active"
)
print(f" PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
print(" This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
print(" can refuse to start another copy until this process stops.")


def _print_other_profiles_gateway_status() -> None:
Expand Down Expand Up @@ -3102,6 +3144,44 @@ def _launchctl_domain_unsupported(returncode: int) -> bool:
return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES


# ── launchd unsupported marker ─────────────────────────────────────────────
# When launchd can't manage the domain on this host (error 5/125, macOS 26+),
# we write a persistent marker so `launchd_status()` can explain that launchd
# supervision is unavailable regardless of whether a fallback process is
# currently running. The marker is cleared when bootstrap/kickstart succeeds,
# so an OS update that fixes the underlying issue allows automatic recovery.


def _launchd_unsupported_marker_path() -> Path:
return get_hermes_home() / ".gateway-launchd-unsupported"


def _write_launchd_unsupported_marker() -> None:
"""Persist that launchd cannot supervise the gateway on this host."""
try:
_launchd_unsupported_marker_path().write_text(
json.dumps({
"written_at": datetime.now(timezone.utc).isoformat(),
"reason": "launchd domain unsupported (exit 5/125)",
}),
encoding="utf-8",
)
except OSError:
pass


def _clear_launchd_unsupported_marker() -> None:
"""Clear the unsupported marker when launchd bootstrap succeeds."""
try:
_launchd_unsupported_marker_path().unlink(missing_ok=True)
except OSError:
pass


def _launchd_unsupported_marker_exists() -> bool:
return _launchd_unsupported_marker_path().exists()


def _gateway_run_command() -> list[str]:
"""Build the `python -m hermes_cli.main [--profile X] gateway run --replace` argv.

Expand Down Expand Up @@ -3159,6 +3239,7 @@ def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True)
"""
from hermes_constants import display_hermes_home as _dhh

_write_launchd_unsupported_marker()
print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
if _spawn_detached_gateway():
print("✓ Started gateway as a background process instead")
Expand Down Expand Up @@ -3347,6 +3428,7 @@ def launchd_install(force: bool = False):

print()
print("✓ Service installed and loaded!")
_clear_launchd_unsupported_marker()
print()
print("Next steps:")
print(" hermes gateway status # Check status")
Expand Down Expand Up @@ -3397,6 +3479,7 @@ def launchd_start():
_launchd_fallback_to_detached(f"launchctl exit {e.returncode}")
return
print("✓ Service started")
_clear_launchd_unsupported_marker()
return

refresh_launchd_plist_if_needed()
Expand Down Expand Up @@ -3430,6 +3513,7 @@ def launchd_start():
_launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
return
print("✓ Service started")
_clear_launchd_unsupported_marker()


def launchd_stop():
Expand Down Expand Up @@ -3539,6 +3623,7 @@ def launchd_restart():
)
subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90)
print("✓ Service restarted")
_clear_launchd_unsupported_marker()
except subprocess.CalledProcessError as e:
if not _launchd_error_indicates_unloaded(e):
# Not a "job unloaded" code. If the domain is fundamentally
Expand All @@ -3564,6 +3649,7 @@ def launchd_restart():
_launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
return
print("✓ Service restarted")
_clear_launchd_unsupported_marker()


def launchd_status(deep: bool = False):
Expand All @@ -3576,26 +3662,69 @@ def launchd_status(deep: bool = False):
text=True,
timeout=10,
)
loaded = result.returncode == 0
loaded_output = result.stdout
service_listed = result.returncode == 0
list_output = result.stdout
except subprocess.TimeoutExpired:
loaded = False
loaded_output = ""
service_listed = False
list_output = ""

# Determine whether launchd is actively supervising a process.
# ``launchctl list`` returns exit 0 whenever the service definition is
# registered — even when ``state = not running`` (macOS 26+ with an
# unmanageable domain). A PID in the output confirms a live process.
launchd_pid = _parse_launchd_pid_from_list_output(list_output) if service_listed else None

# Hermes PID tracking — may be a detached fallback process spawned when
# launchd cannot manage the domain on this host.
from gateway.status import get_running_pid
fallback_pid = get_running_pid(cleanup_stale=False)

# Avoid double-counting: when launchd IS supervising, fallback_pid and
# launchd_pid point at the same process (the gateway writes both the
# launchd PID and the Hermes PID file).
if launchd_pid is not None and fallback_pid == launchd_pid:
fallback_pid = None

# Persistent marker written when launchd bootstrap/kickstart fails with
# exit 5/125 on this host. Lets us explain *why* launchd can't supervise
# even when no fallback process is currently running.
launchd_unsupported = _launchd_unsupported_marker_exists()

# ── Report ──
print(f"Launchd plist: {plist_path}")
if launchd_plist_is_current():
print("✓ Service definition matches the current Hermes install")
else:
print("⚠ Service definition is stale relative to the current Hermes install")
print(" Run: hermes gateway start")

if loaded:
print("✓ Gateway service is loaded")
print(loaded_output)
if service_listed:
if launchd_pid is not None:
print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})")
print(" Auto-start at login and auto-restart on crash are available.")
if launchd_unsupported:
print(" (launchd domain was previously unavailable but is now working)")
elif launchd_unsupported:
print("⚠ Gateway service is registered but launchd is not supervising it")
print(" launchd cannot manage the gateway on this macOS version.")
if fallback_pid:
print(f"✓ Detached fallback process is running (PID {fallback_pid})")
print(" Cron jobs will fire. Stop with: hermes gateway stop")
else:
print("✗ No fallback process is running")
print(" Run: hermes gateway start")
print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.")
else:
print("✓ Gateway service is registered with launchd")
print(list_output)
if fallback_pid:
print(f" Detached gateway process is running (PID {fallback_pid})")
else:
print("✗ Gateway service is not loaded")
print(" Service definition exists locally but launchd has not loaded it.")
print(" Run: hermes gateway start")
if fallback_pid:
print(f" Note: a detached gateway process is running (PID {fallback_pid})")

if deep:
log_file = get_hermes_home() / "logs" / "gateway.log"
Expand Down
Loading