fix(gateway): multiplex secondary profiles show as unreachable in per-profile status checks

gateway/run.py

@@ -17225,6 +17225,76 @@ async def _start_secondary_profile_adapters(self) -> int:
         except Exception:
             logger.debug("could not record served_profiles", exc_info=True)
 
+        # Refresh EACH secondary profile's OWN gateway_state.json so the
+        # dashboard's per-profile liveness check (gateway/status.py
+        # resolve_gateway_liveness, scoped to that profile's home) reports
+        # it running instead of falling through to a stale/dead-PID record
+        # left over from before multiplexing took over (or before this
+        # profile was ever brought up standalone). Without this, only the
+        # ACTIVE profile's status file is ever refreshed post-startup — a
+        # secondary profile's own file just rots at whatever it last said,
+        # so a genuinely healthy secondary profile can show "not running"
+        # in any UI that scopes its liveness check to that profile
+        # specifically (reported 2026-09-02 as "can't reach a secondary
+        # profile's bot in the desktop app" while its jobs/bot were fine).
+        #
+        # write_runtime_status's ``path=`` is required here, NOT
+        # _profile_runtime_scope: that scope only overrides the HERMES_HOME
+        # CONTEXTVAR, and gateway/status.py's identity-file paths
+        # deliberately read the process-level HERMES_HOME instead (#56986),
+        # so the contextvar override would silently no-op and rewrite the
+        # ACTIVE profile's own file again.
+        for profile_name, profile_home in profile_homes:
+            if profile_name == active:
+                continue
+            secondary_state_path = Path(profile_home) / "gateway_state.json"
+            try:
+                write_runtime_status(
+                    gateway_state="running",
+                    multiplex_secondary=True,
+                    path=secondary_state_path,
+                )
+            except Exception:
+                logger.debug(
+                    "could not refresh gateway_state.json for secondary "
+                    "profile '%s'",
+                    profile_name,
+                    exc_info=True,
+                )
+                continue
+            # Also re-stamp each of this profile's CONNECTED platforms with
+            # the live process's (writer_pid, writer_start_time) identity.
+            # The top-level refresh above only touches gateway_state/pid —
+            # each platform sub-entry carries its OWN writer_pid/
+            # writer_start_time fingerprint, and /api/status's cross-profile
+            # aggregation (_owned_platform_entries in hermes_cli/web_server.py)
+            # requires an EXACT match against the live gateway process before
+            # including that platform in the aggregated view. Without this,
+            # a secondary profile's platform entries stay fingerprinted to
+            # whatever process last wrote them (e.g. a dead pre-multiplex
+            # PID), so the profile shows as running but with NO connected
+            # platforms anywhere that reads the aggregation — exactly the
+            # "can't reach this profile's chat" symptom the gateway_state
+            # top-level fix alone does not cover (found 2026-09-02 while
+            # investigating a recurrence of the same desktop-connectivity
+            # report).
+            try:
+                profile_platform_map = self._profile_adapters.get(profile_name) or {}
+                for platform_enum in profile_platform_map:
+                    platform_name = getattr(platform_enum, "value", platform_enum)
+                    write_runtime_status(
+                        platform=platform_name,
+                        platform_state="connected",
+                        path=secondary_state_path,
+                    )
+            except Exception:
+                logger.debug(
+                    "could not refresh platform writer identity for "
+                    "secondary profile '%s'",
+                    profile_name,
+                    exc_info=True,
+                )
+
         return connected
 
     async def _start_one_profile_adapters(

gateway/status.py

@@ -683,11 +683,26 @@ def _record_matches_live_gateway_pid(
     profile's live gateway would make the dead profile look alive.  When the
     live command line cannot be read (Windows/permission), fall back to the
     persisted record so cross-platform behavior is preserved.
+
+    Exception: a record stamped ``multiplex_secondary: true`` describes a
+    SECONDARY profile served by a multiplex gateway's single SHARED process
+    (``gateway.multiplex_profiles: true`` — one bare ``gateway run`` command
+    with no ``-p``/``--profile`` flag serves every profile). The per-profile
+    ``-p <name>`` substring check below is written for the "one dedicated
+    process per profile" deployment and can never pass for a multiplex
+    secondary profile's shared process — that command line names no single
+    profile because it names all of them. Skip the substring check for a
+    record carrying that marker; the live-cmdline "looks like a gateway"
+    check above it (in the caller) plus the (pid, start_time) match already
+    guard against PID reuse, and the marker is only ever written by the
+    gateway's own multiplex startup path (never user-controllable input).
     """
     live_cmdline = _read_process_cmdline(pid)
     if live_cmdline:
         if not looks_like_gateway_runtime_command_line(live_cmdline):
             return False
+        if record.get("multiplex_secondary"):
+            return True
         if expected_home is not None and not _command_line_belongs_to_profile(
             live_cmdline, expected_home
         ):
@@ -1217,9 +1232,25 @@ def write_runtime_status(
     served_profiles: Any = _UNSET,
     session_store: Any = _UNSET,
     clear_profile_platforms: bool = False,
+    multiplex_secondary: Any = _UNSET,
+    path: Optional[Path] = None,
 ) -> None:
-    """Persist gateway runtime health information for diagnostics/status."""
-    path = _get_runtime_status_path()
+    """Persist gateway runtime health information for diagnostics/status.
+
+    ``path``: write to this file instead of the resolved
+    ``_get_runtime_status_path()``. Needed for a multiplex gateway's
+    secondary-profile refresh (see ``gateway/run.py``'s
+    ``_start_secondary_profile_adapters``): ``_get_runtime_status_path()``
+    deliberately resolves via ``_get_process_hermes_home()``, which SKIPS
+    the ``HERMES_HOME`` override contextvar on purpose (#56986 — gateway
+    identity files must not follow an active per-session profile-dispatch
+    override into the wrong directory). That means wrapping this call in
+    ``_profile_runtime_scope(profile_home)`` does NOT redirect the write —
+    it silently writes the ACTIVE profile's file again. An explicit
+    ``path`` is the only way to deliberately target another profile's own
+    ``gateway_state.json`` from the active profile's process.
+    """
+    path = path if path is not None else _get_runtime_status_path()
     payload = _read_json_file(path) or _build_runtime_status_record()
     previous_payload = copy.deepcopy(payload)
     current_record = _build_pid_record()
@@ -1261,6 +1292,14 @@ def write_runtime_status(
         # for a single-profile gateway. Lets `hermes status` show per-profile
         # coverage without a second probe.
         payload["served_profiles"] = list(served_profiles or [])
+    if multiplex_secondary is not _UNSET:
+        # Stamped True when this file is a SECONDARY profile's own
+        # gateway_state.json under a multiplex gateway (one shared process
+        # serving several profiles, no per-profile ``-p`` command line).
+        # Read by _record_matches_live_gateway_pid to skip its per-profile
+        # ``-p <name>`` command-line check, which can never match a shared
+        # multiplex process's bare argv. See that function's docstring.
+        payload["multiplex_secondary"] = bool(multiplex_secondary)
     if session_store is not _UNSET:
         state = "unknown"
         if isinstance(session_store, dict):

tests/gateway/test_multiplex_secondary_gateway_state.py

@@ -0,0 +1,196 @@
+"""Multiplex secondary profiles must show as "running" in per-profile
+liveness checks, not stuck on a stale/dead-PID gateway_state.json.
+
+Regression coverage for two layered bugs:
+
+1. ``_start_secondary_profile_adapters`` (gateway/run.py) never refreshed a
+   SECONDARY profile's own ``gateway_state.json`` after startup — only the
+   active/default profile's file got rewritten, so a secondary profile's
+   file just rotted at whatever it last said (often a dead PID from before
+   multiplexing, or before that profile was ever run standalone).
+
+2. Even after refreshing it, ``_record_matches_live_gateway_pid`` (gateway/
+   status.py) validates a *live* PID's command line against the profile's
+   ``-p <name>``/``--profile <name>`` flag — which a multiplex gateway's
+   single SHARED process never carries (it serves every profile from one
+   bare command). Without the ``multiplex_secondary`` marker this check
+   fails even for a freshly-written, genuinely-live record.
+"""
+
+from pathlib import Path
+
+from gateway.status import (
+    _record_matches_live_gateway_pid,
+    get_runtime_status_running_pid,
+    read_runtime_status,
+    write_runtime_status,
+)
+
+
+def _write_and_read(tmp_path, monkeypatch, *, multiplex_secondary):
+    """Write a runtime-status record scoped to tmp_path, as the gateway's
+    secondary-profile refresh path would, then read it back."""
+    write_runtime_status(
+        gateway_state="running",
+        multiplex_secondary=multiplex_secondary,
+        path=tmp_path / "gateway_state.json",
+    )
+    return read_runtime_status(tmp_path / "gateway_state.json")
+
+
+def test_write_runtime_status_stamps_multiplex_secondary_marker(tmp_path):
+    record = _write_and_read(tmp_path, None, multiplex_secondary=True)
+    assert record is not None
+    assert record.get("multiplex_secondary") is True
+    assert record.get("gateway_state") == "running"
+
+
+def test_write_runtime_status_omits_marker_when_not_passed(tmp_path):
+    write_runtime_status(
+        gateway_state="running", path=tmp_path / "gateway_state.json"
+    )
+    record = read_runtime_status(tmp_path / "gateway_state.json")
+    assert record is not None
+    assert "multiplex_secondary" not in record
+
+
+def test_multiplex_secondary_record_skips_per_profile_cmdline_check(
+    tmp_path, monkeypatch
+):
+    """A multiplex-secondary record must match a live PID even when that
+    PID's command line carries NO ``-p <profile>`` flag (the shared-process
+    case) — this is exactly what a non-multiplex per-profile record would
+    correctly REJECT (see the sibling assertion below)."""
+    record = {"multiplex_secondary": True}
+    monkeypatch.setattr(
+        "gateway.status._read_process_cmdline",
+        lambda pid: "/usr/bin/python -m hermes_cli.main gateway run",
+    )
+    monkeypatch.setattr(
+        "gateway.status.looks_like_gateway_runtime_command_line",
+        lambda cmdline: True,
+    )
+    assert _record_matches_live_gateway_pid(
+        record, 12345, expected_home=Path("/home/hermes/.hermes/profiles/summer")
+    )
+
+
+def test_non_multiplex_record_still_requires_matching_profile_cmdline(
+    tmp_path, monkeypatch
+):
+    """Sibling/contrast case: WITHOUT the marker, a shared-process command
+    line (no ``-p <profile>``) must still be rejected for a named profile's
+    expected_home — preserves the existing PID-reuse protection for the
+    "one dedicated process per profile" deployment this check was built for.
+    """
+    record = {}  # no multiplex_secondary marker
+    monkeypatch.setattr(
+        "gateway.status._read_process_cmdline",
+        lambda pid: "/usr/bin/python -m hermes_cli.main gateway run",
+    )
+    monkeypatch.setattr(
+        "gateway.status.looks_like_gateway_runtime_command_line",
+        lambda cmdline: True,
+    )
+    assert not _record_matches_live_gateway_pid(
+        record, 12345, expected_home=Path("/home/hermes/.hermes/profiles/summer")
+    )
+
+
+def test_resolve_gateway_liveness_reports_running_for_refreshed_secondary_profile(
+    tmp_path, monkeypatch
+):
+    """End-to-end: after the gateway's secondary-profile refresh writes a
+    fresh, multiplex_secondary-stamped record naming THIS process's own
+    PID, resolve_gateway_liveness (the exact function the desktop dashboard
+    calls, scoped to that profile's home) must report it running — closing
+    the loop on the actual reported symptom (a healthy secondary profile
+    showing as unreachable in the desktop app)."""
+    import os
+
+    from gateway.status import resolve_gateway_liveness
+
+    profile_home = tmp_path / "profiles" / "summer"
+    profile_home.mkdir(parents=True)
+    state_path = profile_home / "gateway_state.json"
+
+    write_runtime_status(
+        gateway_state="running",
+        multiplex_secondary=True,
+        path=state_path,
+    )
+
+    # write_runtime_status stamps pid/start_time from THIS test process —
+    # a real live PID, so the (pid, start_time) reuse guard passes and only
+    # the multiplex_secondary bypass under test is what makes the
+    # per-profile command-line check pass too.
+    monkeypatch.setattr(
+        "gateway.status._read_process_cmdline",
+        lambda pid: "/usr/bin/python -m hermes_cli.main gateway run",
+    )
+    monkeypatch.setattr(
+        "gateway.status.looks_like_gateway_runtime_command_line",
+        lambda cmdline: True,
+    )
+
+    liveness = resolve_gateway_liveness(
+        profile_dir=profile_home,
+        pid_probe=lambda *a, **k: None,  # no gateway.pid file for this profile
+        health_probe=None,
+    )
+    assert liveness.running is True
+    assert liveness.pid == os.getpid()
+
+
+def test_platform_entries_are_restamped_to_the_live_process(tmp_path):
+    """The gateway_state top-level refresh alone is NOT enough: each
+    platform sub-entry (e.g. "slack") carries its OWN writer_pid/
+    writer_start_time fingerprint, separate from the top-level pid.
+
+    hermes_cli/web_server.py's cross-profile /api/status aggregation
+    (``_owned_profile_platforms``) only includes a platform entry when its
+    writer_pid/writer_start_time EXACTLY match the profile's live gateway
+    process. If a secondary profile's platform entry is left stamped with
+    a stale/dead writer identity (e.g. from before multiplexing took over),
+    the profile's gateway_state.json can correctly say "running" while
+    still showing ZERO connected platforms to any caller that reads the
+    aggregation -- reproducing "can't reach this profile's chat in the
+    desktop app" even after the top-level gateway_state fix lands.
+
+    This asserts the fix: re-stamping a platform via
+    write_runtime_status(platform=..., path=...) after the top-level
+    refresh makes that entry pass the SAME ownership check
+    _owned_profile_platforms uses.
+    """
+    import os
+
+    state_path = tmp_path / "gateway_state.json"
+
+    # Simulate the old, stale writer identity a secondary profile's platform
+    # entry could be left with (a long-dead pre-multiplex PID/start_time).
+    write_runtime_status(
+        gateway_state="running",
+        multiplex_secondary=True,
+        path=state_path,
+    )
+
+    # This is the fix under test: re-stamp the platform entry via the live
+    # process (this test process stands in for the live gateway).
+    write_runtime_status(
+        platform="slack",
+        platform_state="connected",
+        path=state_path,
+    )
+
+    record = read_runtime_status(state_path)
+    slack_entry = record["platforms"]["slack"]
+
+    # Mirrors hermes_cli.web_server._owned_profile_platforms's exact-match
+    # ownership check against the live process's own identity.
+    live_pid = os.getpid()
+    from gateway.status import _get_process_start_time
+
+    live_start = _get_process_start_time(live_pid)
+    assert slack_entry.get("writer_pid") == live_pid
+    assert slack_entry.get("writer_start_time") == live_start
+    assert slack_entry.get("state") == "connected"