fix(kanban): bootstrap managed Hermes relaunches

gateway/run.py

@@ -2910,21 +2910,22 @@ def _get_channel_override(
 
 
 def _resolve_hermes_bin() -> Optional[list[str]]:
-    """Hermes update/restart argv: the running interpreter's ``python -m hermes_cli.main``
-    (exactly this install), else ``hermes`` on PATH, else None. The module argv must win: a
-    PATH-first lookup lets an attacker-planted ``hermes`` shadow the running install when
-    /update or /restart re-execs it (#111569)."""
+    """Build an installation-bound Hermes update/restart command.
+
+    Re-enter the current managed checkout through its durable launcher or
+    isolated bootstrap before consulting PATH. A plain ``python -m`` loses the
+    source/dependency roots that the launcher activated, especially after the
+    process changes CWD or starts from an isolated child environment.
+    """
     try:
-        import importlib.util
-        if importlib.util.find_spec("hermes_cli") is not None:
-            return [sys.executable, "-m", "hermes_cli.main"]
+        from hermes_cli._launchers import current_installation_command
+
+        return current_installation_command()
     except Exception:
-        pass
-    import shutil
-    hermes_bin = shutil.which("hermes")
-    if hermes_bin:
-        return [hermes_bin]
-    return None
+        import shutil
+
+        hermes_bin = shutil.which("hermes")
+        return [hermes_bin] if hermes_bin else None
 
 
 _PROFILE_ID_KEY_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")

hermes_cli/_launchers.py

@@ -76,6 +76,17 @@ def installation_command(repo_root: Path, args=(), *, module: str = "hermes_cli.
     prefix = [] if module == "hermes_cli.main" else ["--run-module", module]
     return [str(root / ".hermes" / "bin" / "hermes"), *prefix, *args]
 
+
+def current_installation_command(args=(), *, module: str = "hermes_cli.main") -> list[str]:
+    """Build an argv for this imported Hermes checkout without consulting CWD or PATH.
+
+    Managed source installs use their durable local launcher; developer/Nix
+    checkouts use ``runtime_command``'s isolated bootstrap with this module's
+    absolute source root. Both paths survive a Kanban workspace CWD and ``-I``.
+    """
+    root = Path(__file__).resolve().parents[1]
+    return installation_command(root, args, module=module)
+
 #: Launcher command names — keep in lockstep with scripts/install.ps1
 #: Publish-UserCommand and hermes_cli/_install_repair.py.
 WINDOWS_BIN_LAUNCHERS = ("hermes", "hermes-acp")

hermes_cli/kanban_db_dispatch.py

@@ -10,6 +10,7 @@
 import contextlib
 import os
 import re
+import shutil
 import signal
 import sqlite3
 import subprocess
@@ -2442,10 +2443,11 @@ def _rotate_worker_log(
         pass
 
 
-def _module_hermes_argv() -> list[str]:
-    """Interpreter-bound Hermes CLI invocation (``hermes_cli.main`` is the
-    console-script target — there is no top-level ``hermes`` package)."""
-    return [sys.executable, "-m", "hermes_cli.main"]
+def _current_installation_argv() -> list[str]:
+    """Bind a child to this Hermes checkout, not the dispatcher's CWD/PATH."""
+    from hermes_cli._launchers import current_installation_command
+
+    return current_installation_command()
 
 
 def _absolute_hermes_path(path: str) -> str:
@@ -2500,46 +2502,43 @@ def _safe_which_no_cwd(command: str) -> Optional[str]:
 def _hermes_path_argv(path: str) -> list[str]:
     """argv for a resolved Hermes executable path. Windows batch shims
     (``.cmd``/``.bat``) are unsafe as argv[0] because the argument vector
-    includes task-derived values; prefer the module form."""
+    includes task-derived values; use the installation-bound bootstrap instead.
+    """
     if _kb._IS_WINDOWS and _is_windows_batch_shim(path):
-        return _module_hermes_argv()
+        return _current_installation_argv()
     return [_absolute_hermes_path(path)]
 
 
 def _resolve_hermes_argv() -> list[str]:
-    """Resolve the ``hermes`` invocation as argv for ``Popen``: ``$HERMES_BIN``
-    (path-like -> absolute; bare names keep PATH semantics, never a
-    same-directory file), then the running interpreter's ``sys.executable -m
-    hermes_cli.main`` (exactly this install; also covers shim-less cron,
-    systemd ``User=``, launchd), then ``which("hermes")`` (Windows: safe PATH
-    search, batch shims fall back to the module form) only when ``hermes_cli``
-    is not importable. The module argv must win over PATH: a PATH-first lookup
-    lets an attacker-planted ``hermes`` shadow the running install (#111569).
-    Mirrors ``gateway.run._resolve_hermes_bin``; local because ``hermes_cli``
-    sits below ``gateway`` in the dependency order.
+    """Resolve the installed Hermes CLI for a dispatcher-spawned worker.
+
+    A valid explicit ``HERMES_BIN`` remains an operator override. Otherwise
+    use the current source install's durable launcher or ``-I`` bootstrap;
+    unlike ``sys.executable -m hermes_cli.main``, that preserves import roots
+    activated by the launcher when the child starts from a Kanban worktree.
+    PATH is only a last-resort compatibility path when the install bootstrap
+    cannot be constructed.
     """
-    import importlib.util
-    import shutil
-
     env_bin = os.environ.get("HERMES_BIN", "").strip()
     if env_bin:
         if _looks_like_path(env_bin):
             return _hermes_path_argv(env_bin)
         resolved_env_bin = _safe_which_no_cwd(env_bin)
         if resolved_env_bin:
             return _hermes_path_argv(resolved_env_bin)
-        return _module_hermes_argv()
 
+    install_error = None
     try:
-        if importlib.util.find_spec("hermes_cli") is not None:
-            return _module_hermes_argv()
-    except Exception:
-        pass
+        return _current_installation_argv()
+    except Exception as exc:
+        install_error = exc
 
     hermes_bin = _safe_which_no_cwd("hermes") if _kb._IS_WINDOWS else shutil.which("hermes")
     if hermes_bin:
         return _hermes_path_argv(hermes_bin)
-    return _module_hermes_argv()
+    raise RuntimeError(
+        "cannot resolve an installation-bound Hermes CLI; set HERMES_BIN to an executable path"
+    ) from install_error
 
 
 def _worker_terminal_timeout_env(

hermes_cli/relaunch.py

@@ -3,7 +3,6 @@
 
 import os
 import pathlib
-import shutil
 import sys
 from typing import Optional, Sequence
 
@@ -60,13 +59,14 @@ def _extract_inherited_flags(argv: Sequence[str]) -> list[str]:
 
 
 def resolve_hermes_bin() -> Optional[str]:
-    """Hermes entry point: ``sys.argv[0]`` if a real executable, else ``which hermes``, else ``None``
-    (caller falls back to ``python -m hermes_cli.main``).
+    """Return an absolute, safe ``sys.argv[0]`` executable, or ``None``.
 
+    ``None`` makes relaunch use :func:`hermes_cli._launchers.current_installation_command`,
+    which is independent of the current directory and inherited PATH/Python paths.
     Python launchers are never returned: on Windows a ``.py`` can't be exec'd directly, and on
     POSIX the git installer runs the extensionless source launcher under the managed venv
     interpreter while its ``#!/usr/bin/env python3`` shebang would pick the system Python and
-    lose the venv. Falling through to ``sys.executable -m hermes_cli.main`` keeps the venv.
+    lose the venv.
     """
     argv0 = sys.argv[0]
     _is_windows = sys.platform == "win32"
@@ -83,28 +83,27 @@ def _is_unsafe_python_launcher(p: str) -> bool:
 
         return _needs_interpreter(pathlib.Path(p))
 
-    # Absolute executable (nix store, venv wrappers, …), then relative-to-CWD, then PATH.
+    # Never interpret a bare argv[0] relative to the worker CWD: that directory
+    # is often an unrelated Kanban worktree which may contain its own `hermes`.
     if (
         os.path.isabs(argv0) and os.path.isfile(argv0) and os.access(argv0, os.X_OK)
         and not _is_unsafe_python_launcher(argv0)
     ):
         return argv0
-    if not argv0.startswith("-") and os.path.isfile(argv0):
-        abs_path = os.path.abspath(argv0)
-        if os.access(abs_path, os.X_OK) and not _is_unsafe_python_launcher(abs_path):
-            return abs_path
-    path_bin = shutil.which("hermes")
-    if path_bin and not _is_unsafe_python_launcher(path_bin):
-        return path_bin
     return None
 
 
 def build_relaunch_argv(
     extra_args: Sequence[str], *, preserve_inherited: bool = True, original_argv: Optional[Sequence[str]] = None
 ) -> list[str]:
-    """Construct an argv list for replacing the current process with hermes."""
+    """Construct an argv list for replacing the current process with this Hermes installation."""
     bin_path = resolve_hermes_bin()
-    argv = [bin_path] if bin_path else [sys.executable, "-m", "hermes_cli.main"]
+    if bin_path:
+        argv = [bin_path]
+    else:
+        from hermes_cli._launchers import current_installation_command
+
+        argv = current_installation_command()
     src = list(original_argv) if original_argv is not None else list(sys.argv[1:])
     if preserve_inherited:
         argv.extend(_extract_inherited_flags(src))
@@ -129,12 +128,12 @@ def relaunch(
         except KeyboardInterrupt:
             sys.exit(130)
         except OSError as exc:
-            # Raw ``[Errno 8] Exec format error`` is cryptic; usual causes are ``hermes`` not on
-            # PATH yet (install hasn't propagated User PATH into this shell) or a stale shim.
+            # A missing install-bound launcher or a damaged runtime bootstrap is more likely than PATH
+            # here: managed relaunches no longer depend on ambient command discovery.
             print(
                 f"\nHermes relaunch failed: {exc}\n"
                 f"Command: {' '.join(new_argv)}\n"
-                f"Fix: open a new terminal so PATH picks up, then re-run hermes.",
+                f"Fix: repair the Hermes installation, then re-run the command.",
                 file=sys.stderr,
             )
             sys.exit(1)

tests/e2e/core/kanban/test_kanban_worker_relaunch_provider.py

@@ -0,0 +1,133 @@
+"""A dispatcher retry must keep its model/provider pin and launch from the task workspace.
+
+The first OpenAI Codex request returns 429; a fresh worker must use the same route,
+complete the card, and persist ``billing_provider=openai-codex``. HERMES_BIN is
+cleared and PATH contains no Hermes executable so dispatch uses the install-bound
+launcher/bootstrap rather than an ambient command.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import sqlite3
+import sys
+import time
+
+import pytest
+
+from tests.e2e.core.kanban._helpers import Board
+from tests.fakes.fake_llm_provider import Error, FakeLLMServer
+from tests.fakes.providers.openai_responses import (
+    FakeResponsesServer, FunctionCall, HttpError, Message, Turn,
+)
+
+pytestmark = [
+    pytest.mark.skipif(not sys.platform.startswith("linux"), reason="worker liveness uses /proc"),
+    pytest.mark.live_system_guard_bypass,
+]
+
+MODEL = "gpt-6-luna"
+PROVIDER = "openai-codex"
+
+
+def _codex_store() -> tuple[dict, str]:
+    """One local fake OAuth grant with enough TTL to avoid contacting the token endpoint."""
+    def segment(value: dict) -> str:
+        return base64.urlsafe_b64encode(json.dumps(value).encode()).rstrip(b"=").decode()
+
+    exp = int(time.time()) + 8 * 60 * 60
+    access_token = (
+        f"{segment({'alg': 'none'})}."
+        f"{segment({'exp': exp, 'sub': 'e2e-codex-user', 'https://api.openai.com/auth': {'chatgpt_account_id': 'e2e-codex-account'}})}.sig"
+    )
+    tokens = {"access_token": access_token, "refresh_token": "e2e-refresh-not-used"}
+    refreshed = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 60))
+    store = {
+        "version": 1,
+        "active_provider": PROVIDER,
+        "providers": {PROVIDER: {"tokens": tokens, "last_refresh": refreshed, "auth_mode": "chatgpt"}},
+        "credential_pool": {PROVIDER: [{
+            "id": "e2e-codex-login", "label": "device_code", "auth_type": "oauth",
+            "priority": 0, "source": "device_code", **tokens, "last_refresh": refreshed,
+        }]},
+    }
+    return store, access_token
+
+
+def _session_routes(board: Board) -> list[tuple[str, str]]:
+    path = board.hermes_home / "state.db"
+    if not path.is_file():
+        return []
+    conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5)
+    try:
+        rows = conn.execute(
+            """SELECT DISTINCT u.model, u.billing_provider
+               FROM session_model_usage AS u
+               JOIN sessions AS s ON s.id = u.session_id
+               WHERE s.source = 'kanban'"""
+        ).fetchall()
+        return [(str(model), str(provider)) for model, provider in rows]
+    finally:
+        conn.close()
+
+
+def test_worker_relaunch_and_rate_limit_retry_keep_pinned_provider(tmp_path) -> None:
+    auth_store, bearer = _codex_store()
+    codex_steps = [
+        HttpError(status=429, message="scripted Codex rate limit", retry_after=0),
+        Turn([FunctionCall("kanban_complete", {"summary": "Codex route survived retry"})]),
+        Turn([Message("card completed through OpenAI Codex")]),
+    ]
+
+    def default_route_must_not_be_used(_record: dict):
+        return Error(429, "default provider was used instead of the task override", retry_after=0)
+
+    with FakeLLMServer(default_route_must_not_be_used) as default_server, \
+            FakeResponsesServer(codex_steps, api_key=bearer) as codex_server:
+        board = Board(
+            tmp_path,
+            default_server.base_url,
+            env_extra={
+                "HERMES_BIN": "",
+                "HERMES_CODEX_BASE_URL": codex_server.base_url,
+                "PATH": "/usr/bin:/bin",
+            },
+        )
+        board.hermes_home.joinpath(".env").write_text(
+            f"OPENAI_API_KEY=sk-fake-default\nHERMES_CODEX_BASE_URL={codex_server.base_url}\n",
+            encoding="utf-8",
+        )
+        board.hermes_home.joinpath("auth.json").write_text(json.dumps(auth_store), encoding="utf-8")
+        try:
+            task_id = board.create(
+                "pinned Codex retry probe",
+                "--model", MODEL,
+                "--provider", PROVIDER,
+            )
+
+            first_tick = board.dispatch()
+            assert task_id in [row["task_id"] for row in first_tick.get("spawned", [])], board.diag(task_id)
+            first_pid = int(board.task(task_id)["worker_pid"])
+            board.wait_worker_exit(task_id, first_pid)
+
+            second_tick = board.dispatch()
+            assert task_id in [row["task_id"] for row in second_tick.get("spawned", [])], board.diag(task_id)
+            second_pid = int(board.task(task_id)["worker_pid"])
+            assert second_pid != first_pid
+            board.wait_worker_exit(task_id, second_pid)
+
+            assert board.task(task_id)["status"] == "done", board.diag(task_id)
+            outcomes = [run["outcome"] for run in board.runs(task_id)]
+            assert outcomes[:2] == ["rate_limited", "completed"], board.diag(task_id)
+            assert default_server.main_requests() == [], "task override fell through to the profile provider"
+            requests = codex_server.main_requests()
+            assert len(requests) == 3, board.diag(task_id)
+            assert codex_server.invalid_requests() == [], codex_server.invalid_requests()
+            assert all(request.get("model") == MODEL for request in requests), requests
+            assert all(
+                request["headers"].get("authorization") == f"Bearer {bearer}"
+                for request in codex_server.requests if request["kind"] == "main"
+            ), codex_server.requests
+            assert (MODEL, PROVIDER) in _session_routes(board), board.diag(task_id)
+        finally:
+            board.kill_workers()