Cron .py scripts import Hermes and its dependencies again on POSIX managed-store installs (#123044, #123440, salvage #123051)

cron/scheduler_script.py

@@ -114,6 +114,54 @@ def _read_windows_pyvenv_cfg(venv_dir: Path) -> dict[str, str]:
     }
 
 
+# Mirrors ``python script.py``: sys.path[0] is the script's directory (none under -P), a real
+# ``__main__`` module that outlives the body (atexit/threads can still pickle its classes; a
+# runpy temp module is swapped out when the body returns), plus the live checkout next.
+_POSIX_SCRIPT_BOOTSTRAP = """\
+import importlib.machinery, os, sys, types
+repo, script = sys.argv[1], sys.argv[2]
+sys.argv = [script] + sys.argv[3:]
+if sys.flags.safe_path:
+    sys.path.insert(0, repo)
+else:
+    sys.path[0:1] = [os.path.dirname(script), repo]
+main = types.ModuleType("__main__")
+main.__file__ = script
+main.__loader__ = importlib.machinery.SourceFileLoader("__main__", script)
+main.__cached__ = None
+main.__builtins__ = __builtins__
+sys.modules["__main__"] = main
+with open(script, "rb") as f:
+    code = compile(f.read(), script, "exec")
+exec(code, main.__dict__)
+"""
+
+
+def _posix_cron_script_argv(script: Path) -> tuple[list[str], dict[str, str]]:
+    """POSIX managed-store installs run cron ``.py`` scripts on the selected dependency venv's
+    interpreter: the store Python has the repo and managed site-packages only on its in-process
+    ``sys.path``, so its children import neither (#123044). No ``PYTHONPATH``: everything the
+    script spawns would inherit it and a foreign interpreter would load the store's compiled
+    extensions (#123440). The venv resolves Hermes from its generation's workspace snapshot,
+    rebuilt only on a dependency change, so the bootstrap puts the live checkout first.
+    Lazy installs are off for the script's process tree: a script importing ``hermes_bootstrap``
+    could otherwise complete a source update and ``execv`` itself onto the bare store Python."""
+    from hermes_cli._launchers import resolve_store_python
+    from pm.environments import project_python
+
+    repo = Path(__file__).resolve().parents[1]
+    if resolve_store_python(repo) is None:
+        return [sys.executable, str(script)], {}
+    # project_python, not committed_venv as on Windows: a pre-PM venv selected before the first
+    # commit runs on its OWN interpreter here, so there is no ABI mix (#122183).
+    python = project_python(repo)
+    if not python.is_file():
+        # The caller's interpreter is the bare store Python here — the #123044 failure mode.
+        raise RuntimeError(f"dependency environment interpreter is missing: {python}")
+    return ([str(python), "-c", _POSIX_SCRIPT_BOOTSTRAP, str(repo), str(script)],
+            {"HERMES_DISABLE_LAZY_INSTALLS": "1"})
+
+
 def _windows_cron_python_invocation(python_exe: str) -> tuple[str, dict[str, str]]:
     """Hidden, output-capable Python invocation for Windows cron scripts. ``pythonw.exe`` loses
     captured output; uv venv launchers can re-exec the base console python and flash a window
@@ -320,7 +368,9 @@ def _resolve_script_path(script_path: str) -> tuple[Optional[Path], Optional[str
 def _script_argv(path: Path) -> tuple[Optional[list[str]], dict[str, str], Optional[str]]:
     """``(argv, env_overlay, error)`` for a validated script. Interpreter by extension — the
     shebang is deliberately NOT honoured (small, auditable surface): ``.sh``/``.bash`` → bash,
-    else ``sys.executable`` (Windows uv-venv overlay gets the .pth bootstrap)."""
+    else a Python chosen by ``_posix_cron_script_argv`` / ``_windows_cron_python_invocation``.
+    Interpreter selection reads PM's install records and may raise; callers run this inside
+    their ``try``."""
     if path.suffix.lower() in {".sh", ".bash"}:
         # which() finds Git Bash on Windows; None there → clear error instead of a "[WinError 2]".
         _bash = shutil.which("bash") or ("/bin/bash" if os.path.isfile("/bin/bash") else None)
@@ -331,6 +381,9 @@ def _script_argv(path: Path) -> tuple[Optional[list[str]], dict[str, str], Optio
                 "or rewrite the script as Python (.py)."
             )
         return [_bash, str(path)], {}, None
+    if sys.platform != "win32":
+        argv, env_overlay = _posix_cron_script_argv(path)
+        return argv, env_overlay, None
     python_exe, env_overlay = _windows_cron_python_invocation(sys.executable)
     if env_overlay:
         return _windows_cron_bootstrap_argv(python_exe, env_overlay, str(path)), env_overlay, None
@@ -355,11 +408,10 @@ def _run_job_script(
     if path is None:
         return False, err
     script_timeout = _get_script_timeout()
-    argv, env_overlay, err = _script_argv(path)
-    if argv is None:
-        return False, err
-
     try:
+        argv, env_overlay, err = _script_argv(path)
+        if argv is None:
+            return False, err
         from tools.environments.local import build_subprocess_env
         # Lossy decode only: keep the platform-default (locale) encoding — gating ``encoding=``
         # to win32 was deliberate (#66566: unconditional UTF-8 leaked into POSIX) — but

tests/cron/conftest.py

@@ -45,6 +45,13 @@ def register_job(self, job):
     return _make
 
 
+@pytest.fixture(autouse=True)
+def _no_managed_store(tmp_path, monkeypatch):
+    """Point PM's store at an empty dir: script runs must not select the host install's
+    dependency venv (POSIX cron scripts run on it when a store is committed)."""
+    monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "no-pm-store"))
+
+
 @pytest.fixture(autouse=True)
 def _default_cron_test_model(monkeypatch):
     """Pin a default HERMES_MODEL so cron run_job tests have a resolvable model."""

tests/cron/test_cron_script.py

@@ -305,8 +305,85 @@ def test_bootstrap_argv_falls_back_without_site_packages(self, cron_env, tmp_pat
         )
         assert argv == [sys.executable, str(script)]
 
+    @pytest.mark.platforms("posix")
+    def test_posix_managed_store_script_runs_on_venv_with_live_checkout(
+        self, cron_env, tmp_path, monkeypatch
+    ):
+        """#123044/#123440: on a POSIX managed-store install a cron ``.py`` script imports the
+        selected venv's packages, resolves Hermes from the LIVE checkout ahead of the venv's
+        workspace snapshot, keeps ``python script.py`` path and ``__main__`` semantics, and
+        leaves no ``PYTHONPATH`` for its own children to inherit."""
+        from cron import scheduler_script
+        from pm.environments import site_packages
+
+        venv = tmp_path / "selected-venv"
+        (venv / "bin").mkdir(parents=True)
+        (venv / "bin" / "python").symlink_to(sys.executable)
+        (venv / "pyvenv.cfg").write_text(
+            f"home = {Path(sys.base_prefix) / 'bin'}\ninclude-system-site-packages = false\n",
+            encoding="utf-8",
+        )
+        deps = site_packages(venv)
+        deps.mkdir(parents=True)
+        (deps / "probe_pkg.py").write_text("VALUE = 42\n", encoding="utf-8")
+        snapshot = tmp_path / "workspace-snapshot"
+        snapshot.mkdir()
+        (snapshot / "hermes_constants.py").write_text("STALE = True\n", encoding="utf-8")
+        (deps / "snapshot.pth").write_text(f"{snapshot}\n", encoding="utf-8")
+
+        monkeypatch.setattr(
+            "hermes_cli._launchers.resolve_store_python", lambda repo: Path(sys.executable)
+        )
+        monkeypatch.setattr("pm.environments.selected_venv", lambda repo: venv)
+        monkeypatch.delenv("PYTHONPATH", raising=False)
 
+        script = cron_env / "scripts" / "probe.py"
+        script.write_text(
+            "import atexit, os, pickle, sys, probe_pkg, hermes_constants\n"
+            "class Probe: pass\n"
+            "atexit.register(lambda: print('pickled', bool(pickle.dumps(Probe()))))\n"
+            "print(probe_pkg.VALUE)\n"
+            "print(hermes_constants.__file__)\n"
+            "print(sys.path[0])\n"
+            "print('PYTHONPATH=' + (os.environ.get('PYTHONPATH') or ''))\n",
+            encoding="utf-8",
+        )
 
+        success, output = scheduler_script._run_job_script("probe.py")
+        assert success is True, output
+        value, constants_file, path0, pythonpath, pickled = output.splitlines()
+        assert value == "42"
+        repo = Path(scheduler_script.__file__).resolve().parents[1]
+        assert Path(constants_file).resolve() == repo / "hermes_constants.py"
+        assert Path(path0).resolve() == script.parent.resolve()
+        assert pythonpath == "PYTHONPATH="
+        assert pickled == "pickled True"  # __main__ outlives the body, as in a plain run
+
+    @pytest.mark.platforms("posix")
+    @pytest.mark.parametrize("broken", ["selection_raises", "interpreter_missing"])
+    def test_posix_unusable_store_selection_fails_the_run_not_the_tick(
+        self, cron_env, tmp_path, monkeypatch, broken
+    ):
+        """An unusable PM selection (broken record, or a venv whose interpreter is gone) is
+        reported as a failed run naming the cause: never a silent run on the bare store Python,
+        and never an exception escaping ``_run_job_script`` to strand the execution row."""
+        from cron.scheduler_script import _run_job_script
+
+        def _broken(repo):
+            raise RuntimeError("dependency environment is missing or outside this install")
+
+        monkeypatch.setattr(
+            "hermes_cli._launchers.resolve_store_python", lambda repo: Path(sys.executable)
+        )
+        monkeypatch.setattr(
+            "pm.environments.selected_venv",
+            _broken if broken == "selection_raises" else (lambda repo: tmp_path / "gone-venv"),
+        )
+        (cron_env / "scripts" / "probe.py").write_text('print("ok")\n', encoding="utf-8")
+
+        success, output = _run_job_script("probe.py")
+        assert success is False
+        assert "dependency environment" in output
 
     def test_emoji_stdout_round_trips_through_script_capture(self, cron_env):
         """Emoji in script stdout must reach the caller intact (#42384).