From ebf137294e8161996970f4934c757403e25b1fc1 Mon Sep 17 00:00:00 2001 From: g3org3yo Date: Fri, 25 Sep 2026 19:03:06 +0300 Subject: [PATCH] fix(gateway): stop Windows gateway boot from adopting another Python's venv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PM install keeps one dependency generation per install (a store interpreter plus its own environment) while the checkout may still hold the legacy pre-PM `venv/`, built by uv for a different CPython. `_ensure_windows_gateway_venv_imports` walked `VIRTUAL_ENV` then `/venv` and inserted whichever site-packages it found at sys.path[0:1] — ahead of the generation the process booted from — so on a cp314 store interpreter `pydantic` resolved from the cp311 tree and its compiled core could not load: ModuleNotFoundError: No module named 'pydantic_core._pydantic_core' The gateway then logged "Supervised task hosted_room_worker died 5 times in rapid succession" and hosted rooms stopped working, on every update that moved the store Python. Reproduced deterministically by importing `hermes_bootstrap`, calling the helper, then importing `tui_gateway.server` with the store interpreter. Two changes: * `_ensure_windows_gateway_venv_imports` now prefers PM's committed environment (`pm.environments.committed_venv`) and skips any candidate whose recorded Python does not match the running interpreter, warning when that leaves nothing. A candidate without `pyvenv.cfg` keeps the historical behavior. * `pm.environments.venv_python_version` also reads `version_info`, the key `uv venv` writes for every Hermes-managed environment. Reading only `version` returned None for all of them, which is what let a mismatched tree look acceptable in the first place (and dated POSIX site-packages from the caller). Tests: tests/gateway/test_windows_gateway_venv_injection.py (leftover skipped, matching venv still injected, marker-less venv unchanged, committed generation outranks the leftover) and tests/pm/test_venv_python_version.py. Both fail on the parent commit and pass with the fix. --- gateway/run.py | 45 ++++++++ pm/environments.py | 8 +- .../test_windows_gateway_venv_injection.py | 103 ++++++++++++++++++ tests/pm/test_venv_python_version.py | 60 ++++++++++ 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 tests/gateway/test_windows_gateway_venv_injection.py create mode 100644 tests/pm/test_venv_python_version.py diff --git a/gateway/run.py b/gateway/run.py index e60de6e07d156..e821c491dcc1e 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -419,6 +419,18 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool: _GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile( "(" + "|".join(_ENDPOINT_UNREACHABLE_MARKERS) + ")", re.IGNORECASE) +def _venv_held_python(venv_dir: Path) -> tuple[int, int] | None: + """The ``(major, minor)`` of the interpreter a venv actually holds, or ``None`` if unknown.""" + try: + from pm.environments import venv_python_version + except Exception: + return None + try: + return venv_python_version(venv_dir) + except Exception: + return None + + def _ensure_windows_gateway_venv_imports() -> None: """Make detached Windows gateway runs see the Hermes venv packages. @@ -428,10 +440,24 @@ def _ensure_windows_gateway_venv_imports() -> None: project_root = Path(__file__).resolve().parent.parent candidates: list[Path] = [] + # PM owns the dependency environment of an install. When it committed one, that is the tree + # this interpreter booted from, so it outranks the checkout's legacy ``venv/`` — a leftover + # from the pre-PM layout, often built for another Python. + pm_venv: Path | None = None + try: + from pm.environments import committed_venv + + pm_venv = committed_venv(project_root) + except Exception: + pm_venv = None + if pm_venv is not None: + candidates.append(pm_venv) if os.environ.get("VIRTUAL_ENV"): candidates.append(Path(os.environ["VIRTUAL_ENV"])) candidates.append(project_root / "venv") + running = (sys.version_info.major, sys.version_info.minor) + mismatched: list[str] = [] seen: set[str] = set() for venv_dir in candidates: try: @@ -443,6 +469,16 @@ def _ensure_windows_gateway_venv_imports() -> None: continue seen.add(venv_key) + held = _venv_held_python(resolved_venv) + if held is not None and held != running: + # Another interpreter's site-packages: it would shadow the correct tree (sys.path + # order decides), and its compiled extensions cannot be imported at all — a cp311 + # ``_pydantic_core`` under cp314 raised + # ``ModuleNotFoundError: No module named 'pydantic_core._pydantic_core'`` and killed + # the gateway's hosted-room workers on every update that moved the store Python. + mismatched.append(f"{resolved_venv} (python {held[0]}.{held[1]})") + continue + site_packages = resolved_venv / "Lib" / "site-packages" if not site_packages.exists(): continue @@ -465,6 +501,15 @@ def _ensure_windows_gateway_venv_imports() -> None: os.environ["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(pythonpath)) return + if mismatched: + logger.warning( + "Windows gateway venv injection skipped for interpreter-mismatched path(s): %s " + "(running Python %s.%s). Their site-packages would shadow the real dependency tree.", + ", ".join(mismatched), + running[0], + running[1], + ) + def _gateway_platform_value(platform: Any) -> str: """Return a normalized gateway platform value for enums or raw strings.""" diff --git a/pm/environments.py b/pm/environments.py index 95e874f7c366b..f73dc5f5cf52e 100644 --- a/pm/environments.py +++ b/pm/environments.py @@ -215,7 +215,7 @@ def project_python(project_root: Path) -> Path: def venv_python_version(venv: Path) -> tuple[int, int] | None: - """The interpreter version a POSIX venv actually holds, or ``None``. + """The interpreter version a venv actually holds, or ``None``. ``site_packages`` must not date the tree from the CALLER's ``sys.version_info``: an update can rebuild the dependency environment with a different Python than @@ -223,11 +223,15 @@ def venv_python_version(venv: Path) -> tuple[int, int] | None: built the environment with CPython 3.14 while the PATH shim ran 3.11, so the shim composed ``lib/python3.11/site-packages`` inside a 3.14 venv, found no tree, and failed *after* a successful update. + + Both marker spellings matter: ``virtualenv``/``venv`` write ``version``, while + ``uv venv`` (every Hermes-managed environment) writes ``version_info``. Reading + only ``version`` left every uv venv unidentifiable -- the exact case above. """ try: for line in (venv / "pyvenv.cfg").read_text(encoding="utf-8-sig").splitlines(): key, _, value = line.partition("=") - if key.strip() != "version": + if key.strip() not in ("version", "version_info"): continue major, _, rest = value.strip().partition(".") minor, _, _ = rest.partition(".") diff --git a/tests/gateway/test_windows_gateway_venv_injection.py b/tests/gateway/test_windows_gateway_venv_injection.py new file mode 100644 index 0000000000000..02926d071c0b7 --- /dev/null +++ b/tests/gateway/test_windows_gateway_venv_injection.py @@ -0,0 +1,103 @@ +"""Windows detached-gateway venv injection must not adopt another interpreter's venv. + +PM installs keep one dependency generation per install (a store interpreter + its own +environment) while the checkout may still hold the legacy pre-PM ``venv/``. Injecting that +leftover put a cp311 ``site-packages`` ahead of the real one on a cp314 interpreter, so +``pydantic`` resolved from the old tree and its compiled core refused to load: + + ModuleNotFoundError: No module named 'pydantic_core._pydantic_core' + +That killed the gateway's ``hosted_room_worker`` on every update that moved the store Python. +""" + +import os +import sys +from pathlib import Path + +import pytest + +import gateway.run as gr + +pytestmark = pytest.mark.skipif(sys.platform != "win32", reason="Windows-only helper") + + +def _fake_venv(root: Path, version: str | None) -> Path: + """A venv-shaped directory; ``version=None`` omits ``pyvenv.cfg`` (unknown interpreter).""" + venv = root / "venv" + (venv / "Lib" / "site-packages").mkdir(parents=True, exist_ok=True) + if version is not None: + (venv / "pyvenv.cfg").write_text( + f"home = C:\\Python\nimplementation = CPython\nversion_info = {version}\n", + encoding="utf-8", + ) + return venv + + +def _running_version() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def _other_version() -> str: + """A Python version that is guaranteed not to be the running one.""" + return "3.12" if _running_version() != "3.12" else "3.11" + + +def _point_project_at(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Fake a checkout root and a host with no PM generation committed.""" + monkeypatch.setattr(gr, "__file__", str(tmp_path / "gateway" / "run.py")) + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.setattr(sys, "path", list(sys.path)) + + import pm.environments as pm_env + + monkeypatch.setattr(pm_env, "committed_venv", lambda project_root=None: None) + + +def test_other_pythons_venv_is_not_injected(monkeypatch, tmp_path): + """The pre-PM leftover is skipped instead of shadowing the interpreter's own packages.""" + venv = _fake_venv(tmp_path, _other_version()) + _point_project_at(monkeypatch, tmp_path) + monkeypatch.setenv("PYTHONPATH", "sentinel") + + gr._ensure_windows_gateway_venv_imports() + + assert str(venv / "Lib" / "site-packages") not in sys.path + assert os.environ["PYTHONPATH"] == "sentinel" + + +def test_matching_python_venv_is_still_injected(monkeypatch, tmp_path): + """Legacy installs (venv built for the running interpreter) keep working unchanged.""" + venv = _fake_venv(tmp_path, _running_version()) + _point_project_at(monkeypatch, tmp_path) + + gr._ensure_windows_gateway_venv_imports() + + site_packages = str(venv / "Lib" / "site-packages") + assert site_packages in sys.path + assert os.environ["VIRTUAL_ENV"] == str(venv) + + +def test_unknown_python_venv_keeps_historical_behavior(monkeypatch, tmp_path): + """No ``pyvenv.cfg`` → nothing to compare, so the candidate is used as before.""" + venv = _fake_venv(tmp_path, None) + _point_project_at(monkeypatch, tmp_path) + + gr._ensure_windows_gateway_venv_imports() + + assert str(venv / "Lib" / "site-packages") in sys.path + + +def test_committed_generation_outranks_the_leftover(monkeypatch, tmp_path): + """PM's committed environment wins, and the mismatched leftover stays off sys.path.""" + leftover = _fake_venv(tmp_path, _other_version()) + committed = _fake_venv(tmp_path / "installs" / "abc", _running_version()) + _point_project_at(monkeypatch, tmp_path) + + import pm.environments as pm_env + + monkeypatch.setattr(pm_env, "committed_venv", lambda project_root=None: committed) + + gr._ensure_windows_gateway_venv_imports() + + assert str(committed / "Lib" / "site-packages") in sys.path + assert str(leftover / "Lib" / "site-packages") not in sys.path diff --git a/tests/pm/test_venv_python_version.py b/tests/pm/test_venv_python_version.py new file mode 100644 index 0000000000000..3d8d545705a65 --- /dev/null +++ b/tests/pm/test_venv_python_version.py @@ -0,0 +1,60 @@ +"""A venv's Python version must be readable whatever wrote its ``pyvenv.cfg``. + +``uv venv`` (every Hermes-managed environment) records ``version_info``; ``venv``/``virtualenv`` +record ``version``. Reading only one spelling made uv venvs unidentifiable, so callers fell back +to the *calling* interpreter's version and composed a site-packages path the venv never had. +""" + +from pathlib import Path + +import pytest + +from pm.environments import venv_python_version + + +def _venv(tmp_path: Path, cfg: str | None) -> Path: + venv = tmp_path / "venv" + venv.mkdir(exist_ok=True) + if cfg is not None: + (venv / "pyvenv.cfg").write_text(cfg, encoding="utf-8") + return venv + + +def test_uv_style_version_info_is_read(tmp_path): + """Real ``uv venv`` output from a Hermes install.""" + venv = _venv( + tmp_path, + "home = C:\\Users\\x\\AppData\\Local\\hermes\\tools\\python-3.14.7-win32-x64\n" + "implementation = CPython\n" + "uv = 0.12.3\n" + "version_info = 3.14.7\n" + "include-system-site-packages = false\n" + "relocatable = true\n", + ) + + assert venv_python_version(venv) == (3, 14) + + +def test_virtualenv_style_version_is_read(tmp_path): + venv = _venv(tmp_path, "home = /usr/bin\nversion = 3.11.9\n") + + assert venv_python_version(venv) == (3, 11) + + +def test_missing_marker_returns_none(tmp_path): + assert venv_python_version(_venv(tmp_path, None)) is None + + +def test_unparsable_value_returns_none(tmp_path): + assert venv_python_version(_venv(tmp_path, "version_info = unknown\n")) is None + + +@pytest.mark.skipif(__import__("os").name == "nt", reason="POSIX path layout only") +def test_site_packages_uses_the_venvs_own_version(tmp_path): + """The regression the helper exists for: not the caller's interpreter version.""" + venv = _venv(tmp_path, "version_info = 3.11.9\n") + (venv / "lib" / "python3.11").mkdir(parents=True) + + from pm.environments import site_packages + + assert site_packages(venv) == venv / "lib" / "python3.11" / "site-packages"