Support Windows CPython interpreters distributed by non-standard orgs (…

…#2504)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/changelog/2504.bugfix.rst

@@ -0,0 +1 @@
+Discover CPython implementations distributed on Windows by any organization - by :user:`faph`.

src/virtualenv/discovery/windows/__init__.py

@@ -2,6 +2,12 @@
 from ..py_spec import PythonSpec
 from .pep514 import discover_pythons
 
+# Map of well-known organizations (as per PEP 514 Company Windows Registry key part) versus Python implementation
+_IMPLEMENTATION_BY_ORG = {
+    "ContinuumAnalytics": "CPython",
+    "PythonCore": "CPython",
+}
+
 
 class Pep514PythonInfo(PythonInfo):
     """A Python information acquired from PEP-514"""
@@ -19,13 +25,17 @@ def propose_interpreters(spec, cache_dir, env):
     )
 
     for name, major, minor, arch, exe, _ in existing:
-        # pre-filter
-        if name in ("PythonCore", "ContinuumAnalytics"):
-            name = "CPython"
-        registry_spec = PythonSpec(None, name, major, minor, None, arch, exe)
-        if registry_spec.satisfies(spec):
+        # Map well-known/most common organizations to a Python implementation, use the org name as a fallback for
+        # backwards compatibility.
+        implementation = _IMPLEMENTATION_BY_ORG.get(name, name)
+
+        # Pre-filtering based on Windows Registry metadata, for CPython only
+        skip_pre_filter = implementation.lower() != "cpython"
+        registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe)
+        if skip_pre_filter or registry_spec.satisfies(spec):
             interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False)
             if interpreter is not None:
+                # Final filtering/matching using interpreter metadata
                 if interpreter.satisfies(spec, impl_must_match=True):
                     yield interpreter
 

tests/unit/discovery/windows/conftest.py

@@ -0,0 +1,100 @@
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture()
+def _mock_registry(mocker):
+    from virtualenv.discovery.windows.pep514 import winreg
+
+    loc, glob = {}, {}
+    mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text()
+    exec(mock_value_str, glob, loc)
+    enum_collect = loc["enum_collect"]
+    value_collect = loc["value_collect"]
+    key_open = loc["key_open"]
+    hive_open = loc["hive_open"]
+
+    def _enum_key(key, at):
+        key_id = key.value if isinstance(key, Key) else key
+        result = enum_collect[key_id][at]
+        if isinstance(result, OSError):
+            raise result
+        return result
+
+    mocker.patch.object(winreg, "EnumKey", side_effect=_enum_key)
+
+    def _query_value_ex(key, value_name):
+        key_id = key.value if isinstance(key, Key) else key
+        result = value_collect[key_id][value_name]
+        if isinstance(result, OSError):
+            raise result
+        return result
+
+    mocker.patch.object(winreg, "QueryValueEx", side_effect=_query_value_ex)
+
+    class Key:
+        def __init__(self, value):
+            self.value = value
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):  # noqa: U100
+            return None
+
+    @contextmanager
+    def _open_key_ex(*args):
+        if len(args) == 2:
+            key, value = args
+            key_id = key.value if isinstance(key, Key) else key
+            result = Key(key_open[key_id][value])  # this needs to be something that can be with-ed, so let's wrap it
+        elif len(args) == 4:
+            result = hive_open[args]
+        else:
+            raise RuntimeError
+        value = result.value if isinstance(result, Key) else result
+        if isinstance(value, OSError):
+            raise value
+        yield result
+
+    mocker.patch.object(winreg, "OpenKeyEx", side_effect=_open_key_ex)
+    mocker.patch("os.path.exists", return_value=True)
+
+
+def _mock_pyinfo(major, minor, arch, exe):
+    """Return PythonInfo objects with essential metadata set for the given args"""
+    from virtualenv.discovery.py_info import PythonInfo, VersionInfo
+
+    info = PythonInfo()
+    info.base_prefix = str(Path(exe).parent)
+    info.executable = info.original_executable = info.system_executable = exe
+    info.implementation = "CPython"
+    info.architecture = arch
+    info.version_info = VersionInfo(major, minor, 0, "final", 0)
+    return info
+
+
+@pytest.fixture()
+def _populate_pyinfo_cache(monkeypatch):
+    """Add metadata to virtualenv.discovery.cached_py_info._CACHE for all (mocked) registry entries"""
+    import virtualenv.discovery.cached_py_info
+
+    # Data matches _mock_registry fixture
+    interpreters = [
+        ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None),
+        ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
+        ("PythonCore", 3, 5, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
+        ("PythonCore", 3, 7, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python37-32\\python.exe", None),
+        ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None),
+        ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None),
+        ("PythonCore", 3, 4, 64, "C:\\Python34\\python.exe", None),
+        ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None),
+    ]
+    for _, major, minor, arch, exe, _ in interpreters:
+        info = _mock_pyinfo(major, minor, arch, exe)
+        monkeypatch.setitem(virtualenv.discovery.cached_py_info._CACHE, Path(info.executable), info)

tests/unit/discovery/windows/test_windows.py

@@ -0,0 +1,33 @@
+import sys
+
+import pytest
+
+from virtualenv.discovery.py_spec import PythonSpec
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry")
+@pytest.mark.usefixtures("_mock_registry")
+@pytest.mark.usefixtures("_populate_pyinfo_cache")
+@pytest.mark.parametrize(
+    ("string_spec", "expected_exe"),
+    [
+        # 64-bit over 32-bit
+        ("python3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"),
+        ("cpython3.10", "C:\\Users\\user\\Miniconda3-64\\python.exe"),
+        # 1 installation of 3.9 available
+        ("python3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"),
+        ("cpython3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"),
+        # resolves to highest available version
+        ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"),
+        ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"),
+        # Non-standard org name
+        ("python3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"),
+        ("cpython3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"),
+    ],
+)
+def test_propose_interpreters(string_spec, expected_exe):
+    from virtualenv.discovery.windows import propose_interpreters
+
+    spec = PythonSpec.from_string_spec(string_spec)
+    interpreter = next(propose_interpreters(spec=spec, cache_dir=None, env=None))
+    assert interpreter.executable == expected_exe

tests/unit/discovery/windows/test_windows_pep514.py

@@ -1,8 +1,5 @@
 import sys
 import textwrap
-from collections import defaultdict
-from contextlib import contextmanager
-from pathlib import Path
 
 import pytest
 
@@ -14,16 +11,17 @@ def test_pep514():
 
     interpreters = list(discover_pythons())
     assert interpreters == [
-        ("ContinuumAnalytics", 3, 7, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None),
-        ("ContinuumAnalytics", 3, 7, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None),
-        ("PythonCore", 3, 6, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
-        ("PythonCore", 3, 6, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
-        ("PythonCore", 3, 5, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe", None),
-        ("PythonCore", 3, 6, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
-        ("PythonCore", 3, 7, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python37-32\\python.exe", None),
-        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None),
+        ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None),
+        ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None),
+        ("PythonCore", 3, 8, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", None),
+        ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None),
+        ("PythonCore", 3, 10, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", None),
+        ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None),
+        ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None),
         ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None),
-        ("PythonCore", 3, 4, 64, "C:\\Python34\\python.exe", None),
+        ("PythonCore", 3, 7, 64, "C:\\Python37\\python.exe", None),
     ]
 
 
@@ -36,16 +34,17 @@ def test_pep514_run(capsys, caplog):
     out, err = capsys.readouterr()
     expected = textwrap.dedent(
         r"""
-    ('ContinuumAnalytics', 3, 7, 32, 'C:\\Users\\user\\Miniconda3\\python.exe', None)
-    ('ContinuumAnalytics', 3, 7, 64, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None)
+    ('CompanyA', 3, 6, 64, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None)
+    ('ContinuumAnalytics', 3, 10, 32, 'C:\\Users\\user\\Miniconda3\\python.exe', None)
+    ('ContinuumAnalytics', 3, 10, 64, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None)
     ('PythonCore', 2, 7, 64, 'C:\\Python27\\python.exe', None)
-    ('PythonCore', 3, 4, 64, 'C:\\Python34\\python.exe', None)
-    ('PythonCore', 3, 5, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe', None)
-    ('PythonCore', 3, 6, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe', None)
-    ('PythonCore', 3, 6, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe', None)
-    ('PythonCore', 3, 6, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe', None)
-    ('PythonCore', 3, 7, 32, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python37-32\\python.exe', None)
-    ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe', None)
+    ('PythonCore', 3, 10, 32, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None)
+    ('PythonCore', 3, 12, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None)
+    ('PythonCore', 3, 7, 64, 'C:\\Python37\\python.exe', None)
+    ('PythonCore', 3, 8, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None)
+    ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None)
+    ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None)
+    ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None)
     """,
     ).strip()
     assert out.strip() == expected
@@ -56,136 +55,9 @@ def test_pep514_run(capsys, caplog):
         f"{prefix}HKEY_CURRENT_USER/PythonCore/3.2/SysArchitecture error: arch is not string: 100",
         f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: no ExecutablePath or default for it",
         f"{prefix}HKEY_CURRENT_USER/PythonCore/3.3 error: could not load exe with value None",
-        f"{prefix}HKEY_CURRENT_USER/PythonCore/3.8/InstallPath error: missing",
-        f"{prefix}HKEY_CURRENT_USER/PythonCore/3.9/SysVersion error: invalid format magic",
+        f"{prefix}HKEY_CURRENT_USER/PythonCore/3.11/InstallPath error: missing",
+        f"{prefix}HKEY_CURRENT_USER/PythonCore/3.12/SysVersion error: invalid format magic",
         f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X/SysVersion error: version is not string: 2778",
         f"{prefix}HKEY_CURRENT_USER/PythonCore/3.X error: invalid format 3.X",
     ]
     assert caplog.messages == expected_logs
-
-
-@pytest.fixture()
-def _mock_registry(mocker):
-    from virtualenv.discovery.windows.pep514 import winreg
-
-    loc, glob = {}, {}
-    mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text()
-    exec(mock_value_str, glob, loc)
-    enum_collect = loc["enum_collect"]
-    value_collect = loc["value_collect"]
-    key_open = loc["key_open"]
-    hive_open = loc["hive_open"]
-
-    def _e(key, at):
-        key_id = key.value if isinstance(key, Key) else key
-        result = enum_collect[key_id][at]
-        if isinstance(result, OSError):
-            raise result
-        return result
-
-    mocker.patch.object(winreg, "EnumKey", side_effect=_e)
-
-    def _v(key, value_name):
-        key_id = key.value if isinstance(key, Key) else key
-        result = value_collect[key_id][value_name]
-        if isinstance(result, OSError):
-            raise result
-        return result
-
-    mocker.patch.object(winreg, "QueryValueEx", side_effect=_v)
-
-    class Key:
-        def __init__(self, value):
-            self.value = value
-
-        def __enter__(self):
-            return self
-
-        def __exit__(self, exc_type, exc_val, exc_tb):  # noqa: U100
-            return None
-
-    @contextmanager
-    def _o(*args):
-        if len(args) == 2:
-            key, value = args
-            key_id = key.value if isinstance(key, Key) else key
-            result = Key(key_open[key_id][value])  # this needs to be something that can be with-ed, so let's wrap it
-        elif len(args) == 4:
-            result = hive_open[args]
-        else:
-            raise RuntimeError
-        value = result.value if isinstance(result, Key) else result
-        if isinstance(value, OSError):
-            raise value
-        yield result
-
-    mocker.patch.object(winreg, "OpenKeyEx", side_effect=_o)
-    mocker.patch("os.path.exists", return_value=True)
-
-
-@pytest.fixture()
-def _collect_winreg_access(mocker):
-    # noinspection PyUnresolvedReferences
-    from winreg import EnumKey, OpenKeyEx, QueryValueEx
-
-    from virtualenv.discovery.windows.pep514 import winreg
-
-    hive_open = {}
-    key_open = defaultdict(dict)
-
-    @contextmanager
-    def _c(*args):
-        res = None
-        key_id = id(args[0]) if len(args) == 2 else None
-        try:
-            with OpenKeyEx(*args) as c:
-                res = id(c)
-                yield c
-        except Exception as exception:
-            res = exception
-            raise exception
-        finally:
-            if len(args) == 4:
-                hive_open[args] = res
-            elif len(args) == 2:
-                key_open[key_id][args[1]] = res
-
-    enum_collect = defaultdict(list)
-
-    def _e(key, at):
-        result = None
-        key_id = id(key)
-        try:
-            result = EnumKey(key, at)
-            return result
-        except Exception as exception:
-            result = exception
-            raise result
-        finally:
-            enum_collect[key_id].append(result)
-
-    value_collect = defaultdict(dict)
-
-    def _v(key, value_name):
-        result = None
-        key_id = id(key)
-        try:
-            result = QueryValueEx(key, value_name)
-            return result
-        except Exception as exception:
-            result = exception
-            raise result
-        finally:
-            value_collect[key_id][value_name] = result
-
-    mocker.patch.object(winreg, "EnumKey", side_effect=_e)
-    mocker.patch.object(winreg, "QueryValueEx", side_effect=_v)
-    mocker.patch.object(winreg, "OpenKeyEx", side_effect=_c)
-
-    yield
-
-    print("")
-    print(f"hive_open = {hive_open}")
-    print(f"key_open = {dict(key_open.items())}")
-    print(f"value_collect = {dict(value_collect.items())}")
-    print(f"enum_collect = {dict(enum_collect.items())}")