Consider system packages as installed if the venv includes them (#8359)

Signed-off-by: Aurélien Bompard <[email protected]>
Co-authored-by: Randy Döring <[email protected]>

src/poetry/utils/env/base_env.py

@@ -219,8 +219,11 @@ def platlib(self) -> Path:
 
         return self._platlib
 
+    def _get_lib_dirs(self) -> list[Path]:
+        return [self.purelib, self.platlib]
+
     def is_path_relative_to_lib(self, path: Path) -> bool:
-        for lib_path in [self.purelib, self.platlib]:
+        for lib_path in self._get_lib_dirs():
             with contextlib.suppress(ValueError):
                 path.relative_to(lib_path)
                 return True

src/poetry/utils/env/system_env.py

@@ -2,6 +2,7 @@
 
 import os
 import platform
+import site
 import sys
 import sysconfig
 
@@ -87,3 +88,6 @@ def get_pip_version(self) -> Version:
 
     def is_venv(self) -> bool:
         return self._path != self._base
+
+    def _get_lib_dirs(self) -> list[Path]:
+        return super()._get_lib_dirs() + [Path(d) for d in site.getsitepackages()]

src/poetry/utils/env/virtual_env.py

@@ -3,9 +3,11 @@
 import json
 import os
 import re
+import sys
 
 from contextlib import contextmanager
 from copy import deepcopy
+from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
@@ -20,6 +22,7 @@
 from poetry.utils.env.script_strings import GET_PYTHON_VERSION
 from poetry.utils.env.script_strings import GET_SYS_PATH
 from poetry.utils.env.script_strings import GET_SYS_TAGS
+from poetry.utils.env.system_env import SystemEnv
 
 
 if TYPE_CHECKING:
@@ -133,3 +136,21 @@ def temp_environ(self) -> Iterator[None]:
 
     def _updated_path(self) -> str:
         return os.pathsep.join([str(self._bin_dir), os.environ.get("PATH", "")])
+
+    @cached_property
+    def includes_system_site_packages(self) -> bool:
+        pyvenv_cfg = self._path / "pyvenv.cfg"
+        return (
+            re.search(
+                r"^\s*include-system-site-packages\s*=\s*true\s*$",
+                pyvenv_cfg.read_text(),
+                re.IGNORECASE | re.MULTILINE,
+            )
+            is not None
+        )
+
+    def is_path_relative_to_lib(self, path: Path) -> bool:
+        return super().is_path_relative_to_lib(path) or (
+            self.includes_system_site_packages
+            and SystemEnv(Path(sys.prefix)).is_path_relative_to_lib(path)
+        )

tests/repositories/test_installed_repository.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import shutil
 import zipfile
 
 from pathlib import Path
@@ -10,14 +11,20 @@
 
 from poetry.repositories.installed_repository import InstalledRepository
 from poetry.utils._compat import metadata
+from poetry.utils.env import EnvManager
 from poetry.utils.env import MockEnv as BaseMockEnv
+from poetry.utils.env import VirtualEnv
 
 
 if TYPE_CHECKING:
     from _pytest.logging import LogCaptureFixture
     from poetry.core.packages.package import Package
     from pytest_mock.plugin import MockerFixture
 
+    from poetry.poetry import Poetry
+    from tests.types import FixtureDirGetter
+    from tests.types import ProjectFactory
+
 FIXTURES_DIR = Path(__file__).parent / "fixtures"
 ENV_DIR = (FIXTURES_DIR / "installed").resolve()
 SITE_PURELIB = ENV_DIR / "lib" / "python3.7" / "site-packages"
@@ -104,6 +111,11 @@ def get_package_from_repository(
     return None
 
 
+@pytest.fixture
+def poetry(project_factory: ProjectFactory, fixture_dir: FixtureDirGetter) -> Poetry:
+    return project_factory("simple", source=fixture_dir("simple_project"))
+
+
 def test_load_successful(repository: InstalledRepository) -> None:
     assert len(repository.packages) == len(INSTALLED_RESULTS)
 
@@ -301,3 +313,27 @@ def test_load_pep_610_compliant_editable_directory_packages(
     assert package.source_type == "directory"
     assert package.source_url == "/path/to/distributions/directory-pep-610"
     assert package.develop
+
+
+def test_system_site_packages_source_type(
+    tmp_path: Path, mocker: MockerFixture, poetry: Poetry
+) -> None:
+    """
+    The source type of system site packages
+    must not be falsely identified as "directory".
+    """
+    venv_path = tmp_path / "venv"
+    site_path = tmp_path / "site"
+    for dist_info in {"cleo-0.7.6.dist-info", "directory_pep_610-1.2.3.dist-info"}:
+        shutil.copytree(SITE_PURELIB / dist_info, site_path / dist_info)
+    mocker.patch("poetry.utils.env.virtual_env.VirtualEnv.sys_path", [str(site_path)])
+    mocker.patch("site.getsitepackages", return_value=[str(site_path)])
+
+    EnvManager(poetry).build_venv(path=venv_path, flags={"system-site-packages": True})
+    env = VirtualEnv(venv_path)
+    installed_repository = InstalledRepository.load(env)
+
+    source_types = {
+        package.name: package.source_type for package in installed_repository.packages
+    }
+    assert source_types == {"cleo": None, "directory-pep-610": "directory"}

tests/utils/test_env.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import contextlib
 import os
+import site
 import subprocess
 import sys
 
@@ -18,6 +20,7 @@
 from poetry.repositories.installed_repository import InstalledRepository
 from poetry.toml.file import TOMLFile
 from poetry.utils._compat import WINDOWS
+from poetry.utils._compat import metadata
 from poetry.utils.env import GET_BASE_PREFIX
 from poetry.utils.env import GET_PYTHON_VERSION_ONELINER
 from poetry.utils.env import EnvCommandError
@@ -1462,8 +1465,25 @@ def test_env_system_packages(tmp_path: Path, poetry: Poetry) -> None:
     pyvenv_cfg = venv_path / "pyvenv.cfg"
 
     EnvManager(poetry).build_venv(path=venv_path, flags={"system-site-packages": True})
+    env = VirtualEnv(venv_path)
 
     assert "include-system-site-packages = true" in pyvenv_cfg.read_text()
+    assert env.includes_system_site_packages
+
+
+def test_env_system_packages_are_relative_to_lib(
+    tmp_path: Path, poetry: Poetry
+) -> None:
+    venv_path = tmp_path / "venv"
+    EnvManager(poetry).build_venv(path=venv_path, flags={"system-site-packages": True})
+    env = VirtualEnv(venv_path)
+    site_dir = Path(site.getsitepackages()[-1])
+    for dist in metadata.distributions():
+        # Emulate is_relative_to, only available in 3.9+
+        with contextlib.suppress(ValueError):
+            dist._path.relative_to(site_dir)  # type: ignore[attr-defined]
+            break
+    assert env.is_path_relative_to_lib(dist._path)  # type: ignore[attr-defined]
 
 
 @pytest.mark.parametrize(