Skip to content

Commit 33eae26

Browse files
committed
utils/env: better support system site packages dir
This change improves handling of site-packages under system env, by gracefully handling fallbacks to user site when required and possible. Resolves: #3079
1 parent 9010821 commit 33eae26

File tree

8 files changed

+218
-40
lines changed

8 files changed

+218
-40
lines changed

poetry/installation/pip_installer.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,9 @@ def remove(self, package):
113113
raise
114114

115115
# This is a workaround for https://github.com/pypa/pip/issues/4176
116-
nspkg_pth_file = self._env.site_packages / "{}-nspkg.pth".format(package.name)
116+
nspkg_pth_file = self._env.site_packages.path / "{}-nspkg.pth".format(
117+
package.name
118+
)
117119
if nspkg_pth_file.exists():
118120
nspkg_pth_file.unlink()
119121

poetry/masonry/builders/editable.py

+32-29
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ def _setup_build(self):
9494
os.remove(str(setup))
9595

9696
def _add_pth(self):
97-
pth_file = Path(self._module.name).with_suffix(".pth")
9897
paths = set()
9998
for include in self._module.includes:
10099
if isinstance(include, PackageInclude) and (
@@ -106,29 +105,25 @@ def _add_pth(self):
106105
for path in paths:
107106
content += decode(path + os.linesep)
108107

109-
for site_package in [self._env.site_packages, self._env.usersite]:
110-
if not site_package:
111-
continue
112-
113-
try:
114-
site_package.mkdir(parents=True, exist_ok=True)
115-
path = site_package.joinpath(pth_file)
116-
self._debug(
117-
" - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
118-
path.name, site_package, self._poetry.file.parent
119-
)
108+
pth_file = Path(self._module.name).with_suffix(".pth")
109+
try:
110+
pth_file = self._env.site_packages.write_text(
111+
pth_file, content, encoding="utf-8"
112+
)
113+
self._debug(
114+
" - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
115+
pth_file.name, pth_file.parent, self._poetry.file.parent
120116
)
121-
path.write_text(content, encoding="utf-8")
122-
return [path]
123-
except PermissionError:
124-
self._debug("- <b>{}</b> is not writable trying next available site")
125-
126-
self._io.error_line(
127-
" - Failed to create <c2>{}</c2> for {}".format(
128-
pth_file.name, self._poetry.file.parent
129117
)
130-
)
131-
return []
118+
return [pth_file]
119+
except OSError:
120+
# TODO: Replace with PermissionError
121+
self._io.error_line(
122+
" - Failed to create <c2>{}</c2> for {}".format(
123+
pth_file.name, self._poetry.file.parent
124+
)
125+
)
126+
return []
132127

133128
def _add_scripts(self):
134129
added = []
@@ -187,19 +182,27 @@ def _add_dist_info(self, added_files):
187182
added_files = added_files[:]
188183

189184
builder = WheelBuilder(self._poetry)
190-
dist_info = self._env.site_packages.joinpath(builder.dist_info)
185+
186+
dist_info_path = Path(builder.dist_info)
187+
for dist_info in self._env.site_packages.find(
188+
dist_info_path, writable_only=True
189+
):
190+
if dist_info.exists():
191+
self._debug(
192+
" - Removing existing <c2>{}</c2> directory from <b>{}</b>".format(
193+
dist_info.name, dist_info.parent
194+
)
195+
)
196+
shutil.rmtree(str(dist_info))
197+
198+
dist_info = self._env.site_packages.mkdir(dist_info_path)
191199

192200
self._debug(
193201
" - Adding the <c2>{}</c2> directory to <b>{}</b>".format(
194-
dist_info.name, self._env.site_packages
202+
dist_info.name, dist_info.parent
195203
)
196204
)
197205

198-
if dist_info.exists():
199-
shutil.rmtree(str(dist_info))
200-
201-
dist_info.mkdir()
202-
203206
with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f:
204207
builder._write_metadata_file(f)
205208

poetry/utils/env.py

+127-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import shutil
88
import sys
99
import sysconfig
10+
import tempfile
1011
import textwrap
1112

1213
from contextlib import contextmanager
@@ -39,6 +40,7 @@
3940
from poetry.utils._compat import encode
4041
from poetry.utils._compat import list_to_shell_command
4142
from poetry.utils._compat import subprocess
43+
from poetry.utils.helpers import paths_csv
4244

4345

4446
GET_ENVIRONMENT_INFO = """\
@@ -143,6 +145,125 @@ def _version_nodot(version):
143145
"""
144146

145147

148+
class SitePackages:
149+
def __init__(
150+
self, path, fallbacks=None, skip_write_checks=False
151+
): # type: (Path, List[Path], bool) -> None
152+
self._path = path
153+
self._fallbacks = fallbacks or []
154+
self._skip_write_checks = skip_write_checks
155+
self._candidates = [self._path] + self._fallbacks
156+
self._writable_candidates = None if not skip_write_checks else self._candidates
157+
158+
@property
159+
def path(self): # type: () -> Path
160+
return self._path
161+
162+
@property
163+
def candidates(self): # type: () -> List[Path]
164+
return self._candidates
165+
166+
@property
167+
def writable_candidates(self): # type: () -> List[Path]
168+
if self._writable_candidates is not None:
169+
return self._writable_candidates
170+
171+
self._writable_candidates = []
172+
for candidate in self._candidates:
173+
try:
174+
if not candidate.exists():
175+
continue
176+
177+
with tempfile.TemporaryFile(dir=str(candidate)):
178+
self._writable_candidates.append(candidate)
179+
except (IOError, OSError):
180+
pass
181+
182+
return self._writable_candidates
183+
184+
def make_candidates(
185+
self, path, writable_only=False
186+
): # type: (Path, bool) -> List[Path]
187+
candidates = self._candidates if not writable_only else self.writable_candidates
188+
if path.is_absolute():
189+
for candidate in candidates:
190+
try:
191+
path.relative_to(candidate)
192+
return [path]
193+
except ValueError:
194+
pass
195+
else:
196+
raise ValueError(
197+
"{} is not relative to any discovered {}sites".format(
198+
path, "writable " if writable_only else ""
199+
)
200+
)
201+
202+
return [candidate / path for candidate in candidates if candidate]
203+
204+
def _path_method_wrapper(
205+
self, path, method, *args, **kwargs
206+
): # type: (Path, str, *Any, **Any) -> Union[Tuple[Path, Any], List[Tuple[Path, Any]]]
207+
208+
# TODO: Move to parameters after dropping Python 2.7
209+
return_first = kwargs.pop("return_first", True)
210+
writable_only = kwargs.pop("writable_only", False)
211+
212+
candidates = self.make_candidates(path, writable_only=writable_only)
213+
214+
if not candidates:
215+
raise RuntimeError(
216+
'Unable to find a suitable destination for "{}" in {}'.format(
217+
str(path), paths_csv(self._candidates)
218+
)
219+
)
220+
221+
results = []
222+
223+
for candidate in candidates:
224+
try:
225+
result = candidate, getattr(candidate, method)(*args, **kwargs)
226+
if return_first:
227+
return result
228+
else:
229+
results.append(result)
230+
except (IOError, OSError):
231+
# TODO: Replace with PermissionError
232+
pass
233+
234+
if results:
235+
return results
236+
237+
raise OSError("Unable to access any of {}".format(paths_csv(candidates)))
238+
239+
def write_text(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path
240+
return self._path_method_wrapper(path, "write_text", *args, **kwargs)[0]
241+
242+
def mkdir(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path
243+
return self._path_method_wrapper(path, "mkdir", *args, **kwargs)[0]
244+
245+
def exists(self, path): # type: (Path) -> bool
246+
return any(
247+
value[-1]
248+
for value in self._path_method_wrapper(path, "exists", return_first=False)
249+
)
250+
251+
def find(self, path, writable_only=False): # type: (Path, bool) -> List[Path]
252+
return [
253+
value[0]
254+
for value in self._path_method_wrapper(
255+
path, "exists", return_first=False, writable_only=writable_only
256+
)
257+
if value[-1] is True
258+
]
259+
260+
def __getattr__(self, item):
261+
try:
262+
return super(SitePackages, self).__getattribute__(item)
263+
except AttributeError:
264+
return getattr(self.path, item)
265+
266+
146267
class EnvError(Exception):
147268

148269
pass
@@ -810,9 +931,13 @@ def pip_version(self):
810931
return self._pip_version
811932

812933
@property
813-
def site_packages(self): # type: () -> Path
934+
def site_packages(self): # type: () -> SitePackages
814935
if self._site_packages is None:
815-
self._site_packages = self.purelib
936+
# we disable write checks if no user site exist
937+
fallbacks = [self.usersite] if self.usersite else []
938+
self._site_packages = SitePackages(
939+
self.purelib, fallbacks, skip_write_checks=False if fallbacks else True
940+
)
816941
return self._site_packages
817942

818943
@property

poetry/utils/helpers.py

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import tempfile
66

77
from contextlib import contextmanager
8+
from typing import List
89
from typing import Optional
910

1011
import requests
@@ -113,3 +114,7 @@ def get_package_version_display_string(
113114
)
114115

115116
return package.full_pretty_version
117+
118+
119+
def paths_csv(paths): # type: (List[Path]) -> str
120+
return ", ".join('"{}"'.format(str(c)) for c in paths)

tests/installation/test_pip_installer.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,9 @@ def test_uninstall_git_package_nspkg_pth_cleanup(mocker, tmp_venv, pool):
189189
)
190190

191191
# we do this here because the virtual env might not be usable if failure case is triggered
192-
pth_file_candidate = tmp_venv.site_packages / "{}-nspkg.pth".format(package.name)
192+
pth_file_candidate = tmp_venv.site_packages.path / "{}-nspkg.pth".format(
193+
package.name
194+
)
193195

194196
# in order to reproduce the scenario where the git source is removed prior to proper
195197
# clean up of nspkg.pth file, we need to make sure the fixture is copied and not

tests/masonry/builders/test_editable_builder.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,14 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_
7676
builder.build()
7777

7878
assert tmp_venv._bin_dir.joinpath("foo").exists()
79-
assert tmp_venv.site_packages.joinpath("simple_project.pth").exists()
80-
assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.joinpath(
79+
assert tmp_venv.site_packages.path.joinpath("simple_project.pth").exists()
80+
assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.path.joinpath(
8181
"simple_project.pth"
8282
).read_text().strip(
8383
os.linesep
8484
)
8585

86-
dist_info = tmp_venv.site_packages.joinpath("simple_project-1.2.3.dist-info")
86+
dist_info = tmp_venv.site_packages.path.joinpath("simple_project-1.2.3.dist-info")
8787
assert dist_info.exists()
8888
assert dist_info.joinpath("INSTALLER").exists()
8989
assert dist_info.joinpath("METADATA").exists()
@@ -130,7 +130,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_
130130
assert metadata == dist_info.joinpath("METADATA").read_text(encoding="utf-8")
131131

132132
records = dist_info.joinpath("RECORD").read_text()
133-
assert str(tmp_venv.site_packages.joinpath("simple_project.pth")) in records
133+
assert str(tmp_venv.site_packages.path.joinpath("simple_project.pth")) in records
134134
assert str(tmp_venv._bin_dir.joinpath("foo")) in records
135135
assert str(tmp_venv._bin_dir.joinpath("baz")) in records
136136
assert str(dist_info.joinpath("METADATA")) in records
@@ -190,7 +190,7 @@ def test_builder_installs_proper_files_when_packages_configured(
190190
builder = EditableBuilder(project_with_include, tmp_venv, NullIO())
191191
builder.build()
192192

193-
pth_file = tmp_venv.site_packages.joinpath("with_include.pth")
193+
pth_file = tmp_venv.site_packages.path.joinpath("with_include.pth")
194194
assert pth_file.is_file()
195195

196196
paths = set()

tests/utils/test_env.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -848,7 +848,7 @@ def test_system_env_has_correct_paths():
848848
assert paths.get("purelib") is not None
849849
assert paths.get("platlib") is not None
850850
assert paths.get("scripts") is not None
851-
assert env.site_packages == Path(paths["purelib"])
851+
assert env.site_packages.path == Path(paths["purelib"])
852852

853853

854854
@pytest.mark.parametrize(
@@ -868,4 +868,4 @@ def test_venv_has_correct_paths(tmp_venv):
868868
assert paths.get("purelib") is not None
869869
assert paths.get("platlib") is not None
870870
assert paths.get("scripts") is not None
871-
assert tmp_venv.site_packages == Path(paths["purelib"])
871+
assert tmp_venv.site_packages.path == Path(paths["purelib"])

tests/utils/test_env_site.py

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import uuid
2+
3+
from poetry.utils._compat import Path
4+
from poetry.utils._compat import decode
5+
from poetry.utils.env import SitePackages
6+
7+
8+
def test_env_site_simple(tmp_dir):
9+
site_packages = SitePackages(Path("/non-existent"), fallbacks=[Path(tmp_dir)])
10+
candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True)
11+
hello = Path(tmp_dir) / "hello.txt"
12+
13+
assert len(candidates) == 1
14+
assert candidates[0].as_posix() == hello.as_posix()
15+
16+
content = decode(str(uuid.uuid4()))
17+
site_packages.write_text(Path("hello.txt"), content, encoding="utf-8")
18+
19+
assert hello.read_text(encoding="utf-8") == content
20+
21+
assert not (site_packages.path / "hello.txt").exists()
22+
23+
24+
def test_env_site_select_first(tmp_dir):
25+
path = Path(tmp_dir)
26+
fallback = path / "fallback"
27+
fallback.mkdir(parents=True)
28+
29+
site_packages = SitePackages(path, fallbacks=[fallback])
30+
candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True)
31+
32+
assert len(candidates) == 2
33+
assert len(site_packages.find(Path("hello.txt"))) == 0
34+
35+
content = decode(str(uuid.uuid4()))
36+
site_packages.write_text(Path("hello.txt"), content, encoding="utf-8")
37+
38+
assert (site_packages.path / "hello.txt").exists()
39+
assert not (fallback / "hello.txt").exists()
40+
41+
assert len(site_packages.find(Path("hello.txt"))) == 1

0 commit comments

Comments
 (0)