From 6af44fb1b8b72c62c608c153b059f8393d6c5a75 Mon Sep 17 00:00:00 2001 From: finswimmer Date: Tue, 15 Sep 2020 06:47:23 +0200 Subject: [PATCH 1/3] change (masonry.builders.builder): check if a file is excluded first before setting source root to the package base fix (masonry.builders.wheel): fix substitution in string formation new (tests.masonry.builders.test_wheel): test to check if file is excluded in wheel, when listed in gitignore and src layout is used --- poetry/core/masonry/builders/builder.py | 18 ++++---- poetry/core/masonry/builders/wheel.py | 2 +- .../default_src_with_excluded_data/LICENSE | 20 +++++++++ .../default_src_with_excluded_data/README.rst | 2 + .../pyproject.toml | 39 +++++++++++++++++ .../src/my_package/__init__.py | 0 .../src/my_package/data/data1.txt | 0 .../src/my_package/data/sub_data/data2.txt | 0 .../src/my_package/data/sub_data/data3.txt | 0 tests/masonry/builders/test_wheel.py | 42 +++++++++++++++++++ 10 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/LICENSE create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/README.rst create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/pyproject.toml create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/__init__.py create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/data1.txt create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data2.txt create mode 100644 tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data3.txt diff --git a/poetry/core/masonry/builders/builder.py b/poetry/core/masonry/builders/builder.py index 7089a1ae3..55c9fd1f2 100644 --- a/poetry/core/masonry/builders/builder.py +++ b/poetry/core/masonry/builders/builder.py @@ -161,22 +161,20 @@ def find_files_to_add( to_add.add(include_file) continue - if ( - isinstance(include, PackageInclude) - and include.source - and self.format == "wheel" - ): - source_root = include.base - else: - source_root = self._path - - include_file = BuildIncludeFile(path=file, source_root=source_root) + include_file = BuildIncludeFile(path=file, source_root=self._path) if self.is_excluded( include_file.relative_to_source_root() ) and isinstance(include, PackageInclude): continue + if ( + isinstance(include, PackageInclude) + and include.source + and self.format == "wheel" + ): + include_file.source_root = include.base + if file.suffix == ".pyc": continue diff --git a/poetry/core/masonry/builders/wheel.py b/poetry/core/masonry/builders/wheel.py index 0f15b1cce..c8698577d 100644 --- a/poetry/core/masonry/builders/wheel.py +++ b/poetry/core/masonry/builders/wheel.py @@ -181,7 +181,7 @@ def _write_metadata(self, wheel): relative_path = "%s/%s" % (self.dist_info, path.relative_to(self._path)) self._add_file(wheel, path, relative_path) else: - logger.debug("Skipping: %s", path.as_posix()) + logger.debug("Skipping: %s" % path.as_posix()) with self._write_to_zip(wheel, self.dist_info + "/WHEEL") as f: self._write_wheel_file(f) diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/LICENSE b/tests/masonry/builders/fixtures/default_src_with_excluded_data/LICENSE new file mode 100644 index 000000000..44cf2b30e --- /dev/null +++ b/tests/masonry/builders/fixtures/default_src_with_excluded_data/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2018 Sébastien Eustace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/README.rst b/tests/masonry/builders/fixtures/default_src_with_excluded_data/README.rst new file mode 100644 index 000000000..f7fe15470 --- /dev/null +++ b/tests/masonry/builders/fixtures/default_src_with_excluded_data/README.rst @@ -0,0 +1,2 @@ +My Package +========== diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/pyproject.toml b/tests/masonry/builders/fixtures/default_src_with_excluded_data/pyproject.toml new file mode 100644 index 000000000..80e86166a --- /dev/null +++ b/tests/masonry/builders/fixtures/default_src_with_excluded_data/pyproject.toml @@ -0,0 +1,39 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Sébastien Eustace " +] +license = "MIT" + +readme = "README.rst" + +homepage = "https://python-poetry.org/" +repository = "https://github.com/python-poetry/poetry" +documentation = "https://python-poetry.org/docs" + +keywords = ["packaging", "dependency", "poetry"] + +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +# Requirements +[tool.poetry.dependencies] +python = "^3.6" +cleo = "^0.6" +cachy = { version = "^0.2.0", extras = ["msgpack"] } + +pendulum = { version = "^1.4", optional = true } + +[tool.poetry.dev-dependencies] +pytest = "~3.4" + +[tool.poetry.extras] +time = ["pendulum"] + +[tool.poetry.scripts] +my-script = "my_package:main" +my-2nd-script = "my_package:main2" diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/__init__.py b/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/data1.txt b/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/data1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data2.txt b/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data3.txt b/tests/masonry/builders/fixtures/default_src_with_excluded_data/src/my_package/data/sub_data/data3.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/masonry/builders/test_wheel.py b/tests/masonry/builders/test_wheel.py index 32fb05169..a8bbdd527 100644 --- a/tests/masonry/builders/test_wheel.py +++ b/tests/masonry/builders/test_wheel.py @@ -7,6 +7,7 @@ from poetry.core.factory import Factory from poetry.core.masonry.builders.wheel import WheelBuilder from poetry.core.utils._compat import Path +from tests.masonry.builders.test_sdist import project fixtures_dir = Path(__file__).parent / "fixtures" @@ -228,3 +229,44 @@ def test_wheel_with_file_with_comma(): with zipfile.ZipFile(str(whl)) as z: records = z.read("comma_file-1.2.3.dist-info/RECORD") assert '\n"comma_file/a,b.py"' in records.decode() + + +def test_default_src_with_excluded_data(mocker): + # Patch git module to return specific excluded files + p = mocker.patch("poetry.core.vcs.git.Git.get_ignored_files") + p.return_value = [ + ( + ( + Path(__file__).parent + / "fixtures" + / "default_src_with_excluded_data" + / "src" + / "my_package" + / "data" + / "sub_data" + / "data2.txt" + ) + .relative_to(project("default_src_with_excluded_data")) + .as_posix() + ) + ] + poetry = Factory().create_poetry(project("default_src_with_excluded_data")) + + builder = WheelBuilder(poetry) + builder.build() + + whl = ( + fixtures_dir + / "default_src_with_excluded_data" + / "dist" + / "my_package-1.2.3-py3-none-any.whl" + ) + + assert whl.exists() + + with zipfile.ZipFile(str(whl)) as z: + names = z.namelist() + assert "my_package/__init__.py" in names + assert "my_package/data/data1.txt" in names + assert "my_package/data/sub_data/data2.txt" not in names + assert "my_package/data/sub_data/data3.txt" in names From 7f4f70fe4cb996ee8a7a455226923f4a13f89714 Mon Sep 17 00:00:00 2001 From: finswimmer Date: Thu, 17 Sep 2020 06:29:21 +0200 Subject: [PATCH 2/3] new (masonry.builders.builder): add attribute `project_root` to `BuildIncludeFile` to store the original path to the project's root folder new (masonry.builders.builder): method to return relative path to project's root for `BuildIncludeFile` change (masonry.builders.builder): use relative path to project's root when checking if BuildIncludeFile is excluded fix (masonry.builders.wheel): fix substitution in string formation --- poetry/core/masonry/builders/builder.py | 34 +++++++++++++++++-------- poetry/core/masonry/builders/sdist.py | 4 ++- poetry/core/masonry/builders/wheel.py | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/poetry/core/masonry/builders/builder.py b/poetry/core/masonry/builders/builder.py index 55c9fd1f2..4e89775b1 100644 --- a/poetry/core/masonry/builders/builder.py +++ b/poetry/core/masonry/builders/builder.py @@ -152,7 +152,9 @@ def find_files_to_add( if self.format in formats: for current_file in file.glob("**/*"): include_file = BuildIncludeFile( - path=current_file, source_root=self._path + path=current_file, + project_root=self._original_path, + source_root=self._path, ) if not current_file.is_dir() and not self.is_excluded( @@ -161,19 +163,23 @@ def find_files_to_add( to_add.add(include_file) continue - include_file = BuildIncludeFile(path=file, source_root=self._path) - - if self.is_excluded( - include_file.relative_to_source_root() - ) and isinstance(include, PackageInclude): - continue - if ( isinstance(include, PackageInclude) and include.source and self.format == "wheel" ): - include_file.source_root = include.base + source_root = include.base + else: + source_root = self._path + + include_file = BuildIncludeFile( + path=file, project_root=self._original_path, source_root=source_root + ) + + if self.is_excluded( + include_file.relative_to_project_root() + ) and isinstance(include, PackageInclude): + continue if file.suffix == ".pyc": continue @@ -189,7 +195,9 @@ def find_files_to_add( if self._package.build_script and not exclude_build: to_add.add( BuildIncludeFile( - path=self._package.build_script, source_root=self._path + path=self._package.build_script, + project_root=self._original_path, + source_root=self._path, ) ) @@ -303,13 +311,16 @@ class BuildIncludeFile: def __init__( self, path, # type: Path + project_root, # type: Path source_root=None, # type: Optional[Path] ): """ + :param project_root: the full path of the project's root :param path: a full path to the file to be included :param source_root: the root path to resolve to """ self.path = Path(path) + self.project_root = Path(project_root).resolve() self.source_root = None if not source_root else Path(source_root).resolve() if not self.path.is_absolute() and self.source_root: self.path = (self.source_root / self.path).resolve() @@ -330,6 +341,9 @@ def __hash__(self): def __repr__(self): # type: () -> str return str(self.path) + def relative_to_project_root(self): # type(): -> Path + return self.path.relative_to(self.project_root) + def relative_to_source_root(self): # type(): -> Path if self.source_root is not None: return self.path.relative_to(self.source_root) diff --git a/poetry/core/masonry/builders/sdist.py b/poetry/core/masonry/builders/sdist.py index ae6cc677b..37d38a2ec 100644 --- a/poetry/core/masonry/builders/sdist.py +++ b/poetry/core/masonry/builders/sdist.py @@ -319,7 +319,9 @@ def find_files_to_add( additional_files.add(self._poetry.local_config["readme"]) for file in additional_files: - file = BuildIncludeFile(path=file, source_root=self._path) + file = BuildIncludeFile( + path=file, project_root=self._original_path, source_root=self._path + ) if file.path.exists(): logger.debug("Adding: {}".format(file.relative_to_source_root())) to_add.add(file) diff --git a/poetry/core/masonry/builders/wheel.py b/poetry/core/masonry/builders/wheel.py index c8698577d..58b798328 100644 --- a/poetry/core/masonry/builders/wheel.py +++ b/poetry/core/masonry/builders/wheel.py @@ -181,7 +181,7 @@ def _write_metadata(self, wheel): relative_path = "%s/%s" % (self.dist_info, path.relative_to(self._path)) self._add_file(wheel, path, relative_path) else: - logger.debug("Skipping: %s" % path.as_posix()) + logger.debug("Skipping: {}".format(path.as_posix())) with self._write_to_zip(wheel, self.dist_info + "/WHEEL") as f: self._write_wheel_file(f) From 0ff4ba5149d63f0d73a3581fe216d38f4e1382e0 Mon Sep 17 00:00:00 2001 From: finswimmer Date: Thu, 17 Sep 2020 17:54:54 +0200 Subject: [PATCH 3/3] change (masonry.builders.builder): remove obsolete `_original_path` attribute of `Builder` fix (masonry.builders.builder): fix some type hints --- poetry/core/masonry/builders/builder.py | 17 ++++++++--------- poetry/core/masonry/builders/sdist.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/poetry/core/masonry/builders/builder.py b/poetry/core/masonry/builders/builder.py index 4e89775b1..92ca60e07 100644 --- a/poetry/core/masonry/builders/builder.py +++ b/poetry/core/masonry/builders/builder.py @@ -45,7 +45,6 @@ def __init__( self._poetry = poetry self._package = poetry.package self._path = poetry.file.parent - self._original_path = self._path self._excluded_files = None packages = [] @@ -93,7 +92,7 @@ def build(self): def find_excluded_files(self): # type: () -> Set[str] if self._excluded_files is None: # Checking VCS - vcs = get_vcs(self._original_path) + vcs = get_vcs(self._path) if not vcs: vcs_ignored_files = set() else: @@ -153,7 +152,7 @@ def find_files_to_add( for current_file in file.glob("**/*"): include_file = BuildIncludeFile( path=current_file, - project_root=self._original_path, + project_root=self._path, source_root=self._path, ) @@ -173,7 +172,7 @@ def find_files_to_add( source_root = self._path include_file = BuildIncludeFile( - path=file, project_root=self._original_path, source_root=source_root + path=file, project_root=self._path, source_root=source_root ) if self.is_excluded( @@ -196,14 +195,14 @@ def find_files_to_add( to_add.add( BuildIncludeFile( path=self._package.build_script, - project_root=self._original_path, + project_root=self._path, source_root=self._path, ) ) return to_add - def get_metadata_content(self): # type: () -> bytes + def get_metadata_content(self): # type: () -> str content = METADATA_BASE.format( name=self._meta.name, version=self._meta.version, @@ -310,9 +309,9 @@ def temporary_directory(cls, *args, **kwargs): class BuildIncludeFile: def __init__( self, - path, # type: Path - project_root, # type: Path - source_root=None, # type: Optional[Path] + path, # type: Union[Path, str] + project_root, # type: Union[Path, str] + source_root=None, # type: Optional[Union[Path, str]] ): """ :param project_root: the full path of the project's root diff --git a/poetry/core/masonry/builders/sdist.py b/poetry/core/masonry/builders/sdist.py index 37d38a2ec..981b63b2a 100644 --- a/poetry/core/masonry/builders/sdist.py +++ b/poetry/core/masonry/builders/sdist.py @@ -320,7 +320,7 @@ def find_files_to_add( for file in additional_files: file = BuildIncludeFile( - path=file, project_root=self._original_path, source_root=self._path + path=file, project_root=self._path, source_root=self._path ) if file.path.exists(): logger.debug("Adding: {}".format(file.relative_to_source_root()))