From 9982b095edb0404b9be98e278b4d889616a3bdb7 Mon Sep 17 00:00:00 2001 From: Tobias Klare Date: Mon, 14 Oct 2019 09:37:29 +0200 Subject: [PATCH 01/13] This PR impliments the feature request #784. When a folder is explicit defined in `pyproject.toml` as excluded, all nested data, including subfolder, are excluded. It is no longer neccessary to use the glob `folder/**/*` --- poetry/masonry/builders/builder.py | 2 ++ tests/masonry/builders/test_sdist.py | 26 ++++++++++++++++++++++++++ tests/masonry/builders/test_wheel.py | 17 +++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index ef38540743f..3341997c4f3 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -86,6 +86,8 @@ def find_excluded_files(self): # type: () -> Set[str] explicitely_excluded = set() for excluded_glob in self._package.exclude: + if Path(self._path.as_posix(), str(excluded_glob)).is_dir(): + excluded_glob = Path(str(excluded_glob), "**/*").as_posix() for excluded in glob( os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True ): diff --git a/tests/masonry/builders/test_sdist.py b/tests/masonry/builders/test_sdist.py index 281016a3653..0c4bae44736 100644 --- a/tests/masonry/builders/test_sdist.py +++ b/tests/masonry/builders/test_sdist.py @@ -410,6 +410,32 @@ def test_default_with_excluded_data(mocker): assert "my-package-1.2.3/PKG-INFO" in names +def test_src_excluded_nested_data(): + module_path = fixtures_dir / "default_with_excluded_data_toml" + poetry = Factory().create_poetry(module_path) + poetry._package.exclude = ["my_package/data"] + + builder = SdistBuilder(poetry, NullEnv(), NullIO()) + builder.build() + + sdist = module_path / "dist" / "my-package-1.2.3.tar.gz" + + assert sdist.exists() + + with tarfile.open(str(sdist), "r") as tar: + names = tar.getnames() + assert len(names) == len(set(names)) + assert "my-package-1.2.3/LICENSE" in names + assert "my-package-1.2.3/README.rst" in names + assert "my-package-1.2.3/my_package/__init__.py" in names + assert "my-package-1.2.3/pyproject.toml" in names + assert "my-package-1.2.3/setup.py" in names + assert "my-package-1.2.3/PKG-INFO" in names + assert "my-package-1.2.3/my_package/data/data1.txt" not in names + assert "my-package-1.2.3/my_package/data/sub_data/data2.txt" not in names + assert "my-package-1.2.3/my_package/data/sub_data/data3.txt" not in names + + def test_proper_python_requires_if_two_digits_precision_version_specified(): poetry = Factory().create_poetry(project("simple_version")) diff --git a/tests/masonry/builders/test_wheel.py b/tests/masonry/builders/test_wheel.py index 7b5ec8611ea..fe4e78489b6 100644 --- a/tests/masonry/builders/test_wheel.py +++ b/tests/masonry/builders/test_wheel.py @@ -77,6 +77,23 @@ def test_wheel_excluded_data(): assert "my_package/data/data1.txt" not in z.namelist() +def test_wheel_excluded_nested_data(): + module_path = fixtures_dir / "default_with_excluded_data_toml" + poetry = Factory().create_poetry(module_path) + poetry._package.exclude = ["my_package/data"] + WheelBuilder.make(poetry, NullEnv(), NullIO()) + + whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" + + assert whl.exists() + + with zipfile.ZipFile(str(whl)) as z: + assert "my_package/__init__.py" in z.namelist() + assert "my_package/data/sub_data/data2.txt" not in z.namelist() + assert "my_package/data/sub_data/data3.txt" not in z.namelist() + assert "my_package/data/data1.txt" not in z.namelist() + + def test_wheel_localversionlabel(): module_path = fixtures_dir / "localversionlabel" WheelBuilder.make(Factory().create_poetry(module_path), NullEnv(), NullIO()) From 35603ac71baeb2b95e407ca00661207fbbdfa264 Mon Sep 17 00:00:00 2001 From: Tobias Klare Date: Mon, 14 Oct 2019 10:53:28 +0200 Subject: [PATCH 02/13] use `Path` instead of `os.path.join` to create string for globbing --- poetry/masonry/builders/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 3341997c4f3..ebdb3373a76 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -89,7 +89,7 @@ def find_excluded_files(self): # type: () -> Set[str] if Path(self._path.as_posix(), str(excluded_glob)).is_dir(): excluded_glob = Path(str(excluded_glob), "**/*").as_posix() for excluded in glob( - os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True + Path(self._path.as_posix(), str(excluded_glob)).as_posix(), recursive=True ): explicitely_excluded.add( Path(excluded).relative_to(self._path).as_posix() From 0834f780f57fcb70d80984e5956cbfd73e902b57 Mon Sep 17 00:00:00 2001 From: Tobias Klare Date: Mon, 14 Oct 2019 10:55:46 +0200 Subject: [PATCH 03/13] try to fix linting error --- poetry/masonry/builders/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index ebdb3373a76..39d7bc1bb89 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -89,7 +89,8 @@ def find_excluded_files(self): # type: () -> Set[str] if Path(self._path.as_posix(), str(excluded_glob)).is_dir(): excluded_glob = Path(str(excluded_glob), "**/*").as_posix() for excluded in glob( - Path(self._path.as_posix(), str(excluded_glob)).as_posix(), recursive=True + Path(self._path.as_posix(), str(excluded_glob)).as_posix(), + recursive=True, ): explicitely_excluded.add( Path(excluded).relative_to(self._path).as_posix() From e5e125a01266778ff5186989c0c01064144a69b7 Mon Sep 17 00:00:00 2001 From: Tobias Klare Date: Mon, 14 Oct 2019 12:16:23 +0200 Subject: [PATCH 04/13] create glob pattern string by concatenating and not using Path --- poetry/masonry/builders/builder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 39d7bc1bb89..3cecd6a6d84 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -87,10 +87,9 @@ def find_excluded_files(self): # type: () -> Set[str] explicitely_excluded = set() for excluded_glob in self._package.exclude: if Path(self._path.as_posix(), str(excluded_glob)).is_dir(): - excluded_glob = Path(str(excluded_glob), "**/*").as_posix() + excluded_glob = str(excluded_glob) + "/**/*" for excluded in glob( - Path(self._path.as_posix(), str(excluded_glob)).as_posix(), - recursive=True, + os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True ): explicitely_excluded.add( Path(excluded).relative_to(self._path).as_posix() From 1df49c8257a21640f7c5f765e3dce48c8db2a646 Mon Sep 17 00:00:00 2001 From: Tobias Klare Date: Mon, 14 Oct 2019 15:17:18 +0200 Subject: [PATCH 05/13] using `os.path.isdir()`` for checking of explicit excluded name is a folder, because pathlib's `is_dir()` raises in exception under windows of name contains globing characters --- poetry/masonry/builders/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 3cecd6a6d84..fdcb06d146f 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -86,8 +86,8 @@ def find_excluded_files(self): # type: () -> Set[str] explicitely_excluded = set() for excluded_glob in self._package.exclude: - if Path(self._path.as_posix(), str(excluded_glob)).is_dir(): - excluded_glob = str(excluded_glob) + "/**/*" + if os.path.isdir(os.path.join(self._path.as_posix(), str(excluded_glob))): + excluded_glob = Path(str(excluded_glob), "**/*") for excluded in glob( os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True ): From 56868de3b0c6093237872d2c10a0af233f3431d3 Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Tue, 15 Oct 2019 06:09:48 +0200 Subject: [PATCH 06/13] Remove nested data when wildcards where used. Steps to do this are: 1. expand any wildcard used 2. if expanded path is a folder append **/* and expand again --- poetry/masonry/builders/builder.py | 16 +++++--- .../fixtures/exclude_nested_data_toml/LICENSE | 20 +++++++++ .../exclude_nested_data_toml/README.rst | 2 + .../my_package/__init__.py | 0 .../my_package/data/data1.txt | 0 .../my_package/data/sub_data/data2.txt | 0 .../my_package/data/sub_data/data3.txt | 0 .../my_package/puplic/item1/itemdata1.txt | 0 .../puplic/item1/subitem/subitemdata.txt | 0 .../my_package/puplic/item2/itemdata2.txt | 0 .../my_package/puplic/publicdata.txt | 0 .../exclude_nested_data_toml/pyproject.toml | 41 +++++++++++++++++++ tests/masonry/builders/test_sdist.py | 11 +++-- tests/masonry/builders/test_wheel.py | 7 +++- 14 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index fdcb06d146f..ef2997885d4 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import itertools import os import re import shutil @@ -85,12 +86,15 @@ def find_excluded_files(self): # type: () -> Set[str] vcs_ignored_files = set(vcs.get_ignored_files()) explicitely_excluded = set() - for excluded_glob in self._package.exclude: - if os.path.isdir(os.path.join(self._path.as_posix(), str(excluded_glob))): - excluded_glob = Path(str(excluded_glob), "**/*") - for excluded in glob( - os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True - ): + for excluded_glob in itertools.chain.from_iterable( + map( + lambda excluded_path: glob(Path(self._path, excluded_path).as_posix(), recursive=True), + self._package.exclude, + ) + ): + if Path(excluded_glob).is_dir(): + excluded_glob = Path(excluded_glob, "**/*") + for excluded in glob(str(excluded_glob), recursive=True): explicitely_excluded.add( Path(excluded).relative_to(self._path).as_posix() ) diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE b/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE new file mode 100644 index 00000000000..44cf2b30e68 --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2018 Sébastien Eustace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst b/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst new file mode 100644 index 00000000000..f7fe15470f9 --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst @@ -0,0 +1,2 @@ +My Package +========== diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml b/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml new file mode 100644 index 00000000000..21efcba30eb --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml @@ -0,0 +1,41 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Sébastien Eustace " +] +license = "MIT" + +readme = "README.rst" + +exclude = ["my_package/data/", "**/*/item*"] + +homepage = "https://poetry.eustace.io/" +repository = "https://github.com/sdispater/poetry" +documentation = "https://poetry.eustace.io/docs" + +keywords = ["packaging", "dependency", "poetry"] + +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +# Requirements +[tool.poetry.dependencies] +python = "^3.6" +cleo = "^0.6" +cachy = { version = "^0.2.0", extras = ["msgpack"] } + +pendulum = { version = "^1.4", optional = true } + +[tool.poetry.dev-dependencies] +pytest = "~3.4" + +[tool.poetry.extras] +time = ["pendulum"] + +[tool.poetry.scripts] +my-script = "my_package:main" +my-2nd-script = "my_package:main2" diff --git a/tests/masonry/builders/test_sdist.py b/tests/masonry/builders/test_sdist.py index 0c4bae44736..496d608a2fc 100644 --- a/tests/masonry/builders/test_sdist.py +++ b/tests/masonry/builders/test_sdist.py @@ -411,9 +411,8 @@ def test_default_with_excluded_data(mocker): def test_src_excluded_nested_data(): - module_path = fixtures_dir / "default_with_excluded_data_toml" + module_path = fixtures_dir / "exclude_nested_data_toml" poetry = Factory().create_poetry(module_path) - poetry._package.exclude = ["my_package/data"] builder = SdistBuilder(poetry, NullEnv(), NullIO()) builder.build() @@ -427,13 +426,17 @@ def test_src_excluded_nested_data(): assert len(names) == len(set(names)) assert "my-package-1.2.3/LICENSE" in names assert "my-package-1.2.3/README.rst" in names - assert "my-package-1.2.3/my_package/__init__.py" in names assert "my-package-1.2.3/pyproject.toml" in names assert "my-package-1.2.3/setup.py" in names assert "my-package-1.2.3/PKG-INFO" in names - assert "my-package-1.2.3/my_package/data/data1.txt" not in names + assert "my-package-1.2.3/my_package/__init__.py" in names assert "my-package-1.2.3/my_package/data/sub_data/data2.txt" not in names assert "my-package-1.2.3/my_package/data/sub_data/data3.txt" not in names + assert "my-package-1.2.3/my_package/data/data1.txt" not in names + assert "my-package-1.2.3/my_package/puplic/publicdata.txt" in names + assert "my-package-1.2.3/my_package/public/item1/itemdata1.txt" not in names + assert "my-package-1.2.3/my_package/public/item1/subitem/subitemdata.txt" not in names + assert "my-package-1.2.3/my_package/public/item2/itemdata2.txt" not in names def test_proper_python_requires_if_two_digits_precision_version_specified(): diff --git a/tests/masonry/builders/test_wheel.py b/tests/masonry/builders/test_wheel.py index fe4e78489b6..33c50a5d6a7 100644 --- a/tests/masonry/builders/test_wheel.py +++ b/tests/masonry/builders/test_wheel.py @@ -78,9 +78,8 @@ def test_wheel_excluded_data(): def test_wheel_excluded_nested_data(): - module_path = fixtures_dir / "default_with_excluded_data_toml" + module_path = fixtures_dir / "exclude_nested_data_toml" poetry = Factory().create_poetry(module_path) - poetry._package.exclude = ["my_package/data"] WheelBuilder.make(poetry, NullEnv(), NullIO()) whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" @@ -92,6 +91,10 @@ def test_wheel_excluded_nested_data(): assert "my_package/data/sub_data/data2.txt" not in z.namelist() assert "my_package/data/sub_data/data3.txt" not in z.namelist() assert "my_package/data/data1.txt" not in z.namelist() + assert "my_package/puplic/publicdata.txt" in z.namelist() + assert "my_package/public/item1/itemdata1.txt" not in z.namelist() + assert "my_package/public/item1/subitem/subitemdata.txt" not in z.namelist() + assert "my_package/public/item2/itemdata2.txt" not in z.namelist() def test_wheel_localversionlabel(): From a8a9c5df1c032482be671d85003b3a6711c9a6ba Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Tue, 15 Oct 2019 06:09:48 +0200 Subject: [PATCH 07/13] fix linting --- poetry/masonry/builders/builder.py | 18 +++++--- .../fixtures/exclude_nested_data_toml/LICENSE | 20 +++++++++ .../exclude_nested_data_toml/README.rst | 2 + .../my_package/__init__.py | 0 .../my_package/data/data1.txt | 0 .../my_package/data/sub_data/data2.txt | 0 .../my_package/data/sub_data/data3.txt | 0 .../my_package/puplic/item1/itemdata1.txt | 0 .../puplic/item1/subitem/subitemdata.txt | 0 .../my_package/puplic/item2/itemdata2.txt | 0 .../my_package/puplic/publicdata.txt | 0 .../exclude_nested_data_toml/pyproject.toml | 41 +++++++++++++++++++ tests/masonry/builders/test_sdist.py | 14 +++++-- tests/masonry/builders/test_wheel.py | 7 +++- 14 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt create mode 100644 tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index fdcb06d146f..69f2cac7fcf 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import itertools import os import re import shutil @@ -85,12 +86,17 @@ def find_excluded_files(self): # type: () -> Set[str] vcs_ignored_files = set(vcs.get_ignored_files()) explicitely_excluded = set() - for excluded_glob in self._package.exclude: - if os.path.isdir(os.path.join(self._path.as_posix(), str(excluded_glob))): - excluded_glob = Path(str(excluded_glob), "**/*") - for excluded in glob( - os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True - ): + for excluded_glob in itertools.chain.from_iterable( + map( + lambda excluded_path: glob( + Path(self._path, excluded_path).as_posix(), recursive=True + ), + self._package.exclude, + ) + ): + if Path(excluded_glob).is_dir(): + excluded_glob = Path(excluded_glob, "**/*") + for excluded in glob(str(excluded_glob), recursive=True): explicitely_excluded.add( Path(excluded).relative_to(self._path).as_posix() ) diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE b/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE new file mode 100644 index 00000000000..44cf2b30e68 --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2018 Sébastien Eustace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst b/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst new file mode 100644 index 00000000000..f7fe15470f9 --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst @@ -0,0 +1,2 @@ +My Package +========== diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/data1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/data/sub_data/data3.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/itemdata1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item1/subitem/subitemdata.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/item2/itemdata2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt b/tests/masonry/builders/fixtures/exclude_nested_data_toml/my_package/puplic/publicdata.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml b/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml new file mode 100644 index 00000000000..21efcba30eb --- /dev/null +++ b/tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml @@ -0,0 +1,41 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Sébastien Eustace " +] +license = "MIT" + +readme = "README.rst" + +exclude = ["my_package/data/", "**/*/item*"] + +homepage = "https://poetry.eustace.io/" +repository = "https://github.com/sdispater/poetry" +documentation = "https://poetry.eustace.io/docs" + +keywords = ["packaging", "dependency", "poetry"] + +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +# Requirements +[tool.poetry.dependencies] +python = "^3.6" +cleo = "^0.6" +cachy = { version = "^0.2.0", extras = ["msgpack"] } + +pendulum = { version = "^1.4", optional = true } + +[tool.poetry.dev-dependencies] +pytest = "~3.4" + +[tool.poetry.extras] +time = ["pendulum"] + +[tool.poetry.scripts] +my-script = "my_package:main" +my-2nd-script = "my_package:main2" diff --git a/tests/masonry/builders/test_sdist.py b/tests/masonry/builders/test_sdist.py index 0c4bae44736..b67dc5ee229 100644 --- a/tests/masonry/builders/test_sdist.py +++ b/tests/masonry/builders/test_sdist.py @@ -411,9 +411,8 @@ def test_default_with_excluded_data(mocker): def test_src_excluded_nested_data(): - module_path = fixtures_dir / "default_with_excluded_data_toml" + module_path = fixtures_dir / "exclude_nested_data_toml" poetry = Factory().create_poetry(module_path) - poetry._package.exclude = ["my_package/data"] builder = SdistBuilder(poetry, NullEnv(), NullIO()) builder.build() @@ -427,13 +426,20 @@ def test_src_excluded_nested_data(): assert len(names) == len(set(names)) assert "my-package-1.2.3/LICENSE" in names assert "my-package-1.2.3/README.rst" in names - assert "my-package-1.2.3/my_package/__init__.py" in names assert "my-package-1.2.3/pyproject.toml" in names assert "my-package-1.2.3/setup.py" in names assert "my-package-1.2.3/PKG-INFO" in names - assert "my-package-1.2.3/my_package/data/data1.txt" not in names + assert "my-package-1.2.3/my_package/__init__.py" in names assert "my-package-1.2.3/my_package/data/sub_data/data2.txt" not in names assert "my-package-1.2.3/my_package/data/sub_data/data3.txt" not in names + assert "my-package-1.2.3/my_package/data/data1.txt" not in names + assert "my-package-1.2.3/my_package/puplic/publicdata.txt" in names + assert "my-package-1.2.3/my_package/public/item1/itemdata1.txt" not in names + assert ( + "my-package-1.2.3/my_package/public/item1/subitem/subitemdata.txt" + not in names + ) + assert "my-package-1.2.3/my_package/public/item2/itemdata2.txt" not in names def test_proper_python_requires_if_two_digits_precision_version_specified(): diff --git a/tests/masonry/builders/test_wheel.py b/tests/masonry/builders/test_wheel.py index fe4e78489b6..33c50a5d6a7 100644 --- a/tests/masonry/builders/test_wheel.py +++ b/tests/masonry/builders/test_wheel.py @@ -78,9 +78,8 @@ def test_wheel_excluded_data(): def test_wheel_excluded_nested_data(): - module_path = fixtures_dir / "default_with_excluded_data_toml" + module_path = fixtures_dir / "exclude_nested_data_toml" poetry = Factory().create_poetry(module_path) - poetry._package.exclude = ["my_package/data"] WheelBuilder.make(poetry, NullEnv(), NullIO()) whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" @@ -92,6 +91,10 @@ def test_wheel_excluded_nested_data(): assert "my_package/data/sub_data/data2.txt" not in z.namelist() assert "my_package/data/sub_data/data3.txt" not in z.namelist() assert "my_package/data/data1.txt" not in z.namelist() + assert "my_package/puplic/publicdata.txt" in z.namelist() + assert "my_package/public/item1/itemdata1.txt" not in z.namelist() + assert "my_package/public/item1/subitem/subitemdata.txt" not in z.namelist() + assert "my_package/public/item2/itemdata2.txt" not in z.namelist() def test_wheel_localversionlabel(): From ce576c7040fcdab4e27b464b85250b02370de77f Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Tue, 15 Oct 2019 20:44:21 +0200 Subject: [PATCH 08/13] only glob a second time if path is dir --- poetry/masonry/builders/builder.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 69f2cac7fcf..4d81f30a237 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -89,16 +89,19 @@ def find_excluded_files(self): # type: () -> Set[str] for excluded_glob in itertools.chain.from_iterable( map( lambda excluded_path: glob( - Path(self._path, excluded_path).as_posix(), recursive=True + str(Path(self._path, excluded_path)), recursive=True ), self._package.exclude, ) ): if Path(excluded_glob).is_dir(): - excluded_glob = Path(excluded_glob, "**/*") - for excluded in glob(str(excluded_glob), recursive=True): + for excluded in glob(str(Path(excluded_glob, "**/*")), recursive=True): + explicitely_excluded.add( + Path(excluded).relative_to(self._path).as_posix() + ) + else: explicitely_excluded.add( - Path(excluded).relative_to(self._path).as_posix() + Path(excluded_glob).relative_to(self._path).as_posix() ) ignored = vcs_ignored_files | explicitely_excluded From 1cd93bdbe0f7fbab179b5d6f1a13dfa6e67f746e Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Fri, 18 Oct 2019 19:07:24 +0200 Subject: [PATCH 09/13] implement @sdispater 's suggestion for better readability --- poetry/masonry/builders/builder.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 4d81f30a237..6a5d433195a 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -86,23 +86,13 @@ def find_excluded_files(self): # type: () -> Set[str] vcs_ignored_files = set(vcs.get_ignored_files()) explicitely_excluded = set() - for excluded_glob in itertools.chain.from_iterable( - map( - lambda excluded_path: glob( - str(Path(self._path, excluded_path)), recursive=True - ), - self._package.exclude, - ) - ): - if Path(excluded_glob).is_dir(): - for excluded in glob(str(Path(excluded_glob, "**/*")), recursive=True): - explicitely_excluded.add( - Path(excluded).relative_to(self._path).as_posix() - ) - else: - explicitely_excluded.add( - Path(excluded_glob).relative_to(self._path).as_posix() - ) + for excluded_glob in self._package.exclude: + excluded_path = Path(self._path, excluded_glob) + if excluded_path.is_dir(): + excluded_glob = Path(excluded_glob, "**/*").as_posix() + + for excluded in self._path.glob(excluded_glob): + explicitely_excluded.add(excluded.relative_to(self._path).as_posix()) ignored = vcs_ignored_files | explicitely_excluded result = set() From fa8c04f8e26228ce941d658767a083807b8081e6 Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Fri, 18 Oct 2019 19:24:59 +0200 Subject: [PATCH 10/13] fix glob for windows? --- poetry/masonry/builders/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 6a5d433195a..4b874a29ead 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -89,7 +89,7 @@ def find_excluded_files(self): # type: () -> Set[str] for excluded_glob in self._package.exclude: excluded_path = Path(self._path, excluded_glob) if excluded_path.is_dir(): - excluded_glob = Path(excluded_glob, "**/*").as_posix() + excluded_glob = str(Path(excluded_glob, "**/*")) for excluded in self._path.glob(excluded_glob): explicitely_excluded.add(excluded.relative_to(self._path).as_posix()) From 09af714e1cfedc95f24eb655acc31727ee7f0086 Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Sat, 19 Oct 2019 09:56:51 +0200 Subject: [PATCH 11/13] On Windows, testing if a path with a glob is a directory will raise an OSError --- poetry/masonry/builders/builder.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 4b874a29ead..3822e17de7f 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -88,8 +88,14 @@ def find_excluded_files(self): # type: () -> Set[str] explicitely_excluded = set() for excluded_glob in self._package.exclude: excluded_path = Path(self._path, excluded_glob) - if excluded_path.is_dir(): - excluded_glob = str(Path(excluded_glob, "**/*")) + + try: + is_dir = excluded_path.is_dir() + except OSError: + # On Windows, testing if a path with a glob is a directory will raise an OSError + is_dir = False + if is_dir: + excluded_glob = Path(excluded_glob, "**/*").as_posix() for excluded in self._path.glob(excluded_glob): explicitely_excluded.add(excluded.relative_to(self._path).as_posix()) From 56179212db2d9dcfad271c9fe3f24f8d7c121774 Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Sat, 19 Oct 2019 10:35:10 +0200 Subject: [PATCH 12/13] pathlibs glob function doesn't return the correct case (https://bugs.python.org/issue26655). So switching back to glob.glob() --- poetry/masonry/builders/builder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index 3822e17de7f..c1952857c33 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -95,10 +95,14 @@ def find_excluded_files(self): # type: () -> Set[str] # On Windows, testing if a path with a glob is a directory will raise an OSError is_dir = False if is_dir: - excluded_glob = Path(excluded_glob, "**/*").as_posix() + excluded_glob = Path(excluded_glob, "**/*") - for excluded in self._path.glob(excluded_glob): - explicitely_excluded.add(excluded.relative_to(self._path).as_posix()) + for excluded in glob( + Path(self._path, excluded_glob).as_posix(), recursive=True + ): + explicitely_excluded.add( + Path(excluded).relative_to(self._path).as_posix() + ) ignored = vcs_ignored_files | explicitely_excluded result = set() From 38ed4d697846cb8c544d4f5d5e616134935bc02b Mon Sep 17 00:00:00 2001 From: "finswimmer77@gmail.com" Date: Sat, 19 Oct 2019 13:18:48 +0200 Subject: [PATCH 13/13] removing obsolete imports --- poetry/masonry/builders/builder.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/poetry/masonry/builders/builder.py b/poetry/masonry/builders/builder.py index c1952857c33..14ca2de7db6 100644 --- a/poetry/masonry/builders/builder.py +++ b/poetry/masonry/builders/builder.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- -import itertools -import os import re import shutil import tempfile - from collections import defaultdict from contextlib import contextmanager from typing import Set @@ -18,12 +15,10 @@ from poetry.utils._compat import lru_cache from poetry.utils._compat import to_str from poetry.vcs import get_vcs - from ..metadata import Metadata from ..utils.module import Module from ..utils.package_include import PackageInclude - AUTHOR_REGEX = re.compile(r"(?u)^(?P[- .,\w\d'’\"()]+) <(?P.+?)>$") METADATA_BASE = """\ @@ -35,7 +30,6 @@ class Builder(object): - AVAILABLE_PYTHONS = {"2", "2.7", "3", "3.4", "3.5", "3.6", "3.7"} format = None