From f8798262044499a29b875769fbd7cb446a88aab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 27 Sep 2021 23:13:21 +0100 Subject: [PATCH] destinations: receive scheme>records mapping in finalize_installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .pre-commit-config.yaml | 2 +- src/installer/_core.py | 18 +++-- src/installer/destinations.py | 9 ++- tests/test_core.py | 141 ++++++++++++++++++++++------------ tests/test_destinations.py | 62 +++++++++------ 5 files changed, 147 insertions(+), 85 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b53d790..c05e3ef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: name: mypy (Python 2) additional_dependencies: ["pathlib2"] exclude: docs/.*|tests/.*|tools/.*|noxfile.py - args: ["-2"] + args: ["-2", "--ignore-missing-imports"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: 'v4.0.1' diff --git a/src/installer/_core.py b/src/installer/_core.py index 6e7ced41..e483a0a6 100644 --- a/src/installer/_core.py +++ b/src/installer/_core.py @@ -1,5 +1,6 @@ """Core wheel installation logic.""" +import collections import posixpath from io import BytesIO @@ -8,13 +9,12 @@ from installer.exceptions import InvalidWheelSource from installer.records import RecordEntry from installer.sources import WheelSource -from installer.utils import SCHEME_NAMES, parse_entrypoints, parse_metadata_file +from installer.utils import SCHEME_NAMES, Scheme, parse_entrypoints, parse_metadata_file if TYPE_CHECKING: - from typing import Dict, Tuple + from typing import DefaultDict, Dict, List, Tuple from installer._compat.typing import Binary, FSPath - from installer.utils import Scheme __all__ = ["install"] @@ -82,7 +82,9 @@ def install(source, destination, additional_metadata): # RECORD handling record_file_path = posixpath.join(source.dist_info_dir, u"RECORD") - written_records = [] + written_records = collections.defaultdict( + list + ) # type: DefaultDict[Scheme, List[RecordEntry]] # Write the entry-points based scripts. if "entry_points.txt" in source.dist_info_filenames: @@ -94,7 +96,7 @@ def install(source, destination, additional_metadata): attr=attr, section=section, ) - written_records.append(record) + written_records[Scheme("scripts")].append(record) # Write all the files from the wheel. for record_elements, stream in source.get_contents(): @@ -115,7 +117,7 @@ def install(source, destination, additional_metadata): path=destination_path, stream=stream, ) - written_records.append(record) + written_records[scheme].append(record) # Write all the installation-specific metadata for filename, contents in additional_metadata.items(): @@ -127,9 +129,9 @@ def install(source, destination, additional_metadata): path=path, stream=other_stream, ) - written_records.append(record) + written_records[root_scheme].append(record) - written_records.append(RecordEntry(record_file_path, None, None)) + written_records[root_scheme].append(RecordEntry(record_file_path, None, None)) destination.finalize_installation( scheme=root_scheme, record_file_path=record_file_path, diff --git a/src/installer/destinations.py b/src/installer/destinations.py index e00bb261..0eec8be8 100644 --- a/src/installer/destinations.py +++ b/src/installer/destinations.py @@ -1,6 +1,7 @@ """Handles all file writing and post-installation processing.""" import io +import itertools import os from installer._compat import FileExistsError @@ -15,6 +16,7 @@ ) if TYPE_CHECKING: + from collections.abc import Mapping from typing import BinaryIO, Dict, Iterable from installer._compat.typing import FSPath, Text @@ -65,7 +67,7 @@ def write_file(self, scheme, path, stream): raise NotImplementedError def finalize_installation(self, scheme, record_file_path, records): - # type: (Scheme, FSPath, Iterable[RecordEntry]) -> None + # type: (Scheme, FSPath, Mapping[Scheme, Iterable[RecordEntry]]) -> None """Finalize installation, after all the files are written. Handles (re)writing of the ``RECORD`` file. @@ -180,12 +182,13 @@ def write_script(self, name, module, attr, section): return entry def finalize_installation(self, scheme, record_file_path, records): - # type: (Scheme, FSPath, Iterable[RecordEntry]) -> None + # type: (Scheme, FSPath, Mapping[Scheme, Iterable[RecordEntry]]) -> None """Finalize installation, by writing the ``RECORD`` file. :param scheme: scheme to write the ``RECORD`` file in :param record_file_path: path of the ``RECORD`` file with that scheme :param records: entries to write to the ``RECORD`` file """ - with construct_record_file(records) as record_stream: + records_set = set(itertools.chain.from_iterable(records.values())) + with construct_record_file(records_set) as record_stream: self.write_to_fs(scheme, record_file_path, record_stream) diff --git a/tests/test_core.py b/tests/test_core.py index c53788f9..206d2803 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import collections import hashlib import textwrap from io import BytesIO @@ -204,18 +205,29 @@ def main(): mock.call.finalize_installation( scheme="purelib", record_file_path="fancy-1.0.0.dist-info/RECORD", - records=[ - ("fancy", "fancy", "main", "console"), - ("fancy-gui", "fancy", "main", "gui"), - ("fancy/__init__.py", "purelib", 0), - ("fancy/__main__.py", "purelib", 0), - ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), - ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), - ("fancy-1.0.0.dist-info/entry_points.txt", "purelib", 0), - ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), - ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), - RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), - ], + records=collections.defaultdict( + list, + { + "scripts": [ + ("fancy", "fancy", "main", "console"), + ("fancy-gui", "fancy", "main", "gui"), + ], + "purelib": [ + ("fancy/__init__.py", "purelib", 0), + ("fancy/__main__.py", "purelib", 0), + ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), + ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), + ( + "fancy-1.0.0.dist-info/entry_points.txt", + "purelib", + 0, + ), + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ], + }, + ), ), ] ) @@ -304,15 +316,20 @@ def main(): mock.call.finalize_installation( scheme="purelib", record_file_path="fancy-1.0.0.dist-info/RECORD", - records=[ - ("fancy/__init__.py", "purelib", 0), - ("fancy/__main__.py", "purelib", 0), - ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), - ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), - ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), - ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), - RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), - ], + records=collections.defaultdict( + list, + { + "purelib": [ + ("fancy/__init__.py", "purelib", 0), + ("fancy/__main__.py", "purelib", 0), + ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), + ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ], + }, + ), ), ] ) @@ -425,18 +442,29 @@ def main(): mock.call.finalize_installation( scheme="platlib", record_file_path="fancy-1.0.0.dist-info/RECORD", - records=[ - ("fancy", "fancy", "main", "console"), - ("fancy-gui", "fancy", "main", "gui"), - ("fancy/__init__.py", "platlib", 0), - ("fancy/__main__.py", "platlib", 0), - ("fancy-1.0.0.dist-info/METADATA", "platlib", 0), - ("fancy-1.0.0.dist-info/WHEEL", "platlib", 0), - ("fancy-1.0.0.dist-info/entry_points.txt", "platlib", 0), - ("fancy-1.0.0.dist-info/top_level.txt", "platlib", 0), - ("fancy-1.0.0.dist-info/fun_file.txt", "platlib", 0), - RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), - ], + records=collections.defaultdict( + list, + { + "scripts": [ + ("fancy", "fancy", "main", "console"), + ("fancy-gui", "fancy", "main", "gui"), + ], + "platlib": [ + ("fancy/__init__.py", "platlib", 0), + ("fancy/__main__.py", "platlib", 0), + ("fancy-1.0.0.dist-info/METADATA", "platlib", 0), + ("fancy-1.0.0.dist-info/WHEEL", "platlib", 0), + ( + "fancy-1.0.0.dist-info/entry_points.txt", + "platlib", + 0, + ), + ("fancy-1.0.0.dist-info/top_level.txt", "platlib", 0), + ("fancy-1.0.0.dist-info/fun_file.txt", "platlib", 0), + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ], + }, + ), ), ] ) @@ -690,21 +718,38 @@ def test_handles_data_properly(self, mock_destination): mock.call.finalize_installation( scheme="purelib", record_file_path="fancy-1.0.0.dist-info/RECORD", - records=[ - ("fancy", "fancy", "main", "console"), - ("fancy-gui", "fancy", "main", "gui"), - ("fancy/data.py", "data", 0), - ("fancy/headers.py", "headers", 0), - ("fancy/platlib.py", "platlib", 0), - ("fancy/purelib.py", "purelib", 0), - ("fancy/scripts.py", "scripts", 0), - ("fancy/__init__.py", "purelib", 0), - ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), - ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), - ("fancy-1.0.0.dist-info/entry_points.txt", "purelib", 0), - ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), - RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), - ], + records=collections.defaultdict( + list, + { + "scripts": [ + ("fancy", "fancy", "main", "console"), + ("fancy-gui", "fancy", "main", "gui"), + ("fancy/scripts.py", "scripts", 0), + ], + "data": [ + ("fancy/data.py", "data", 0), + ], + "headers": [ + ("fancy/headers.py", "headers", 0), + ], + "platlib": [ + ("fancy/platlib.py", "platlib", 0), + ], + "purelib": [ + ("fancy/purelib.py", "purelib", 0), + ("fancy/__init__.py", "purelib", 0), + ("fancy-1.0.0.dist-info/METADATA", "purelib", 0), + ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0), + ( + "fancy-1.0.0.dist-info/entry_points.txt", + "purelib", + 0, + ), + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ], + }, + ), ), ] ) diff --git a/tests/test_destinations.py b/tests/test_destinations.py index d9d9fc03..20fd0dd4 100644 --- a/tests/test_destinations.py +++ b/tests/test_destinations.py @@ -94,32 +94,44 @@ def test_write_script(self, destination): assert record.path == "my_entrypoint" def test_finalize_write_record(self, destination): - records = [ - destination.write_file("data", "my_data1.bin", io.BytesIO(b"my data 1")), - destination.write_file("data", "my_data2.bin", io.BytesIO(b"my data 2")), - destination.write_file("data", "my_data3.bin", io.BytesIO(b"my data 3")), - destination.write_file("scripts", "my_script", io.BytesIO(b"my script")), - destination.write_file( - "scripts", "my_script2", io.BytesIO(b"#!python\nmy script") - ), - destination.write_script( - "my_entrypoint", "my_module", "my_function", "console" - ), - RecordEntry("RECORD", None, None), - ] + records = { + "purelib": [ + destination.write_file( + "data", "my_data1.bin", io.BytesIO(b"my data 1") + ), + destination.write_file( + "data", "my_data2.bin", io.BytesIO(b"my data 2") + ), + destination.write_file( + "data", "my_data3.bin", io.BytesIO(b"my data 3") + ), + destination.write_file( + "scripts", "my_script", io.BytesIO(b"my script") + ), + RecordEntry("RECORD", None, None), + ], + "scripts": [ + destination.write_file( + "scripts", "my_script2", io.BytesIO(b"#!python\nmy script") + ), + destination.write_script( + "my_entrypoint", "my_module", "my_function", "console" + ), + ], + } destination.finalize_installation("purelib", "RECORD", records) file_path = os.path.join(destination.scheme_dict["purelib"], "RECORD") - with open(file_path, "rb") as f: - data = f.read() - - assert data == ( - b"my_data1.bin,sha256=355d00f8ce0e3eea93b078de0fa5ad87ff94aaba40000772a6572eb2d159f2ce,9\n" - b"my_data2.bin,sha256=94fed5f2858baa0c9709b74048d88f76c5288333d466186dffb17c4f96c2dde4,9\n" - b"my_data3.bin,sha256=d7c92baeebb582bd35c7e58cffd0a14804a81efd267d1015ebe0766ddf6cc69a,9\n" - b"my_script,sha256=33ad1f5af51230990fb70d9aa54be3596c0e72744f715cbfccee3ee25a47d3ca,9\n" - b"my_script2,sha256=93dffdf7b9136d36109bb11714b7255592f59b637df2b53dd105f8e9778cbe36,22\n" - b"my_entrypoint,sha256=fe9ffd9f099e21ea0c05f4346a486bd4a6ca9f795a0f2760d09edccb416ce892,216\n" - b"RECORD,,\n" - ) + with open(file_path, "r") as f: + records = set(f) + + assert records == { + "my_data1.bin,sha256=355d00f8ce0e3eea93b078de0fa5ad87ff94aaba40000772a6572eb2d159f2ce,9\n", + "my_data2.bin,sha256=94fed5f2858baa0c9709b74048d88f76c5288333d466186dffb17c4f96c2dde4,9\n", + "my_data3.bin,sha256=d7c92baeebb582bd35c7e58cffd0a14804a81efd267d1015ebe0766ddf6cc69a,9\n", + "my_script,sha256=33ad1f5af51230990fb70d9aa54be3596c0e72744f715cbfccee3ee25a47d3ca,9\n", + "my_script2,sha256=93dffdf7b9136d36109bb11714b7255592f59b637df2b53dd105f8e9778cbe36,22\n", + "my_entrypoint,sha256=fe9ffd9f099e21ea0c05f4346a486bd4a6ca9f795a0f2760d09edccb416ce892,216\n", + "RECORD,,\n", + }