From 27d857cba5c5e61057b8c9b4246d0284bc03297b Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 19:32:01 -0400 Subject: [PATCH 01/12] Some performance optimizations after analyizing benchmarks. --- pipenv/resolver.py | 5 ++++- pipenv/utils/resolver.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 08c16e5726..4cb89c985a 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,6 +3,8 @@ import os import sys +from pipenv.patched.pip._vendor.pyparsing.core import cached_property + def _ensure_modules(): spec = importlib.util.spec_from_file_location( @@ -226,6 +228,7 @@ def marker_to_str(marker): return marker_str return None + @cached_property def get_cleaned_dict(self): self.validate_constraints() if self.entry.extras != self.lockfile_entry.extras: @@ -550,7 +553,7 @@ def clean_results(results, resolver, project, category): reverse_deps=reverse_deps, category=category, ) - entry_dict = translate_markers(entry.get_cleaned_dict()) + entry_dict = translate_markers(entry.get_cleaned_dict) new_results.append(entry_dict) return new_results diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 4919942298..315e74d087 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -326,7 +326,7 @@ def prepare_index_lookup(self): alt_index_lookup[req_name] = index_mapping[index] return alt_index_lookup - @property + @cached_property def finder(self): finder = get_package_finder( install_cmd=self.pip_command, @@ -338,7 +338,7 @@ def finder(self): finder._link_collector.search_scope.index_lookup = index_lookup return finder - @property + @cached_property def parsed_constraints(self): pip_options = self.pip_options pip_options.extra_index_urls = [] @@ -512,6 +512,7 @@ def collect_hashes(self, ireq): return {self.project.get_hash_from_link(self.hash_cache, link)} return set() + @cached_property def resolve_hashes(self): if self.results is not None: for ireq in self.results: @@ -612,7 +613,7 @@ def actually_resolve_deps( category, ) resolver.resolve() - hashes = resolver.resolve_hashes() + hashes = resolver.resolve_hashes resolver.resolve_constraints() results = resolver.clean_results() for warning in warning_list: From d877bb95d1677b0145096f2bfdee2e2134873540 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 19:33:47 -0400 Subject: [PATCH 02/12] add news fragment --- news/5841.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/5841.bugfix.rst diff --git a/news/5841.bugfix.rst b/news/5841.bugfix.rst new file mode 100644 index 0000000000..13ab1ef5bc --- /dev/null +++ b/news/5841.bugfix.rst @@ -0,0 +1 @@ +Add back some relevant caching to increase performance after the major refactor released with ``2023.8.19`` From ecd948ea0f2630c2ff5092bd4dbe2e820e3843aa Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 19:42:02 -0400 Subject: [PATCH 03/12] attempt to fix build by moving import inside class def --- pipenv/resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 4cb89c985a..8c284b0bd7 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,8 +3,6 @@ import os import sys -from pipenv.patched.pip._vendor.pyparsing.core import cached_property - def _ensure_modules(): spec = importlib.util.spec_from_file_location( @@ -87,6 +85,8 @@ def handle_parsed_args(parsed): class Entry: """A resolved entry from a resolver run""" + from pipenv.patched.pip._vendor.pyparsing.core import cached_property + def __init__( self, name, entry_dict, project, resolver, reverse_deps=None, category=None ): From a0342ee294c82cb6a760c3fc52de86e880b857ae Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 19:45:07 -0400 Subject: [PATCH 04/12] Target this performance optimization for after we drop 3.7 --- pipenv/resolver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 8c284b0bd7..ea2a771e03 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -2,6 +2,7 @@ import json import os import sys +from functools import cached_property def _ensure_modules(): @@ -85,8 +86,6 @@ def handle_parsed_args(parsed): class Entry: """A resolved entry from a resolver run""" - from pipenv.patched.pip._vendor.pyparsing.core import cached_property - def __init__( self, name, entry_dict, project, resolver, reverse_deps=None, category=None ): From 7a914a2ba5facd5c0cfc54f1014bdd6af819fa1d Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 19:52:58 -0400 Subject: [PATCH 05/12] Fallback to property for python 3.7 --- pipenv/resolver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ea2a771e03..3a7b1a3509 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -2,7 +2,11 @@ import json import os import sys -from functools import cached_property + +try: + from functools import cached_property +except ImportError: + cached_property = property def _ensure_modules(): From 3b1f082124c9385074b52377b6bd4732b7d221d3 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 20:01:02 -0400 Subject: [PATCH 06/12] I don't think this one can be cached --- pipenv/utils/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 315e74d087..443a9f7c98 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -338,7 +338,7 @@ def finder(self): finder._link_collector.search_scope.index_lookup = index_lookup return finder - @cached_property + @property def parsed_constraints(self): pip_options = self.pip_options pip_options.extra_index_urls = [] From ec3f27939545901cd093f73aa2e461864ec8f935 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 20:27:36 -0400 Subject: [PATCH 07/12] Revert "I don't think this one can be cached" This reverts commit 3b1f082124c9385074b52377b6bd4732b7d221d3. --- pipenv/utils/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 443a9f7c98..315e74d087 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -338,7 +338,7 @@ def finder(self): finder._link_collector.search_scope.index_lookup = index_lookup return finder - @property + @cached_property def parsed_constraints(self): pip_options = self.pip_options pip_options.extra_index_urls = [] From c07591d397253bb2a99caf8fea4b324f9f64f0d1 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 20:30:40 -0400 Subject: [PATCH 08/12] caching finder is the problem --- pipenv/utils/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 315e74d087..01dc86a520 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -326,7 +326,7 @@ def prepare_index_lookup(self): alt_index_lookup[req_name] = index_mapping[index] return alt_index_lookup - @cached_property + @property def finder(self): finder = get_package_finder( install_cmd=self.pip_command, From b21d6d88b44827ea5fbd332a7bcc39d3e67cd27b Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 20 Aug 2023 21:28:10 -0400 Subject: [PATCH 09/12] Iterate on trying to cache the finder --- pipenv/utils/resolver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 01dc86a520..ecf0606fa7 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -326,7 +326,7 @@ def prepare_index_lookup(self): alt_index_lookup[req_name] = index_mapping[index] return alt_index_lookup - @property + @cached_property def finder(self): finder = get_package_finder( install_cmd=self.pip_command, @@ -434,9 +434,10 @@ def get_resolver(self, clear=False): yield resolver def resolve(self): + constraints = self.constraints with temp_environ(), self.get_resolver() as resolver: try: - results = resolver.resolve(self.constraints, check_supported_wheels=False) + results = resolver.resolve(constraints, check_supported_wheels=False) except InstallationError as e: raise ResolutionFailure(message=str(e)) else: From d65dc33726cc1b0be51e8752b1ac5415e92e09c2 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 21 Aug 2023 08:09:00 -0400 Subject: [PATCH 10/12] Try to re-use package finder, but re-prepare index lookups --- pipenv/utils/resolver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index ecf0606fa7..fbe2e7f1da 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -327,12 +327,17 @@ def prepare_index_lookup(self): return alt_index_lookup @cached_property - def finder(self): + def package_finder(self): finder = get_package_finder( install_cmd=self.pip_command, options=self.pip_options, session=self.session, ) + return finder + + @property + def finder(self): + finder = self.package_finder index_lookup = self.prepare_index_lookup() finder._link_collector.index_lookup = index_lookup finder._link_collector.search_scope.index_lookup = index_lookup From c9238dbb04239031a77d4f976794b19ad281e835 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 21 Aug 2023 08:19:15 -0400 Subject: [PATCH 11/12] Only validate constraints once when cleaning the results --- pipenv/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 3a7b1a3509..dc77d967cd 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -233,7 +233,6 @@ def marker_to_str(marker): @cached_property def get_cleaned_dict(self): - self.validate_constraints() if self.entry.extras != self.lockfile_entry.extras: entry_extras = list(self.entry.extras) if self.lockfile_entry.extras: @@ -539,6 +538,7 @@ def clean_results(results, resolver, project, category): if not project.lockfile_exists: return results + resolver.validate_constraints() lockfile = project.lockfile_content lockfile_section = get_lockfile_section_using_pipfile_category(category) reverse_deps = project.environment.reverse_dependencies() From bccb124962e1b779345b9b61acfda1e09a95e73d Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 21 Aug 2023 08:21:54 -0400 Subject: [PATCH 12/12] Revert "Only validate constraints once when cleaning the results" This reverts commit c9238dbb04239031a77d4f976794b19ad281e835. --- pipenv/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index dc77d967cd..3a7b1a3509 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -233,6 +233,7 @@ def marker_to_str(marker): @cached_property def get_cleaned_dict(self): + self.validate_constraints() if self.entry.extras != self.lockfile_entry.extras: entry_extras = list(self.entry.extras) if self.lockfile_entry.extras: @@ -538,7 +539,6 @@ def clean_results(results, resolver, project, category): if not project.lockfile_exists: return results - resolver.validate_constraints() lockfile = project.lockfile_content lockfile_section = get_lockfile_section_using_pipfile_category(category) reverse_deps = project.environment.reverse_dependencies()