Skip to content

Use new pip resolver for pip-compile #5358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jurre merged 2 commits into from
Jul 14, 2022
Merged

Use new pip resolver for pip-compile #5358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7969ac8
Use new pip resolver for pip-compile
jurre Jul 12, 2022
65fe503
Update specs and error handling to match new resolver
jurre Jul 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 10 additions & 27 deletions python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ class PipCompileVersionResolver
"pip._internal.exceptions.InstallationSubprocessError: Command errored out with exit status 1:"
# See https://packaging.python.org/en/latest/tutorials/packaging-projects/#configuring-metadata
PYTHON_PACKAGE_NAME_REGEX = /[A-Za-z0-9_\-]+/.freeze
RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
ERROR_REGEX = /(?<=ERROR\:\W).*$/.freeze

attr_reader :dependency, :dependency_files, :credentials

Expand Down Expand Up @@ -76,16 +78,13 @@ def fetch_latest_resolvable_version_string(requirement:)
# Shell out to pip-compile.
# This is slow, as pip-compile needs to do installs.
run_pip_compile_command(
"pyenv exec pip-compile --allow-unsafe -v "\
"#{pip_compile_options(filename)} -P #{dependency.name} "\
"#{filename}"
"pyenv exec pip-compile -v #{pip_compile_options(filename)} -P #{dependency.name} #{filename}"
)
# Run pip-compile a second time, without an update argument,
# to ensure it handles markers correctly
write_original_manifest_files unless dependency.top_level?
run_pip_compile_command(
"pyenv exec pip-compile --allow-unsafe "\
"#{pip_compile_options(filename)} #{filename}"
"pyenv exec pip-compile #{pip_compile_options(filename)} #{filename}"
)
end

Expand Down Expand Up @@ -114,7 +113,7 @@ def compilation_error?(error)
# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/PerceivedComplexity
def handle_pip_compile_errors(error)
if error.message.include?("Could not find a version")
if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
check_original_requirements_resolvable
# If the original requirements are resolvable but we get an
# incompatibility error after unlocking then it's likely to be
Expand All @@ -138,7 +137,7 @@ def handle_pip_compile_errors(error)
return
end

if error.message.include?("Could not find a version ") &&
if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
!error.message.match?(/#{Regexp.quote(dependency.name)}/i)
# Sometimes pip-tools gets confused and can't work around
# sub-dependency incompatibilities. Ignore those cases.
Expand Down Expand Up @@ -179,7 +178,7 @@ def check_original_requirements_resolvable

filenames_to_compile.each do |filename|
run_pip_compile_command(
"pyenv exec pip-compile #{pip_compile_options(filename)} --allow-unsafe #{filename}"
"pyenv exec pip-compile #{pip_compile_options(filename)} #{filename}"
)
end

Expand All @@ -188,7 +187,7 @@ def check_original_requirements_resolvable
# Pick the error message that includes resolvability errors, this might be the cause from
# handle_pip_compile_errors (it's unclear if we should always pick the cause here)
error_message = [e.message, e.cause&.message].compact.find do |msg|
["UnsupportedConstraint", "Could not find a version"].any? { |err| msg.include?(err) }
msg.include?(RESOLUTION_IMPOSSIBLE_ERROR)
end

cleaned_message = clean_error_message(error_message || "")
Expand Down Expand Up @@ -220,6 +219,7 @@ def run_command(command, env: python_env)
def pip_compile_options(filename)
options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"]
options += pip_compile_index_options
options += ["--resolver backtracking", "--allow-unsafe"]

if (requirements_file = compiled_file_for_filename(filename))
options << "--output-file=#{requirements_file.name}"
Expand Down Expand Up @@ -353,25 +353,8 @@ def normalise(name)
NameNormaliser.normalise(name)
end

VERBOSE_ERROR_OUTPUT_LINES = [
"Traceback",
"Using indexes:",
"Current constraints:",
"Finding the best candidates:",
"Finding secondary dependencies:",
"\n",
" "
].freeze

def clean_error_message(message)
msg_lines = message.lines
msg = msg_lines.
take_while { |l| !l.start_with?("During handling of") }.
drop_while { |l| l.start_with?(*VERBOSE_ERROR_OUTPUT_LINES) }.
join.strip

# Redact any URLs, as they may include credentials
msg.gsub(/http.*?(?=\s)/, "<redacted>")
message.scan(ERROR_REGEX).last
end

def filenames_to_compile
Expand Down
67 changes: 5 additions & 62 deletions python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,41 +140,6 @@
end
end

context "when unlocking causes a conflict (in the sub-dependencies)" do
let(:manifest_fixture_name) { "unresolvable_if_unpinned.in" }
let(:generated_fixture_name) do
"pip_compile_unresolvable_if_unpinned.txt"
end
let(:dependency_name) { "boto3" }
let(:dependency_version) { "1.7.84" }
let(:updated_requirement) { ">= 1.7.84, <= 1.9.28" }
let(:dependency_requirements) do
[{
file: "requirements/test.in",
requirement: ">=1.7,<1.8",
groups: [],
source: nil
}]
end
it { is_expected.to be nil }

context "and updating would cause a conflict" do
let(:dependency_name) { "moto" }
let(:dependency_version) { "1.3.6" }
let(:updated_requirement) { ">= 1.3.6, <= 1.3.7" }

let(:dependency_requirements) do
[{
file: "requirements/test.in",
requirement: nil,
groups: [],
source: nil
}]
end
it { is_expected.to be nil }
end
end

context "with multiple requirement.in files" do
let(:dependency_files) do
[manifest_file, manifest_file2, generated_file, generated_file2]
Expand Down Expand Up @@ -219,37 +184,14 @@
expect { subject }.
to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
expect(error.message).
to include("Could not find a version that matches boto3")
to include("Cannot install -r requirements/dev.in (line 1) and botocore==1.10.84 because these "\
"package versions have conflicting dependencies.")
end
end
end
end
end

context "with an unresolvable requirement" do
let(:manifest_fixture_name) { "unresolvable.in" }
let(:dependency_files) { [manifest_file] }
let(:dependency_name) { "boto3" }
let(:dependency_version) { nil }
let(:updated_requirement) { ">= 0, <= 1.9.28" }
let(:dependency_requirements) do
[{
file: "requirements/test.in",
requirement: "==1.9.27",
groups: [],
source: nil
}]
end

it "raises a helpful error" do
expect { subject }.
to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
expect(error.message).
to include("Could not find a version that matches boto3")
end
end
end

context "with an unresolvable project" do
let(:dependency_files) { project_dependency_files("unresolvable") }
let(:dependency) do
Expand All @@ -273,7 +215,7 @@
expect { subject }.
to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
expect(error.message).
to include("Could not find a version that matches jupyter-server")
to include("Could not find a version that satisfies the requirement jupyter-server<=18.1.0,>=17.3.0")
end
end
end
Expand Down Expand Up @@ -438,7 +380,8 @@
expect { subject }.
to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
expect(error.message).
to include("Could not find a version that matches boto3")
to include("Cannot install -r requirements/test.in (line 1) and botocore==1.10.84 because these "\
"package versions have conflicting dependencies.")
end
end
end
Expand Down
3 changes: 0 additions & 3 deletions python/spec/fixtures/pip_compile_files/unresolvable_if_unpinned.in
View file
Open in desktop

This file was deleted.

101 changes: 0 additions & 101 deletions python/spec/fixtures/pip_compile_files/unresolvable_if_unpinned.txt
View file
Open in desktop

This file was deleted.