diff --git a/Dockerfile b/Dockerfile index 4d832e74bd4..a8f75c052d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,21 +98,37 @@ RUN mkdir -p /tmp/ruby-install \ && rm -rf /var/lib/gems/*/cache/* \ && rm -rf /tmp/ruby-install -### PYTHON +### PYTHON +COPY --chown=dependabot:dependabot python/helpers /opt/python/helpers # Install Python with pyenv. +USER root ENV PYENV_ROOT=/usr/local/.pyenv \ PATH="/usr/local/.pyenv/bin:$PATH" RUN mkdir -p "$PYENV_ROOT" && chown dependabot:dependabot "$PYENV_ROOT" USER dependabot +ENV DEPENDABOT_NATIVE_HELPERS_PATH="/opt" RUN git -c advice.detachedHead=false clone https://github.com/pyenv/pyenv.git --branch v2.3.6 --single-branch --depth=1 /usr/local/.pyenv \ # This is the version of CPython that gets installed && pyenv install 3.11.0 \ && pyenv global 3.11.0 \ - && rm -Rf /tmp/python-build* -USER root - + && pyenv install 3.10.8 \ + && pyenv install 3.9.15 \ + && pyenv install 3.8.15 \ + && pyenv install 3.7.15 \ + && rm -Rf /tmp/python-build* \ + && bash /opt/python/helpers/build \ + && cd /usr/local/.pyenv \ + && tar czf 3.10.tar.gz versions/3.10.8 \ + && tar czf 3.9.tar.gz versions/3.9.15 \ + && tar czf 3.8.tar.gz versions/3.8.15 \ + && tar czf 3.7.tar.gz versions/3.7.15 \ + && rm -Rf versions/3.10.8 \ + && rm -Rf versions/3.9.15 \ + && rm -Rf versions/3.8.15 \ + && rm -Rf versions/3.7.15 +USER root ### JAVASCRIPT # Install Node and npm @@ -303,9 +319,6 @@ RUN bash /opt/npm_and_yarn/helpers/build # 3.2.3. RUN corepack prepare yarn@3.2.3 --activate -COPY --chown=dependabot:dependabot python/helpers /opt/python/helpers -RUN bash /opt/python/helpers/build - COPY --chown=dependabot:dependabot terraform/helpers /opt/terraform/helpers RUN bash /opt/terraform/helpers/build diff --git a/python/helpers/build b/python/helpers/build index d28e0d92962..f47502610e9 100755 --- a/python/helpers/build +++ b/python/helpers/build @@ -19,3 +19,7 @@ cp -r \ cd "$install_dir" PYENV_VERSION=3.11.0 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt" +PYENV_VERSION=3.10.8 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt" +PYENV_VERSION=3.9.15 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt" +PYENV_VERSION=3.8.15 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt" +PYENV_VERSION=3.7.15 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt" diff --git a/python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb b/python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb index 0dcd0767733..ddde890c99a 100644 --- a/python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb +++ b/python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb @@ -168,7 +168,7 @@ def handle_pip_errors(stdout, command, time_taken, exit_value) end def run_pip_compile_command(command, allow_unsafe_shell_command: false) - run_command("pyenv local #{python_version}") + run_command("pyenv local #{Helpers.python_major_minor(python_version)}") run_command( command, allow_unsafe_shell_command: allow_unsafe_shell_command diff --git a/python/lib/dependabot/python/file_updater/pipfile_file_updater.rb b/python/lib/dependabot/python/file_updater/pipfile_file_updater.rb index ba179a226a0..c3304b7ca7c 100644 --- a/python/lib/dependabot/python/file_updater/pipfile_file_updater.rb +++ b/python/lib/dependabot/python/file_updater/pipfile_file_updater.rb @@ -133,6 +133,7 @@ def prepared_pipfile_content content = freeze_other_dependencies(content) content = freeze_dependencies_being_updated(content) content = add_private_sources(content) + content = update_python_requirement(content) content end @@ -142,6 +143,12 @@ def freeze_other_dependencies(pipfile_content) freeze_top_level_dependencies_except(dependencies) end + def update_python_requirement(pipfile_content) + PipfilePreparer. + new(pipfile_content: pipfile_content). + update_python_requirement(Helpers.python_major_minor(python_version)) + end + # rubocop:disable Metrics/PerceivedComplexity def freeze_dependencies_being_updated(pipfile_content) pipfile_object = TomlRB.parse(pipfile_content) @@ -264,7 +271,7 @@ def run_command(command, env: {}) end def run_pipenv_command(command, env: pipenv_env_variables) - run_command("pyenv local #{python_version}") + run_command("pyenv local #{Helpers.python_major_minor(python_version)}") run_command(command, env: env) end @@ -276,7 +283,7 @@ def write_temporary_dependency_files(pipfile_content) end # Overwrite the .python-version with updated content - File.write(".python-version", python_version) + File.write(".python-version", Helpers.python_major_minor(python_version)) setup_files.each do |file| path = file.name diff --git a/python/lib/dependabot/python/file_updater/pipfile_preparer.rb b/python/lib/dependabot/python/file_updater/pipfile_preparer.rb index fc19b4716d7..6f16e6af34a 100644 --- a/python/lib/dependabot/python/file_updater/pipfile_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pipfile_preparer.rb @@ -70,10 +70,12 @@ def update_python_requirement(requirement) pipfile_object = TomlRB.parse(pipfile_content) pipfile_object["requires"] ||= {} - pipfile_object["requires"].delete("python_full_version") - pipfile_object["requires"].delete("python_version") - pipfile_object["requires"]["python_full_version"] = requirement - + if pipfile_object.dig("requires", "python_full_version") && pipfile_object.dig("requires", "python_version") + pipfile_object["requires"].delete("python_full_version") + elsif pipfile_object.dig("requires", "python_full_version") + pipfile_object["requires"].delete("python_full_version") + pipfile_object["requires"]["python_version"] = requirement + end TomlRB.dump(pipfile_object) end diff --git a/python/lib/dependabot/python/helpers.rb b/python/lib/dependabot/python/helpers.rb index baa49f30f1d..9689573bfe0 100644 --- a/python/lib/dependabot/python/helpers.rb +++ b/python/lib/dependabot/python/helpers.rb @@ -1,19 +1,36 @@ # frozen_string_literal: true require "dependabot/logger" +require "dependabot/python/version" module Dependabot module Python module Helpers def self.install_required_python(python_version) # The leading space is important in the version check - return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_version}") + return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_major_minor(python_version)}.") + + if File.exist?("/usr/local/.pyenv/#{python_major_minor(python_version)}.tar.gz") + SharedHelpers.run_shell_command( + "tar xzf /usr/local/.pyenv/#{python_major_minor(python_version)}.tar.gz -C /usr/local/.pyenv/" + ) + return if SharedHelpers.run_shell_command("pyenv versions"). + include?(" #{python_major_minor(python_version)}.") + end Dependabot.logger.info("Installing required Python #{python_version}.") + start = Time.now SharedHelpers.run_shell_command("pyenv install -s #{python_version}") SharedHelpers.run_shell_command("pyenv exec pip install --upgrade pip") SharedHelpers.run_shell_command("pyenv exec pip install -r" \ "#{NativeHelpers.python_requirements_path}") + time_taken = Time.now - start + Dependabot.logger.info("Installing Python #{python_version} took #{time_taken}s.") + end + + def self.python_major_minor(python_version) + python = Python::Version.new(python_version) + "#{python.segments[0]}.#{python.segments[1]}" end end end diff --git a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb index f3e5983a135..cbbd95d8e25 100644 --- a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +++ b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb @@ -254,7 +254,7 @@ def pip_compile_index_options end def run_pip_compile_command(command) - run_command("pyenv local #{python_version}") + run_command("pyenv local #{Helpers.python_major_minor(python_version)}") run_command(command) end diff --git a/python/lib/dependabot/python/update_checker/pipenv_version_resolver.rb b/python/lib/dependabot/python/update_checker/pipenv_version_resolver.rb index c5bb2f9848a..88944c79a05 100644 --- a/python/lib/dependabot/python/update_checker/pipenv_version_resolver.rb +++ b/python/lib/dependabot/python/update_checker/pipenv_version_resolver.rb @@ -290,7 +290,7 @@ def write_temporary_dependency_files(updated_req: nil, end # Overwrite the .python-version with updated content - File.write(".python-version", python_version) + File.write(".python-version", Helpers.python_major_minor(python_version)) setup_files.each do |file| path = file.name @@ -341,6 +341,7 @@ def pipfile_content(updated_requirement:) content = freeze_other_dependencies(content) content = set_target_dependency_req(content, updated_requirement) content = add_private_sources(content) + content = update_python_requirement(content) content end @@ -350,6 +351,12 @@ def freeze_other_dependencies(pipfile_content) freeze_top_level_dependencies_except([dependency]) end + def update_python_requirement(pipfile_content) + Python::FileUpdater::PipfilePreparer. + new(pipfile_content: pipfile_content). + update_python_requirement(Helpers.python_major_minor(python_version)) + end + # rubocop:disable Metrics/PerceivedComplexity def set_target_dependency_req(pipfile_content, updated_requirement) return pipfile_content unless updated_requirement @@ -461,7 +468,7 @@ def run_command(command, env: {}) end def run_pipenv_command(command, env: pipenv_env_variables) - run_command("pyenv local #{python_version}") + run_command("pyenv local #{Helpers.python_major_minor(python_version)}") run_command(command, env: env) end diff --git a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb index 345d2120886..3076c64cd30 100644 --- a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb @@ -133,7 +133,7 @@ ) end - it "updates both files correctly", :slow do + it "updates both files correctly" do expect(updated_files.map(&:name)).to eq(%w(Pipfile Pipfile.lock)) updated_lockfile = updated_files.find { |f| f.name == "Pipfile.lock" }