From aebe284e3e97406a5636e195a3cb9a57cd6c32ee Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Mon, 29 Aug 2016 13:47:46 -0500 Subject: [PATCH 1/3] =?UTF-8?q?[RubyGems]=20Stop=20fetching=20all=20specs?= =?UTF-8?q?=20=F0=9F=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/bundler/source/rubygems.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 602b4386a32..9619d5f4972 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -241,9 +241,9 @@ def double_check_for(unmet_dependency_names, override_dupes = false, index = spe return unless api_fetchers.any? unmet_dependency_names = unmet_dependency_names.call - Bundler.ui.debug "#{self}: 2x check for #{unmet_dependency_names}" + return if unmet_dependency_names.nil? || unmet_dependency_names.empty? - return if unmet_dependency_names && unmet_dependency_names.empty? + Bundler.ui.debug "Double checking for #{unmet_dependency_names} in #{self}" fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes) end From 20cbbf0d5d86a30b8afef8906bdf7536b0565367 Mon Sep 17 00:00:00 2001 From: Homu Date: Tue, 30 Aug 2016 05:00:36 +0900 Subject: [PATCH 2/3] Auto merge of #4928 - bundler:seg-update-compact-index, r=segiddins Update specs to use compact_index 0.11 Also removed the penalty for adding requirements to dependencies! \c @indirect # Conflicts: # spec/support/artifice/compact_index_concurrent_download.rb --- spec/support/artifice/compact_index.rb | 4 ++-- .../support/artifice/compact_index_extra_api.rb | 4 ++-- spec/support/rubygems_ext.rb | 17 +++++++++++------ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/spec/support/artifice/compact_index.rb b/spec/support/artifice/compact_index.rb index 88706c35908..458c81b4b1c 100644 --- a/spec/support/artifice/compact_index.rb +++ b/spec/support/artifice/compact_index.rb @@ -90,8 +90,8 @@ def gems(gem_repo = gem_repo1) file = tmp("versions.list") file.delete if file.file? file = CompactIndex::VersionsFile.new(file.to_s) - file.update_with(gems) - CompactIndex.versions(file, nil, {}) + file.create(gems) + file.contents end end diff --git a/spec/support/artifice/compact_index_extra_api.rb b/spec/support/artifice/compact_index_extra_api.rb index 063e5589d43..844a9ca9f25 100644 --- a/spec/support/artifice/compact_index_extra_api.rb +++ b/spec/support/artifice/compact_index_extra_api.rb @@ -15,8 +15,8 @@ class CompactIndexExtraApi < CompactIndexAPI file = tmp("versions.list") file.delete if file.file? file = CompactIndex::VersionsFile.new(file.to_s) - file.update_with(gems(gem_repo4)) - CompactIndex.versions(file, nil, {}) + file.create(gems(gem_repo4)) + file.contents end end diff --git a/spec/support/rubygems_ext.rb b/spec/support/rubygems_ext.rb index 85ed1bdf0f2..dda77ec7124 100644 --- a/spec/support/rubygems_ext.rb +++ b/spec/support/rubygems_ext.rb @@ -9,7 +9,8 @@ module Rubygems # rack 2.x requires Ruby version >= 2.2.2. # artifice doesn't support rack 2.x now. "rack" => "< 2", - "fakeweb artifice compact_index" => nil, + "fakeweb artifice" => nil, + "compact_index" => "~> 0.11.0", "sinatra" => "1.2.7", # Rake version has to be consistent for tests to pass "rake" => "10.0.2", @@ -35,7 +36,7 @@ def self.setup FileUtils.rm_rf(Path.base_system_gems) FileUtils.mkdir_p(Path.base_system_gems) puts "installing gems for the tests to use..." - DEPS.sort {|a, _| a[1].nil? ? 1 : -1 }.each {|n, v| install_gem(n, v) } + install_gems(DEPS) File.open(manifest_path, "w") {|f| f << manifest.join } end @@ -44,10 +45,14 @@ def self.setup Gem::DefaultUserInteraction.ui = Gem::SilentUI.new end - def self.install_gem(name, version = nil) - cmd = "gem install #{name} --no-rdoc --no-ri" - cmd += " --version '#{version}'" if version - system(cmd) || raise("Installing gem #{name} for the tests to use failed!") + def self.install_gems(gems) + reqs, no_reqs = gems.partition {|_, req| !req.nil? && !req.split(" ").empty? } + no_reqs.map!(&:first) + reqs.map! {|name, req| "'#{name}:#{req}'" } + deps = reqs.concat(no_reqs).join(" ") + cmd = "gem install #{deps} --no-rdoc --no-ri" + puts cmd + system(cmd) || raise("Installing gems #{deps} for the tests to use failed!") end def gem_command(command, args = "", options = {}) From bdb853450492e723a519dfeabf5408f8070b11aa Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 30 Aug 2016 14:44:37 -0500 Subject: [PATCH 3/3] [Index] Allow #dependency_names when all specs already have dependencies --- lib/bundler/definition.rb | 7 +++++- lib/bundler/fetcher.rb | 2 +- lib/bundler/index.rb | 12 +++++++++ lib/bundler/source/rubygems.rb | 4 +-- spec/install/gems/compact_index_spec.rb | 25 +++++++++++++++++++ .../compact_index_extra_api_missing.rb | 16 ++++++++++++ 6 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 spec/support/artifice/compact_index_extra_api_missing.rb diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index ab754ae24d0..83d975d46d5 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -230,7 +230,12 @@ def index names = :names # do this so we only have to traverse to get dependency_names from the index once unmet_dependency_names = proc do if names == :names - names = dependency_names.+(idx.dependency_names).uniq unless idx.size > Source::Rubygems::API_REQUEST_LIMIT + names = if idx.size > Source::Rubygems::API_REQUEST_LIMIT + new_names = idx.dependency_names_if_available + new_names && dependency_names.+(new_names).uniq + else + dependency_names.+(idx.dependency_names).uniq + end else names end diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index 13d8ce81196..8620c6d5ab4 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -205,7 +205,7 @@ def http_proxy end def inspect - "#<#{self.class}:0x#{object_id} uri=#{uri}>" + "#<#{self.class}:0x#{object_id} uri=#{uri} fetchers=#{fetchers.map {|f| f.class.name }}>" end private diff --git a/lib/bundler/index.rb b/lib/bundler/index.rb index 833276b2514..339e30bb577 100644 --- a/lib/bundler/index.rb +++ b/lib/bundler/index.rb @@ -114,6 +114,18 @@ def dependency_names names.uniq end + def dependency_names_if_available + reduce([]) do |names, spec| + case spec + when EndpointSpecification, Gem::Specification, LazySpecification, StubSpecification + names.concat(spec.dependencies) + else + # RemoteSpecification from the full index + return nil + end + end.tap {|n| n && n.map!(&:name) } + end + def use(other, override_dupes = false) return unless other other.each do |s| diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 9619d5f4972..8078b06d625 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -241,9 +241,9 @@ def double_check_for(unmet_dependency_names, override_dupes = false, index = spe return unless api_fetchers.any? unmet_dependency_names = unmet_dependency_names.call - return if unmet_dependency_names.nil? || unmet_dependency_names.empty? + return if !unmet_dependency_names.nil? && unmet_dependency_names.empty? - Bundler.ui.debug "Double checking for #{unmet_dependency_names} in #{self}" + Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}" fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes) end diff --git a/spec/install/gems/compact_index_spec.rb b/spec/install/gems/compact_index_spec.rb index 1cd0f370ab9..7c26e84046f 100644 --- a/spec/install/gems/compact_index_spec.rb +++ b/spec/install/gems/compact_index_spec.rb @@ -334,6 +334,31 @@ def require(*args) should_be_installed "back_deps 1.0" end + it "does not fetch every spec if the index of gems is large when doing back deps & everything is the compact index" do + build_repo4 do + build_gem "back_deps" do |s| + s.add_dependency "foo" + end + build_gem "missing" + # need to hit the limit + 1.upto(Bundler::Source::Rubygems::API_REQUEST_LIMIT) do |i| + build_gem "gem#{i}" + end + + FileUtils.rm_rf Dir[gem_repo4("gems/foo-*.gem")] + end + + gemfile <<-G + source "#{source_uri}" + source "#{source_uri}/extra" do + gem "back_deps" + end + G + + bundle! :install, :artifice => "compact_index_extra_api_missing" + should_be_installed "back_deps 1.0" + end + it "uses the endpoint if all sources support it" do gemfile <<-G source "#{source_uri}" diff --git a/spec/support/artifice/compact_index_extra_api_missing.rb b/spec/support/artifice/compact_index_extra_api_missing.rb new file mode 100644 index 00000000000..d11793fc2b3 --- /dev/null +++ b/spec/support/artifice/compact_index_extra_api_missing.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +require File.expand_path("../compact_index_extra_api", __FILE__) + +Artifice.deactivate + +class CompactIndexExtraAPIMissing < CompactIndexExtraApi + get "/extra/fetch/actual/gem/:id" do + if params[:id] == "missing-1.0.gemspec.rz" + halt 404 + else + File.read("#{gem_repo2}/quick/Marshal.4.8/#{params[:id]}") + end + end +end + +Artifice.activate_with(CompactIndexExtraAPIMissing)