[RubyGems] Stop fetching all specs 😁

lib/bundler/definition.rb

@@ -230,7 +230,12 @@ def index
             names = :names # do this so we only have to traverse to get dependency_names from the index once
             unmet_dependency_names = proc do
               if names == :names
-                names = dependency_names.+(idx.dependency_names).uniq unless idx.size > Source::Rubygems::API_REQUEST_LIMIT
+                names = if idx.size > Source::Rubygems::API_REQUEST_LIMIT
+                  new_names = idx.dependency_names_if_available
+                  new_names && dependency_names.+(new_names).uniq
+                else
+                  dependency_names.+(idx.dependency_names).uniq
+                end
               else
                 names
               end

lib/bundler/fetcher.rb

@@ -205,7 +205,7 @@ def http_proxy
     end
 
     def inspect
-      "#<#{self.class}:0x#{object_id} uri=#{uri}>"
+      "#<#{self.class}:0x#{object_id} uri=#{uri} fetchers=#{fetchers.map {|f| f.class.name }}>"
     end
 
   private

lib/bundler/index.rb

@@ -114,6 +114,18 @@ def dependency_names
       names.uniq
     end
 
+    def dependency_names_if_available
+      reduce([]) do |names, spec|
+        case spec
+        when EndpointSpecification, Gem::Specification, LazySpecification, StubSpecification
+          names.concat(spec.dependencies)
+        else
+          # RemoteSpecification from the full index
+          return nil
+        end
+      end.tap {|n| n && n.map!(&:name) }
+    end
+
     def use(other, override_dupes = false)
       return unless other
       other.each do |s|

lib/bundler/source/rubygems.rb

@@ -241,9 +241,9 @@ def double_check_for(unmet_dependency_names, override_dupes = false, index = spe
         return unless api_fetchers.any?
 
         unmet_dependency_names = unmet_dependency_names.call
-        Bundler.ui.debug "#{self}: 2x check for #{unmet_dependency_names}"
+        return if !unmet_dependency_names.nil? && unmet_dependency_names.empty?
 
-        return if unmet_dependency_names && unmet_dependency_names.empty?
+        Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}"
 
         fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes)
       end

spec/install/gems/compact_index_spec.rb

@@ -334,6 +334,31 @@ def require(*args)
     should_be_installed "back_deps 1.0"
   end
 
+  it "does not fetch every spec if the index of gems is large when doing back deps & everything is the compact index" do
+    build_repo4 do
+      build_gem "back_deps" do |s|
+        s.add_dependency "foo"
+      end
+      build_gem "missing"
+      # need to hit the limit
+      1.upto(Bundler::Source::Rubygems::API_REQUEST_LIMIT) do |i|
+        build_gem "gem#{i}"
+      end
+
+      FileUtils.rm_rf Dir[gem_repo4("gems/foo-*.gem")]
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra" do
+        gem "back_deps"
+      end
+    G
+
+    bundle! :install, :artifice => "compact_index_extra_api_missing"
+    should_be_installed "back_deps 1.0"
+  end
+
   it "uses the endpoint if all sources support it" do
     gemfile <<-G
       source "#{source_uri}"

spec/support/artifice/compact_index.rb

@@ -90,8 +90,8 @@ def gems(gem_repo = gem_repo1)
       file = tmp("versions.list")
       file.delete if file.file?
       file = CompactIndex::VersionsFile.new(file.to_s)
-      file.update_with(gems)
-      CompactIndex.versions(file, nil, {})
+      file.create(gems)
+      file.contents
     end
   end
 

spec/support/artifice/compact_index_extra_api.rb

@@ -15,8 +15,8 @@ class CompactIndexExtraApi < CompactIndexAPI
       file = tmp("versions.list")
       file.delete if file.file?
       file = CompactIndex::VersionsFile.new(file.to_s)
-      file.update_with(gems(gem_repo4))
-      CompactIndex.versions(file, nil, {})
+      file.create(gems(gem_repo4))
+      file.contents
     end
   end
 

spec/support/artifice/compact_index_extra_api_missing.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+require File.expand_path("../compact_index_extra_api", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexExtraAPIMissing < CompactIndexExtraApi
+  get "/extra/fetch/actual/gem/:id" do
+    if params[:id] == "missing-1.0.gemspec.rz"
+      halt 404
+    else
+      File.read("#{gem_repo2}/quick/Marshal.4.8/#{params[:id]}")
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexExtraAPIMissing)

spec/support/rubygems_ext.rb

@@ -9,7 +9,8 @@ module Rubygems
         # rack 2.x requires Ruby version >= 2.2.2.
         # artifice doesn't support rack 2.x now.
         "rack" => "< 2",
-        "fakeweb artifice compact_index" => nil,
+        "fakeweb artifice" => nil,
+        "compact_index" => "~> 0.11.0",
         "sinatra" => "1.2.7",
         # Rake version has to be consistent for tests to pass
         "rake" => "10.0.2",
@@ -35,7 +36,7 @@ def self.setup
         FileUtils.rm_rf(Path.base_system_gems)
         FileUtils.mkdir_p(Path.base_system_gems)
         puts "installing gems for the tests to use..."
-        DEPS.sort {|a, _| a[1].nil? ? 1 : -1 }.each {|n, v| install_gem(n, v) }
+        install_gems(DEPS)
         File.open(manifest_path, "w") {|f| f << manifest.join }
       end
 
@@ -44,10 +45,14 @@ def self.setup
       Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
     end
 
-    def self.install_gem(name, version = nil)
-      cmd = "gem install #{name} --no-rdoc --no-ri"
-      cmd += " --version '#{version}'" if version
-      system(cmd) || raise("Installing gem #{name} for the tests to use failed!")
+    def self.install_gems(gems)
+      reqs, no_reqs = gems.partition {|_, req| !req.nil? && !req.split(" ").empty? }
+      no_reqs.map!(&:first)
+      reqs.map! {|name, req| "'#{name}:#{req}'" }
+      deps = reqs.concat(no_reqs).join(" ")
+      cmd = "gem install #{deps} --no-rdoc --no-ri"
+      puts cmd
+      system(cmd) || raise("Installing gems #{deps} for the tests to use failed!")
     end
 
     def gem_command(command, args = "", options = {})