[2.0] Remove RubyGems Aggregate & support transitive source pinning

lib/bundler/definition.rb

@@ -218,15 +218,36 @@ def index
           dependency_names -= pinned_spec_names(source.specs)
           dependency_names.push(*source.unmet_deps).uniq!
         end
+
+        # Suppose the gem Foo depends on the gem Bar.  Foo exists in Source A.  Bar has some versions that exist in both
+        # sources A and B.  At this point, the API request will have found all the versions of Bar in source A,
+        # but will not have found any versions of Bar from source B, which is a problem if the requested version
+        # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for
+        # each spec we found, we add all possible versions from all sources to the index.
+        loop do
+          idxcount = idx.size
+          sources.all_sources.each do |source|
+            names = :names # do this so we only have to traverse to get dependency_names from the index once
+            unmet_dependency_names = proc do
+              if names == :names
+                names = dependency_names.+(idx.dependency_names).uniq unless idx.size > Source::Rubygems::API_REQUEST_LIMIT
+              else
+                names
+              end
+            end
+            source.double_check_for(unmet_dependency_names, :override_dupes)
+          end
+          break if idxcount == idx.size
+        end
       end
     end
 
     # used when frozen is enabled so we can find the bundler
     # spec, even if (say) a git gem is not checked out.
     def rubygems_index
       @rubygems_index ||= Index.build do |idx|
-        sources.rubygems_sources.each do |rubygems|
-          idx.add_source rubygems.specs
+        sources.rubygems_sources.each do |s|
+          idx.add_source s.specs
         end
       end
     end
@@ -496,19 +517,6 @@ def converge_paths
     def converge_sources
       changes = false
 
-      # Get the Rubygems sources from the gems.locked
-      locked_gem_sources = @locked_sources.select {|s| s.is_a?(Source::Rubygems) }
-      # Get the Rubygems remotes from the gems.rb
-      actual_remotes = sources.rubygems_remotes
-
-      # If there is a Rubygems source in both
-      if !locked_gem_sources.empty? && !actual_remotes.empty?
-        locked_gem_sources.each do |locked_gem|
-          # Merge the remotes from the gems.rb into the gems.locked
-          changes |= locked_gem.replace_remotes(actual_remotes)
-        end
-      end
-
       # Replace the sources from the gems.rb with the sources from the gems.locked,
       # if they exist in the gems.locked and are `==`. If you can't find an equivalent
       # source in the gems.locked, use the one from the gems.rb.
@@ -654,10 +662,11 @@ def source_requirements
       # Record the specs available in each gem's source, so that those
       # specs will be available later when the resolver knows where to
       # look for that gemspec (or its dependencies)
-      source_requirements = {}
+      default = sources.default_source
+      source_requirements = { :default => default }
       dependencies.each do |dep|
-        next unless dep.source
-        source_requirements[dep.name] = dep.source.specs
+        next unless source = dep.source || default
+        source_requirements[dep.name] = source
       end
       source_requirements
     end

lib/bundler/dsl.rb

@@ -132,7 +132,7 @@ def source(source, &blk)
         with_source(@sources.add_rubygems_source("remotes" => source), &blk)
       else
         check_primary_source_safety(@sources)
-        @sources.add_rubygems_remote(source)
+        @sources.global_rubygems_remote = source
       end
     end
 
@@ -150,6 +150,16 @@ def git_source(name, &block)
     end
 
     def path(path, options = {}, &blk)
+      unless block_given?
+        msg = "You can no longer specify a path source by itself. Instead, \n" \
+              "either use the :path option on a gem, or specify the gems that \n" \
+              "bundler should find in the path source by passing a block to \n" \
+              "the path method, like: \n\n" \
+              "  path 'dir/containing/rails' do\n" \
+              "    gem 'rails'\n" \
+              "  end"
+        raise DeprecatedError, msg
+      end
       source_options = normalize_hash(options).merge("path" => Pathname.new(path), "root_path" => gemfile_root)
       source = @sources.add_path_source(source_options)
       with_source(source, &blk)
@@ -384,8 +394,8 @@ def normalize_source(source)
       end
     end
 
-    def check_primary_source_safety(source)
-      return unless source.rubygems_primary_remotes.any?
+    def check_primary_source_safety(source_list)
+      return if source_list.global_rubygems_source.nil?
 
       raise GemspecError, "This #{SharedHelpers.gemfile_name} contains multiple primary sources. " \
         "Each source after the first must include a block to indicate which gems " \

lib/bundler/fetcher/downloader.rb

@@ -31,7 +31,7 @@ def fetch(uri, options = {}, counter = 0)
         when Net::HTTPUnauthorized
           raise AuthenticationRequiredError, uri.host
         when Net::HTTPNotFound
-          raise FallbackError, "Net::HTTPNotFound"
+          raise FallbackError, "Net::HTTPNotFound (#{uri})"
         else
           raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
         end

lib/bundler/index.rb

@@ -98,6 +98,7 @@ def each(&blk)
       specs.values.each do |spec_sets|
         spec_sets.values.each(&blk)
       end
+      sources.each {|s| s.each(&blk) }
     end
 
     # returns a list of the dependencies

lib/bundler/lockfile_parser.rb

@@ -62,8 +62,6 @@ def initialize(lockfile)
       @state        = nil
       @specs        = {}
 
-      @rubygems_aggregate = Source::Rubygems.new
-
       if lockfile.match(/<<<<<<<|=======|>>>>>>>|\|\|\|\|\|\|\|/)
         raise LockfileError, "Your #{SharedHelpers.lockfile_name} contains merge conflicts.\n" \
           "Run `git checkout HEAD -- #{SharedHelpers.lockfile_name}` first to get a clean lock."
@@ -87,7 +85,6 @@ def initialize(lockfile)
           send("parse_#{@state}", line)
         end
       end
-      @sources << @rubygems_aggregate
       @specs = @specs.values
       warn_for_outdated_bundler_version
     rescue ArgumentError => e
@@ -129,23 +126,10 @@ def parse_source(line)
         @type = line
       when SPECS
         case @type
-        when PATH
-          @current_source = TYPES[@type].from_lock(@opts)
-          @sources << @current_source
-        when GIT
-          @current_source = TYPES[@type].from_lock(@opts)
-          # Strip out duplicate GIT sections
-          if @sources.include?(@current_source)
-            @current_source = @sources.find {|s| s == @current_source }
-          else
-            @sources << @current_source
-          end
         when GEM
-          Array(@opts["remote"]).each do |url|
-            @rubygems_aggregate.add_remote(url)
-          end
-          @current_source = @rubygems_aggregate
+          @opts["remotes"] = @opts.delete("remote")
         end
+        add_source(TYPES[@type].from_lock(@opts))
       when OPTIONS
         value = $2
         value = true if value == "true"
@@ -164,6 +148,16 @@ def parse_source(line)
       end
     end
 
+    def add_source(source)
+      # Strip out duplicate sections
+      if @sources.include?(source) && !source.instance_of?(Bundler::Source::Path)
+        @current_source = @sources.find {|s| s == @current_source }
+      else
+        @current_source = source
+        @sources << @current_source
+      end
+    end
+
     NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
     NAME_VERSION_2 = /^ {2}#{NAME_VERSION}(!)?$/
     NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/

lib/bundler/resolver.rb

@@ -54,11 +54,20 @@ def message
             o << %(the gems in your #{SharedHelpers.gemfile_name}, which may resolve the conflict.\n)
           elsif !conflict.existing
             o << "\n"
+
+            relevant_sources = if conflict.requirement.source
+              [conflict.requirement.source]
+            elsif conflict.requirement.all_sources
+              conflict.requirement.all_sources
+            else
+              raise "no source set for #{conflict}"
+            end.compact.uniq
+
             if conflict.requirement_trees.first.size > 1
-              o << "Could not find gem '#{conflict.requirement}', which is required by "
-              o << "gem '#{conflict.requirement_trees.first[-2]}', in any of the sources."
+              o << "Could not find gem '#{printable_dep(conflict.requirement)}', which is required by "
+              o << "gem '#{printable_dep(conflict.requirement_trees.first[-2])}', in any of the relevant sources:\n  #{relevant_sources * "\n  "}\n"
             else
-              o << "Could not find gem '#{conflict.requirement}' in any of the sources\n"
+              o << "Could not find gem '#{printable_dep(conflict.requirement)}' in any of the relevant sources:\n   #{relevant_sources * "\n  "}\n"
             end
           end
           o
@@ -259,7 +268,17 @@ def search_for(dependency)
       platform = dependency.__platform
       dependency = dependency.dep unless dependency.is_a? Gem::Dependency
       search = @search_for[dependency] ||= begin
-        index = @source_requirements[dependency.name] || @index
+        index = Index.build do |idx|
+          if source = @source_requirements[dependency.name]
+            idx.add_source source.specs
+          elsif dependency.all_sources
+            dependency.all_sources.each {|s| idx.add_source(s.specs) if s }
+          else
+            idx.add_source @source_requirements[:default].specs
+          end
+        end
+        # source = @source_requirements[dependency.name]
+        # index = (source && source.specs) || @index
         results = index.search(dependency, @base[dependency.name])
         if vertex = @base_dg.vertex_named(dependency.name)
           locked_requirement = vertex.payload.requirement
@@ -303,8 +322,19 @@ def requirement_satisfied_by?(requirement, activated, spec)
       requirement.matches_spec?(spec)
     end
 
+    def relevant_sources_for_vertex(vertex)
+      if vertex.root?
+        [@source_requirements[vertex.name]]
+      else
+        vertex.recursive_predecessors.map do |v|
+          @source_requirements[v.name]
+        end << @source_requirements[:default]
+      end
+    end
+
     def sort_dependencies(dependencies, activated, conflicts)
       dependencies.sort_by do |dependency|
+        dependency.all_sources = relevant_sources_for_vertex(activated.vertex_named(dependency.name))
         name = name_for(dependency)
         [
           activated.vertex_named(name).payload ? 0 : 1,
@@ -336,26 +366,26 @@ def amount_constrained(dependency)
 
     def verify_gemfile_dependencies_are_found!(requirements)
       requirements.each do |requirement|
-        next if requirement.name == "bundler"
+        name = requirement.name
+        next if name == "bundler"
         next unless search_for(requirement).empty?
 
-        if (base = @base[requirement.name]) && !base.empty?
+        if (base = @base[name]) && !base.empty?
           version = base.first.version
           message = "You have requested:\n" \
-            "  #{requirement.name} #{requirement.requirement}\n\n" \
-            "The bundle currently has #{requirement.name} locked at #{version}.\n" \
-            "Try running `bundle update #{requirement.name}`\n\n" \
+            "  #{name} #{requirement.requirement}\n\n" \
+            "The bundle currently has #{name} locked at #{version}.\n" \
+            "Try running `bundle update #{name}`\n\n" \
             "If you are updating multiple gems in your Gemfile at once,\n" \
             "try passing them all to `bundle update`"
-        elsif requirement.source
-          name = requirement.name
-          specs = @source_requirements[name][name]
+        elsif source = @source_requirements[name]
+          specs = source.specs[name]
           versions_with_platforms = specs.map {|s| [s.version, s.platform] }
-          message = String.new("Could not find gem '#{requirement}' in #{requirement.source}.\n")
+          message = String.new("Could not find gem '#{requirement}' in #{source}.\n")
           message << if versions_with_platforms.any?
-                       "Source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}"
+                       "The source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}"
                      else
-                       "Source does not contain any versions of '#{requirement}'"
+                       "The source does not contain any versions of '#{requirement}'."
                      end
         else
           message = "Could not find gem '#{requirement}' in any of the gem sources " \
@@ -370,7 +400,8 @@ def formatted_versions_with_platforms(versions_with_platforms)
         version = vwp.first
         platform = vwp.last
         version_platform_str = String.new(version.to_s)
-        version_platform_str << " #{platform}" unless platform.nil?
+        version_platform_str << " #{platform}" unless platform.nil? || platform == Gem::Platform::RUBY
+        version_platform_str
       end
       version_platform_strs.join(", ")
     end

lib/bundler/rubygems_ext.rb

@@ -112,7 +112,7 @@ def dependencies_to_gemfile(dependencies, group = nil)
   end
 
   class Dependency
-    attr_accessor :source, :groups
+    attr_accessor :source, :groups, :all_sources
 
     alias_method :eql?, :==
 

lib/bundler/source.rb

@@ -29,6 +29,12 @@ def can_lock?(spec)
       spec.source == self
     end
 
+    # it's possible that gems from one source depend on gems from some
+    # other source, so now we download gemspecs and iterate over those
+    # dependencies, looking for gems we don't have info on yet.
+    def double_check_for(*)
+    end
+
     def include?(other)
       other == self
     end