Fix back deps with pinned sources

Unpinned back deps still pull from an arbitrary source at this point, will have to fix that later

8 files changed, 86 insertions, 66 deletions

diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 022c22e8ea..cd3e25862d 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -218,6 +218,33 @@ module Bundler
           dependency_names -= pinned_spec_names(source.specs)
           dependency_names.push(*source.unmet_deps).uniq!
         end
+
+        memoize = proc do |&work|
+          ret = memoize
+          proc do
+            if memoize.equal?(ret)
+              ret = work.call
+            else
+              ret
+            end
+          end
+        end
+
+        # Suppose the gem Foo depends on the gem Bar.  Foo exists in Source A.  Bar has some versions that exist in both
+        # sources A and B.  At this point, the API request will have found all the versions of Bar in source A,
+        # but will not have found any versions of Bar from source B, which is a problem if the requested version
+        # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for
+        # each spec we found, we add all possible versions from all sources to the index.
+        loop do
+          idxcount = idx.size
+          sources.all_sources.each do |source|
+            unmet_dependency_names = memoize.call do
+              dependency_names.+(idx.dependency_names).uniq unless idx.size > Source::Rubygems::API_REQUEST_LIMIT
+            end
+            source.double_check_for(unmet_dependency_names, :override_dupes)
+          end
+          break if idxcount == idx.size
+        end
       end
     end
 
@@ -644,7 +671,7 @@ module Bundler
       source_requirements = {}
       dependencies.each do |dep|
         next unless source = dep.source || sources.rubygems_global
-        source_requirements[dep.name] = source.specs
+        source_requirements[dep.name] = source
       end
       source_requirements
     end
diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb
index 204e33387e..5a638a4b27 100644
--- a/lib/bundler/fetcher/downloader.rb
+++ b/lib/bundler/fetcher/downloader.rb
@@ -31,7 +31,7 @@ module Bundler
         when Net::HTTPUnauthorized
           raise AuthenticationRequiredError, uri.host
         when Net::HTTPNotFound
-          raise FallbackError, "Net::HTTPNotFound"
+          raise FallbackError, "Net::HTTPNotFound (#{uri})"
         else
           raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
         end
diff --git a/lib/bundler/index.rb b/lib/bundler/index.rb
index f2defe1cff..833276b251 100644
--- a/lib/bundler/index.rb
+++ b/lib/bundler/index.rb
@@ -98,6 +98,7 @@ module Bundler
       specs.values.each do |spec_sets|
         spec_sets.values.each(&blk)
       end
+      sources.each {|s| s.each(&blk) }
     end
 
     # returns a list of the dependencies
diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb
index b888133ce6..0afd4b6944 100644
--- a/lib/bundler/resolver.rb
+++ b/lib/bundler/resolver.rb
@@ -259,7 +259,8 @@ module Bundler
       platform = dependency.__platform
       dependency = dependency.dep unless dependency.is_a? Gem::Dependency
       search = @search_for[dependency] ||= begin
-        index = @source_requirements[dependency.name] || @index
+        source = @source_requirements[dependency.name]
+        index = (source && source.specs) || @index
         results = index.search(dependency, @base[dependency.name])
         if vertex = @base_dg.vertex_named(dependency.name)
           locked_requirement = vertex.payload.requirement
@@ -349,13 +350,13 @@ module Bundler
             "If you are updating multiple gems in your Gemfile at once,\n" \
             "try passing them all to `bundle update`"
         elsif source = @source_requirements[name]
-          specs = source[name]
+          specs = source.specs[name]
           versions_with_platforms = specs.map {|s| [s.version, s.platform] }
-          message = String.new("Could not find gem '#{requirement}' in #{requirement.source || "the global source or on this machine"}.\n")
+          message = String.new("Could not find gem '#{requirement}' in #{source}.\n")
           message << if versions_with_platforms.any?
-                       "Source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}"
+                       "The source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}"
                      else
-                       "Source does not contain any versions of '#{requirement}'."
+                       "The source does not contain any versions of '#{requirement}'."
                      end
         else
           message = "Could not find gem '#{requirement}' in any of the gem sources " \
diff --git a/lib/bundler/source.rb b/lib/bundler/source.rb
index eee4ade45b..56ce30af10 100644
--- a/lib/bundler/source.rb
+++ b/lib/bundler/source.rb
@@ -29,6 +29,12 @@ module Bundler
       spec.source == self
     end
 
+    # it's possible that gems from one source depend on gems from some
+    # other source, so now we download gemspecs and iterate over those
+    # dependencies, looking for gems we don't have info on yet.
+    def double_check_for(*)
+    end
+
     def include?(other)
       other == self
     end
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index 5eb42c9010..602b4386a3 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -65,8 +65,12 @@ module Bundler
       end
 
       def to_s
-        remote_names = remotes.map(&:to_s).join(", ")
-        "rubygems repository #{remote_names}"
+        if remotes.empty?
+          "locally installed gems"
+        else
+          remote_names = remotes.map(&:to_s).join(", ")
+          "rubygems repository #{remote_names} or installed locally"
+        end
       end
       alias_method :name, :to_s
 
@@ -230,6 +234,20 @@ module Bundler
         end
       end
 
+      def double_check_for(unmet_dependency_names, override_dupes = false, index = specs)
+        return unless @allow_remote
+        raise ArgumentError, "missing index" unless index
+
+        return unless api_fetchers.any?
+
+        unmet_dependency_names = unmet_dependency_names.call
+        Bundler.ui.debug "#{self}: 2x check for #{unmet_dependency_names}"
+
+        return if unmet_dependency_names && unmet_dependency_names.empty?
+
+        fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes)
+      end
+
     protected
 
       def credless_remotes
@@ -333,62 +351,28 @@ module Bundler
           index_fetchers = fetchers - api_fetchers
 
           # gather lists from non-api sites
-          index_fetchers.each do |f|
-            Bundler.ui.info "Fetching source index from #{f.uri}"
-            idx.use f.specs_with_retry(nil, self)
-          end
+          fetch_names(index_fetchers, nil, idx, false)
 
           # because ensuring we have all the gems we need involves downloading
           # the gemspecs of those gems, if the non-api sites contain more than
-          # about 100 gems, we treat all sites as non-api for speed.
+          # about 500 gems, we treat all sites as non-api for speed.
           allow_api = idx.size < API_REQUEST_LIMIT && dependency_names.size < API_REQUEST_LIMIT
           Bundler.ui.debug "Need to query more than #{API_REQUEST_LIMIT} gems." \
             " Downloading full index instead..." unless allow_api
 
-          if allow_api
-            api_fetchers.each do |f|
-              Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug?
-              idx.use f.specs_with_retry(dependency_names, self)
-              Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
-            end
-
-            # Suppose the gem Foo depends on the gem Bar.  Foo exists in Source A.  Bar has some versions that exist in both
-            # sources A and B.  At this point, the API request will have found all the versions of Bar in source A,
-            # but will not have found any versions of Bar from source B, which is a problem if the requested version
-            # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for
-            # each spec we found, we add all possible versions from all sources to the index.
-            loop do
-              idxcount = idx.size
-              api_fetchers.each do |f|
-                Bundler.ui.info "Fetching version metadata from #{f.uri}", Bundler.ui.debug?
-                idx.use f.specs_with_retry(idx.dependency_names, self), true
-                Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
-              end
-              break if idxcount == idx.size
-            end
-
-            if api_fetchers.any?
-              # it's possible that gems from one source depend on gems from some
-              # other source, so now we download gemspecs and iterate over those
-              # dependencies, looking for gems we don't have info on yet.
-              unmet = idx.unmet_dependency_names
-
-              # if there are any cross-site gems we missed, get them now
-              api_fetchers.each do |f|
-                Bundler.ui.info "Fetching dependency metadata from #{f.uri}", Bundler.ui.debug?
-                idx.use f.specs_with_retry(unmet, self)
-                Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
-              end if unmet.any?
-            else
-              allow_api = false
-            end
-          end
+          fetch_names(api_fetchers, allow_api && dependency_names, idx, false)
+        end
+      end
 
-          unless allow_api
-            api_fetchers.each do |f|
-              Bundler.ui.info "Fetching source index from #{f.uri}"
-              idx.use f.specs_with_retry(nil, self)
-            end
+      def fetch_names(fetchers, dependency_names, index, override_dupes)
+        fetchers.each do |f|
+          if dependency_names
+            Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug?
+            index.use f.specs_with_retry(dependency_names, self), override_dupes
+            Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+          else
+            Bundler.ui.info "Fetching source index from #{f.uri}"
+            index.use f.specs_with_retry(nil, self), override_dupes
           end
         end
       end
diff --git a/spec/install/gemfile/sources_spec.rb b/spec/install/gemfile/sources_spec.rb
index 889e6b5e12..69e8c12ee9 100644
--- a/spec/install/gemfile/sources_spec.rb
+++ b/spec/install/gemfile/sources_spec.rb
@@ -59,6 +59,10 @@ describe "bundle install with gems on multiple sources" do
           build_gem "rack", "1.0.0" do |s|
             s.write "lib/rack.rb", "RACK = 'FAIL'"
           end
+
+          build_gem "rack-obama" do |s|
+            s.add_dependency "rack"
+          end
         end
 
         gemfile <<-G
@@ -72,7 +76,7 @@ describe "bundle install with gems on multiple sources" do
       end
 
       it "installs the gems without any warning" do
-        bundle :install
+        bundle! :install
         expect(out).not_to include("Warning")
         should_be_installed("rack-obama 1.0.0", "rack 1.0.0")
       end
@@ -86,6 +90,10 @@ describe "bundle install with gems on multiple sources" do
           build_gem "rack", "1.0.0" do |s|
             s.write "lib/rack.rb", "RACK = 'FAIL'"
           end
+
+          build_gem "rack-obama" do |s|
+            s.add_dependency "rack"
+          end
         end
 
         gemfile <<-G
@@ -96,7 +104,7 @@ describe "bundle install with gems on multiple sources" do
       end
 
       it "installs the gems without any warning" do
-        bundle :install
+        bundle! :install
         expect(out).not_to include("Warning")
         should_be_installed("rack-obama 1.0.0", "rack 1.0.0")
       end
diff --git a/spec/install/gems/compact_index_spec.rb b/spec/install/gems/compact_index_spec.rb
index f2a9042cb8..1cd0f370ab 100644
--- a/spec/install/gems/compact_index_spec.rb
+++ b/spec/install/gems/compact_index_spec.rb
@@ -222,8 +222,6 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   it "fetches again when more dependencies are found in subsequent sources" do
-    pending "This test only passed with multiple primary sources. We need to " \
-      "fix it to work with pinned sources, too."
     build_repo2 do
       build_gem "back_deps" do |s|
         s.add_dependency "foo"
@@ -256,7 +254,6 @@ The checksum of /versions does not match the checksum provided by the server! So
       end
     end
 
-    pending "this should not be ambiguous. rack 1.2 should come from the extra source."
     gemfile <<-G
       source "#{source_uri}" do; end
       source "#{source_uri}/extra"
@@ -267,7 +264,6 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   it "considers all possible versions of dependencies from all api gem sources" do
-    pending "this is currently broken with a pinned source. we need to fix it."
     # In this scenario, the gem "somegem" only exists in repo4.  It depends on specific version of activesupport that
     # exists only in repo1.  There happens also be a version of activesupport in repo4, but not the one that version 1.0.0
     # of somegem wants. This test makes sure that bundler actually finds version 1.2.3 of active support in the other
@@ -293,7 +289,6 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   it "prints API output properly with back deps" do
-    pending "back deps are currently broken with pinned sources :'("
     build_repo2 do
       build_gem "back_deps" do |s|
         s.add_dependency "foo"
@@ -315,7 +310,6 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   it "does not fetch every spec if the index of gems is large when doing back deps" do
-    pending "back deps: still broken"
     build_repo2 do
       build_gem "back_deps" do |s|
         s.add_dependency "foo"
@@ -352,7 +346,6 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   it "fetches again when more dependencies are found in subsequent sources using --deployment" do
-    pending "back deps also broken in deployment mode"
     build_repo2 do
       build_gem "back_deps" do |s|
         s.add_dependency "foo"