always try the API, don't re-fetch unmet deps

this adds much more aggressive attempts to use the dependency API (with the
assumption that the server will deny requests that are too large), and also
stops telling sources to fetch dependencies that were already not there.

1 files changed, 26 insertions, 39 deletions

diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index f929c9d674..4be7b45600 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -216,51 +216,38 @@ module Bundler
           old = Bundler.rubygems.sources
           idx = Index.new
 
-          fetchers       = remotes.map { |uri| Bundler::Fetcher.new(uri) }
-          api_fetchers   = fetchers.select { |f| f.use_api }
-          index_fetchers = fetchers - api_fetchers
+          fetchers   = remotes.map { |uri| Bundler::Fetcher.new(uri) }
+          api, index = fetchers.partition { |f| f.use_api }
+          indexes    = {}
+
+          api.each do |f|
+            Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug?
+            indexes[f] = f.specs(dependency_names, self)
+            idx.use indexes[f]
+            Bundler.ui.info "" if !Bundler.ui.debug? # new line now that the dots are over
+          end
+
+          # If there was an error, some api fetchers may now be index fetchers
+          api, index = fetchers.partition { |f| f.use_api }
 
           # gather lists from non-api sites
-          index_fetchers.each do |f|
+          index.each do |f|
             Bundler.ui.info "Fetching source index from #{f.uri}"
-            idx.use f.specs(nil, self)
+            indexes[f] = f.specs(nil, self)
+            idx.use indexes[f]
           end
-          return idx if api_fetchers.empty?
-
-          # because ensuring we have all the gems we need involves downloading
-          # the gemspecs of those gems, if the non-api sites contain more than
-          # about 100 gems, we just treat all sites as non-api for speed.
-          allow_api = idx.size < API_REQUEST_LIMIT && dependency_names.size < API_REQUEST_LIMIT
-
-          if allow_api
-            api_fetchers.each do |f|
-              Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug?
-              idx.use f.specs(dependency_names, self)
+          return idx if api.empty?
+
+          # it's possible that gems from one source depend on gems from some
+          # other source, so now we download gemspecs and iterate over those
+          # dependencies, looking for gems we don't have info on yet.
+          indexes.each do |f, i|
+            unmet = i.unmet_dependency_names
+            api.reject{|af| af == f }.each do |f|
+              Bundler.ui.info "Fetching additional metadata from #{f.uri}", Bundler.ui.debug?
+              idx.use f.specs(unmet, self)
               Bundler.ui.info "" if !Bundler.ui.debug? # new line now that the dots are over
             end
-
-            if api_fetchers.all?{|f| f.use_api }
-              # it's possible that gems from one source depend on gems from some
-              # other source, so now we download gemspecs and iterate over those
-              # dependencies, looking for gems we don't have info on yet.
-              unmet = idx.unmet_dependency_names
-
-              # if there are any cross-site gems we missed, get them now
-              api_fetchers.each do |f|
-                Bundler.ui.info "Fetching additional metadata from #{f.uri}", Bundler.ui.debug?
-                idx.use f.specs(unmet, self)
-                Bundler.ui.info "" if !Bundler.ui.debug? # new line now that the dots are over
-              end if unmet.any?
-            else
-              allow_api = false
-            end
-          end
-
-          if !allow_api
-            api_fetchers.each do |f|
-              Bundler.ui.info "Fetching source index from #{f.uri}"
-              idx.use f.specs(nil, self)
-            end
           end
 
           return idx