diff options
author | Samuel Giddins <segiddins@segiddins.me> | 2016-06-23 18:36:04 -0500 |
---|---|---|
committer | Samuel Giddins <segiddins@segiddins.me> | 2016-06-27 16:27:22 -0500 |
commit | 051e0691d1eec48a98e53b141c8a1dac22d4fe68 (patch) | |
tree | 868a787a853400fb0c0088c2a930e05f72df1bad | |
parent | c98e4fbfd8fab7078826d854aec831956f3900c4 (diff) | |
download | bundler-051e0691d1eec48a98e53b141c8a1dac22d4fe68.tar.gz |
Fix back deps with pinned sources
Unpinned back deps still pull from an arbitrary source at this point, will have to fix that later
-rw-r--r-- | lib/bundler/definition.rb | 29 | ||||
-rw-r--r-- | lib/bundler/fetcher/downloader.rb | 2 | ||||
-rw-r--r-- | lib/bundler/index.rb | 1 | ||||
-rw-r--r-- | lib/bundler/resolver.rb | 11 | ||||
-rw-r--r-- | lib/bundler/source.rb | 6 | ||||
-rw-r--r-- | lib/bundler/source/rubygems.rb | 84 | ||||
-rw-r--r-- | spec/install/gemfile/sources_spec.rb | 12 | ||||
-rw-r--r-- | spec/install/gems/compact_index_spec.rb | 7 |
8 files changed, 86 insertions, 66 deletions
diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 022c22e8ea..cd3e25862d 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -218,6 +218,33 @@ module Bundler dependency_names -= pinned_spec_names(source.specs) dependency_names.push(*source.unmet_deps).uniq! end + + memoize = proc do |&work| + ret = memoize + proc do + if memoize.equal?(ret) + ret = work.call + else + ret + end + end + end + + # Suppose the gem Foo depends on the gem Bar. Foo exists in Source A. Bar has some versions that exist in both + # sources A and B. At this point, the API request will have found all the versions of Bar in source A, + # but will not have found any versions of Bar from source B, which is a problem if the requested version + # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for + # each spec we found, we add all possible versions from all sources to the index. + loop do + idxcount = idx.size + sources.all_sources.each do |source| + unmet_dependency_names = memoize.call do + dependency_names.+(idx.dependency_names).uniq unless idx.size > Source::Rubygems::API_REQUEST_LIMIT + end + source.double_check_for(unmet_dependency_names, :override_dupes) + end + break if idxcount == idx.size + end end end @@ -644,7 +671,7 @@ module Bundler source_requirements = {} dependencies.each do |dep| next unless source = dep.source || sources.rubygems_global - source_requirements[dep.name] = source.specs + source_requirements[dep.name] = source end source_requirements end diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb index 204e33387e..5a638a4b27 100644 --- a/lib/bundler/fetcher/downloader.rb +++ b/lib/bundler/fetcher/downloader.rb @@ -31,7 +31,7 @@ module Bundler when Net::HTTPUnauthorized raise AuthenticationRequiredError, uri.host when Net::HTTPNotFound - raise FallbackError, "Net::HTTPNotFound" + raise FallbackError, "Net::HTTPNotFound (#{uri})" else raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}" end diff --git a/lib/bundler/index.rb b/lib/bundler/index.rb index f2defe1cff..833276b251 100644 --- a/lib/bundler/index.rb +++ b/lib/bundler/index.rb @@ -98,6 +98,7 @@ module Bundler specs.values.each do |spec_sets| spec_sets.values.each(&blk) end + sources.each {|s| s.each(&blk) } end # returns a list of the dependencies diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index b888133ce6..0afd4b6944 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -259,7 +259,8 @@ module Bundler platform = dependency.__platform dependency = dependency.dep unless dependency.is_a? Gem::Dependency search = @search_for[dependency] ||= begin - index = @source_requirements[dependency.name] || @index + source = @source_requirements[dependency.name] + index = (source && source.specs) || @index results = index.search(dependency, @base[dependency.name]) if vertex = @base_dg.vertex_named(dependency.name) locked_requirement = vertex.payload.requirement @@ -349,13 +350,13 @@ module Bundler "If you are updating multiple gems in your Gemfile at once,\n" \ "try passing them all to `bundle update`" elsif source = @source_requirements[name] - specs = source[name] + specs = source.specs[name] versions_with_platforms = specs.map {|s| [s.version, s.platform] } - message = String.new("Could not find gem '#{requirement}' in #{requirement.source || "the global source or on this machine"}.\n") + message = String.new("Could not find gem '#{requirement}' in #{source}.\n") message << if versions_with_platforms.any? - "Source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}" + "The source contains '#{name}' at: #{formatted_versions_with_platforms(versions_with_platforms)}" else - "Source does not contain any versions of '#{requirement}'." + "The source does not contain any versions of '#{requirement}'." end else message = "Could not find gem '#{requirement}' in any of the gem sources " \ diff --git a/lib/bundler/source.rb b/lib/bundler/source.rb index eee4ade45b..56ce30af10 100644 --- a/lib/bundler/source.rb +++ b/lib/bundler/source.rb @@ -29,6 +29,12 @@ module Bundler spec.source == self end + # it's possible that gems from one source depend on gems from some + # other source, so now we download gemspecs and iterate over those + # dependencies, looking for gems we don't have info on yet. + def double_check_for(*) + end + def include?(other) other == self end diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 5eb42c9010..602b4386a3 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -65,8 +65,12 @@ module Bundler end def to_s - remote_names = remotes.map(&:to_s).join(", ") - "rubygems repository #{remote_names}" + if remotes.empty? + "locally installed gems" + else + remote_names = remotes.map(&:to_s).join(", ") + "rubygems repository #{remote_names} or installed locally" + end end alias_method :name, :to_s @@ -230,6 +234,20 @@ module Bundler end end + def double_check_for(unmet_dependency_names, override_dupes = false, index = specs) + return unless @allow_remote + raise ArgumentError, "missing index" unless index + + return unless api_fetchers.any? + + unmet_dependency_names = unmet_dependency_names.call + Bundler.ui.debug "#{self}: 2x check for #{unmet_dependency_names}" + + return if unmet_dependency_names && unmet_dependency_names.empty? + + fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes) + end + protected def credless_remotes @@ -333,62 +351,28 @@ module Bundler index_fetchers = fetchers - api_fetchers # gather lists from non-api sites - index_fetchers.each do |f| - Bundler.ui.info "Fetching source index from #{f.uri}" - idx.use f.specs_with_retry(nil, self) - end + fetch_names(index_fetchers, nil, idx, false) # because ensuring we have all the gems we need involves downloading # the gemspecs of those gems, if the non-api sites contain more than - # about 100 gems, we treat all sites as non-api for speed. + # about 500 gems, we treat all sites as non-api for speed. allow_api = idx.size < API_REQUEST_LIMIT && dependency_names.size < API_REQUEST_LIMIT Bundler.ui.debug "Need to query more than #{API_REQUEST_LIMIT} gems." \ " Downloading full index instead..." unless allow_api - if allow_api - api_fetchers.each do |f| - Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(dependency_names, self) - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end - - # Suppose the gem Foo depends on the gem Bar. Foo exists in Source A. Bar has some versions that exist in both - # sources A and B. At this point, the API request will have found all the versions of Bar in source A, - # but will not have found any versions of Bar from source B, which is a problem if the requested version - # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for - # each spec we found, we add all possible versions from all sources to the index. - loop do - idxcount = idx.size - api_fetchers.each do |f| - Bundler.ui.info "Fetching version metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(idx.dependency_names, self), true - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end - break if idxcount == idx.size - end - - if api_fetchers.any? - # it's possible that gems from one source depend on gems from some - # other source, so now we download gemspecs and iterate over those - # dependencies, looking for gems we don't have info on yet. - unmet = idx.unmet_dependency_names - - # if there are any cross-site gems we missed, get them now - api_fetchers.each do |f| - Bundler.ui.info "Fetching dependency metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(unmet, self) - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end if unmet.any? - else - allow_api = false - end - end + fetch_names(api_fetchers, allow_api && dependency_names, idx, false) + end + end - unless allow_api - api_fetchers.each do |f| - Bundler.ui.info "Fetching source index from #{f.uri}" - idx.use f.specs_with_retry(nil, self) - end + def fetch_names(fetchers, dependency_names, index, override_dupes) + fetchers.each do |f| + if dependency_names + Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug? + index.use f.specs_with_retry(dependency_names, self), override_dupes + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + else + Bundler.ui.info "Fetching source index from #{f.uri}" + index.use f.specs_with_retry(nil, self), override_dupes end end end diff --git a/spec/install/gemfile/sources_spec.rb b/spec/install/gemfile/sources_spec.rb index 889e6b5e12..69e8c12ee9 100644 --- a/spec/install/gemfile/sources_spec.rb +++ b/spec/install/gemfile/sources_spec.rb @@ -59,6 +59,10 @@ describe "bundle install with gems on multiple sources" do build_gem "rack", "1.0.0" do |s| s.write "lib/rack.rb", "RACK = 'FAIL'" end + + build_gem "rack-obama" do |s| + s.add_dependency "rack" + end end gemfile <<-G @@ -72,7 +76,7 @@ describe "bundle install with gems on multiple sources" do end it "installs the gems without any warning" do - bundle :install + bundle! :install expect(out).not_to include("Warning") should_be_installed("rack-obama 1.0.0", "rack 1.0.0") end @@ -86,6 +90,10 @@ describe "bundle install with gems on multiple sources" do build_gem "rack", "1.0.0" do |s| s.write "lib/rack.rb", "RACK = 'FAIL'" end + + build_gem "rack-obama" do |s| + s.add_dependency "rack" + end end gemfile <<-G @@ -96,7 +104,7 @@ describe "bundle install with gems on multiple sources" do end it "installs the gems without any warning" do - bundle :install + bundle! :install expect(out).not_to include("Warning") should_be_installed("rack-obama 1.0.0", "rack 1.0.0") end diff --git a/spec/install/gems/compact_index_spec.rb b/spec/install/gems/compact_index_spec.rb index f2a9042cb8..1cd0f370ab 100644 --- a/spec/install/gems/compact_index_spec.rb +++ b/spec/install/gems/compact_index_spec.rb @@ -222,8 +222,6 @@ The checksum of /versions does not match the checksum provided by the server! So end it "fetches again when more dependencies are found in subsequent sources" do - pending "This test only passed with multiple primary sources. We need to " \ - "fix it to work with pinned sources, too." build_repo2 do build_gem "back_deps" do |s| s.add_dependency "foo" @@ -256,7 +254,6 @@ The checksum of /versions does not match the checksum provided by the server! So end end - pending "this should not be ambiguous. rack 1.2 should come from the extra source." gemfile <<-G source "#{source_uri}" do; end source "#{source_uri}/extra" @@ -267,7 +264,6 @@ The checksum of /versions does not match the checksum provided by the server! So end it "considers all possible versions of dependencies from all api gem sources" do - pending "this is currently broken with a pinned source. we need to fix it." # In this scenario, the gem "somegem" only exists in repo4. It depends on specific version of activesupport that # exists only in repo1. There happens also be a version of activesupport in repo4, but not the one that version 1.0.0 # of somegem wants. This test makes sure that bundler actually finds version 1.2.3 of active support in the other @@ -293,7 +289,6 @@ The checksum of /versions does not match the checksum provided by the server! So end it "prints API output properly with back deps" do - pending "back deps are currently broken with pinned sources :'(" build_repo2 do build_gem "back_deps" do |s| s.add_dependency "foo" @@ -315,7 +310,6 @@ The checksum of /versions does not match the checksum provided by the server! So end it "does not fetch every spec if the index of gems is large when doing back deps" do - pending "back deps: still broken" build_repo2 do build_gem "back_deps" do |s| s.add_dependency "foo" @@ -352,7 +346,6 @@ The checksum of /versions does not match the checksum provided by the server! So end it "fetches again when more dependencies are found in subsequent sources using --deployment" do - pending "back deps also broken in deployment mode" build_repo2 do build_gem "back_deps" do |s| s.add_dependency "foo" |