Auto merge of #4770 - bundler:seg-gem-dep-api-compatibility, r=indirect

[RubygemsIntegration] Add support for reversing hooks

This is a pre-req for being able to use bundler as the backend for RubyGems' Gemfile support, as RubyGems tests run in a single process and the hooks installed (particularly by `replace_entrypoints`) are rather destructive to those tests. More changes to come to get `Gem.use_gemdeps` using bundler (i.e. to get the RG suite green), but this is likely to be the grossest of them.

11 files changed, 156 insertions, 87 deletions

diff --git a/lib/bundler.rb b/lib/bundler.rb
index b8eed25d44..6d64a2eb28 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -89,7 +89,7 @@ module Bundler
 
     def setup(*groups)
       # Return if all groups are already loaded
-      return @setup if defined?(@setup)
+      return @setup if defined?(@setup) && @setup
 
       definition.validate_ruby!
 
@@ -130,13 +130,11 @@ module Bundler
     end
 
     def locked_gems
-      return @locked_gems if defined?(@locked_gems)
-      if Bundler.default_lockfile.exist?
-        lock = Bundler.read_file(Bundler.default_lockfile)
-        @locked_gems = LockfileParser.new(lock)
-      else
-        @locked_gems = nil
-      end
+      @locked_gems ||=
+        if Bundler.default_lockfile.exist?
+          lock = Bundler.read_file(Bundler.default_lockfile)
+          LockfileParser.new(lock)
+        end
     end
 
     def ruby_scope
@@ -386,6 +384,15 @@ module Bundler
       @root = nil
       @settings = nil
       @definition = nil
+      @setup = nil
+      @load = nil
+      @locked_gems = nil
+      @bundle_path = nil
+      @bin_path = nil
+      return unless defined?(@rubygems) && @rubygems
+      rubygems.undo_replacements
+      rubygems.reset
+      @rubygems = nil
     end
 
   private
diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index b7db1179a0..10e8e517f9 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -7,7 +7,7 @@ module Bundler
   class Definition
     include GemHelpers
 
-    attr_reader :dependencies, :platforms, :ruby_version, :locked_deps, :gem_version_promoter
+    attr_reader :dependencies, :platforms, :ruby_version, :locked_deps, :gem_version_promoter, :requires
 
     # Given a gemfile and lockfile creates a Bundler definition
     #
@@ -107,6 +107,8 @@ module Bundler
       @dependency_changes = converge_dependencies
       @local_changes = converge_locals
 
+      @requires = compute_requires
+
       fixup_dependency_types!
     end
 
@@ -215,7 +217,7 @@ module Bundler
     end
 
     def current_dependencies
-      dependencies.reject {|d| !d.should_include? }
+      dependencies.select(&:should_include?)
     end
 
     def specs_for(groups)
@@ -768,5 +770,16 @@ module Bundler
       # to an array. The first element will be the gem name (e.g. foo), the second will be the version number.
       error.message.scan(/Could not find (\w+)-(\d+(?:\.\d+)+)/).flatten
     end
+
+    def compute_requires
+      dependencies.reduce({}) do |requires, dep|
+        next requires unless dep.should_include?
+        requires[dep.name] = Array(dep.autorequire || dep.name).map do |file|
+          # Allow `require: true` as an alias for `require: <name>`
+          file == true ? dep.name : file
+        end
+        requires
+      end
+    end
   end
 end
diff --git a/lib/bundler/dsl.rb b/lib/bundler/dsl.rb
index 0436b58f3a..4cd56e5e8d 100644
--- a/lib/bundler/dsl.rb
+++ b/lib/bundler/dsl.rb
@@ -38,7 +38,7 @@ module Bundler
       original_gemfile = @gemfile
       @gemfile = expanded_gemfile_path
       contents ||= Bundler.read_file(gemfile.to_s)
-      instance_eval(contents, gemfile.to_s, 1)
+      instance_eval(contents.dup.untaint, gemfile.to_s, 1)
     rescue Exception => e
       message = "There was an error " \
         "#{e.is_a?(GemfileEvalError) ? "evaluating" : "parsing"} " \
diff --git a/lib/bundler/environment.rb b/lib/bundler/environment.rb
index b12a146a77..fedc6b298f 100644
--- a/lib/bundler/environment.rb
+++ b/lib/bundler/environment.rb
@@ -31,6 +31,10 @@ module Bundler
       @definition.current_dependencies
     end
 
+    def requires
+      @definition.requires
+    end
+
     def lock(opts = {})
       @definition.lock(Bundler.default_lockfile, opts[:preserve_unknown_sections])
     end
diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb
index 88c446e11a..fc8eadd186 100644
--- a/lib/bundler/rubygems_ext.rb
+++ b/lib/bundler/rubygems_ext.rb
@@ -16,8 +16,15 @@ module Gem
   class Specification
     attr_accessor :remote, :location, :relative_loaded_from
 
-    remove_method :source if instance_methods(false).include?(:source)
-    attr_accessor :source
+    if instance_methods(false).map(&:to_sym).include?(:source)
+      remove_method :source
+      attr_writer :source
+      def source
+        (defined?(@source) && @source) || Gem::Source::Installed.new
+      end
+    else
+      attr_accessor :source
+    end
 
     alias_method :rg_full_gem_path, :full_gem_path
     alias_method :rg_loaded_from,   :loaded_from
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index af2e2f2fca..a083247adc 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -19,6 +19,10 @@ module Bundler
       Gem::Requirement.new(req_str).satisfied_by?(version)
     end
 
+    def initialize
+      @replaced_methods = {}
+    end
+
     def version
       self.class.version
     end
@@ -132,6 +136,14 @@ module Bundler
       Gem.path
     end
 
+    def reset
+      Gem::Specification.reset
+    end
+
+    def post_reset_hooks
+      Gem.post_reset_hooks
+    end
+
     def gem_cache
       gem_path.map {|p| File.expand_path("cache", p) }
     end
@@ -283,13 +295,11 @@ module Bundler
 
     def reverse_rubygems_kernel_mixin
       # Disable rubygems' gem activation system
-      ::Kernel.class_eval do
-        if private_method_defined?(:gem_original_require)
-          alias_method :rubygems_require, :require
-          alias_method :require, :gem_original_require
+      kernel = (class << ::Kernel; self; end)
+      [kernel, ::Kernel].each do |k|
+        if k.private_method_defined?(:gem_original_require)
+          redefine_method(k, :require, k.instance_method(:gem_original_require))
         end
-
-        undef gem
       end
     end
 
@@ -298,41 +308,44 @@ module Bundler
 
       executables = specs.map(&:executables).flatten
 
-      ::Kernel.send(:define_method, :gem) do |dep, *reqs|
-        if executables.include? File.basename(caller.first.split(":").first)
-          break
-        end
-        reqs.pop if reqs.last.is_a?(Hash)
-
-        unless dep.respond_to?(:name) && dep.respond_to?(:requirement)
-          dep = Gem::Dependency.new(dep, reqs)
-        end
-
-        spec = specs.find {|s| s.name == dep.name }
-
-        if spec.nil?
+      kernel = (class << ::Kernel; self; end)
+      [kernel, ::Kernel].each do |kernel_class|
+        redefine_method(kernel_class, :gem) do |dep, *reqs|
+          if executables.include? File.basename(caller.first.split(":").first)
+            break
+          end
+          reqs.pop if reqs.last.is_a?(Hash)
 
-          e = Gem::LoadError.new "#{dep.name} is not part of the bundle. Add it to Gemfile."
-          e.name = dep.name
-          if e.respond_to?(:requirement=)
-            e.requirement = dep.requirement
-          else
-            e.version_requirement = dep.requirement
+          unless dep.respond_to?(:name) && dep.respond_to?(:requirement)
+            dep = Gem::Dependency.new(dep, reqs)
           end
-          raise e
-        elsif dep !~ spec
-          e = Gem::LoadError.new "can't activate #{dep}, already activated #{spec.full_name}. " \
-                                 "Make sure all dependencies are added to Gemfile."
-          e.name = dep.name
-          if e.respond_to?(:requirement=)
-            e.requirement = dep.requirement
-          else
-            e.version_requirement = dep.requirement
+
+          spec = specs.find {|s| s.name == dep.name }
+
+          if spec.nil?
+
+            e = Gem::LoadError.new "#{dep.name} is not part of the bundle. Add it to Gemfile."
+            e.name = dep.name
+            if e.respond_to?(:requirement=)
+              e.requirement = dep.requirement
+            else
+              e.version_requirement = dep.requirement
+            end
+            raise e
+          elsif dep !~ spec
+            e = Gem::LoadError.new "can't activate #{dep}, already activated #{spec.full_name}. " \
+                                   "Make sure all dependencies are added to Gemfile."
+            e.name = dep.name
+            if e.respond_to?(:requirement=)
+              e.requirement = dep.requirement
+            else
+              e.version_requirement = dep.requirement
+            end
+            raise e
           end
-          raise e
-        end
 
-        true
+          true
+        end
       end
     end
 
@@ -463,9 +476,19 @@ module Bundler
       end
     end
 
-    def redefine_method(klass, method, &block)
+    def undo_replacements
+      @replaced_methods.each do |(sym, klass), method|
+        redefine_method(klass, sym, method)
+      end
+      post_reset_hooks.reject! do |proc|
+        proc.binding.eval("__FILE__") == __FILE__
+      end
+      @replaced_methods.clear
+    end
+
+    def redefine_method(klass, method, unbound_method = nil, &block)
       begin
-        if klass.instance_method(method) && method != :initialize
+        if (instance_method = klass.instance_method(method)) && method != :initialize
           # doing this to ensure we also get private methods
           klass.send(:remove_method, method)
         end
@@ -473,7 +496,12 @@ module Bundler
         # method isn't defined
         nil
       end
-      klass.send(:define_method, method, &block)
+      @replaced_methods[[method, klass]] = instance_method
+      if unbound_method
+        klass.send(:define_method, method, unbound_method)
+      elsif block
+        klass.send(:define_method, method, &block)
+      end
     end
 
     # Rubygems 1.4 through 1.6
@@ -489,11 +517,11 @@ module Bundler
       def stub_rubygems(specs)
         # Rubygems versions lower than 1.7 use SourceIndex#from_gems_in
         source_index_class = (class << Gem::SourceIndex; self; end)
-        source_index_class.send(:define_method, :from_gems_in) do |*args|
-          source_index = Gem::SourceIndex.new
-          source_index.spec_dirs = *args
-          source_index.add_specs(*specs)
-          source_index
+        redefine_method(source_index_class, :from_gems_in) do |*args|
+          Gem::SourceIndex.new.tap do |source_index|
+            source_index.spec_dirs = *args
+            source_index.add_specs(*specs)
+          end
         end
       end
 
@@ -510,6 +538,13 @@ module Bundler
         # which is too strict for the kinds of checks we care about. As a
         # result, validation is disabled on versions of RubyGems below 1.7.
       end
+
+      def post_reset_hooks
+        []
+      end
+
+      def reset
+      end
     end
 
     # Rubygems versions 1.3.6 and 1.3.7
@@ -688,25 +723,23 @@ module Bundler
     end
   end
 
-  if RubygemsIntegration.provides?(">= 2.1.0")
-    @rubygems = RubygemsIntegration::MoreFuture.new
-  elsif RubygemsIntegration.provides?(">= 1.99.99")
-    @rubygems = RubygemsIntegration::Future.new
-  elsif RubygemsIntegration.provides?(">= 1.8.20")
-    @rubygems = RubygemsIntegration::MoreModern.new
-  elsif RubygemsIntegration.provides?(">= 1.8.5")
-    @rubygems = RubygemsIntegration::Modern.new
-  elsif RubygemsIntegration.provides?(">= 1.8.0")
-    @rubygems = RubygemsIntegration::AlmostModern.new
-  elsif RubygemsIntegration.provides?(">= 1.7.0")
-    @rubygems = RubygemsIntegration::Transitional.new
-  elsif RubygemsIntegration.provides?(">= 1.4.0")
-    @rubygems = RubygemsIntegration::Legacy.new
-  else # Rubygems 1.3.6 and 1.3.7
-    @rubygems = RubygemsIntegration::Ancient.new
-  end
-
-  class << self
-    attr_reader :rubygems
+  def self.rubygems
+    @rubygems ||= if RubygemsIntegration.provides?(">= 2.1.0")
+      RubygemsIntegration::MoreFuture.new
+    elsif RubygemsIntegration.provides?(">= 1.99.99")
+      RubygemsIntegration::Future.new
+    elsif RubygemsIntegration.provides?(">= 1.8.20")
+      RubygemsIntegration::MoreModern.new
+    elsif RubygemsIntegration.provides?(">= 1.8.5")
+      RubygemsIntegration::Modern.new
+    elsif RubygemsIntegration.provides?(">= 1.8.0")
+      RubygemsIntegration::AlmostModern.new
+    elsif RubygemsIntegration.provides?(">= 1.7.0")
+      RubygemsIntegration::Transitional.new
+    elsif RubygemsIntegration.provides?(">= 1.4.0")
+      RubygemsIntegration::Legacy.new
+    else # Rubygems 1.3.6 and 1.3.7
+      RubygemsIntegration::Ancient.new
+    end
   end
 end
diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb
index ff0b146054..67ae20ff8a 100644
--- a/lib/bundler/settings.rb
+++ b/lib/bundler/settings.rb
@@ -276,11 +276,12 @@ module Bundler
     }xo
 
     def load_config(config_file)
-      SharedHelpers.filesystem_access(config_file, :read) do
-        valid_file = config_file && config_file.exist? && !config_file.size.zero?
+      return unless config_file
+      SharedHelpers.filesystem_access(config_file, :read) do |file|
+        valid_file = file.exist? && !file.size.zero?
         return {} if ignore_config? || !valid_file
         require "bundler/yaml_serializer"
-        YAMLSerializer.load config_file.read
+        YAMLSerializer.load file.read
       end
     end
 
diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb
index efbedeb374..69543356a2 100644
--- a/lib/bundler/shared_helpers.rb
+++ b/lib/bundler/shared_helpers.rb
@@ -23,7 +23,7 @@ module Bundler
     def default_gemfile
       gemfile = find_gemfile
       raise GemfileNotFound, "Could not locate Gemfile" unless gemfile
-      Pathname.new(gemfile)
+      Pathname.new(gemfile).untaint
     end
 
     def default_lockfile
@@ -32,7 +32,7 @@ module Bundler
       case gemfile.basename.to_s
       when "gems.rb" then Pathname.new(gemfile.sub(/.rb$/, ".locked"))
       else Pathname.new("#{gemfile}.lock")
-      end
+      end.untaint
     end
 
     def default_bundle_dir
@@ -102,7 +102,7 @@ module Bundler
     #
     # @see {Bundler::PermissionError}
     def filesystem_access(path, action = :write)
-      yield path
+      yield path.dup.untaint
     rescue Errno::EACCES
       raise PermissionError.new(path, action)
     rescue Errno::EAGAIN
@@ -158,7 +158,7 @@ module Bundler
 
     def search_up(*names)
       previous = nil
-      current  = File.expand_path(SharedHelpers.pwd)
+      current  = File.expand_path(SharedHelpers.pwd).untaint
 
       until !File.directory?(current) || current == previous
         if ENV["BUNDLE_SPEC_RUN"]
diff --git a/spec/bundler/dsl_spec.rb b/spec/bundler/dsl_spec.rb
index 77d663d754..00d36dd55f 100644
--- a/spec/bundler/dsl_spec.rb
+++ b/spec/bundler/dsl_spec.rb
@@ -80,7 +80,7 @@ describe Bundler::Dsl do
     it "handles syntax errors with a useful message" do
       expect(Bundler).to receive(:read_file).with("Gemfile").and_return("}")
       expect { subject.eval_gemfile("Gemfile") }.
-        to raise_error(Bundler::GemfileError, /There was an error parsing `Gemfile`: (syntax error, unexpected tSTRING_DEND|(compile error - )?syntax error, unexpected '}'). Bundler cannot continue./)
+        to raise_error(Bundler::GemfileError, /There was an error parsing `Gemfile`: (syntax error, unexpected tSTRING_DEND|(compile error - )?syntax error, unexpected '\}'). Bundler cannot continue./)
     end
 
     it "distinguishes syntax errors from evaluation errors" do
diff --git a/spec/bundler/shared_helpers_spec.rb b/spec/bundler/shared_helpers_spec.rb
index 2ad0c98dba..39ad2957ce 100644
--- a/spec/bundler/shared_helpers_spec.rb
+++ b/spec/bundler/shared_helpers_spec.rb
@@ -209,6 +209,10 @@ describe Bundler::SharedHelpers do
   end
 
   describe "#set_bundle_environment" do
+    before do
+      ENV["BUNDLE_GEMFILE"] = "Gemfile"
+    end
+
     shared_examples_for "ENV['PATH'] gets set correctly" do
       before { Dir.mkdir ".bundle" }
 
diff --git a/spec/support/helpers.rb b/spec/support/helpers.rb
index 34335ee1a3..bd88fdac68 100644
--- a/spec/support/helpers.rb
+++ b/spec/support/helpers.rb
@@ -13,7 +13,7 @@ module Spec
       FileUtils.mkdir_p(tmp)
       FileUtils.mkdir_p(home)
       ENV["BUNDLE_DISABLE_POSTIT"] = "1"
-      Bundler.send(:remove_instance_variable, :@settings) if Bundler.send(:instance_variable_defined?, :@settings)
+      Bundler.reset!
       Bundler.ui = nil
       Bundler.ui # force it to initialize
     end