Make gemfile_util capable of copying groups over

1 files changed, 321 insertions, 155 deletions

diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb
index 96dfcd78a2..62d8cfdf0b 100644
--- a/tasks/gemfile_util.rb
+++ b/tasks/gemfile_util.rb
@@ -1,4 +1,6 @@
+require "rubygems"
 require "bundler"
+require "shellwords"
 require "set"
 
 module GemfileUtil
@@ -7,212 +9,376 @@ module GemfileUtil
   # gem statement about the same gem in the Gemfile.
   #
   def gem(name, *args)
-    Bundler.ui.debug "gem #{name}, #{args.join(", ")}"
-    current_dep = dependencies.find { |dep| dep.name == name }
-
-    # Set path to absolute in case this is an included Gemfile in bundler 1.11.2 and below
     options = args[-1].is_a?(Hash) ? args[-1] : {}
-    if options[:path]
-      # path sourced gems are assumed to be overrides.
-      options[:override] = true
-      # options[:path] = File.expand_path(options[:path], Bundler.default_gemfile.dirname)
-    end
-    # Handle override
-    if options[:override]
-      override = true
-      options.delete(:override)
-      if current_dep
-        dependencies.delete(current_dep)
-      end
-    else
-      # If an override gem already exists, and we're not an override gem,
-      # ignore this gem in favor of the override (but warn if they don't match)
-      if overridden_gems.include?(name)
-        args.pop if args[-1].is_a?(Hash)
-        version = args || [">=0"]
-        desired_dep = Bundler::Dependency.new(name, version, options.dup)
-        if desired_dep =~ current_dep
-          Bundler.ui.debug "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with override gem #{current_dep} (#{current_dep.source})"
-        else
-          Bundler.ui.warn "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with incompatible override gem #{current_dep} (#{current_dep.source})"
-        end
+
+    # Unless we're finished with everything, ignore gems that are being overridden
+    unless overridden_gems == :finished
+      # If it's a path or override gem, it overrides whatever else is there.
+      if options[:path] || options[:override]
+        options.delete(:override)
+        warn_if_replacing(name, overridden_gems[name], args)
+        overridden_gems[name] = args
+        return
+
+      # If there's an override gem, and we're *not* an override gem, don't do anything
+      elsif overridden_gems[name]
+        warn_if_replacing(name, args, overridden_gems[name])
         return
       end
     end
 
-    # Add the gem normally
+    # Otherwise, add the gem normally
     super
+  rescue
+    puts $!.backtrace
+    raise
+  end
 
-    overridden_gems << name if override
+  def overridden_gems
+    @overridden_gems ||= {}
+  end
 
-    # Emit a warning if we're replacing a dep that doesn't match
-    if current_dep && override
-      added_dep = dependencies.find { |dep| dep.name == name }
-      if added_dep =~ current_dep
-        Bundler.ui.debug "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with override gem #{added_dep} (#{added_dep.source})"
-      else
-        Bundler.ui.warn "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with incompatible override gem #{added_dep} (#{added_dep.source})"
-      end
-    end
+  #
+  # Just before we finish the Gemfile, finish up the override gems
+  #
+  def to_definition(*args)
+    complete_overrides
+    super
   end
 
-  def overridden_gems
-    @overridden_gems ||= Set.new
+  def complete_overrides
+    to_override = overridden_gems
+    unless to_override == :finished
+      @overridden_gems = :finished
+      to_override.each do |name, args|
+        gem name, *args
+      end
+    end
   end
 
   #
   # Include all gems in the locked gemfile.
   #
-  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile)
+  # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile).
+  #          Defaults to <gemfile_path>.lock.
   # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
+  #          to nil), all gems will be selected.
   # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  #          the results if all groups they belong to are ignored. This matches
+  #          bundler's `without` behavior.
   # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
+  #          Gems in this list must be explicitly included in the Gemfile
+  #          with a `gem "gem_name", ...` line or they will be silently
+  #          ignored.
+  # @param copy_groups Whether to copy the groups over from the old lockfile to
+  #          the new. Use this when the new lockfile has the same convention for
+  #          groups as the old. Defaults to `false`.
   #
-  def include_locked_gemfile(gemfile, groups: nil, without_groups: nil, gems: [])
-    gemfile = File.expand_path(gemfile, Bundler.default_gemfile.dirname)
-    gems = Set.new(gems) + GemfileUtil.select_gems(gemfile, groups: nil, without_groups: nil)
-    specs = GemfileUtil.locked_gems("#{gemfile}.lock", gems)
-    specs.each do |name, version: nil, **options|
-      options = options.merge(override: true)
-      Bundler.ui.debug("Adding gem #{name}, #{version}, #{options} from #{gemfile}")
-      gem name, version, options
+  def include_locked_gemfile(gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false)
+    # Parse the desired lockfile
+    gemfile_path = Pathname.new(gemfile_path).expand_path(Bundler.default_gemfile.dirname).realpath
+    lockfile_path = Pathname.new(lockfile_path).expand_path(Bundler.default_gemfile.dirname).realpath
+
+    # Calculate relative_to
+    relative_to = Bundler.default_gemfile.dirname.realpath
+
+    # Call out to create-override-gemfile to read the Gemfile+Gemfile.lock (bundler does not work well if you do two things in one process)
+    create_override_gemfile_bin = File.expand_path("../bin/create-override-gemfile", __FILE__)
+    arguments = [
+      "--gemfile", gemfile_path,
+      "--lockfile", lockfile_path,
+      "--override"
+    ]
+    arguments += [ "--relative-to", relative_to ] if relative_to != "."
+    arguments += Array(groups).flat_map { |group| [ "--group", group ] }
+    arguments += Array(without_groups).flat_map { |without| [ "--without", without ] }
+    arguments += Array(gems).flat_map { |name| [ "--gem", name ] }
+    arguments << "--copy-groups" if copy_groups
+    cmd = Shellwords.join([ Gem.ruby, "-S", create_override_gemfile_bin, *arguments ])
+    output = nil
+    Bundler.ui.info("> #{cmd}")
+    Bundler.with_clean_env do
+      output = `#{cmd}`
     end
-  rescue
-    puts "ERROR: #{$!}"
-    puts $!.backtrace
-    raise
+    instance_eval(output, cmd, 1)
   end
 
   #
   # Include all gems in the locked gemfile.
   #
   # @param current_gemfile The Gemfile you are currently loading (`self`).
-  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile)
+  # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile).
+  #          Defaults to <gemfile_path>.lock.
   # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
+  #          to nil), all gems will be selected.
   # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  #          the results if all groups they belong to are ignored. This matches
+  #          bundler's `without` behavior.
   # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
+  #          Gems in this list must be explicitly included in the Gemfile
+  #          with a `gem "gem_name", ...` line or they will be silently
+  #          ignored.
+  # @param copy_groups Whether to copy the groups over from the old lockfile to
+  #          the new. Use this when the new lockfile has the same convention for
+  #          groups as the old. Defaults to `false`.
   #
-  def self.include_locked_gemfile(current_gemfile, gemfile, groups: nil, without_groups: nil, gems: [])
+  def self.include_locked_gemfile(current_gemfile, gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false)
     current_gemfile.instance_eval do
       extend GemfileUtil
-      include_locked_gemfile(gemfile, groups: groups, without_groups: without_groups, gems: [])
+      include_locked_gemfile(gemfile_path, lockfile_path, groups: groups, without_groups: without_groups, gems: gems, copy_groups: copy_groups)
     end
   end
 
+  def warn_if_replacing(name, old_args, new_args)
+    return if !old_args || !new_args
+    if args_to_dep(name, *old_args) =~ args_to_dep(name, *new_args)
+      Bundler.ui.debug "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})"
+    else
+      Bundler.ui.warn "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})"
+    end
+  end
+
+  def args_to_dep(name, *version, **options)
+    version = [">= 0"] if version.empty?
+    Bundler::Dependency.new(name, version, options)
+  end
+
   #
-  # Select the desired gems, sans dependencies, from the gemfile.
-  #
-  # @param gemfile Path to the Gemfile to load
-  # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
-  # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  # Reads a bundle, including a gemfile and lockfile.
   #
-  # @return An array of strings with the names of the given gems.
+  # Does no validation, does not update the lockfile or its gems in any way.
   #
-  def self.select_gems(gemfile, groups: nil, without_groups: nil)
-    Bundler.with_clean_env do
-      # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work
-      # This works around some issues in bundler 1.11.2.
-      ENV["BUNDLE_GEMFILE"] = gemfile
-
-      parsed_gemfile = Bundler::Dsl.new
-      parsed_gemfile.eval_gemfile(gemfile)
-      deps = parsed_gemfile.dependencies.select do |dep|
-        dep_groups = dep.groups
+  class Bundle
+    #
+    # Parse the given gemfile/lockfile pair.
+    #
+    # @return [Bundle] The parsed bundle.
+    #
+    def self.parse(gemfile_path, lockfile_path = "#{gemfile_path}.lock")
+      result = new(gemfile_path, lockfile_path)
+      result.gems
+      result
+    end
+
+    #
+    # Create a new Bundle to parse the given gemfile/lockfile pair.
+    #
+    def initialize(gemfile_path, lockfile_path = "#{gemfile_path}.lock")
+      @gemfile_path = gemfile_path
+      @lockfile_path = lockfile_path
+    end
+
+    #
+    # The path to the Gemfile
+    #
+    attr_reader :gemfile_path
+
+    #
+    # The path to the Lockfile
+    #
+    attr_reader :lockfile_path
+
+    #
+    # The list of gems.
+    #
+    # @return [Hash<String, Hash>] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path) from the lockfile
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies *include*
+    #             runtime dependencies.
+    #           - groups: The list of groups (symbols) this gem is in. Groups
+    #             are transitive: if A has a runtime dependency on B, and A is
+    #             in group X, then B is also in group X.
+    #           - declared_groups: The list of groups (symbols) this gem was
+    #             declared in the Gemfile.
+    #
+    def gems
+      @gems ||= begin
+        gems = locks.dup
+        gems.each do |name, g|
+          if gem_declarations.has_key?(name)
+            g[:declared_groups] = gem_declarations[name][:groups]
+          else
+            g[:declared_groups] = []
+          end
+          g[:groups] = g[:declared_groups].dup
+        end
+        # Transitivize groups (since dependencies are already transitive, this is easy)
+        gems.each do |name, g|
+          g[:dependencies].each do |dep|
+            gems[dep][:groups] |= gems[name][:declared_groups].dup
+          end
+        end
+        gems
+      end
+    end
+
+    #
+    # Get the gems (and their deps) in the given group.
+    #
+    # @param groups A list of groups to include (whitelist). If not passed (or set
+    #          to nil), all gems will be selected.
+    # @param without_groups A list of groups to ignore. Gems will be excluded from
+    #          the results if all groups they belong to are ignored.
+    #          This matches bundler's `without` behavior.
+    # @param gems A list of gems to include regardless of what groups are included.
+    #
+    # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path) from the lockfile
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies
+    #             *include* runtime dependencies.
+    #           - groups: The list of groups (symbols) this gem is in. Groups
+    #             are transitive: if A has a runtime dependency on B, and A is
+    #             in group X, then B is also in group X.
+    #           - declared_groups: The list of groups (symbols) this gem was
+    #             declared in the Gemfile.
+    #
+    def select_gems(groups: nil, without_groups: nil)
+      # First, select the gems that match
+      result = {}
+      gems.each do |name, g|
+        dep_groups = g[:declared_groups] - [ :only_a_runtime_dependency_of_other_gems ]
         dep_groups = dep_groups & groups if groups
         dep_groups = dep_groups - without_groups if without_groups
-        dep_groups.any?
+        if dep_groups.any?
+          result[name] ||= g
+          g[:dependencies].each do |dep|
+            result[dep] ||= gems[dep]
+          end
+        end
       end
-      deps.map { |dep| dep.name }
+      result
     end
-  end
 
-  #
-  # Get all gems in the locked gemfile that start from the given gem set.
-  #
-  # @param lockfile Path to the Gemfile to load
-  # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
-  # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
-  # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
-  # @param include_development_deps Whether to include development dependencies
-  #                                 or runtime only.
-  #
-  # @return Hash[String, Hash] A hash from gem_name ->  { version: <version>, source: <source>, git: <git>, path: <path>, ref: <ref> }
-  #
-  def self.locked_gems(lockfile, gems, include_development_deps: false)
-    # Grab all the specs from the lockfile
-    parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile))
-    specs = {}
-    parsed_lockfile.specs.each { |s| specs[s.name] = s }
-
-    # Select the desired gems, as well as their dependencies
-    to_process = Array(gems)
-    results = {}
-    while to_process.any?
-      gem_name = to_process.pop
-      next if gem_name == "bundler" # can't be bundled. Messes things up. Stop it.
-      # Only process each gem once
-      unless results.has_key?(gem_name)
-        spec = specs[gem_name]
-        unless spec
-          raise "Gem #{gem_name.inspect} was requested but was not in #{lockfile}! Gems in lockfile: #{specs.keys}"
+    #
+    # Get all locks from the given lockfile.
+    #
+    # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path)
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies *include*
+    #             runtime dependencies.
+    #
+    def locks
+      @locks ||= begin
+        # Grab all the specs from the lockfile
+        locks = {}
+        parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile_path))
+        parsed_lockfile.specs.each do |spec|
+          # Never include bundler, it can't be bundled and doesn't put itself in
+          # the lockfile correctly anyway
+          next if spec.name == "bundler"
+          lock = lock_source_metadata(spec)
+          lock[:version] = spec.version.to_s
+          runtime = spec.dependencies.select { |dep| dep.type == :runtime }
+          lock[:dependencies] = Set.new(runtime.map { |dep| dep.name })
+          lock[:development_dependencies] = Set.new(spec.dependencies.map { |dep| dep.name })
+          lock[:dependencies].delete("bundler")
+          lock[:development_dependencies].delete("bundler")
+          locks[spec.name] = lock
+        end
+
+        # Transitivize the deps.
+        locks.each do |name, lock|
+          lock[:dependencies] = transitive_dependencies(locks, name, :dependencies)
+          lock[:development_dependencies] = transitive_dependencies(locks, name, :development_dependencies)
         end
-        results[gem_name] = gem_metadata(spec, lockfile)
-        spec.dependencies.each do |dep|
-          if dep.type == :runtime || include_development_deps
-            to_process << dep.name
+
+        locks
+      end
+    end
+
+    #
+    # Get all desired gems, sans dependencies, from the gemfile.
+    #
+    # @param gemfile Path to the Gemfile to load
+    #
+    # @return Hash<String, Hash> An array of hashes where key = gem name and value
+    #           has :groups (an array of symbols representing the groups the gem
+    #           is in). :groups are not transitive, since we don't know the
+    #           dependency tree yet.
+    #
+    def gem_declarations
+      @gem_declarations ||= begin
+        Bundler.with_clean_env do
+          # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work
+          # This works around some issues in bundler 1.11.2.
+          ENV["BUNDLE_GEMFILE"] = gemfile_path
+
+          parsed_gemfile = Bundler::Dsl.new
+          parsed_gemfile.eval_gemfile(gemfile_path)
+          parsed_gemfile.complete_overrides if parsed_gemfile.respond_to?(:complete_overrides)
+
+          result = {}
+          parsed_gemfile.dependencies.each do |dep|
+            groups = dep.groups.empty? ? [:default] : dep.groups
+            result[dep.name] = { groups: groups, platforms: dep.platforms }
           end
+          result
         end
       end
     end
 
-    results
-  end
+    private
 
-  private
+    #
+    # Given a bunch of locks (name -> { dependencies: [name,name] }) and a
+    # dependency name, add its dependencies to the result transitively.
+    #
+    def transitive_dependencies(locks, name, dep_key, result = Set.new)
+      locks[name][dep_key].each do |dep|
+        # Only ever add a dep once, so we don't infinitely recurse
+        if result.add?(dep)
+          transitive_dependencies(locks, dep, dep_key, result)
+        end
+      end
+      result
+    end
 
-  #
-  # Get metadata for the given Bundler spec (coming from a lockfile).
-  #
-  # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> }
-  #
-  def self.gem_metadata(spec, lockfile)
-    # Copy source information from included Gemfile
-    result = {}
-    case spec.source
-    when Bundler::Source::Rubygems
-      result[:source] = spec.source.remotes.first.to_s
-      result[:version] = spec.version.to_s
-    when Bundler::Source::Git
-      result[:git] = spec.source.uri.to_s
-      result[:ref] = spec.source.revision
-    when Bundler::Source::Path
-      # Path is relative to the lockfile (if it's relative at all)
-      result[:path] = File.expand_path(spec.source.path.to_s, File.dirname(lockfile))
-    else
-      raise "Unknown source #{spec.source} for gem #{spec.name}"
+    #
+    # Get source and version metadata for the given Bundler spec (coming from a lockfile).
+    #
+    # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> }
+    #
+    def lock_source_metadata(spec)
+      # Copy source information from included Gemfile
+      result = {}
+      case spec.source
+      when Bundler::Source::Rubygems
+        result[:source] = spec.source.remotes.first.to_s
+      when Bundler::Source::Git
+        result[:git] = spec.source.uri.to_s
+        result[:ref] = spec.source.revision
+      when Bundler::Source::Path
+        result[:path] = spec.source.path.to_s
+      else
+        raise "Unknown source #{spec.source} for gem #{spec.name}"
+      end
+      result
     end
-    result
   end
-
 end