Make gemfile_util capable of copying groups over

7 files changed, 443 insertions, 168 deletions

diff --git a/tasks/bin/bundle-platform.bat b/tasks/bin/bundle-platform.bat
new file mode 100644
index 0000000000..d193eb0c05
--- /dev/null
+++ b/tasks/bin/bundle-platform.bat
@@ -0,0 +1,2 @@
+@ECHO OFF
+ruby "%~dpn0" %*
diff --git a/tasks/bin/create-override-gemfile b/tasks/bin/create-override-gemfile
new file mode 100755
index 0000000000..b67da025d2
--- /dev/null
+++ b/tasks/bin/create-override-gemfile
@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler"
+
+Bundler.with_clean_env do
+  require_relative "../gemfile_util"
+
+  options = {}
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: create-override-gemfile [OPTIONS]"
+
+    opts.on("--gemfile GEMFILE", "The Gemfile to read (default: Gemfile).") { |path| options[:gemfile_path] = path }
+    opts.on("--lockfile GEMFILE", "The lockfile to read (default: <gemfile>.lock).") { |path| options[:lockfile_path] = path }
+
+    opts.on("--group GROUP", "Groups to include (whitelist).") do |group|
+      options[:groups] ||= []
+      options[:groups] << group.to_sym
+    end
+
+    opts.on("--without GROUP", "Groups to exclude.") do |group|
+      options[:without_groups] ||= []
+      options[:without_groups] << group.to_sym
+    end
+
+    opts.on("--gem GEM", "Gems to include regardless of groups.") do |name|
+      options[:gems] ||= []
+      options[:gems] << name
+    end
+
+    opts.on("--relative-to PATH", "A path to prepend to any relative paths in the Gemfile.") do |path|
+      unless Pathname.new(path).absolute?
+        puts opts
+        raise "--relative-to #{path} was not an absolute path!"
+      end
+      options[:relative_to] = path
+    end
+
+    opts.on("--[no-]copy-groups", "Whether to copy groups over from the original Gemfile or not (default: false).") { |val| options[:copy_groups] = val }
+
+    opts.on("--[no-]override", "Whether to emit override: true on each gem line (default: false).") { |val| options[:override] = val }
+
+    opts.on("-h", "--help", "Print this message.") do
+      puts opts
+      exit(0)
+    end
+  end
+
+  args = opts.parse(ARGV)
+
+  if args.size > 0
+    puts opts
+    raise "Invalid arguments #{args}"
+  end
+
+  def create_override_gemfile(gemfile_path: "Gemfile", lockfile_path: "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false, relative_to: ".", override: false)
+    relative_to = Pathname.new(relative_to).realpath
+    # Select the gems we want
+    bundle = GemfileUtil::Bundle.parse(gemfile_path, lockfile_path)
+    gems_to_include = bundle.select_gems(groups: groups, without_groups: without_groups)
+    gems.each do |name|
+      raise "Requested gem #{name} is not in #{gemfile_path}.lock!" if !bundle.gems[name]
+      gems_to_include[name] ||= bundle.gems[name]
+      gems_to_include[name][:dependencies].each do |dep|
+        gems_to_include[name] ||= bundle.gems[dep]
+      end
+    end
+
+    # Add the gems to the Gemfile
+    gem_root = Pathname.new(gemfile_path).dirname.realpath
+    gems_to_include.sort_by { |name, options| options[:declared_groups].empty? ? 1 : 0 }.each do |name, options|
+      comment = nil
+      options = options.dup
+      version = options.delete(:version)
+      if copy_groups
+        # Some dependencies have no groups (are not in the Gemfile--just runtime
+        # dependencies). If we actually record that they have no groups, they
+        # will *always* be installed (or perhaps never). We only want them to
+        # install if their other deps do, so we mark them with the groups of the
+        # things that brought them in (the gems that depended on them). To do
+        # this, we just leave :groups intact.
+        if options[:declared_groups].empty?
+          options.delete(:declared_groups)
+          comment = " # Transitive dependency, not actually in original Gemfile"
+        else
+          # For other things, we want to copy the actual declared_groups--the
+          # ones that were in the Gemfile. We want the same --with and --without
+          # options to include and exclude them as worked with the original
+          # Gemfile.
+          options[:groups] = options.delete(:declared_groups)
+        end
+      else
+        options.delete(:groups)
+        options.delete(:declared_groups)
+      end
+      options.delete(:dependencies)
+      options.delete(:development_dependencies)
+      options[:override] = true if override
+      options[:path] = Pathname.new(options[:path]).expand_path(gem_root).relative_path_from(relative_to).to_s if options[:path]
+      line = "gem #{name.inspect}, #{version.inspect}"
+      options.each do |name, value|
+        line << ", #{name}: #{value.inspect}"
+      end
+      line << comment if comment
+      puts line
+    end
+  end
+
+  create_override_gemfile(options)
+end
diff --git a/tasks/bin/run_external_test b/tasks/bin/run_external_test
index f1cefb9138..74f76d3229 100755
--- a/tasks/bin/run_external_test
+++ b/tasks/bin/run_external_test
@@ -20,7 +20,7 @@ TEST_GEMFILE=$TEST_GEM_ROOT/Gemfile
 MODIFIED_TEST_GEMFILE=$TEST_GEMFILE.externaltest
 cat <<EOM > $MODIFIED_TEST_GEMFILE
 require_relative "$PROJECT_ROOT/tasks/gemfile_util"
-GemfileUtil.include_locked_gemfile(self, "$PROJECT_ROOT/Gemfile", groups: [:default], gems: ["$TEST_GEM"] + "$TEST_WITH_GEMS".split(/\s+/))
+GemfileUtil.include_locked_gemfile(self, "$PROJECT_ROOT/Gemfile", gems: ["$TEST_GEM"] + "$TEST_WITH_GEMS".split(/\s+/))
 $TEST_GEM_OVERRIDES
 EOM
 cat $TEST_GEMFILE >> $MODIFIED_TEST_GEMFILE
@@ -38,17 +38,10 @@ export BUNDLE_IGNORE_CONFIG=true
 # Use the top level bundle cache so we don't have to reinstall their packages
 if [ -n "$PROJECT_BUNDLE_PATH" ]; then
   export BUNDLE_PATH=$PROJECT_BUNDLE_PATH
-  export BUNDLE_DISABLE_SHARED_GEMS=1
-  export BUNDLE_NO_PRUNE=true
 fi
 export BUNDLE_FROZEN=
 bundle install
 export BUNDLE_FROZEN=true
 
 bundle config
-
-# Iterate through the remaining arguments as commands
-while test ${#} -gt 0; do
-  bundle exec $1
-  shift
-done
+bundle exec $@
diff --git a/tasks/bundle.rb b/tasks/bundle.rb
index b0cec5f580..349f83124c 100644
--- a/tasks/bundle.rb
+++ b/tasks/bundle.rb
@@ -17,6 +17,7 @@
 
 require_relative "bundle_util"
 require_relative "../version_policy"
+require "fileutils"
 
 desc "Tasks to work with the main Gemfile and Gemfile.<platform>"
 namespace :bundle do
@@ -35,7 +36,9 @@ namespace :bundle do
         puts "-------------------------------------------------------------------"
         puts "Updating Gemfile.#{platform}.lock ..."
         puts "-------------------------------------------------------------------"
-        bundle "lock", gemfile: "Gemfile.#{platform}", platform: platform, delete_gemfile_lock: true
+        puts "Copy Gemfile.lock to Gemfile.#{platform}.lock ..."
+        FileUtils.cp(File.join(project_root, "Gemfile.lock"), File.join(project_root, "Gemfile.#{platform}.lock"))
+        bundle "lock", gemfile: "Gemfile.#{platform}", platform: platform
       end
     end
   end
@@ -55,6 +58,8 @@ namespace :bundle do
         puts "-------------------------------------------------------------------"
         puts "Updating Gemfile.#{platform}.lock (conservatively) ..."
         puts "-------------------------------------------------------------------"
+        puts "Copy Gemfile.lock to Gemfile.#{platform}.lock ..."
+        FileUtils.cp(File.join(project_root, "Gemfile.lock"), File.join(project_root, "Gemfile.#{platform}.lock"))
         bundle "lock", gemfile: "Gemfile.#{platform}", platform: platform
       end
     end
diff --git a/tasks/bundle_util.rb b/tasks/bundle_util.rb
index 91ffa1f317..ee75610cac 100644
--- a/tasks/bundle_util.rb
+++ b/tasks/bundle_util.rb
@@ -1,3 +1,4 @@
+require "bundler"
 require "shellwords"
 
 module BundleUtil
@@ -56,7 +57,7 @@ module BundleUtil
 
         # Run the bundle command
         ruby_platforms = platform ? PLATFORMS[platform].join(" ") : "ruby"
-        cmd = Shellwords.join([bundle_platform, ruby_platforms, *args])
+        cmd = Shellwords.join([Gem.ruby, "-S", bundle_platform, ruby_platforms, *args])
         puts "#{prefix}#{Shellwords.join(["bundle", *args])}#{platform ? " for #{platform} platform" : ""}:"
         with_gemfile(gemfile) do
           puts "#{prefix}BUNDLE_GEMFILE=#{gemfile}"
diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb
index b2eabffb09..0fe3907ce1 100644
--- a/tasks/dependencies.rb
+++ b/tasks/dependencies.rb
@@ -29,7 +29,6 @@ namespace :dependencies do
                     dependencies:update_omnibus_gemfile_lock
                     dependencies:update_acceptance_gemfile_lock
                     dependencies:update_kitchen_tests_gemfile_lock
-                    dependencies:update_omnibus_berksfile_lock
                     dependencies:update_kitchen_tests_berksfile_lock
                   }
 
@@ -81,7 +80,6 @@ namespace :dependencies do
   end
 
   gemfile_lock_task :update_omnibus_gemfile_lock, dirs: %w{omnibus}
-  berksfile_lock_task :update_omnibus_berksfile_lock, dirs: %w{omnibus}
   gemfile_lock_task :update_acceptance_gemfile_lock, dirs: %w{
     acceptance
     acceptance/fips/test/integration/fips/serverspec
diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb
index 96dfcd78a2..62d8cfdf0b 100644
--- a/tasks/gemfile_util.rb
+++ b/tasks/gemfile_util.rb
@@ -1,4 +1,6 @@
+require "rubygems"
 require "bundler"
+require "shellwords"
 require "set"
 
 module GemfileUtil
@@ -7,212 +9,376 @@ module GemfileUtil
   # gem statement about the same gem in the Gemfile.
   #
   def gem(name, *args)
-    Bundler.ui.debug "gem #{name}, #{args.join(", ")}"
-    current_dep = dependencies.find { |dep| dep.name == name }
-
-    # Set path to absolute in case this is an included Gemfile in bundler 1.11.2 and below
     options = args[-1].is_a?(Hash) ? args[-1] : {}
-    if options[:path]
-      # path sourced gems are assumed to be overrides.
-      options[:override] = true
-      # options[:path] = File.expand_path(options[:path], Bundler.default_gemfile.dirname)
-    end
-    # Handle override
-    if options[:override]
-      override = true
-      options.delete(:override)
-      if current_dep
-        dependencies.delete(current_dep)
-      end
-    else
-      # If an override gem already exists, and we're not an override gem,
-      # ignore this gem in favor of the override (but warn if they don't match)
-      if overridden_gems.include?(name)
-        args.pop if args[-1].is_a?(Hash)
-        version = args || [">=0"]
-        desired_dep = Bundler::Dependency.new(name, version, options.dup)
-        if desired_dep =~ current_dep
-          Bundler.ui.debug "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with override gem #{current_dep} (#{current_dep.source})"
-        else
-          Bundler.ui.warn "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with incompatible override gem #{current_dep} (#{current_dep.source})"
-        end
+
+    # Unless we're finished with everything, ignore gems that are being overridden
+    unless overridden_gems == :finished
+      # If it's a path or override gem, it overrides whatever else is there.
+      if options[:path] || options[:override]
+        options.delete(:override)
+        warn_if_replacing(name, overridden_gems[name], args)
+        overridden_gems[name] = args
+        return
+
+      # If there's an override gem, and we're *not* an override gem, don't do anything
+      elsif overridden_gems[name]
+        warn_if_replacing(name, args, overridden_gems[name])
         return
       end
     end
 
-    # Add the gem normally
+    # Otherwise, add the gem normally
     super
+  rescue
+    puts $!.backtrace
+    raise
+  end
 
-    overridden_gems << name if override
+  def overridden_gems
+    @overridden_gems ||= {}
+  end
 
-    # Emit a warning if we're replacing a dep that doesn't match
-    if current_dep && override
-      added_dep = dependencies.find { |dep| dep.name == name }
-      if added_dep =~ current_dep
-        Bundler.ui.debug "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with override gem #{added_dep} (#{added_dep.source})"
-      else
-        Bundler.ui.warn "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with incompatible override gem #{added_dep} (#{added_dep.source})"
-      end
-    end
+  #
+  # Just before we finish the Gemfile, finish up the override gems
+  #
+  def to_definition(*args)
+    complete_overrides
+    super
   end
 
-  def overridden_gems
-    @overridden_gems ||= Set.new
+  def complete_overrides
+    to_override = overridden_gems
+    unless to_override == :finished
+      @overridden_gems = :finished
+      to_override.each do |name, args|
+        gem name, *args
+      end
+    end
   end
 
   #
   # Include all gems in the locked gemfile.
   #
-  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile)
+  # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile).
+  #          Defaults to <gemfile_path>.lock.
   # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
+  #          to nil), all gems will be selected.
   # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  #          the results if all groups they belong to are ignored. This matches
+  #          bundler's `without` behavior.
   # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
+  #          Gems in this list must be explicitly included in the Gemfile
+  #          with a `gem "gem_name", ...` line or they will be silently
+  #          ignored.
+  # @param copy_groups Whether to copy the groups over from the old lockfile to
+  #          the new. Use this when the new lockfile has the same convention for
+  #          groups as the old. Defaults to `false`.
   #
-  def include_locked_gemfile(gemfile, groups: nil, without_groups: nil, gems: [])
-    gemfile = File.expand_path(gemfile, Bundler.default_gemfile.dirname)
-    gems = Set.new(gems) + GemfileUtil.select_gems(gemfile, groups: nil, without_groups: nil)
-    specs = GemfileUtil.locked_gems("#{gemfile}.lock", gems)
-    specs.each do |name, version: nil, **options|
-      options = options.merge(override: true)
-      Bundler.ui.debug("Adding gem #{name}, #{version}, #{options} from #{gemfile}")
-      gem name, version, options
+  def include_locked_gemfile(gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false)
+    # Parse the desired lockfile
+    gemfile_path = Pathname.new(gemfile_path).expand_path(Bundler.default_gemfile.dirname).realpath
+    lockfile_path = Pathname.new(lockfile_path).expand_path(Bundler.default_gemfile.dirname).realpath
+
+    # Calculate relative_to
+    relative_to = Bundler.default_gemfile.dirname.realpath
+
+    # Call out to create-override-gemfile to read the Gemfile+Gemfile.lock (bundler does not work well if you do two things in one process)
+    create_override_gemfile_bin = File.expand_path("../bin/create-override-gemfile", __FILE__)
+    arguments = [
+      "--gemfile", gemfile_path,
+      "--lockfile", lockfile_path,
+      "--override"
+    ]
+    arguments += [ "--relative-to", relative_to ] if relative_to != "."
+    arguments += Array(groups).flat_map { |group| [ "--group", group ] }
+    arguments += Array(without_groups).flat_map { |without| [ "--without", without ] }
+    arguments += Array(gems).flat_map { |name| [ "--gem", name ] }
+    arguments << "--copy-groups" if copy_groups
+    cmd = Shellwords.join([ Gem.ruby, "-S", create_override_gemfile_bin, *arguments ])
+    output = nil
+    Bundler.ui.info("> #{cmd}")
+    Bundler.with_clean_env do
+      output = `#{cmd}`
     end
-  rescue
-    puts "ERROR: #{$!}"
-    puts $!.backtrace
-    raise
+    instance_eval(output, cmd, 1)
   end
 
   #
   # Include all gems in the locked gemfile.
   #
   # @param current_gemfile The Gemfile you are currently loading (`self`).
-  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile)
+  # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile).
+  #          Defaults to <gemfile_path>.lock.
   # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
+  #          to nil), all gems will be selected.
   # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  #          the results if all groups they belong to are ignored. This matches
+  #          bundler's `without` behavior.
   # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
+  #          Gems in this list must be explicitly included in the Gemfile
+  #          with a `gem "gem_name", ...` line or they will be silently
+  #          ignored.
+  # @param copy_groups Whether to copy the groups over from the old lockfile to
+  #          the new. Use this when the new lockfile has the same convention for
+  #          groups as the old. Defaults to `false`.
   #
-  def self.include_locked_gemfile(current_gemfile, gemfile, groups: nil, without_groups: nil, gems: [])
+  def self.include_locked_gemfile(current_gemfile, gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false)
     current_gemfile.instance_eval do
       extend GemfileUtil
-      include_locked_gemfile(gemfile, groups: groups, without_groups: without_groups, gems: [])
+      include_locked_gemfile(gemfile_path, lockfile_path, groups: groups, without_groups: without_groups, gems: gems, copy_groups: copy_groups)
     end
   end
 
+  def warn_if_replacing(name, old_args, new_args)
+    return if !old_args || !new_args
+    if args_to_dep(name, *old_args) =~ args_to_dep(name, *new_args)
+      Bundler.ui.debug "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})"
+    else
+      Bundler.ui.warn "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})"
+    end
+  end
+
+  def args_to_dep(name, *version, **options)
+    version = [">= 0"] if version.empty?
+    Bundler::Dependency.new(name, version, options)
+  end
+
   #
-  # Select the desired gems, sans dependencies, from the gemfile.
-  #
-  # @param gemfile Path to the Gemfile to load
-  # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
-  # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
+  # Reads a bundle, including a gemfile and lockfile.
   #
-  # @return An array of strings with the names of the given gems.
+  # Does no validation, does not update the lockfile or its gems in any way.
   #
-  def self.select_gems(gemfile, groups: nil, without_groups: nil)
-    Bundler.with_clean_env do
-      # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work
-      # This works around some issues in bundler 1.11.2.
-      ENV["BUNDLE_GEMFILE"] = gemfile
-
-      parsed_gemfile = Bundler::Dsl.new
-      parsed_gemfile.eval_gemfile(gemfile)
-      deps = parsed_gemfile.dependencies.select do |dep|
-        dep_groups = dep.groups
+  class Bundle
+    #
+    # Parse the given gemfile/lockfile pair.
+    #
+    # @return [Bundle] The parsed bundle.
+    #
+    def self.parse(gemfile_path, lockfile_path = "#{gemfile_path}.lock")
+      result = new(gemfile_path, lockfile_path)
+      result.gems
+      result
+    end
+
+    #
+    # Create a new Bundle to parse the given gemfile/lockfile pair.
+    #
+    def initialize(gemfile_path, lockfile_path = "#{gemfile_path}.lock")
+      @gemfile_path = gemfile_path
+      @lockfile_path = lockfile_path
+    end
+
+    #
+    # The path to the Gemfile
+    #
+    attr_reader :gemfile_path
+
+    #
+    # The path to the Lockfile
+    #
+    attr_reader :lockfile_path
+
+    #
+    # The list of gems.
+    #
+    # @return [Hash<String, Hash>] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path) from the lockfile
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies *include*
+    #             runtime dependencies.
+    #           - groups: The list of groups (symbols) this gem is in. Groups
+    #             are transitive: if A has a runtime dependency on B, and A is
+    #             in group X, then B is also in group X.
+    #           - declared_groups: The list of groups (symbols) this gem was
+    #             declared in the Gemfile.
+    #
+    def gems
+      @gems ||= begin
+        gems = locks.dup
+        gems.each do |name, g|
+          if gem_declarations.has_key?(name)
+            g[:declared_groups] = gem_declarations[name][:groups]
+          else
+            g[:declared_groups] = []
+          end
+          g[:groups] = g[:declared_groups].dup
+        end
+        # Transitivize groups (since dependencies are already transitive, this is easy)
+        gems.each do |name, g|
+          g[:dependencies].each do |dep|
+            gems[dep][:groups] |= gems[name][:declared_groups].dup
+          end
+        end
+        gems
+      end
+    end
+
+    #
+    # Get the gems (and their deps) in the given group.
+    #
+    # @param groups A list of groups to include (whitelist). If not passed (or set
+    #          to nil), all gems will be selected.
+    # @param without_groups A list of groups to ignore. Gems will be excluded from
+    #          the results if all groups they belong to are ignored.
+    #          This matches bundler's `without` behavior.
+    # @param gems A list of gems to include regardless of what groups are included.
+    #
+    # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path) from the lockfile
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies
+    #             *include* runtime dependencies.
+    #           - groups: The list of groups (symbols) this gem is in. Groups
+    #             are transitive: if A has a runtime dependency on B, and A is
+    #             in group X, then B is also in group X.
+    #           - declared_groups: The list of groups (symbols) this gem was
+    #             declared in the Gemfile.
+    #
+    def select_gems(groups: nil, without_groups: nil)
+      # First, select the gems that match
+      result = {}
+      gems.each do |name, g|
+        dep_groups = g[:declared_groups] - [ :only_a_runtime_dependency_of_other_gems ]
         dep_groups = dep_groups & groups if groups
         dep_groups = dep_groups - without_groups if without_groups
-        dep_groups.any?
+        if dep_groups.any?
+          result[name] ||= g
+          g[:dependencies].each do |dep|
+            result[dep] ||= gems[dep]
+          end
+        end
       end
-      deps.map { |dep| dep.name }
+      result
     end
-  end
 
-  #
-  # Get all gems in the locked gemfile that start from the given gem set.
-  #
-  # @param lockfile Path to the Gemfile to load
-  # @param groups A list of groups to include (whitelist). If not passed (or set
-  #               to nil), all gems will be selected.
-  # @param without_groups A list of groups to ignore. Gems will be excluded from
-  #                       the results if all groups they belong to are ignored.
-  #                       This matches bundler's `without` behavior.
-  # @param gems A list of gems to include above and beyond the given groups.
-  #             Gems in this list must be explicitly included in the Gemfile
-  #             with a `gem "gem_name", ...` line or they will be silently
-  #             ignored.
-  # @param include_development_deps Whether to include development dependencies
-  #                                 or runtime only.
-  #
-  # @return Hash[String, Hash] A hash from gem_name ->  { version: <version>, source: <source>, git: <git>, path: <path>, ref: <ref> }
-  #
-  def self.locked_gems(lockfile, gems, include_development_deps: false)
-    # Grab all the specs from the lockfile
-    parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile))
-    specs = {}
-    parsed_lockfile.specs.each { |s| specs[s.name] = s }
-
-    # Select the desired gems, as well as their dependencies
-    to_process = Array(gems)
-    results = {}
-    while to_process.any?
-      gem_name = to_process.pop
-      next if gem_name == "bundler" # can't be bundled. Messes things up. Stop it.
-      # Only process each gem once
-      unless results.has_key?(gem_name)
-        spec = specs[gem_name]
-        unless spec
-          raise "Gem #{gem_name.inspect} was requested but was not in #{lockfile}! Gems in lockfile: #{specs.keys}"
+    #
+    # Get all locks from the given lockfile.
+    #
+    # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the
+    #           hash has:
+    #           - version: version of the gem.
+    #           - source info (:source/:git/:ref/:path)
+    #           - dependencies: A list of gem names this gem has a runtime
+    #             dependency on. Dependencies are transitive: if A depends on B,
+    #             and B depends on C, then A has C in its :dependencies list.
+    #           - development_dependencies: - A list of gem names this gem has a
+    #             development dependency on. Dependencies are transitive: if A
+    #             depends on B, and B depends on C, then A has C in its
+    #             :development_dependencies list. development dependencies *include*
+    #             runtime dependencies.
+    #
+    def locks
+      @locks ||= begin
+        # Grab all the specs from the lockfile
+        locks = {}
+        parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile_path))
+        parsed_lockfile.specs.each do |spec|
+          # Never include bundler, it can't be bundled and doesn't put itself in
+          # the lockfile correctly anyway
+          next if spec.name == "bundler"
+          lock = lock_source_metadata(spec)
+          lock[:version] = spec.version.to_s
+          runtime = spec.dependencies.select { |dep| dep.type == :runtime }
+          lock[:dependencies] = Set.new(runtime.map { |dep| dep.name })
+          lock[:development_dependencies] = Set.new(spec.dependencies.map { |dep| dep.name })
+          lock[:dependencies].delete("bundler")
+          lock[:development_dependencies].delete("bundler")
+          locks[spec.name] = lock
+        end
+
+        # Transitivize the deps.
+        locks.each do |name, lock|
+          lock[:dependencies] = transitive_dependencies(locks, name, :dependencies)
+          lock[:development_dependencies] = transitive_dependencies(locks, name, :development_dependencies)
         end
-        results[gem_name] = gem_metadata(spec, lockfile)
-        spec.dependencies.each do |dep|
-          if dep.type == :runtime || include_development_deps
-            to_process << dep.name
+
+        locks
+      end
+    end
+
+    #
+    # Get all desired gems, sans dependencies, from the gemfile.
+    #
+    # @param gemfile Path to the Gemfile to load
+    #
+    # @return Hash<String, Hash> An array of hashes where key = gem name and value
+    #           has :groups (an array of symbols representing the groups the gem
+    #           is in). :groups are not transitive, since we don't know the
+    #           dependency tree yet.
+    #
+    def gem_declarations
+      @gem_declarations ||= begin
+        Bundler.with_clean_env do
+          # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work
+          # This works around some issues in bundler 1.11.2.
+          ENV["BUNDLE_GEMFILE"] = gemfile_path
+
+          parsed_gemfile = Bundler::Dsl.new
+          parsed_gemfile.eval_gemfile(gemfile_path)
+          parsed_gemfile.complete_overrides if parsed_gemfile.respond_to?(:complete_overrides)
+
+          result = {}
+          parsed_gemfile.dependencies.each do |dep|
+            groups = dep.groups.empty? ? [:default] : dep.groups
+            result[dep.name] = { groups: groups, platforms: dep.platforms }
           end
+          result
         end
       end
     end
 
-    results
-  end
+    private
 
-  private
+    #
+    # Given a bunch of locks (name -> { dependencies: [name,name] }) and a
+    # dependency name, add its dependencies to the result transitively.
+    #
+    def transitive_dependencies(locks, name, dep_key, result = Set.new)
+      locks[name][dep_key].each do |dep|
+        # Only ever add a dep once, so we don't infinitely recurse
+        if result.add?(dep)
+          transitive_dependencies(locks, dep, dep_key, result)
+        end
+      end
+      result
+    end
 
-  #
-  # Get metadata for the given Bundler spec (coming from a lockfile).
-  #
-  # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> }
-  #
-  def self.gem_metadata(spec, lockfile)
-    # Copy source information from included Gemfile
-    result = {}
-    case spec.source
-    when Bundler::Source::Rubygems
-      result[:source] = spec.source.remotes.first.to_s
-      result[:version] = spec.version.to_s
-    when Bundler::Source::Git
-      result[:git] = spec.source.uri.to_s
-      result[:ref] = spec.source.revision
-    when Bundler::Source::Path
-      # Path is relative to the lockfile (if it's relative at all)
-      result[:path] = File.expand_path(spec.source.path.to_s, File.dirname(lockfile))
-    else
-      raise "Unknown source #{spec.source} for gem #{spec.name}"
+    #
+    # Get source and version metadata for the given Bundler spec (coming from a lockfile).
+    #
+    # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> }
+    #
+    def lock_source_metadata(spec)
+      # Copy source information from included Gemfile
+      result = {}
+      case spec.source
+      when Bundler::Source::Rubygems
+        result[:source] = spec.source.remotes.first.to_s
+      when Bundler::Source::Git
+        result[:git] = spec.source.uri.to_s
+        result[:ref] = spec.source.revision
+      when Bundler::Source::Path
+        result[:path] = spec.source.path.to_s
+      else
+        raise "Unknown source #{spec.source} for gem #{spec.name}"
+      end
+      result
     end
-    result
   end
-
 end