Simplify external tests and make them work with Gemfile.lock

7 files changed, 274 insertions, 140 deletions

diff --git a/tasks/bin/run_chef_tests b/tasks/bin/run_chef_tests
new file mode 100755
index 0000000000..567c6a9587
--- /dev/null
+++ b/tasks/bin/run_chef_tests
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Fail fast (e) and echo commands (vx)
+set -evx
+
+echo --color > .rspec
+echo -fp >> .rspec
+sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers;
+sudo -E $(which bundle) exec rake spec;
+bundle exec rake style;
+bundle exec bundle-audit check --update;
diff --git a/tasks/bin/run_external_test b/tasks/bin/run_external_test
new file mode 100755
index 0000000000..f1cefb9138
--- /dev/null
+++ b/tasks/bin/run_external_test
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Fail fast (e) and echo commands (vx)
+set -evx
+
+# Arguments
+TEST_GEM=$1
+shift
+
+PROJECT_ROOT=$(pwd)
+PROJECT_BUNDLE_PATH=${BUNDLE_PATH:-$(grep BUNDLE_PATH: $PROJECT_ROOT/.bundle/config | cut -d' ' -f2-)}
+if [ -n "$PROJECT_BUNDLE_PATH" ]; then
+  PROJECT_BUNDLE_PATH=$PROJECT_ROOT/$PROJECT_BUNDLE_PATH
+fi
+
+TEST_GEM_ROOT=$(bundle show $TEST_GEM)
+
+# Make a copy of the original Gemfile and stitch in our Gemfile.lock
+TEST_GEMFILE=$TEST_GEM_ROOT/Gemfile
+MODIFIED_TEST_GEMFILE=$TEST_GEMFILE.externaltest
+cat <<EOM > $MODIFIED_TEST_GEMFILE
+require_relative "$PROJECT_ROOT/tasks/gemfile_util"
+GemfileUtil.include_locked_gemfile(self, "$PROJECT_ROOT/Gemfile", groups: [:default], gems: ["$TEST_GEM"] + "$TEST_WITH_GEMS".split(/\s+/))
+$TEST_GEM_OVERRIDES
+EOM
+cat $TEST_GEMFILE >> $MODIFIED_TEST_GEMFILE
+if [ -f $TEST_GEMFILE.lock ]; then
+  cp $TEST_GEMFILE.lock $MODIFIED_TEST_GEMFILE.lock
+elif [ -f $MODIFIED_TEST_GEMFILE.lock ]; then
+  rm -f $MODIFIED_TEST_GEMFILE.lock
+fi
+
+# Run the bundle install
+cd $TEST_GEM_ROOT
+export BUNDLE_GEMFILE=$MODIFIED_TEST_GEMFILE
+# Don't read from the project .bundle/config, just our env vars
+export BUNDLE_IGNORE_CONFIG=true
+# Use the top level bundle cache so we don't have to reinstall their packages
+if [ -n "$PROJECT_BUNDLE_PATH" ]; then
+  export BUNDLE_PATH=$PROJECT_BUNDLE_PATH
+  export BUNDLE_DISABLE_SHARED_GEMS=1
+  export BUNDLE_NO_PRUNE=true
+fi
+export BUNDLE_FROZEN=
+bundle install
+export BUNDLE_FROZEN=true
+
+bundle config
+
+# Iterate through the remaining arguments as commands
+while test ${#} -gt 0; do
+  bundle exec $1
+  shift
+done
diff --git a/tasks/changelog.rb b/tasks/changelog.rb
new file mode 100644
index 0000000000..fda94764ae
--- /dev/null
+++ b/tasks/changelog.rb
@@ -0,0 +1,12 @@
+begin
+  require "github_changelog_generator/task"
+
+  GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+    config.future_release = Chef::VERSION
+    config.enhancement_labels = "enhancement,Enhancement,New Feature,Feature".split(",")
+    config.bug_labels = "bug,Bug,Improvement,Upstream Bug".split(",")
+    config.exclude_labels = "duplicate,question,invalid,wontfix,no_changelog,Exclude From Changelog,Question,Discussion".split(",")
+  end
+rescue LoadError
+  puts "github_changelog_generator is not available. gem install github_changelog_generator to generate changelogs"
+end
diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb
index 7955686963..b2eabffb09 100644
--- a/tasks/dependencies.rb
+++ b/tasks/dependencies.rb
@@ -111,10 +111,10 @@ namespace :dependencies do
       # Replace the bundler and rubygems versions
       OMNIBUS_RUBYGEMS_AT_LATEST_VERSION.each do |override_name, gem_name|
         # Get the latest bundler version
-        puts "Running gem list -re #{gem_name} ..."
-        gem_list = `gem list -re #{gem_name}`
+        puts "Running gem list -r #{gem_name} ..."
+        gem_list = `gem list -r #{gem_name}`
         unless gem_list =~ /^#{gem_name}\s*\(([^)]*)\)$/
-          raise "gem list -re #{gem_name} failed with output:\n#{gem_list}"
+          raise "gem list -r #{gem_name} failed with output:\n#{gem_list}"
         end
 
         # Emit it
@@ -141,6 +141,7 @@ namespace :dependencies do
   # Find out if we're using the latest gems we can (so we don't regress versions)
   desc "Check for gems that are not at the latest released version, and report if anything not in ACCEPTABLE_OUTDATED_GEMS (version_policy.rb) is out of date."
   task :check_outdated do
+    extend BundleUtil
     puts ""
     puts "-------------------------------------------------------------------"
     puts "Checking for outdated gems ..."
diff --git a/tasks/external_tests.rb b/tasks/external_tests.rb
deleted file mode 100644
index a909ec2178..0000000000
--- a/tasks/external_tests.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-require "tempfile"
-require "bundler"
-
-CURRENT_GEM_NAME = "chef"
-CURRENT_GEM_PATH = File.expand_path("../..", __FILE__)
-
-def bundle_exec_with_chef(test_gem, commands)
-  gem_path = Bundler.environment.specs[test_gem].first.full_gem_path
-  gemfile_path = File.join(gem_path, "Gemfile.#{CURRENT_GEM_NAME}-external-test")
-  gemfile = File.open(gemfile_path, "w")
-  begin
-    IO.read(File.join(gem_path, "Gemfile")).each_line do |line|
-      if line =~ /^\s*gemspec/
-        next
-      elsif line =~ /^\s*gem '#{CURRENT_GEM_NAME}'|\s*gem "#{CURRENT_GEM_NAME}"/
-        next
-      elsif line =~ /^\s*dev_gem\s*['"](.+)['"]\s*$/
-        line = "gem '#{$1}', github: 'poise/#{$1}'"
-      elsif line =~ /\s*gem\s*['"]#{test_gem}['"]/ # foodcritic      end
-        next
-      end
-      gemfile.puts(line)
-    end
-    gemfile.puts("gem #{CURRENT_GEM_NAME.inspect}, path: #{CURRENT_GEM_PATH.inspect}")
-    gemfile.puts("gemspec path: #{gem_path.inspect}")
-    gemfile.close
-    Dir.chdir(gem_path) do
-      Bundler.with_clean_env do
-        unless system({ "BUNDLE_GEMFILE" => gemfile_path, "RUBYOPT" => nil, "GEMFILE_MOD" => nil }, "bundle update")
-          raise "Error running bundle update of #{gemfile_path} in #{gem_path}: #{$?.exitstatus}\nGemfile:\n#{IO.read(gemfile_path)}"
-        end
-        Array(commands).each do |command|
-          unless system({ "BUNDLE_GEMFILE" => gemfile_path, "RUBYOPT" => nil, "GEMFILE_MOD" => nil }, "bundle exec #{command}")
-            raise "Error running bundle exec #{command} in #{gem_path} with BUNDLE_GEMFILE=#{gemfile_path}: #{$?.exitstatus}\nGemfile:\n#{IO.read(gemfile_path)}"
-          end
-        end
-      end
-    end
-  ensure
-    File.delete(gemfile_path) if File.exist?(gemfile_path)
-  end
-end
-
-EXTERNAL_PROJECTS = {
-  "chef-zero"             => [ "rake spec", "rake cheffs" ],
-  "cheffish"              => "rake spec",
-  "chef-provisioning"     => "rake spec",
-  "chef-provisioning-aws" => "rake spec",
-  "chef-sugar"            => "rake",
-  "foodcritic"            => "rake test",
-  "chefspec"              => "rake",
-  "chef-rewind"           => "rake spec",
-  "poise"                 => "rake spec",
-  "halite"                => "rake spec",
-  "knife-windows"         => "rake unit_spec",
-}
-
-task :external_specs => EXTERNAL_PROJECTS.keys.map { |g| :"#{g.sub("-", "_")}_spec" }
-
-EXTERNAL_PROJECTS.each do |test_gem, commands|
-  task :"#{test_gem.tr("-", "_")}_spec" do
-    bundle_exec_with_chef(test_gem, commands)
-  end
-end
diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb
index 60d1e2ff31..96dfcd78a2 100644
--- a/tasks/gemfile_util.rb
+++ b/tasks/gemfile_util.rb
@@ -1,99 +1,218 @@
 require "bundler"
+require "set"
 
 module GemfileUtil
   #
-  # Given a set of dependencies with groups in them, and a resolved set of
-  # gemspecs (with dependency info in them), creates a full set of specs
-  # with group information on it. If A is in groups x and y, and A depends on
-  # B and C, then B and C are also in groups x and y.
+  # Adds `override: true`, which allows your statement to override any other
+  # gem statement about the same gem in the Gemfile.
   #
-  class GemGroups < Hash
-    def initialize(resolved)
-      @resolved = resolved
-    end
-    attr_reader :resolved
+  def gem(name, *args)
+    Bundler.ui.debug "gem #{name}, #{args.join(", ")}"
+    current_dep = dependencies.find { |dep| dep.name == name }
 
-    def add_dependency(dep)
-      add_gem_groups(dep.name, dep.groups)
+    # Set path to absolute in case this is an included Gemfile in bundler 1.11.2 and below
+    options = args[-1].is_a?(Hash) ? args[-1] : {}
+    if options[:path]
+      # path sourced gems are assumed to be overrides.
+      options[:override] = true
+      # options[:path] = File.expand_path(options[:path], Bundler.default_gemfile.dirname)
     end
-
-    private
-
-    def add_gem_groups(name, groups)
-      self[name] ||= []
-      difference = groups - self[name]
-      unless difference.empty?
-        self[name] += difference
-        spec = resolved.find { |spec| spec.name == name }
-        if spec
-          spec.dependencies.each do |spec|
-            add_gem_groups(spec.name, difference)
-          end
+    # Handle override
+    if options[:override]
+      override = true
+      options.delete(:override)
+      if current_dep
+        dependencies.delete(current_dep)
+      end
+    else
+      # If an override gem already exists, and we're not an override gem,
+      # ignore this gem in favor of the override (but warn if they don't match)
+      if overridden_gems.include?(name)
+        args.pop if args[-1].is_a?(Hash)
+        version = args || [">=0"]
+        desired_dep = Bundler::Dependency.new(name, version, options.dup)
+        if desired_dep =~ current_dep
+          Bundler.ui.debug "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with override gem #{current_dep} (#{current_dep.source})"
+        else
+          Bundler.ui.warn "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with incompatible override gem #{current_dep} (#{current_dep.source})"
         end
+        return
       end
     end
+
+    # Add the gem normally
+    super
+
+    overridden_gems << name if override
+
+    # Emit a warning if we're replacing a dep that doesn't match
+    if current_dep && override
+      added_dep = dependencies.find { |dep| dep.name == name }
+      if added_dep =~ current_dep
+        Bundler.ui.debug "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with override gem #{added_dep} (#{added_dep.source})"
+      else
+        Bundler.ui.warn "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with incompatible override gem #{added_dep} (#{added_dep.source})"
+      end
+    end
+  end
+
+  def overridden_gems
+    @overridden_gems ||= Set.new
   end
 
-  def calculate_dependents(spec_set)
-    dependents = {}
-    spec_set.each do |spec|
-      dependents[spec] ||= []
+  #
+  # Include all gems in the locked gemfile.
+  #
+  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param groups A list of groups to include (whitelist). If not passed (or set
+  #               to nil), all gems will be selected.
+  # @param without_groups A list of groups to ignore. Gems will be excluded from
+  #                       the results if all groups they belong to are ignored.
+  #                       This matches bundler's `without` behavior.
+  # @param gems A list of gems to include above and beyond the given groups.
+  #             Gems in this list must be explicitly included in the Gemfile
+  #             with a `gem "gem_name", ...` line or they will be silently
+  #             ignored.
+  #
+  def include_locked_gemfile(gemfile, groups: nil, without_groups: nil, gems: [])
+    gemfile = File.expand_path(gemfile, Bundler.default_gemfile.dirname)
+    gems = Set.new(gems) + GemfileUtil.select_gems(gemfile, groups: nil, without_groups: nil)
+    specs = GemfileUtil.locked_gems("#{gemfile}.lock", gems)
+    specs.each do |name, version: nil, **options|
+      options = options.merge(override: true)
+      Bundler.ui.debug("Adding gem #{name}, #{version}, #{options} from #{gemfile}")
+      gem name, version, options
     end
-    spec_set.each do |spec|
-      spec.dependencies.each do |dep|
-        puts "#{dep.class} -> #{spec.class}"
-        dependents[dep] << spec
-      end
+  rescue
+    puts "ERROR: #{$!}"
+    puts $!.backtrace
+    raise
+  end
+
+  #
+  # Include all gems in the locked gemfile.
+  #
+  # @param current_gemfile The Gemfile you are currently loading (`self`).
+  # @param gemfile Path to the Gemfile to load (relative to your Gemfile)
+  # @param groups A list of groups to include (whitelist). If not passed (or set
+  #               to nil), all gems will be selected.
+  # @param without_groups A list of groups to ignore. Gems will be excluded from
+  #                       the results if all groups they belong to are ignored.
+  #                       This matches bundler's `without` behavior.
+  # @param gems A list of gems to include above and beyond the given groups.
+  #             Gems in this list must be explicitly included in the Gemfile
+  #             with a `gem "gem_name", ...` line or they will be silently
+  #             ignored.
+  #
+  def self.include_locked_gemfile(current_gemfile, gemfile, groups: nil, without_groups: nil, gems: [])
+    current_gemfile.instance_eval do
+      extend GemfileUtil
+      include_locked_gemfile(gemfile, groups: groups, without_groups: without_groups, gems: [])
     end
-    dependents
   end
 
-  def include_locked_gemfile(gemfile)
-    #
-    # Read the gemfile and inject its locks as first-class dependencies
-    #
-    current_source = nil
-    bundle = Bundler::Definition.build(gemfile, "#{gemfile}.lock", nil)
-
-    # Go through and create the actual gemfile from the given locks and
-    # groups.
-    bundle.resolve.sort_by { |spec| spec.name }.each do |spec|
-      # bundler can't be installed by bundler so don't pin it.
-      next if spec.name == "bundler"
-      dep = bundle.dependencies.find { |d| d.name == spec.name }
-      gem_metadata = ""
-      if dep
-        gem_metadata << ", groups: #{dep.groups.inspect}" if dep.groups != [:default]
-        gem_metadata << ", platforms: #{dep.platforms.inspect}" if dep.platforms && !dep.platforms.empty?
+  #
+  # Select the desired gems, sans dependencies, from the gemfile.
+  #
+  # @param gemfile Path to the Gemfile to load
+  # @param groups A list of groups to include (whitelist). If not passed (or set
+  #               to nil), all gems will be selected.
+  # @param without_groups A list of groups to ignore. Gems will be excluded from
+  #                       the results if all groups they belong to are ignored.
+  #                       This matches bundler's `without` behavior.
+  #
+  # @return An array of strings with the names of the given gems.
+  #
+  def self.select_gems(gemfile, groups: nil, without_groups: nil)
+    Bundler.with_clean_env do
+      # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work
+      # This works around some issues in bundler 1.11.2.
+      ENV["BUNDLE_GEMFILE"] = gemfile
+
+      parsed_gemfile = Bundler::Dsl.new
+      parsed_gemfile.eval_gemfile(gemfile)
+      deps = parsed_gemfile.dependencies.select do |dep|
+        dep_groups = dep.groups
+        dep_groups = dep_groups & groups if groups
+        dep_groups = dep_groups - without_groups if without_groups
+        dep_groups.any?
       end
-      case spec.source
-      when Bundler::Source::Rubygems
-        if current_source
-          if current_source != spec.source
-            raise "Gem #{spec.name} has source #{spec.source}, but other gems have #{current_source}. Multiple rubygems sources are not supported."
+      deps.map { |dep| dep.name }
+    end
+  end
+
+  #
+  # Get all gems in the locked gemfile that start from the given gem set.
+  #
+  # @param lockfile Path to the Gemfile to load
+  # @param groups A list of groups to include (whitelist). If not passed (or set
+  #               to nil), all gems will be selected.
+  # @param without_groups A list of groups to ignore. Gems will be excluded from
+  #                       the results if all groups they belong to are ignored.
+  #                       This matches bundler's `without` behavior.
+  # @param gems A list of gems to include above and beyond the given groups.
+  #             Gems in this list must be explicitly included in the Gemfile
+  #             with a `gem "gem_name", ...` line or they will be silently
+  #             ignored.
+  # @param include_development_deps Whether to include development dependencies
+  #                                 or runtime only.
+  #
+  # @return Hash[String, Hash] A hash from gem_name ->  { version: <version>, source: <source>, git: <git>, path: <path>, ref: <ref> }
+  #
+  def self.locked_gems(lockfile, gems, include_development_deps: false)
+    # Grab all the specs from the lockfile
+    parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile))
+    specs = {}
+    parsed_lockfile.specs.each { |s| specs[s.name] = s }
+
+    # Select the desired gems, as well as their dependencies
+    to_process = Array(gems)
+    results = {}
+    while to_process.any?
+      gem_name = to_process.pop
+      next if gem_name == "bundler" # can't be bundled. Messes things up. Stop it.
+      # Only process each gem once
+      unless results.has_key?(gem_name)
+        spec = specs[gem_name]
+        unless spec
+          raise "Gem #{gem_name.inspect} was requested but was not in #{lockfile}! Gems in lockfile: #{specs.keys}"
+        end
+        results[gem_name] = gem_metadata(spec, lockfile)
+        spec.dependencies.each do |dep|
+          if dep.type == :runtime || include_development_deps
+            to_process << dep.name
           end
-        else
-          current_source = spec.source
-          add_gemfile_line("source #{spec.source.remotes.first.to_s.inspect}", __LINE__)
         end
-        add_gemfile_line("gem #{spec.name.inspect}, #{spec.version.to_s.inspect}#{gem_metadata}", __LINE__)
-      when Bundler::Source::Git
-        add_gemfile_line("gem #{spec.name.inspect}, git: #{spec.source.uri.to_s.inspect}, ref: #{spec.source.revision.inspect}#{gem_metadata}", __LINE__)
-      when Bundler::Source::Path
-        add_gemfile_line("gem #{spec.name.inspect}, path: #{spec.source.path.to_s.inspect}#{gem_metadata}", __LINE__)
-      else
-        raise "Unknown source #{spec.source} for gem #{spec.name}"
       end
     end
-  rescue
-    puts $!
-    puts $!.backtrace
-    raise
+
+    results
   end
 
   private
 
-  def add_gemfile_line(line, lineno)
-    instance_eval(line, __FILE__, lineno)
+  #
+  # Get metadata for the given Bundler spec (coming from a lockfile).
+  #
+  # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> }
+  #
+  def self.gem_metadata(spec, lockfile)
+    # Copy source information from included Gemfile
+    result = {}
+    case spec.source
+    when Bundler::Source::Rubygems
+      result[:source] = spec.source.remotes.first.to_s
+      result[:version] = spec.version.to_s
+    when Bundler::Source::Git
+      result[:git] = spec.source.uri.to_s
+      result[:ref] = spec.source.revision
+    when Bundler::Source::Path
+      # Path is relative to the lockfile (if it's relative at all)
+      result[:path] = File.expand_path(spec.source.path.to_s, File.dirname(lockfile))
+    else
+      raise "Unknown source #{spec.source} for gem #{spec.name}"
+    end
+    result
   end
+
 end
diff --git a/tasks/maintainers.rb b/tasks/maintainers.rb
index 535edda248..91742854bb 100644
--- a/tasks/maintainers.rb
+++ b/tasks/maintainers.rb
@@ -31,9 +31,10 @@ begin
   require "tomlrb"
   require "octokit"
   require "pp"
-  task :default => :generate
 
   namespace :maintainers do
+    task :default => :generate
+
     desc "Generate MarkDown version of MAINTAINERS file"
     task :generate do
       out = "<!-- This is a generated file. Please do not edit directly -->\n\n"