From bbc1d6ebccbd7f22fea72a9c2b76ef19f3b5beb3 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 12 Apr 2016 16:04:27 -0700 Subject: Simplify external tests and make them work with Gemfile.lock --- tasks/bin/run_chef_tests | 11 ++ tasks/bin/run_external_test | 54 +++++++++ tasks/changelog.rb | 12 ++ tasks/dependencies.rb | 7 +- tasks/external_tests.rb | 64 ----------- tasks/gemfile_util.rb | 263 ++++++++++++++++++++++++++++++++------------ tasks/maintainers.rb | 3 +- 7 files changed, 274 insertions(+), 140 deletions(-) create mode 100755 tasks/bin/run_chef_tests create mode 100755 tasks/bin/run_external_test create mode 100644 tasks/changelog.rb delete mode 100644 tasks/external_tests.rb (limited to 'tasks') diff --git a/tasks/bin/run_chef_tests b/tasks/bin/run_chef_tests new file mode 100755 index 0000000000..567c6a9587 --- /dev/null +++ b/tasks/bin/run_chef_tests @@ -0,0 +1,11 @@ +#!/bin/bash + +# Fail fast (e) and echo commands (vx) +set -evx + +echo --color > .rspec +echo -fp >> .rspec +sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers; +sudo -E $(which bundle) exec rake spec; +bundle exec rake style; +bundle exec bundle-audit check --update; diff --git a/tasks/bin/run_external_test b/tasks/bin/run_external_test new file mode 100755 index 0000000000..f1cefb9138 --- /dev/null +++ b/tasks/bin/run_external_test @@ -0,0 +1,54 @@ +#!/bin/bash + +# Fail fast (e) and echo commands (vx) +set -evx + +# Arguments +TEST_GEM=$1 +shift + +PROJECT_ROOT=$(pwd) +PROJECT_BUNDLE_PATH=${BUNDLE_PATH:-$(grep BUNDLE_PATH: $PROJECT_ROOT/.bundle/config | cut -d' ' -f2-)} +if [ -n "$PROJECT_BUNDLE_PATH" ]; then + PROJECT_BUNDLE_PATH=$PROJECT_ROOT/$PROJECT_BUNDLE_PATH +fi + +TEST_GEM_ROOT=$(bundle show $TEST_GEM) + +# Make a copy of the original Gemfile and stitch in our Gemfile.lock +TEST_GEMFILE=$TEST_GEM_ROOT/Gemfile +MODIFIED_TEST_GEMFILE=$TEST_GEMFILE.externaltest +cat < $MODIFIED_TEST_GEMFILE +require_relative "$PROJECT_ROOT/tasks/gemfile_util" +GemfileUtil.include_locked_gemfile(self, "$PROJECT_ROOT/Gemfile", groups: [:default], gems: ["$TEST_GEM"] + "$TEST_WITH_GEMS".split(/\s+/)) +$TEST_GEM_OVERRIDES +EOM +cat $TEST_GEMFILE >> $MODIFIED_TEST_GEMFILE +if [ -f $TEST_GEMFILE.lock ]; then + cp $TEST_GEMFILE.lock $MODIFIED_TEST_GEMFILE.lock +elif [ -f $MODIFIED_TEST_GEMFILE.lock ]; then + rm -f $MODIFIED_TEST_GEMFILE.lock +fi + +# Run the bundle install +cd $TEST_GEM_ROOT +export BUNDLE_GEMFILE=$MODIFIED_TEST_GEMFILE +# Don't read from the project .bundle/config, just our env vars +export BUNDLE_IGNORE_CONFIG=true +# Use the top level bundle cache so we don't have to reinstall their packages +if [ -n "$PROJECT_BUNDLE_PATH" ]; then + export BUNDLE_PATH=$PROJECT_BUNDLE_PATH + export BUNDLE_DISABLE_SHARED_GEMS=1 + export BUNDLE_NO_PRUNE=true +fi +export BUNDLE_FROZEN= +bundle install +export BUNDLE_FROZEN=true + +bundle config + +# Iterate through the remaining arguments as commands +while test ${#} -gt 0; do + bundle exec $1 + shift +done diff --git a/tasks/changelog.rb b/tasks/changelog.rb new file mode 100644 index 0000000000..fda94764ae --- /dev/null +++ b/tasks/changelog.rb @@ -0,0 +1,12 @@ +begin + require "github_changelog_generator/task" + + GitHubChangelogGenerator::RakeTask.new :changelog do |config| + config.future_release = Chef::VERSION + config.enhancement_labels = "enhancement,Enhancement,New Feature,Feature".split(",") + config.bug_labels = "bug,Bug,Improvement,Upstream Bug".split(",") + config.exclude_labels = "duplicate,question,invalid,wontfix,no_changelog,Exclude From Changelog,Question,Discussion".split(",") + end +rescue LoadError + puts "github_changelog_generator is not available. gem install github_changelog_generator to generate changelogs" +end diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb index 7955686963..b2eabffb09 100644 --- a/tasks/dependencies.rb +++ b/tasks/dependencies.rb @@ -111,10 +111,10 @@ namespace :dependencies do # Replace the bundler and rubygems versions OMNIBUS_RUBYGEMS_AT_LATEST_VERSION.each do |override_name, gem_name| # Get the latest bundler version - puts "Running gem list -re #{gem_name} ..." - gem_list = `gem list -re #{gem_name}` + puts "Running gem list -r #{gem_name} ..." + gem_list = `gem list -r #{gem_name}` unless gem_list =~ /^#{gem_name}\s*\(([^)]*)\)$/ - raise "gem list -re #{gem_name} failed with output:\n#{gem_list}" + raise "gem list -r #{gem_name} failed with output:\n#{gem_list}" end # Emit it @@ -141,6 +141,7 @@ namespace :dependencies do # Find out if we're using the latest gems we can (so we don't regress versions) desc "Check for gems that are not at the latest released version, and report if anything not in ACCEPTABLE_OUTDATED_GEMS (version_policy.rb) is out of date." task :check_outdated do + extend BundleUtil puts "" puts "-------------------------------------------------------------------" puts "Checking for outdated gems ..." diff --git a/tasks/external_tests.rb b/tasks/external_tests.rb deleted file mode 100644 index a909ec2178..0000000000 --- a/tasks/external_tests.rb +++ /dev/null @@ -1,64 +0,0 @@ -require "tempfile" -require "bundler" - -CURRENT_GEM_NAME = "chef" -CURRENT_GEM_PATH = File.expand_path("../..", __FILE__) - -def bundle_exec_with_chef(test_gem, commands) - gem_path = Bundler.environment.specs[test_gem].first.full_gem_path - gemfile_path = File.join(gem_path, "Gemfile.#{CURRENT_GEM_NAME}-external-test") - gemfile = File.open(gemfile_path, "w") - begin - IO.read(File.join(gem_path, "Gemfile")).each_line do |line| - if line =~ /^\s*gemspec/ - next - elsif line =~ /^\s*gem '#{CURRENT_GEM_NAME}'|\s*gem "#{CURRENT_GEM_NAME}"/ - next - elsif line =~ /^\s*dev_gem\s*['"](.+)['"]\s*$/ - line = "gem '#{$1}', github: 'poise/#{$1}'" - elsif line =~ /\s*gem\s*['"]#{test_gem}['"]/ # foodcritic end - next - end - gemfile.puts(line) - end - gemfile.puts("gem #{CURRENT_GEM_NAME.inspect}, path: #{CURRENT_GEM_PATH.inspect}") - gemfile.puts("gemspec path: #{gem_path.inspect}") - gemfile.close - Dir.chdir(gem_path) do - Bundler.with_clean_env do - unless system({ "BUNDLE_GEMFILE" => gemfile_path, "RUBYOPT" => nil, "GEMFILE_MOD" => nil }, "bundle update") - raise "Error running bundle update of #{gemfile_path} in #{gem_path}: #{$?.exitstatus}\nGemfile:\n#{IO.read(gemfile_path)}" - end - Array(commands).each do |command| - unless system({ "BUNDLE_GEMFILE" => gemfile_path, "RUBYOPT" => nil, "GEMFILE_MOD" => nil }, "bundle exec #{command}") - raise "Error running bundle exec #{command} in #{gem_path} with BUNDLE_GEMFILE=#{gemfile_path}: #{$?.exitstatus}\nGemfile:\n#{IO.read(gemfile_path)}" - end - end - end - end - ensure - File.delete(gemfile_path) if File.exist?(gemfile_path) - end -end - -EXTERNAL_PROJECTS = { - "chef-zero" => [ "rake spec", "rake cheffs" ], - "cheffish" => "rake spec", - "chef-provisioning" => "rake spec", - "chef-provisioning-aws" => "rake spec", - "chef-sugar" => "rake", - "foodcritic" => "rake test", - "chefspec" => "rake", - "chef-rewind" => "rake spec", - "poise" => "rake spec", - "halite" => "rake spec", - "knife-windows" => "rake unit_spec", -} - -task :external_specs => EXTERNAL_PROJECTS.keys.map { |g| :"#{g.sub("-", "_")}_spec" } - -EXTERNAL_PROJECTS.each do |test_gem, commands| - task :"#{test_gem.tr("-", "_")}_spec" do - bundle_exec_with_chef(test_gem, commands) - end -end diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb index 60d1e2ff31..96dfcd78a2 100644 --- a/tasks/gemfile_util.rb +++ b/tasks/gemfile_util.rb @@ -1,99 +1,218 @@ require "bundler" +require "set" module GemfileUtil # - # Given a set of dependencies with groups in them, and a resolved set of - # gemspecs (with dependency info in them), creates a full set of specs - # with group information on it. If A is in groups x and y, and A depends on - # B and C, then B and C are also in groups x and y. + # Adds `override: true`, which allows your statement to override any other + # gem statement about the same gem in the Gemfile. # - class GemGroups < Hash - def initialize(resolved) - @resolved = resolved - end - attr_reader :resolved + def gem(name, *args) + Bundler.ui.debug "gem #{name}, #{args.join(", ")}" + current_dep = dependencies.find { |dep| dep.name == name } - def add_dependency(dep) - add_gem_groups(dep.name, dep.groups) + # Set path to absolute in case this is an included Gemfile in bundler 1.11.2 and below + options = args[-1].is_a?(Hash) ? args[-1] : {} + if options[:path] + # path sourced gems are assumed to be overrides. + options[:override] = true + # options[:path] = File.expand_path(options[:path], Bundler.default_gemfile.dirname) end - - private - - def add_gem_groups(name, groups) - self[name] ||= [] - difference = groups - self[name] - unless difference.empty? - self[name] += difference - spec = resolved.find { |spec| spec.name == name } - if spec - spec.dependencies.each do |spec| - add_gem_groups(spec.name, difference) - end + # Handle override + if options[:override] + override = true + options.delete(:override) + if current_dep + dependencies.delete(current_dep) + end + else + # If an override gem already exists, and we're not an override gem, + # ignore this gem in favor of the override (but warn if they don't match) + if overridden_gems.include?(name) + args.pop if args[-1].is_a?(Hash) + version = args || [">=0"] + desired_dep = Bundler::Dependency.new(name, version, options.dup) + if desired_dep =~ current_dep + Bundler.ui.debug "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with override gem #{current_dep} (#{current_dep.source})" + else + Bundler.ui.warn "Replaced Gemfile dependency #{desired_dep} (#{desired_dep.source}) with incompatible override gem #{current_dep} (#{current_dep.source})" end + return end end + + # Add the gem normally + super + + overridden_gems << name if override + + # Emit a warning if we're replacing a dep that doesn't match + if current_dep && override + added_dep = dependencies.find { |dep| dep.name == name } + if added_dep =~ current_dep + Bundler.ui.debug "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with override gem #{added_dep} (#{added_dep.source})" + else + Bundler.ui.warn "Replaced Gemfile dependency #{current_dep} (#{current_dep.source}) with incompatible override gem #{added_dep} (#{added_dep.source})" + end + end + end + + def overridden_gems + @overridden_gems ||= Set.new end - def calculate_dependents(spec_set) - dependents = {} - spec_set.each do |spec| - dependents[spec] ||= [] + # + # Include all gems in the locked gemfile. + # + # @param gemfile Path to the Gemfile to load (relative to your Gemfile) + # @param groups A list of groups to include (whitelist). If not passed (or set + # to nil), all gems will be selected. + # @param without_groups A list of groups to ignore. Gems will be excluded from + # the results if all groups they belong to are ignored. + # This matches bundler's `without` behavior. + # @param gems A list of gems to include above and beyond the given groups. + # Gems in this list must be explicitly included in the Gemfile + # with a `gem "gem_name", ...` line or they will be silently + # ignored. + # + def include_locked_gemfile(gemfile, groups: nil, without_groups: nil, gems: []) + gemfile = File.expand_path(gemfile, Bundler.default_gemfile.dirname) + gems = Set.new(gems) + GemfileUtil.select_gems(gemfile, groups: nil, without_groups: nil) + specs = GemfileUtil.locked_gems("#{gemfile}.lock", gems) + specs.each do |name, version: nil, **options| + options = options.merge(override: true) + Bundler.ui.debug("Adding gem #{name}, #{version}, #{options} from #{gemfile}") + gem name, version, options end - spec_set.each do |spec| - spec.dependencies.each do |dep| - puts "#{dep.class} -> #{spec.class}" - dependents[dep] << spec - end + rescue + puts "ERROR: #{$!}" + puts $!.backtrace + raise + end + + # + # Include all gems in the locked gemfile. + # + # @param current_gemfile The Gemfile you are currently loading (`self`). + # @param gemfile Path to the Gemfile to load (relative to your Gemfile) + # @param groups A list of groups to include (whitelist). If not passed (or set + # to nil), all gems will be selected. + # @param without_groups A list of groups to ignore. Gems will be excluded from + # the results if all groups they belong to are ignored. + # This matches bundler's `without` behavior. + # @param gems A list of gems to include above and beyond the given groups. + # Gems in this list must be explicitly included in the Gemfile + # with a `gem "gem_name", ...` line or they will be silently + # ignored. + # + def self.include_locked_gemfile(current_gemfile, gemfile, groups: nil, without_groups: nil, gems: []) + current_gemfile.instance_eval do + extend GemfileUtil + include_locked_gemfile(gemfile, groups: groups, without_groups: without_groups, gems: []) end - dependents end - def include_locked_gemfile(gemfile) - # - # Read the gemfile and inject its locks as first-class dependencies - # - current_source = nil - bundle = Bundler::Definition.build(gemfile, "#{gemfile}.lock", nil) - - # Go through and create the actual gemfile from the given locks and - # groups. - bundle.resolve.sort_by { |spec| spec.name }.each do |spec| - # bundler can't be installed by bundler so don't pin it. - next if spec.name == "bundler" - dep = bundle.dependencies.find { |d| d.name == spec.name } - gem_metadata = "" - if dep - gem_metadata << ", groups: #{dep.groups.inspect}" if dep.groups != [:default] - gem_metadata << ", platforms: #{dep.platforms.inspect}" if dep.platforms && !dep.platforms.empty? + # + # Select the desired gems, sans dependencies, from the gemfile. + # + # @param gemfile Path to the Gemfile to load + # @param groups A list of groups to include (whitelist). If not passed (or set + # to nil), all gems will be selected. + # @param without_groups A list of groups to ignore. Gems will be excluded from + # the results if all groups they belong to are ignored. + # This matches bundler's `without` behavior. + # + # @return An array of strings with the names of the given gems. + # + def self.select_gems(gemfile, groups: nil, without_groups: nil) + Bundler.with_clean_env do + # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work + # This works around some issues in bundler 1.11.2. + ENV["BUNDLE_GEMFILE"] = gemfile + + parsed_gemfile = Bundler::Dsl.new + parsed_gemfile.eval_gemfile(gemfile) + deps = parsed_gemfile.dependencies.select do |dep| + dep_groups = dep.groups + dep_groups = dep_groups & groups if groups + dep_groups = dep_groups - without_groups if without_groups + dep_groups.any? end - case spec.source - when Bundler::Source::Rubygems - if current_source - if current_source != spec.source - raise "Gem #{spec.name} has source #{spec.source}, but other gems have #{current_source}. Multiple rubygems sources are not supported." + deps.map { |dep| dep.name } + end + end + + # + # Get all gems in the locked gemfile that start from the given gem set. + # + # @param lockfile Path to the Gemfile to load + # @param groups A list of groups to include (whitelist). If not passed (or set + # to nil), all gems will be selected. + # @param without_groups A list of groups to ignore. Gems will be excluded from + # the results if all groups they belong to are ignored. + # This matches bundler's `without` behavior. + # @param gems A list of gems to include above and beyond the given groups. + # Gems in this list must be explicitly included in the Gemfile + # with a `gem "gem_name", ...` line or they will be silently + # ignored. + # @param include_development_deps Whether to include development dependencies + # or runtime only. + # + # @return Hash[String, Hash] A hash from gem_name -> { version: , source: , git: , path: , ref: } + # + def self.locked_gems(lockfile, gems, include_development_deps: false) + # Grab all the specs from the lockfile + parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile)) + specs = {} + parsed_lockfile.specs.each { |s| specs[s.name] = s } + + # Select the desired gems, as well as their dependencies + to_process = Array(gems) + results = {} + while to_process.any? + gem_name = to_process.pop + next if gem_name == "bundler" # can't be bundled. Messes things up. Stop it. + # Only process each gem once + unless results.has_key?(gem_name) + spec = specs[gem_name] + unless spec + raise "Gem #{gem_name.inspect} was requested but was not in #{lockfile}! Gems in lockfile: #{specs.keys}" + end + results[gem_name] = gem_metadata(spec, lockfile) + spec.dependencies.each do |dep| + if dep.type == :runtime || include_development_deps + to_process << dep.name end - else - current_source = spec.source - add_gemfile_line("source #{spec.source.remotes.first.to_s.inspect}", __LINE__) end - add_gemfile_line("gem #{spec.name.inspect}, #{spec.version.to_s.inspect}#{gem_metadata}", __LINE__) - when Bundler::Source::Git - add_gemfile_line("gem #{spec.name.inspect}, git: #{spec.source.uri.to_s.inspect}, ref: #{spec.source.revision.inspect}#{gem_metadata}", __LINE__) - when Bundler::Source::Path - add_gemfile_line("gem #{spec.name.inspect}, path: #{spec.source.path.to_s.inspect}#{gem_metadata}", __LINE__) - else - raise "Unknown source #{spec.source} for gem #{spec.name}" end end - rescue - puts $! - puts $!.backtrace - raise + + results end private - def add_gemfile_line(line, lineno) - instance_eval(line, __FILE__, lineno) + # + # Get metadata for the given Bundler spec (coming from a lockfile). + # + # @return Hash { version: , git: , path: , source: , ref: } + # + def self.gem_metadata(spec, lockfile) + # Copy source information from included Gemfile + result = {} + case spec.source + when Bundler::Source::Rubygems + result[:source] = spec.source.remotes.first.to_s + result[:version] = spec.version.to_s + when Bundler::Source::Git + result[:git] = spec.source.uri.to_s + result[:ref] = spec.source.revision + when Bundler::Source::Path + # Path is relative to the lockfile (if it's relative at all) + result[:path] = File.expand_path(spec.source.path.to_s, File.dirname(lockfile)) + else + raise "Unknown source #{spec.source} for gem #{spec.name}" + end + result end + end diff --git a/tasks/maintainers.rb b/tasks/maintainers.rb index 535edda248..91742854bb 100644 --- a/tasks/maintainers.rb +++ b/tasks/maintainers.rb @@ -31,9 +31,10 @@ begin require "tomlrb" require "octokit" require "pp" - task :default => :generate namespace :maintainers do + task :default => :generate + desc "Generate MarkDown version of MAINTAINERS file" task :generate do out = "\n\n" -- cgit v1.2.1