Pin everything down with Gemfile.lock, add rake dependencies to update

5 files changed, 451 insertions, 0 deletions

diff --git a/tasks/bin/bundle-platform b/tasks/bin/bundle-platform
new file mode 100755
index 0000000000..7c77393cb1
--- /dev/null
+++ b/tasks/bin/bundle-platform
@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+platforms = ARGV.shift
+old_platforms = Gem.platforms
+Gem.platforms = platforms.split(" ").map { |p| Gem::Platform.new(p) }
+puts "bundle-platform set Gem.platforms to #{Gem.platforms.map { |p| p.to_s }} (was #{old_platforms.map { |p| p.to_s } })"
+
+# The rest of this is a normal bundler binstub
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+
+load Gem.bin_path("bundler", "bundle")
diff --git a/tasks/bundle.rb b/tasks/bundle.rb
new file mode 100644
index 0000000000..b0cec5f580
--- /dev/null
+++ b/tasks/bundle.rb
@@ -0,0 +1,80 @@
+#
+# Copyright:: Copyright (c) 2016 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "bundle_util"
+require_relative "../version_policy"
+
+desc "Tasks to work with the main Gemfile and Gemfile.<platform>"
+namespace :bundle do
+  desc "Update Gemfile.lock and all Gemfile.<platform>.locks (or one or more gems via bundle:update gem1 gem2 ...)."
+  task :update, [:args] do |t, rake_args|
+    extend BundleUtil
+    args = rake_args[:args] || ""
+    with_bundle_unfrozen do
+      puts ""
+      puts "-------------------------------------------------------------------"
+      puts "Updating Gemfile.lock ..."
+      puts "-------------------------------------------------------------------"
+      bundle "install #{args}", delete_gemfile_lock: true
+      platforms.each do |platform|
+        puts ""
+        puts "-------------------------------------------------------------------"
+        puts "Updating Gemfile.#{platform}.lock ..."
+        puts "-------------------------------------------------------------------"
+        bundle "lock", gemfile: "Gemfile.#{platform}", platform: platform, delete_gemfile_lock: true
+      end
+    end
+  end
+
+  desc "Conservatively update Gemfile.lock and all Gemfile.<platform>.locks"
+  task :install, [:args] do |t, rake_args|
+    extend BundleUtil
+    args = rake_args[:args] || ""
+    with_bundle_unfrozen do
+      puts ""
+      puts "-------------------------------------------------------------------"
+      puts "Updating Gemfile.lock ..."
+      puts "-------------------------------------------------------------------"
+      bundle "install #{args}"
+      platforms.each do |platform|
+        puts ""
+        puts "-------------------------------------------------------------------"
+        puts "Updating Gemfile.#{platform}.lock (conservatively) ..."
+        puts "-------------------------------------------------------------------"
+        bundle "lock", gemfile: "Gemfile.#{platform}", platform: platform
+      end
+    end
+  end
+end
+
+desc "Run bundle with arbitrary args against the given platform; e.g. rake bundle[show]. No platform to run against the main bundle; bundle[show,windows] to run the windows one; bundle[show,*] to run against all non-default platforms."
+task :bundle, [:args, :platform] do |t, rake_args|
+  extend BundleUtil
+  args = rake_args[:args] || ""
+  platform = rake_args[:platform]
+  with_bundle_unfrozen do
+    if platform == "*"
+      platforms.each do |platform|
+        bundle args, platform: platform
+      end
+    elsif platform
+      bundle args, platform: platform
+    else
+      bundle args
+    end
+  end
+end
diff --git a/tasks/bundle_util.rb b/tasks/bundle_util.rb
new file mode 100644
index 0000000000..91ffa1f317
--- /dev/null
+++ b/tasks/bundle_util.rb
@@ -0,0 +1,93 @@
+require "shellwords"
+
+module BundleUtil
+  PLATFORMS = { "windows" => %w{ruby x86-mingw32} }
+
+  def project_root
+    File.expand_path("../..", __FILE__)
+  end
+
+  def bundle_platform
+    File.join(project_root, "tasks", "bin", "bundle-platform")
+  end
+
+  # Parse the output of "bundle outdated" and get the list of gems that
+  # were outdated
+  def parse_bundle_outdated(bundle_outdated_output)
+    result = []
+    bundle_outdated_output.each_line do |line|
+      if line =~ /^\s*\* (.+) \(newest ([^,]+), installed ([^,)])*/
+        gem_name, newest_version, installed_version = $1, $2, $3
+        result << [ line, gem_name ]
+      end
+    end
+    result
+  end
+
+  def with_bundle_unfrozen
+    bundle "config --delete frozen"
+    begin
+      yield
+    ensure
+      bundle "config --local frozen 1"
+    end
+  end
+
+  # Run bundle-platform with the given ruby platform(s)
+  def bundle(args, gemfile: nil, platform: nil, cwd: nil, extract_output: false, delete_gemfile_lock: false)
+    args = args.split(/\s+/)
+    if cwd
+      prefix = "[#{cwd}] "
+    end
+    cwd = File.expand_path(cwd || ".", project_root)
+    Bundler.with_clean_env do
+      Dir.chdir(cwd) do
+        gemfile ||= "Gemfile"
+        gemfile = File.expand_path(gemfile, cwd)
+        raise "No platform #{platform} (supported: #{PLATFORMS.keys.join(", ")})" if platform && !PLATFORMS[platform]
+
+        # First delete the gemfile.lock
+        if delete_gemfile_lock
+          if File.exist?("#{gemfile}.lock")
+            puts "Deleting #{gemfile}.lock ..."
+            File.delete("#{gemfile}.lock")
+          end
+        end
+
+        # Run the bundle command
+        ruby_platforms = platform ? PLATFORMS[platform].join(" ") : "ruby"
+        cmd = Shellwords.join([bundle_platform, ruby_platforms, *args])
+        puts "#{prefix}#{Shellwords.join(["bundle", *args])}#{platform ? " for #{platform} platform" : ""}:"
+        with_gemfile(gemfile) do
+          puts "#{prefix}BUNDLE_GEMFILE=#{gemfile}"
+          puts "#{prefix}> #{cmd}"
+          if extract_output
+            `#{cmd}`
+          else
+            unless system(bundle_platform, ruby_platforms, *args)
+              raise "#{bundle_platform} failed: exit code #{$?}"
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def with_gemfile(gemfile)
+    old_gemfile = ENV["BUNDLE_GEMFILE"]
+    ENV["BUNDLE_GEMFILE"] = gemfile
+    begin
+      yield
+    ensure
+      if old_gemfile
+        ENV["BUNDLE_GEMFILE"] = old_gemfile
+      else
+        ENV.delete("BUNDLE_GEMFILE")
+      end
+    end
+  end
+
+  def platforms
+    PLATFORMS.keys
+  end
+end
diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb
new file mode 100644
index 0000000000..7955686963
--- /dev/null
+++ b/tasks/dependencies.rb
@@ -0,0 +1,164 @@
+#
+# Copyright:: Copyright (c) 2016 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "bundle_util"
+require_relative "bundle"
+require_relative "../version_policy"
+
+desc "Tasks to update and check dependencies"
+namespace :dependencies do
+  # Update all dependencies to the latest constraint-matching version
+  desc "Update all dependencies. dependencies:update[conservative] to update as little as possible."
+  task :update, [:conservative] => %w{
+                    dependencies:update_gemfile_lock
+                    dependencies:update_omnibus_overrides
+                    dependencies:update_omnibus_gemfile_lock
+                    dependencies:update_acceptance_gemfile_lock
+                    dependencies:update_kitchen_tests_gemfile_lock
+                    dependencies:update_omnibus_berksfile_lock
+                    dependencies:update_kitchen_tests_berksfile_lock
+                  }
+
+  desc "Update Gemfile.lock and all Gemfile.<platform>.locks. update_gemfile_lock[conservative] to update as little as possible."
+  task :update_gemfile_lock, [:conservative] do |t, rake_args|
+    conservative = rake_args[:conservative]
+    if conservative
+      Rake::Task["bundle:install"].invoke
+    else
+      Rake::Task["bundle:update"].invoke
+    end
+  end
+
+  def gemfile_lock_task(task_name, dirs: [])
+    dirs.each do |dir|
+      desc "Update #{dir}/Gemfile.lock. #{task_name}[conservative] to update as little as possible."
+      task task_name, [:conservative] do |t, rake_args|
+        extend BundleUtil
+        conservative = rake_args[:conservative]
+        puts ""
+        puts "-------------------------------------------------------------------"
+        puts "Updating #{dir}/Gemfile.lock#{conservative ? " (conservatively)" : ""} ..."
+        puts "-------------------------------------------------------------------"
+        bundle "install", cwd: dir, delete_gemfile_lock: !conservative
+      end
+    end
+  end
+
+  def berksfile_lock_task(task_name, dirs: [])
+    dirs.each do |dir|
+      desc "Update #{dir}/Berksfile.lock. #{task_name}[conservative] to update as little as possible."
+      task task_name, [:conservative] do |t, rake_args|
+        extend BundleUtil
+        conservative = rake_args[:conservative]
+        puts ""
+        puts "-------------------------------------------------------------------"
+        puts "Updating #{dir}/Berksfile.lock#{conservative ? " (conservatively)" : ""} ..."
+        puts "-------------------------------------------------------------------"
+        if !conservative && File.exist?("#{project_root}/#{dir}/Berksfile.lock")
+          File.delete("#{project_root}/#{dir}/Berksfile.lock")
+        end
+        Dir.chdir("#{project_root}/#{dir}") do
+          Bundler.with_clean_env do
+            sh "bundle exec berks install"
+          end
+        end
+      end
+    end
+  end
+
+  gemfile_lock_task :update_omnibus_gemfile_lock, dirs: %w{omnibus}
+  berksfile_lock_task :update_omnibus_berksfile_lock, dirs: %w{omnibus}
+  gemfile_lock_task :update_acceptance_gemfile_lock, dirs: %w{
+    acceptance
+    acceptance/fips/test/integration/fips/serverspec
+  }
+  gemfile_lock_task :update_kitchen_tests_gemfile_lock, dirs: %w{
+    kitchen-tests
+    kitchen-tests/test/integration/webapp/serverspec
+  }
+  berksfile_lock_task :update_kitchen_tests_berksfile_lock, dirs: %w{
+    kitchen-tests
+    kitchen-tests/cookbooks/audit_test
+  }
+  # kitchen-tests/cookbooks/webapp isn't solving right now ....
+
+  desc "Update omnibus overrides, including versions in version_policy.rb and latest version of gems: #{OMNIBUS_RUBYGEMS_AT_LATEST_VERSION.keys}. update_omnibus_overrides[conservative] does nothing."
+  task :update_omnibus_overrides, [:conservative] do |t, rake_args|
+    conservative = rake_args[:conservative]
+    unless conservative
+      puts ""
+      puts "-------------------------------------------------------------------"
+      puts "Updating omnibus_overrides.rb ..."
+      puts "-------------------------------------------------------------------"
+
+      # Generate the new overrides file
+      overrides = "# Generated by \"rake dependencies\". Do not edit.\n"
+
+      # Replace the bundler and rubygems versions
+      OMNIBUS_RUBYGEMS_AT_LATEST_VERSION.each do |override_name, gem_name|
+        # Get the latest bundler version
+        puts "Running gem list -re #{gem_name} ..."
+        gem_list = `gem list -re #{gem_name}`
+        unless gem_list =~ /^#{gem_name}\s*\(([^)]*)\)$/
+          raise "gem list -re #{gem_name} failed with output:\n#{gem_list}"
+        end
+
+        # Emit it
+        puts "Latest version of #{gem_name} is #{$1}"
+        overrides << "override #{override_name.inspect}, version: #{$1.inspect}\n"
+      end
+
+      # Add explicit overrides
+      OMNIBUS_OVERRIDES.each do |override_name, version|
+        overrides << "override #{override_name.inspect}, version: #{version.inspect}\n"
+      end
+
+      # Write the file out (if changed)
+      overrides_path = File.expand_path("../../omnibus_overrides.rb", __FILE__)
+      if overrides != IO.read(overrides_path)
+        puts "Overrides changed!"
+        puts `git diff #{overrides_path}`
+        puts "Writing modified #{overrides_path} ..."
+        IO.write(overrides_path, overrides)
+      end
+    end
+  end
+
+  # Find out if we're using the latest gems we can (so we don't regress versions)
+  desc "Check for gems that are not at the latest released version, and report if anything not in ACCEPTABLE_OUTDATED_GEMS (version_policy.rb) is out of date."
+  task :check_outdated do
+    puts ""
+    puts "-------------------------------------------------------------------"
+    puts "Checking for outdated gems ..."
+    puts "-------------------------------------------------------------------"
+    # TODO check for outdated windows gems too
+    bundle_outdated = bundle("outdated", extract_output: true)
+    puts bundle_outdated
+    outdated_gems = parse_bundle_outdated(bundle_outdated).map { |line, gem_name| gem_name }
+    # Weed out the acceptable ones
+    outdated_gems = outdated_gems.reject { |gem_name| ACCEPTABLE_OUTDATED_GEMS.include?(gem_name) }
+    if outdated_gems.empty?
+      puts ""
+      puts "SUCCESS!"
+    else
+      raise "ERROR: outdated gems: #{outdated_gems.join(", ")}. Either fix them or add them to ACCEPTABLE_OUTDATED_GEMS in #{__FILE__}."
+    end
+  end
+end
+desc "Update all dependencies and check for outdated gems. Call dependencies[conservative] to update as little as possible."
+task :dependencies, [:conservative] => [ "dependencies:update", "dependencies:check_outdated" ]
+task :update, [:conservative] => [ "dependencies:update", "dependencies:check_outdated"]
diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb
new file mode 100644
index 0000000000..60d1e2ff31
--- /dev/null
+++ b/tasks/gemfile_util.rb
@@ -0,0 +1,99 @@
+require "bundler"
+
+module GemfileUtil
+  #
+  # Given a set of dependencies with groups in them, and a resolved set of
+  # gemspecs (with dependency info in them), creates a full set of specs
+  # with group information on it. If A is in groups x and y, and A depends on
+  # B and C, then B and C are also in groups x and y.
+  #
+  class GemGroups < Hash
+    def initialize(resolved)
+      @resolved = resolved
+    end
+    attr_reader :resolved
+
+    def add_dependency(dep)
+      add_gem_groups(dep.name, dep.groups)
+    end
+
+    private
+
+    def add_gem_groups(name, groups)
+      self[name] ||= []
+      difference = groups - self[name]
+      unless difference.empty?
+        self[name] += difference
+        spec = resolved.find { |spec| spec.name == name }
+        if spec
+          spec.dependencies.each do |spec|
+            add_gem_groups(spec.name, difference)
+          end
+        end
+      end
+    end
+  end
+
+  def calculate_dependents(spec_set)
+    dependents = {}
+    spec_set.each do |spec|
+      dependents[spec] ||= []
+    end
+    spec_set.each do |spec|
+      spec.dependencies.each do |dep|
+        puts "#{dep.class} -> #{spec.class}"
+        dependents[dep] << spec
+      end
+    end
+    dependents
+  end
+
+  def include_locked_gemfile(gemfile)
+    #
+    # Read the gemfile and inject its locks as first-class dependencies
+    #
+    current_source = nil
+    bundle = Bundler::Definition.build(gemfile, "#{gemfile}.lock", nil)
+
+    # Go through and create the actual gemfile from the given locks and
+    # groups.
+    bundle.resolve.sort_by { |spec| spec.name }.each do |spec|
+      # bundler can't be installed by bundler so don't pin it.
+      next if spec.name == "bundler"
+      dep = bundle.dependencies.find { |d| d.name == spec.name }
+      gem_metadata = ""
+      if dep
+        gem_metadata << ", groups: #{dep.groups.inspect}" if dep.groups != [:default]
+        gem_metadata << ", platforms: #{dep.platforms.inspect}" if dep.platforms && !dep.platforms.empty?
+      end
+      case spec.source
+      when Bundler::Source::Rubygems
+        if current_source
+          if current_source != spec.source
+            raise "Gem #{spec.name} has source #{spec.source}, but other gems have #{current_source}. Multiple rubygems sources are not supported."
+          end
+        else
+          current_source = spec.source
+          add_gemfile_line("source #{spec.source.remotes.first.to_s.inspect}", __LINE__)
+        end
+        add_gemfile_line("gem #{spec.name.inspect}, #{spec.version.to_s.inspect}#{gem_metadata}", __LINE__)
+      when Bundler::Source::Git
+        add_gemfile_line("gem #{spec.name.inspect}, git: #{spec.source.uri.to_s.inspect}, ref: #{spec.source.revision.inspect}#{gem_metadata}", __LINE__)
+      when Bundler::Source::Path
+        add_gemfile_line("gem #{spec.name.inspect}, path: #{spec.source.path.to_s.inspect}#{gem_metadata}", __LINE__)
+      else
+        raise "Unknown source #{spec.source} for gem #{spec.name}"
+      end
+    end
+  rescue
+    puts $!
+    puts $!.backtrace
+    raise
+  end
+
+  private
+
+  def add_gemfile_line(line, lineno)
+    instance_eval(line, __FILE__, lineno)
+  end
+end