Use locked dependencies to build chef

4 files changed, 494 insertions, 0 deletions

diff --git a/omnibus/files/chef-appbundle/build-chef-appbundle.rb b/omnibus/files/chef-appbundle/build-chef-appbundle.rb
new file mode 100644
index 0000000000..97a94ef5a7
--- /dev/null
+++ b/omnibus/files/chef-appbundle/build-chef-appbundle.rb
@@ -0,0 +1,93 @@
+require_relative "../chef-gem/build-chef-gem"
+
+module BuildChefAppbundle
+  include BuildChefGem
+
+  def lockdown_gem(gem_name)
+    shared_gemfile = self.shared_gemfile
+
+    # Update the Gemfile to restrict to built versions so that bundle installs
+    # will do the right thing
+    block "Lock down the #{gem_name} gem" do
+      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
+      installed_gemfile = File.join(installed_path, "Gemfile")
+
+      #
+      # Include the main distribution Gemfile in the gem's Gemfile
+      #
+      # NOTE: if this fails and the build retries, you will see this multiple
+      # times in the file.
+      #
+      distribution_gemfile = Pathname(shared_gemfile).relative_path_from(Pathname(installed_gemfile)).to_s
+      gemfile_text = IO.read(installed_gemfile)
+      gemfile_text << <<-EOM.gsub(/^\s+/, "")
+        # Lock gems that are part of the distribution
+        distribution_gemfile = File.expand_path(#{distribution_gemfile.inspect}, __FILE__)
+        instance_eval(IO.read(distribution_gemfile), distribution_gemfile)
+      EOM
+      create_file(installed_gemfile) { gemfile_text }
+
+      # Remove the gemfile.lock
+      remove_file("#{installed_gemfile}.lock") if File.exist?("#{installed_gemfile}.lock")
+
+      # If it's frozen, make it not be.
+      shellout!("#{bundle_bin} config --delete frozen")
+
+      # This could be changed to `bundle install` if we wanted to actually
+      # install extra deps out of their gemfile ...
+      shellout!("#{bundle_bin} lock", env: env, cwd: installed_path)
+      # bundle lock doesn't always tell us when it fails, so we have to check :/
+      unless File.exist?("#{installed_gemfile}.lock")
+        raise "bundle lock failed: no #{installed_gemfile}.lock created!"
+      end
+
+      # Ensure all the gems we need are actually installed (if the bundle adds
+      # something, we need to know about it so we can include it in the main
+      # solve).
+      # Save bundle config and modify to use --without development before checking
+      bundle_config = File.expand_path("../.bundle/config", installed_gemfile)
+      orig_config = IO.read(bundle_config) if File.exist?(bundle_config)
+      # "test", "changelog" and "guard" come from berkshelf, "maintenance" comes from chef
+      # "tools" and "integration" come from inspec
+      shellout!("#{bundle_bin} config --local without #{without_groups.join(":")}", env: env, cwd: installed_path)
+      shellout!("#{bundle_bin} config --local frozen 1")
+
+      shellout!("#{bundle_bin} check", env: env, cwd: installed_path)
+
+      # Restore bundle config
+      if orig_config
+        create_file(bundle_config) { orig_config }
+      else
+        remove_file bundle_config
+      end
+    end
+  end
+
+  # appbundle the gem, making /opt/chef/bin/<binary> do the superfast pinning
+  # thing.
+  #
+  # To protect the app from loading the wrong versions of things, it uses
+  # appbundler against the resulting file.
+  #
+  # Relocks the Gemfiles inside the specified gems (e.g. berkshelf, test-kitchen,
+  # chef) to use the chef distribution's chosen gems.
+  def appbundle_gem(gem_name)
+    # First lock the gemfile down.
+    lockdown_gem(gem_name)
+
+    shared_gemfile = self.shared_gemfile
+
+    # Ensure the main bin dir exists
+    bin_dir = File.join(install_dir, "bin")
+    mkdir(bin_dir)
+
+    block "Lock down the #{gem_name} gem" do
+      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
+
+      # appbundle the gem
+      appbundler_args = [ installed_path, bin_dir, gem_name ]
+      appbundler_args = appbundler_args.map { |a| ::Shellwords.escape(a) }
+      shellout!("#{appbundler_bin} #{appbundler_args.join(" ")}", env: env, cwd: installed_path)
+    end
+  end
+end
diff --git a/omnibus/files/chef-gem/build-chef-gem.rb b/omnibus/files/chef-gem/build-chef-gem.rb
new file mode 100644
index 0000000000..82c0ba08e2
--- /dev/null
+++ b/omnibus/files/chef-gem/build-chef-gem.rb
@@ -0,0 +1,123 @@
+require "shellwords"
+require "pathname"
+require "bundler"
+require_relative "../../../version_policy"
+
+# Common definitions and helpers (like compile environment and binary
+# locations) for all chef software definitions.
+module BuildChefGem
+  PLATFORM_FAMILY_FAMILIES = {
+    "linux" => %w{wrlinux debian fedora rhel suse gentoo slackware arch exherbo alpine},
+    "bsd" => %w{dragonflybsd freebsd netbsd openbsd},
+    "solaris" => %w{smartos omnios openindiana opensolaris solaris2 nextentacore},
+    "aix" => %w{aix},
+    "windows" => %w{windows},
+    "mac_os_x" => %w{mac_os_x},
+  }
+  def platform_family_families
+    PLATFORM_FAMILY_FAMILIES.keys
+  end
+
+  def platform_family_family
+    PLATFORM_FAMILY_FAMILIES.
+      select { |key, families| families.include?(Omnibus::Ohai["platform_family"]) }.
+      first[0]
+  end
+
+  def embedded_bin(binary)
+    windows_safe_path("#{install_dir}/embedded/bin/#{binary}")
+  end
+
+  def appbundler_bin
+    embedded_bin("appbundler")
+  end
+
+  def bundle_bin
+    embedded_bin("bundle")
+  end
+
+  def gem_bin
+    embedded_bin("gem")
+  end
+
+  def rake_bin
+    embedded_bin("rake")
+  end
+
+  def without_groups
+    # Add --without for every known OS except the one we're in.
+    exclude_os_groups = platform_family_families - [ platform_family_family ]
+    (INSTALL_WITHOUT_GROUPS + exclude_os_groups).map { |g| g.to_sym }
+  end
+
+  #
+  # Get the path to the top level shared Gemfile included by all individual
+  # Gemfiles
+  #
+  def shared_gemfile
+    File.join(install_dir, "Gemfile")
+  end
+
+  # A common env for building everything including nokogiri and dep-selector-libgecode
+  def env
+    env = with_standard_compiler_flags(with_embedded_path, bfd_flags: true)
+
+    # From dep-selector-libgecode
+    # On some RHEL-based systems, the default GCC that's installed is 4.1. We
+    # need to use 4.4, which is provided by the gcc44 and gcc44-c++ packages.
+    # These do not use the gcc binaries so we set the flags to point to the
+    # correct version here.
+    if File.exist?("/usr/bin/gcc44")
+      env["CC"]  = "gcc44"
+      env["CXX"] = "g++44"
+    end
+
+    # From dep-selector-libgecode
+    # Ruby DevKit ships with BSD Tar
+    env["PROG_TAR"] = "bsdtar" if windows?
+    env["ARFLAGS"] = "rv #{env["ARFLAGS"]}" if env["ARFLAGS"]
+
+    # Set up nokogiri environment and args
+    env["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true"
+    env
+  end
+
+  #
+  # Install arguments for various gems (to be passed to `gem install` or set in
+  # `bundle config build.<gemname>`).
+  #
+  def all_install_args
+    @all_install_args = {
+      "nokogiri" => %W{
+        --use-system-libraries
+        --with-xml2-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
+        --with-xml2-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxml2")}
+        --with-xslt-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
+        --with-xslt-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxslt")}
+        --with-iconv-dir=#{Shellwords.escape("#{install_dir}/embedded")}
+        --with-zlib-dir=#{Shellwords.escape("#{install_dir}/embedded")}
+      }.join(" "),
+    }
+  end
+
+  # gem install arguments for a particular gem. "" if no special args.
+  def install_args_for(gem_name)
+    all_install_args[gem_name] || ""
+  end
+
+  # Give block all the variables
+  def block(*args, &block)
+    super do
+      extend BuildChefGem
+      instance_eval(&block)
+    end
+  end
+
+  # Give build all the variables
+  def build(*args, &block)
+    super do
+      extend BuildChefGem
+      instance_eval(&block)
+    end
+  end
+end
diff --git a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
new file mode 100644
index 0000000000..5992ae8057
--- /dev/null
+++ b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
@@ -0,0 +1,132 @@
+require "bundler"
+require "omnibus"
+require_relative "../build-chef-gem"
+
+module BuildChefGem
+  class GemInstallSoftwareDef
+    def self.define(software, software_filename)
+      new(software, software_filename).send(:define)
+    end
+
+    include BuildChefGem
+    include Omnibus::Logging
+
+    protected
+
+    def initialize(software, software_filename)
+      @software = software
+      @software_filename = software_filename
+    end
+
+    attr_reader :software, :software_filename
+
+    def define
+      software.name "#{File.basename(software_filename)[0..-4]}"
+      software.default_version gem_version
+
+      # If the source directory for building stuff changes, tell omnibus to
+      # de-cache us
+      software.source path: File.expand_path("../..", __FILE__)
+
+      # ruby and bundler and friends
+      software.dependency "ruby"
+      software.dependency "rubygems"
+
+      gem_name = self.gem_name
+      gem_version = self.gem_version
+      gemspec = self.gemspec
+      lockfile_path = self.lockfile_path
+
+      software.build do
+        extend BuildChefGem
+
+        if gem_version == "<skip>"
+          if gemspec
+            block do
+              log.info(log_key) { "#{gem_name} has source #{gemspec.source.name} in #{lockfile_path}. We only cache rubygems.org installs in omnibus to keep things simple. The chef step will build #{gem_name} ..." }
+            end
+          else
+            block do
+              log.info(log_key) { "#{gem_name} is not in the #{lockfile_path}. This can happen if your OS doesn't build it, or if chef no longer depends on it. Skipping ..." }
+            end
+          end
+        else
+          block do
+            log.info(log_key) { "Found version #{gem_version} of #{gem_name} in #{lockfile_path}. Building early to take advantage of omnibus caching ..." }
+          end
+          gem "install #{gem_name} -v #{gem_version} --no-doc --no-ri --ignore-dependencies --verbose -- #{install_args_for(gem_name)}", env: env
+        end
+      end
+    end
+
+    # Path above omnibus (where Gemfile is)
+    def root_path
+      File.expand_path("../../../../..", __FILE__)
+    end
+
+    def gemfile_path
+      # gemfile path could be relative to software filename (and often is)
+      @gemfile_path ||= begin
+        # Grab the version (and maybe source) from the lockfile so omnibus knows whether
+        # to toss the cache or not
+        gemfile_path = File.join(root_path, "Gemfile")
+        platform_gemfile_path = "#{gemfile_path}.#{Omnibus::Ohai["platform"]}"
+        if File.exist?(platform_gemfile_path)
+          gemfile_path = platform_gemfile_path
+        end
+        gemfile_path
+      end
+    end
+
+    def lockfile_path
+      @lockfile_path ||= "#{gemfile_path}.lock"
+    end
+
+    def gem_name
+      @gem_name ||= begin
+        # File must be named chef-<gemname>.rb
+        # Will look at chef/Gemfile.lock and install that version of the gem using "gem install"
+        # (and only that version)
+        if File.basename(software_filename) =~ /^chef-gem-(.+)\.rb$/
+          $1
+        else
+          raise "#{software_filename} must be named chef-<gemname>.rb to build a gem automatically"
+        end
+      end
+    end
+
+    def gemspec
+      @gemspec ||= begin
+        old_frozen = Bundler.settings[:frozen]
+        Bundler.settings[:frozen] = true
+        begin
+          bundle = Bundler::Definition.build(gemfile_path, lockfile_path, nil)
+          dependencies = bundle.dependencies.select { |d| (d.groups - without_groups).any? }
+          # This is sacrilege: figure out a way we can grab the list of dependencies *without*
+          # requiring everything to be installed or calling private methods ...
+          gemspec = bundle.resolve.for(bundle.send(:expand_dependencies, dependencies)).find { |s| s.name == gem_name }
+          if gemspec
+            log.info(software.name) { "Using #{gem_name} version #{gemspec.version} from #{gemfile_path}" }
+          elsif bundle.resolve.find { |s| s.name == gem_name }
+            log.info(software.name) { "#{gem_name} not loaded from #{gemfile_path}, skipping" }
+          else
+            raise "#{gem_name} not found in #{gemfile_path} or #{lockfile_path}"
+          end
+          gemspec
+        ensure
+          Bundler.settings[:frozen] = old_frozen
+        end
+      end
+    end
+
+    def gem_version
+      @gem_version ||= begin
+        if gemspec && gemspec.source.name == "rubygems repository https://rubygems.org/"
+          gemspec.version.to_s
+        else
+          "<skip>"
+        end
+      end
+    end
+  end
+end
diff --git a/omnibus/files/chef/build-chef.rb b/omnibus/files/chef/build-chef.rb
new file mode 100644
index 0000000000..d3e68d4e8a
--- /dev/null
+++ b/omnibus/files/chef/build-chef.rb
@@ -0,0 +1,146 @@
+require "shellwords"
+require "pathname"
+require "bundler"
+require_relative "../chef-gem/build-chef-gem"
+require_relative "../../../version_policy"
+
+# We use this to break up the `build` method into readable parts
+module BuildChef
+  include BuildChefGem
+
+  def create_bundle_config(gemfile, without: without_groups, retries: nil, jobs: nil, frozen: nil)
+    if without
+      without = without.dup
+      # no_aix, no_windows groups
+      without << "no_#{Omnibus::Ohai["platform"]}"
+    end
+
+    bundle_config = File.expand_path("../.bundle/config", gemfile)
+
+    block "Put build config into #{bundle_config}: #{ { without: without, retries: retries, jobs: jobs, frozen: frozen } }" do
+      # bundle config build.nokogiri #{nokogiri_build_config} messes up the line,
+      # so we write it directly ourselves.
+      new_bundle_config = "---\n"
+      new_bundle_config << "BUNDLE_WITHOUT: #{Array(without).join(":")}\n" if without
+      new_bundle_config << "BUNDLE_RETRY: #{retries}\n" if retries
+      new_bundle_config << "BUNDLE_JOBS: #{jobs}\n" if jobs
+      new_bundle_config << "BUNDLE_FROZEN: '1'\n" if frozen
+      all_install_args.each do |gem_name, install_args|
+        new_bundle_config << "BUNDLE_BUILD__#{gem_name.upcase}: #{install_args}\n"
+      end
+      create_file(bundle_config) { new_bundle_config }
+    end
+  end
+
+  #
+  # Get the (possibly platform-specific) path to the Gemfile.
+  # /var/omnibus/cache/src/chef/Gemfile or
+  # /var/omnibus/cache/src/chef/Gemfile.windows
+  #
+  def chef_gemfile
+    gemfile = File.join(project_dir, "Gemfile")
+    # Check for platform specific version
+    platform_gemfile = "#{gemfile}.#{Omnibus::Ohai["platform"]}"
+    if File.exist?(platform_gemfile)
+      gemfile = platform_gemfile
+    end
+    gemfile
+  end
+
+  #
+  # Some gems we installed don't end up in the `gem list` due to the fact that
+  # they have git sources (`gem 'chef', github: 'chef/chef'`) or paths (`gemspec`
+  # or `gem 'chef-config', path: 'chef-config'`). To get them in there, we need
+  # to go through these gems, run `rake install` from their top level, and
+  # then delete the git cached versions.
+  #
+  # Once we finish with all this, we update the Gemfile that will end up in the
+  # chef so that it doesn't have git or path references anymore.
+  #
+  def properly_reinstall_git_and_path_sourced_gems
+    # Emit blank line to separate different tasks
+    block { log.info(log_key) { "" } }
+    chef_env = env.dup.merge("BUNDLE_GEMFILE" => chef_gemfile)
+
+    # Reinstall git-sourced or path-sourced gems, and delete the originals
+    block "Reinstall git-sourced gems properly" do
+      # Grab info about the gem environment so we can make decisions
+      gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp
+      gem_install_dir = File.join(gemdir, "gems")
+
+      # bundle list --paths gets us the list of gem install paths. Get the ones
+      # that are installed local (git and path sources like `gem :x, github: 'chef/x'`
+      # or `gem :x, path: '.'` or `gemspec`). To do this, we just detect which ones
+      # have properly-installed paths (in the `gems` directory that shows up when
+      # you run `gem list`).
+      locally_installed_gems = shellout!("#{bundle_bin} list --paths", env: chef_env, cwd: project_dir).
+        stdout.lines.select { |gem_path| !gem_path.start_with?(gem_install_dir) }
+
+      # Install the gems for real using `rake install` in their directories
+      locally_installed_gems.each do |gem_path|
+        gem_path = gem_path.chomp
+        # We use the already-installed bundle to rake install, because (hopefully)
+        # just rake installing doesn't require anything special.
+        # Emit blank line to separate different tasks
+        log.info(log_key) { "" }
+        log.info(log_key) { "Properly installing git or path sourced gem #{gem_path} using rake install" }
+        shellout!("#{bundle_bin} exec #{rake_bin} install", env: chef_env, cwd: gem_path)
+      end
+    end
+  end
+
+  def install_shared_gemfile
+    # Emit blank line to separate different tasks
+    block { log.info(log_key) { "" } }
+
+    shared_gemfile = self.shared_gemfile
+    chef_env = env.dup.merge("BUNDLE_GEMFILE" => chef_gemfile)
+
+    # Show the config for good measure
+    bundle "config", env: chef_env
+
+    # Make `Gemfile` point to these by removing path and git sources and pinning versions.
+    block "Rewrite Gemfile using all properly-installed gems" do
+      gem_pins = ""
+      result = []
+      shellout!("#{bundle_bin} list", env: chef_env).stdout.lines.map do |line|
+        if line =~ /^\s*\*\s*(\S+)\s+\((\S+).*\)\s*$/
+          name, version = $1, $2
+          # rubocop is an exception, since different projects disagree
+          next if GEMS_ALLOWED_TO_FLOAT.include?(name)
+          gem_pins << "override_gem #{name.inspect}, #{version.inspect}\n"
+        end
+      end
+
+      create_file(shared_gemfile) { <<-EOM }
+        # Meant to be included in component Gemfiles at the end with:
+        #
+        #     instance_eval(IO.read("#{install_dir}/Gemfile"), "#{install_dir}/Gemfile")
+        #
+        # Override any existing gems with our own.
+        def override_gem(name, *args, &block)
+          # If the Gemfile re-specifies something in our lockfile, ignore it.
+          current = dependencies.find { |dep| dep.name == name }
+          dependencies.delete(current) if current
+          gem(name, *args, &block)
+        end
+        #{gem_pins}
+      EOM
+    end
+
+    shared_gemfile_env = env.dup.merge("BUNDLE_GEMFILE" => shared_gemfile)
+
+    # Create a `Gemfile.lock` at the final location
+    bundle "lock", env: shared_gemfile_env
+
+    # Freeze the location's Gemfile.lock.
+    create_bundle_config(shared_gemfile, frozen: true)
+
+    # Clear the now-unnecessary git caches, cached gems, and git-checked-out gems
+    block "Delete bundler git cache and git installs" do
+      gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp
+      remove_file "#{gemdir}/cache"
+      remove_file "#{gemdir}/bundler"
+    end
+  end
+end