simplify omnibus config and greenify builds again

this is also necessary for bundler-1.14.x

i'm still not entirely clear why we ever needed all the fussy software gem
configs or what the build-chef / build-chef-gem infrastructure ever
did for us.  it seems to have been mostly micro-optimization around
building the software gems before bundle installing the project in order
to take advantage of git caching.  i aggressively don't care about that,
this is quite fast enough.  we can install nokogiri and libgecode early
and that should take care of 98% of the build optimization issue.

Signed-off-by: Lamont Granquist <lamont@scriptkiddie.org>

4 files changed, 0 insertions, 503 deletions

diff --git a/omnibus/files/chef-appbundle/build-chef-appbundle.rb b/omnibus/files/chef-appbundle/build-chef-appbundle.rb
deleted file mode 100644
index eaf4904501..0000000000
--- a/omnibus/files/chef-appbundle/build-chef-appbundle.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-require_relative "../chef-gem/build-chef-gem"
-
-module BuildChefAppbundle
-  include BuildChefGem
-
-  def lockdown_gem(gem_name)
-    shared_gemfile = self.shared_gemfile
-
-    # Update the Gemfile to restrict to built versions so that bundle installs
-    # will do the right thing
-    block "Lock down the #{gem_name} gem" do
-      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
-      installed_gemfile = File.join(installed_path, "Gemfile")
-
-      #
-      # Include the main distribution Gemfile in the gem's Gemfile
-      #
-      # NOTE: if this fails and the build retries, you will see this multiple
-      # times in the file.
-      #
-      distribution_gemfile = Pathname(shared_gemfile).relative_path_from(Pathname(installed_gemfile)).to_s
-      gemfile_text = <<-EOM.gsub(/^\s+/, "")
-        # Lock gems that are part of the distribution
-        distribution_gemfile = File.expand_path(#{distribution_gemfile.inspect}, __FILE__)
-        instance_eval(IO.read(distribution_gemfile), distribution_gemfile)
-      EOM
-      gemfile_text << IO.read(installed_gemfile)
-      create_file(installed_gemfile) { gemfile_text }
-
-      # Remove the gemfile.lock
-      remove_file("#{installed_gemfile}.lock") if File.exist?("#{installed_gemfile}.lock")
-
-      # If it's frozen, make it not be.
-      shellout!("#{bundle_bin} config --delete frozen")
-
-      # This could be changed to `bundle install` if we wanted to actually
-      # install extra deps out of their gemfile ...
-      shellout!("#{bundle_bin} lock", env: env, cwd: installed_path)
-      # bundle lock doesn't always tell us when it fails, so we have to check :/
-      unless File.exist?("#{installed_gemfile}.lock")
-        raise "bundle lock failed: no #{installed_gemfile}.lock created!"
-      end
-
-      # Ensure all the gems we need are actually installed (if the bundle adds
-      # something, we need to know about it so we can include it in the main
-      # solve).
-      # Save bundle config and modify to use --without development before checking
-      bundle_config = File.expand_path("../.bundle/config", installed_gemfile)
-      orig_config = IO.read(bundle_config) if File.exist?(bundle_config)
-      # "test", "changelog" and "guard" come from berkshelf, "maintenance" comes from chef
-      # "tools" and "integration" come from inspec
-      shellout!("#{bundle_bin} config --local without #{without_groups.join(":")}", env: env, cwd: installed_path)
-      shellout!("#{bundle_bin} config --local frozen 1")
-
-      shellout!("#{bundle_bin} check", env: env, cwd: installed_path)
-
-      # Restore bundle config
-      if orig_config
-        create_file(bundle_config) { orig_config }
-      else
-        remove_file bundle_config
-      end
-    end
-  end
-
-  # appbundle the gem, making /opt/chef/bin/<binary> do the superfast pinning
-  # thing.
-  #
-  # To protect the app from loading the wrong versions of things, it uses
-  # appbundler against the resulting file.
-  #
-  # Relocks the Gemfiles inside the specified gems (e.g. berkshelf, test-kitchen,
-  # chef) to use the distribution's chosen gems.
-  def appbundle_gem(gem_name)
-    # First lock the gemfile down.
-    lockdown_gem(gem_name)
-
-    shared_gemfile = self.shared_gemfile
-
-    # Ensure the main bin dir exists
-    bin_dir = File.join(install_dir, "bin")
-    mkdir(bin_dir)
-
-    block "Lock down the #{gem_name} gem" do
-      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
-
-      # appbundle the gem
-      appbundler_args = [ installed_path, bin_dir, gem_name ]
-      appbundler_args = appbundler_args.map { |a| ::Shellwords.escape(a) }
-      shellout!("#{appbundler_bin} #{appbundler_args.join(" ")}", env: env, cwd: installed_path)
-    end
-  end
-end
diff --git a/omnibus/files/chef-gem/build-chef-gem.rb b/omnibus/files/chef-gem/build-chef-gem.rb
deleted file mode 100644
index c9aaaada1d..0000000000
--- a/omnibus/files/chef-gem/build-chef-gem.rb
+++ /dev/null
@@ -1,128 +0,0 @@
-require "shellwords"
-require "pathname"
-require "bundler"
-require_relative "../../../version_policy"
-
-# Common definitions and helpers (like compile environment and binary
-# locations) for all software definitions.
-module BuildChefGem
-  PLATFORM_FAMILY_FAMILIES = {
-    "linux" => %w{wrlinux debian fedora rhel suse gentoo slackware arch exherbo alpine},
-    "bsd" => %w{dragonflybsd freebsd netbsd openbsd},
-    "solaris" => %w{smartos omnios openindiana opensolaris solaris2 nextentacore},
-    "aix" => %w{aix},
-    "windows" => %w{windows},
-    "mac_os_x" => %w{mac_os_x},
-  }
-  def platform_family_families
-    PLATFORM_FAMILY_FAMILIES.keys
-  end
-
-  def platform_family_family
-    PLATFORM_FAMILY_FAMILIES.
-      select { |key, families| families.include?(Omnibus::Ohai["platform_family"]) }.
-      first[0]
-  end
-
-  def embedded_bin(binary)
-    windows_safe_path("#{install_dir}/embedded/bin/#{binary}")
-  end
-
-  def appbundler_bin
-    embedded_bin("appbundler")
-  end
-
-  def bundle_bin
-    embedded_bin("bundle")
-  end
-
-  def gem_bin
-    embedded_bin("gem")
-  end
-
-  def rake_bin
-    embedded_bin("rake")
-  end
-
-  def without_groups
-    # Add --without for every known OS except the one we're in.
-    exclude_os_groups = platform_family_families - [ platform_family_family ]
-    (INSTALL_WITHOUT_GROUPS + exclude_os_groups).map { |g| g.to_sym }
-  end
-
-  #
-  # Get the path to the top level shared Gemfile included by all individual
-  # Gemfiles
-  #
-  def shared_gemfile
-    File.join(install_dir, "Gemfile")
-  end
-
-  # A common env for building everything including nokogiri and dep-selector-libgecode
-  def env
-    env = with_standard_compiler_flags(with_embedded_path, bfd_flags: true)
-
-    # From dep-selector-libgecode
-    # On some RHEL-based systems, the default GCC that's installed is 4.1. We
-    # need to use 4.4, which is provided by the gcc44 and gcc44-c++ packages.
-    # These do not use the gcc binaries so we set the flags to point to the
-    # correct version here.
-    if File.exist?("/usr/bin/gcc44")
-      env["CC"]  = "gcc44"
-      env["CXX"] = "g++44"
-    end
-
-    if solaris_11?
-      env["CFLAGS"] << " -std=c99"
-      env["CPPFLAGS"] << " -D_XOPEN_SOURCE=600 -D_XPG6"
-    end
-
-    # From dep-selector-libgecode
-    # Ruby DevKit ships with BSD Tar
-    env["PROG_TAR"] = "bsdtar" if windows?
-    env["ARFLAGS"] = "rv #{env["ARFLAGS"]}" if env["ARFLAGS"]
-
-    # Set up nokogiri environment and args
-    env["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true"
-    env
-  end
-
-  #
-  # Install arguments for various gems (to be passed to `gem install` or set in
-  # `bundle config build.<gemname>`).
-  #
-  def all_install_args
-    @all_install_args = {
-      "nokogiri" => %W{
-        --use-system-libraries
-        --with-xml2-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
-        --with-xml2-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxml2")}
-        --with-xslt-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
-        --with-xslt-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxslt")}
-        --without-iconv-dir
-        --with-zlib-dir=#{Shellwords.escape("#{install_dir}/embedded")}
-      }.join(" "),
-    }
-  end
-
-  # gem install arguments for a particular gem. "" if no special args.
-  def install_args_for(gem_name)
-    all_install_args[gem_name] || ""
-  end
-
-  # Give block all the variables
-  def block(*args, &block)
-    super do
-      extend BuildChefGem
-      instance_eval(&block)
-    end
-  end
-
-  # Give build all the variables
-  def build(*args, &block)
-    super do
-      extend BuildChefGem
-      instance_eval(&block)
-    end
-  end
-end
diff --git a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
deleted file mode 100644
index ea6c32e94a..0000000000
--- a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
+++ /dev/null
@@ -1,155 +0,0 @@
-require "bundler"
-require "omnibus"
-require_relative "../build-chef-gem"
-require_relative "../../../../tasks/gemfile_util"
-
-module BuildChefGem
-  class GemInstallSoftwareDef
-    def self.define(software, software_filename)
-      new(software, software_filename).send(:define)
-    end
-
-    include BuildChefGem
-    include Omnibus::Logging
-
-    protected
-
-    def initialize(software, software_filename)
-      @software = software
-      @software_filename = software_filename
-    end
-
-    attr_reader :software, :software_filename
-
-    # XXX: why are we programmatically defining config that is already expressed as code-as-configuration?
-
-    def define
-      # this has to come first because gem_metadata depends on it
-      software.name "#{File.basename(software_filename)[0..-4]}"
-      if installing_from_git?
-        define_git
-      else
-        define_gem
-      end
-    end
-
-    def define_git
-      software.default_version gem_metadata[:ref]
-
-      # If the source directory for building stuff changes, tell omnibus to de-cache us
-      software.source git: gem_metadata[:git]
-
-      # ruby and bundler and friends
-      software.dependency "ruby"
-      software.dependency "rubygems"
-
-      software.relative_path gem_name
-
-      gem_name = self.gem_name
-
-      software.build do
-        extend BuildChefGem
-        gem "build #{gem_name}.gemspec", env: env
-        gem "install #{gem_name}*.gem --no-ri --no-rdoc", env: env
-      end
-    end
-
-    def define_gem
-      software.default_version gem_version
-
-      # If the source directory for building stuff changes, tell omnibus to
-      # de-cache us
-      software.source path: File.expand_path("../..", __FILE__)
-
-      # ruby and bundler and friends
-      software.dependency "ruby"
-      software.dependency "rubygems"
-
-      gem_name = self.gem_name
-      gem_version = self.gem_version
-      gem_metadata = self.gem_metadata
-      lockfile_path = self.lockfile_path
-
-      software.build do
-        extend BuildChefGem
-
-        if gem_version == "<skip>"
-          if gem_metadata
-            block do
-              raise "can we just remove this use case?  what is it for?"
-              #log.info(log_key) { "#{gem_name} has source #{gem_metadata} in #{lockfile_path}. We only cache rubygems.org installs in omnibus to keep things simple. The chef step will build #{gem_name} ..." }
-            end
-          else
-            block do
-              log.info(log_key) { "#{gem_name} is not in the #{lockfile_path}. This can happen if your OS doesn't build it, or if chef no longer depends on it. Skipping ..." }
-            end
-          end
-        else
-          block do
-            log.info(log_key) { "Found version #{gem_version} of #{gem_name} in #{lockfile_path}. Building early to take advantage of omnibus caching ..." }
-          end
-          gem "install #{gem_name} -v #{gem_version} --no-doc --no-ri --ignore-dependencies --verbose -- #{install_args_for(gem_name)}", env: env
-        end
-      end
-    end
-
-    # Path above omnibus (where Gemfile is)
-    def root_path
-      File.expand_path("../../../../..", __FILE__)
-    end
-
-    def gemfile_path
-      File.join(root_path, "Gemfile")
-    end
-
-    def lockfile_path
-      "#{gemfile_path}.lock"
-    end
-
-    def gem_name
-      @gem_name ||= begin
-        # File must be named chef-<gemname>.rb
-        # Will look at chef/Gemfile.lock and install that version of the gem using "gem install"
-        # (and only that version)
-        if File.basename(software_filename) =~ /^chef-gem-(.+)\.rb$/
-          $1
-        else
-          raise "#{software_filename} must be named chef-<gemname>.rb to build a gem automatically"
-        end
-      end
-    end
-
-    def gem_metadata
-      @gem_metadata ||= begin
-        bundle = GemfileUtil::Bundle.parse(gemfile_path, lockfile_path)
-        result = bundle.gems[gem_name]
-        if result
-          if bundle.select_gems(without_groups: without_groups).include?(gem_name)
-            log.info(software.name) { "Using #{gem_name} version #{result[:version]} from #{gemfile_path}" }
-            result
-          else
-            log.info(software.name) { "#{gem_name} not loaded from #{gemfile_path} because it was only in groups #{without_groups.join(", ")}. Skipping ..." }
-            nil
-          end
-        else
-          log.info(software.name) { "#{gem_name} was not found in #{lockfile_path}. Skipping ..." }
-          nil
-        end
-      end
-    end
-
-    def installing_from_git?
-      gem_metadata && gem_metadata[:git] && gem_metadata[:ref]
-    end
-
-    def gem_version
-      @gem_version ||= begin
-        if gem_metadata && URI(gem_metadata[:source]) == URI("https://rubygems.org/")
-          gem_metadata[:version]
-        else
-          "<skip>"
-        end
-      end
-    end
-  end
-end
diff --git a/omnibus/files/chef/build-chef.rb b/omnibus/files/chef/build-chef.rb
deleted file mode 100644
index 4b8ec78054..0000000000
--- a/omnibus/files/chef/build-chef.rb
+++ /dev/null
@@ -1,127 +0,0 @@
-require "shellwords"
-require "pathname"
-require "bundler"
-require_relative "../chef-gem/build-chef-gem"
-require_relative "../../../version_policy"
-
-# We use this to break up the `build` method into readable parts
-module BuildChef
-  include BuildChefGem
-
-  def create_bundle_config(gemfile, without: without_groups, retries: nil, jobs: nil, frozen: nil)
-    bundle_config = File.expand_path("../.bundle/config", gemfile)
-
-    block "Put build config into #{bundle_config}: #{{ without: without, retries: retries, jobs: jobs, frozen: frozen }}" do
-      # bundle config build.nokogiri #{nokogiri_build_config} messes up the line,
-      # so we write it directly ourselves.
-      new_bundle_config = "---\n"
-      new_bundle_config << "BUNDLE_WITHOUT: #{Array(without).join(":")}\n" if without
-      new_bundle_config << "BUNDLE_RETRY: #{retries}\n" if retries
-      new_bundle_config << "BUNDLE_JOBS: #{jobs}\n" if jobs
-      new_bundle_config << "BUNDLE_FROZEN: '1'\n" if frozen
-      all_install_args.each do |gem_name, install_args|
-        new_bundle_config << "BUNDLE_BUILD__#{gem_name.upcase}: #{install_args}\n"
-      end
-      create_file(bundle_config) { new_bundle_config }
-    end
-  end
-
-  #
-  # Get the (possibly platform-specific) path to the Gemfile.
-  #
-  def project_gemfile
-    File.join(project_dir, "Gemfile")
-  end
-
-  #
-  # Some gems we installed don't end up in the `gem list` due to the fact that
-  # they have git sources (`gem 'chef', github: 'chef/chef'`) or paths (`gemspec`
-  # or `gem 'chef-config', path: 'chef-config'`). To get them in there, we need
-  # to go through these gems, run `rake install` from their top level, and
-  # then delete the git cached versions.
-  #
-  # Once we finish with all this, we update the Gemfile that will end up in the
-  # top-level install so that it doesn't have git or path references anymore.
-  #
-  def properly_reinstall_git_and_path_sourced_gems
-    # Emit blank line to separate different tasks
-    block { log.info(log_key) { "" } }
-    project_env = env.dup.merge("BUNDLE_GEMFILE" => project_gemfile)
-
-    # Reinstall git-sourced or path-sourced gems, and delete the originals
-    block "Reinstall git-sourced gems properly" do
-      # Grab info about the gem environment so we can make decisions
-      gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp
-      gem_install_dir = File.join(gemdir, "gems")
-
-      # bundle list --paths gets us the list of gem install paths. Get the ones
-      # that are installed local (git and path sources like `gem :x, github: 'chef/x'`
-      # or `gem :x, path: '.'` or `gemspec`). To do this, we just detect which ones
-      # have properly-installed paths (in the `gems` directory that shows up when
-      # you run `gem list`).
-      locally_installed_gems = shellout!("#{bundle_bin} list --paths", env: project_env, cwd: project_dir).
-        stdout.lines.select { |gem_path| !gem_path.start_with?(gem_install_dir) }
-
-      # Install the gems for real using `rake install` in their directories
-      locally_installed_gems.each do |gem_path|
-        gem_path = gem_path.chomp
-        # We use the already-installed bundle to rake install, because (hopefully)
-        # just rake installing doesn't require anything special.
-        # Emit blank line to separate different tasks
-        log.info(log_key) { "" }
-        log.info(log_key) { "Properly installing git or path sourced gem #{gem_path} using rake install" }
-        shellout!("#{bundle_bin} exec #{rake_bin} install", env: project_env, cwd: gem_path)
-      end
-    end
-  end
-
-  def install_shared_gemfile
-    # Emit blank line to separate different tasks
-    block { log.info(log_key) { "" } }
-
-    shared_gemfile = self.shared_gemfile
-    project_env = env.dup.merge("BUNDLE_GEMFILE" => project_gemfile)
-
-    # Show the config for good measure
-    bundle "config", env: project_env
-
-    # Make `Gemfile` point to these by removing path and git sources and pinning versions.
-    block "Rewrite Gemfile using all properly-installed gems" do
-      gem_pins = ""
-      result = []
-      shellout!("#{bundle_bin} list", env: project_env).stdout.lines.map do |line|
-        if line =~ /^\s*\*\s*(\S+)\s+\((\S+).*\)\s*$/
-          name, version = $1, $2
-          # rubocop is an exception, since different projects disagree
-          next if GEMS_ALLOWED_TO_FLOAT.include?(name)
-          gem_pins << "gem #{name.inspect}, #{version.inspect}, override: true\n"
-        end
-      end
-
-      # Find the installed chef gem by looking for lib/chef.rb
-      chef_gem = File.expand_path("../..", shellout!("#{gem_bin} which chef", env: project_env).stdout.chomp)
-      # Figure out the path to gemfile_util from there
-      gemfile_util = Pathname.new(File.join(chef_gem, "tasks", "gemfile_util"))
-      gemfile_util = gemfile_util.relative_path_from(Pathname.new(shared_gemfile).dirname)
-
-      create_file(shared_gemfile) { <<-EOM }
-        # Meant to be included in component Gemfiles at the beginning with:
-        #
-        #     instance_eval(IO.read("#{install_dir}/Gemfile"), "#{install_dir}/Gemfile")
-        #
-        # Override any existing gems with our own.
-        require_relative "#{gemfile_util}"
-        extend GemfileUtil
-        #{gem_pins}
-      EOM
-    end
-
-    shared_gemfile_env = env.dup.merge("BUNDLE_GEMFILE" => shared_gemfile)
-
-    # Create a `Gemfile.lock` at the final location
-    bundle "lock", env: shared_gemfile_env
-
-    # Freeze the location's Gemfile.lock.
-    create_bundle_config(shared_gemfile, frozen: true)
-  end
-end