Use locked dependencies to build chef

20 files changed, 728 insertions, 48 deletions

diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb
index aec14bfe0e..ce79c5cd1d 100644
--- a/omnibus/config/projects/chef.rb
+++ b/omnibus/config/projects/chef.rb
@@ -36,61 +36,20 @@ else
   install_dir "#{default_root}/#{name}"
 end
 
-override :ruby, version: "2.1.8"
-# Leave dev-kit pinned to 4.5 because 4.7 is 20MB larger and we don't want
-# to unnecessarily make the client any fatter.
-override :'ruby-windows-devkit', version: "4.5.2-20111229-1559" if windows? && windows_arch_i386?
-override :bundler,      version: "1.11.2"
-override :rubygems,     version: "2.5.2"
-
-# Chef Release version pinning
-override :chef, version: "local_source"
-override :ohai, version: "master"
-
 # Global FIPS override flag.
 if windows? || rhel?
   override :fips, enabled: true
 end
 
-dependency "preparation"
-dependency "rb-readline"
-dependency "nokogiri"
-dependency "pry"
-dependency "chef"
-dependency "shebang-cleanup"
-dependency "version-manifest"
-dependency "openssl-customization"
-
-if windows?
-  dependency "ruby-windows-devkit"
-  dependency "ruby-windows-devkit-bash"
-end
-
-# Lower level library pins
-override :xproto,             version: "7.0.28"
-override :"util-macros",      version: "1.19.0"
-override :makedepend,         version: "1.0.5"
-
-## We are currently on the latest of these:
-#override :"ncurses",          version: "5.9"
-#override :"zlib",             version: "1.2.8"
-#override :"pkg-config-lite",  version: "0.28-1"
-#override :"libffi",           version: "3.2.1"
-#override :"libyaml",          version: "0.1.6"
-#override :"libiconv",         version: "1.14"
-#override :"liblzma",          version: "5.2.2"
-#override :"libxml2",          version: "2.9.3"
-#override :"libxslt",          version: "1.1.28"
+# Load dynamically updated overrides
+overrides_path = File.expand_path("../../../../omnibus_overrides.rb", __FILE__)
+instance_eval(IO.read(overrides_path), overrides_path)
 
-## according to comment in omnibus-sw, latest versions don't work on solaris
-# https://github.com/chef/omnibus-software/blob/aefb7e79d29ca746c3f843673ef5e317fa3cba54/config/software/libtool.rb#L23
-#override :"libtool"
-
-## These can float as they are frequently updated in a way that works for us
-#override :"cacerts",                             # probably best to float?
-#override :"openssl"                              # leave this?
+dependency "preparation"
 
-dependency "clean-static-libs"
+# All actual dependencies are in chef-complete, so that the addition
+# or removal of a dependency doesn't dirty the entire project file
+dependency "chef-complete"
 
 package :rpm do
   signing_passphrase ENV["OMNIBUS_RPM_SIGNING_PASSPHRASE"]
diff --git a/omnibus/config/software/chef-appbundle.rb b/omnibus/config/software/chef-appbundle.rb
new file mode 100644
index 0000000000..19228738c3
--- /dev/null
+++ b/omnibus/config/software/chef-appbundle.rb
@@ -0,0 +1,14 @@
+name "chef-appbundle"
+default_version "local_source"
+source path: project.files_path
+
+dependency "chef"
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef-appbundle/build-chef-appbundle"
+  extend BuildChefAppbundle
+
+  appbundle_gem "chef"
+  appbundle_gem "ohai"
+end
diff --git a/omnibus/config/software/chef-complete.rb b/omnibus/config/software/chef-complete.rb
new file mode 100644
index 0000000000..46fc8046e4
--- /dev/null
+++ b/omnibus/config/software/chef-complete.rb
@@ -0,0 +1,20 @@
+name "chef-complete"
+
+license :project_license
+
+dependency "chef"
+dependency "chef-appbundle"
+dependency "chef-remove-docs"
+
+dependency "shebang-cleanup"
+dependency "version-manifest"
+dependency "openssl-customization"
+
+if windows?
+  # TODO can this be safely moved to before the chef?
+  # It would make caching better ...
+  dependency "ruby-windows-devkit"
+  dependency "ruby-windows-devkit-bash"
+end
+
+dependency "clean-static-libs"
diff --git a/omnibus/config/software/chef-gem-binding_of_caller.rb b/omnibus/config/software/chef-gem-binding_of_caller.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-binding_of_caller.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-byebug.rb b/omnibus/config/software/chef-gem-byebug.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-byebug.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-debug_inspector.rb b/omnibus/config/software/chef-gem-debug_inspector.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-debug_inspector.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-ffi-yajl.rb b/omnibus/config/software/chef-gem-ffi-yajl.rb
new file mode 100644
index 0000000000..e8279f7d81
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ffi-yajl.rb
@@ -0,0 +1,8 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
+
+dependency "chef-gem-libyajl2"
diff --git a/omnibus/config/software/chef-gem-ffi.rb b/omnibus/config/software/chef-gem-ffi.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ffi.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-json.rb b/omnibus/config/software/chef-gem-json.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-json.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-libyajl2.rb b/omnibus/config/software/chef-gem-libyajl2.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-libyajl2.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-mini_portile2.rb b/omnibus/config/software/chef-gem-mini_portile2.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-mini_portile2.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-nokogiri.rb b/omnibus/config/software/chef-gem-nokogiri.rb
new file mode 100644
index 0000000000..a18d156e11
--- /dev/null
+++ b/omnibus/config/software/chef-gem-nokogiri.rb
@@ -0,0 +1,8 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
+
+dependency "chef-gem-mini_portile2"
diff --git a/omnibus/config/software/chef-gem-ruby-prof.rb b/omnibus/config/software/chef-gem-ruby-prof.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ruby-prof.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-ruby-shadow.rb b/omnibus/config/software/chef-gem-ruby-shadow.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ruby-shadow.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-remove-docs.rb b/omnibus/config/software/chef-remove-docs.rb
new file mode 100644
index 0000000000..2e71e63792
--- /dev/null
+++ b/omnibus/config/software/chef-remove-docs.rb
@@ -0,0 +1,33 @@
+#
+# Copyright 2012-2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "chef-remove-docs"
+
+license :project_license
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef/build-chef"
+  extend BuildChef
+
+  delete "#{install_dir}/embedded/docs"
+  delete "#{install_dir}/embedded/share/man"
+  delete "#{install_dir}/embedded/share/doc"
+  delete "#{install_dir}/embedded/share/gtk-doc"
+  delete "#{install_dir}/embedded/ssl/man"
+  delete "#{install_dir}/embedded/man"
+  delete "#{install_dir}/embedded/info"
+end
diff --git a/omnibus/config/software/chef.rb b/omnibus/config/software/chef.rb
new file mode 100644
index 0000000000..02f4162313
--- /dev/null
+++ b/omnibus/config/software/chef.rb
@@ -0,0 +1,90 @@
+name "chef"
+default_version "local_source"
+
+license :project_license
+
+# For the specific super-special version "local_source", build the source from
+# the local git checkout. This is what you'd want to occur by default if you
+# just ran omnibus build locally.
+version("local_source") do
+  source path: File.expand_path("../..", project.files_path),
+         # Since we are using the local repo, we try to not copy any files
+         # that are generated in the process of bundle installing omnibus.
+         # If the install steps are well-behaved, this should not matter
+         # since we only perform bundle and gem installs from the
+         # omnibus cache source directory, but we do this regardless
+         # to maintain consistency between what a local build sees and
+         # what a github based build will see.
+         options: { exclude: [ "omnibus/vendor" ] }
+end
+
+# For any version other than "local_source", fetch from github.
+if version != "local_source"
+  source git: "git://github.com/chef/chef.git"
+end
+
+# For nokogiri
+dependency "libxml2"
+dependency "libxslt"
+dependency "libiconv"
+dependency "liblzma"
+dependency "zlib"
+
+# ruby and bundler and friends
+dependency "ruby"
+dependency "rubygems"
+dependency "bundler"
+
+# Install all the native gems separately
+# Worst offenders first to take best advantage of cache:
+dependency "chef-gem-ffi-yajl"
+dependency "chef-gem-nokogiri"
+dependency "chef-gem-libyajl2"
+dependency "chef-gem-ruby-prof"
+dependency "chef-gem-byebug"
+dependency "chef-gem-debug_inspector"
+dependency "chef-gem-binding_of_caller"
+
+# Now everyone else, in alphabetical order because we don't care THAT much
+Dir.entries(File.dirname(__FILE__)).sort.each do |gem_software|
+  if gem_software =~ /^(chef-gem-.+)\.rb$/
+    dependency $1
+  end
+end
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef/build-chef"
+  extend BuildChef
+
+  chef_build_env = env.dup
+  chef_build_env["BUNDLE_GEMFILE"] = chef_gemfile
+
+  # Prepare to install: build config, retries, job, frozen=true
+  # TODO Windows install seems to sometimes install already-installed gems such
+  # as gherkin (and fail as a result) if you use jobs: 4.
+  create_bundle_config(chef_gemfile, retries: 4, jobs: 1, frozen: true)
+
+  # Install all the things. Arguments are specified in .bundle/config (see create_bundle_config)
+  block { log.info(log_key) { "" } }
+  bundle "install --verbose", env: chef_build_env
+
+  # For whatever reason, nokogiri software def deletes this (rather small) directory
+  block { log.info(log_key) { "" } }
+  block "Remove mini_portile test dir" do
+    mini_portile = shellout!("#{bundle_bin} show mini_portile").stdout.chomp
+    remove_directory File.join(mini_portile, "test")
+  end
+
+  # Check that it worked
+  block { log.info(log_key) { "" } }
+  bundle "check", env: chef_build_env
+
+  # fix up git-sourced gems
+  properly_reinstall_git_and_path_sourced_gems
+  install_shared_gemfile
+
+  # Check that the final gemfile worked
+  block { log.info(log_key) { "" } }
+  bundle "check", env: env, cwd: File.dirname(shared_gemfile)
+end
diff --git a/omnibus/files/chef-appbundle/build-chef-appbundle.rb b/omnibus/files/chef-appbundle/build-chef-appbundle.rb
new file mode 100644
index 0000000000..97a94ef5a7
--- /dev/null
+++ b/omnibus/files/chef-appbundle/build-chef-appbundle.rb
@@ -0,0 +1,93 @@
+require_relative "../chef-gem/build-chef-gem"
+
+module BuildChefAppbundle
+  include BuildChefGem
+
+  def lockdown_gem(gem_name)
+    shared_gemfile = self.shared_gemfile
+
+    # Update the Gemfile to restrict to built versions so that bundle installs
+    # will do the right thing
+    block "Lock down the #{gem_name} gem" do
+      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
+      installed_gemfile = File.join(installed_path, "Gemfile")
+
+      #
+      # Include the main distribution Gemfile in the gem's Gemfile
+      #
+      # NOTE: if this fails and the build retries, you will see this multiple
+      # times in the file.
+      #
+      distribution_gemfile = Pathname(shared_gemfile).relative_path_from(Pathname(installed_gemfile)).to_s
+      gemfile_text = IO.read(installed_gemfile)
+      gemfile_text << <<-EOM.gsub(/^\s+/, "")
+        # Lock gems that are part of the distribution
+        distribution_gemfile = File.expand_path(#{distribution_gemfile.inspect}, __FILE__)
+        instance_eval(IO.read(distribution_gemfile), distribution_gemfile)
+      EOM
+      create_file(installed_gemfile) { gemfile_text }
+
+      # Remove the gemfile.lock
+      remove_file("#{installed_gemfile}.lock") if File.exist?("#{installed_gemfile}.lock")
+
+      # If it's frozen, make it not be.
+      shellout!("#{bundle_bin} config --delete frozen")
+
+      # This could be changed to `bundle install` if we wanted to actually
+      # install extra deps out of their gemfile ...
+      shellout!("#{bundle_bin} lock", env: env, cwd: installed_path)
+      # bundle lock doesn't always tell us when it fails, so we have to check :/
+      unless File.exist?("#{installed_gemfile}.lock")
+        raise "bundle lock failed: no #{installed_gemfile}.lock created!"
+      end
+
+      # Ensure all the gems we need are actually installed (if the bundle adds
+      # something, we need to know about it so we can include it in the main
+      # solve).
+      # Save bundle config and modify to use --without development before checking
+      bundle_config = File.expand_path("../.bundle/config", installed_gemfile)
+      orig_config = IO.read(bundle_config) if File.exist?(bundle_config)
+      # "test", "changelog" and "guard" come from berkshelf, "maintenance" comes from chef
+      # "tools" and "integration" come from inspec
+      shellout!("#{bundle_bin} config --local without #{without_groups.join(":")}", env: env, cwd: installed_path)
+      shellout!("#{bundle_bin} config --local frozen 1")
+
+      shellout!("#{bundle_bin} check", env: env, cwd: installed_path)
+
+      # Restore bundle config
+      if orig_config
+        create_file(bundle_config) { orig_config }
+      else
+        remove_file bundle_config
+      end
+    end
+  end
+
+  # appbundle the gem, making /opt/chef/bin/<binary> do the superfast pinning
+  # thing.
+  #
+  # To protect the app from loading the wrong versions of things, it uses
+  # appbundler against the resulting file.
+  #
+  # Relocks the Gemfiles inside the specified gems (e.g. berkshelf, test-kitchen,
+  # chef) to use the chef distribution's chosen gems.
+  def appbundle_gem(gem_name)
+    # First lock the gemfile down.
+    lockdown_gem(gem_name)
+
+    shared_gemfile = self.shared_gemfile
+
+    # Ensure the main bin dir exists
+    bin_dir = File.join(install_dir, "bin")
+    mkdir(bin_dir)
+
+    block "Lock down the #{gem_name} gem" do
+      installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp
+
+      # appbundle the gem
+      appbundler_args = [ installed_path, bin_dir, gem_name ]
+      appbundler_args = appbundler_args.map { |a| ::Shellwords.escape(a) }
+      shellout!("#{appbundler_bin} #{appbundler_args.join(" ")}", env: env, cwd: installed_path)
+    end
+  end
+end
diff --git a/omnibus/files/chef-gem/build-chef-gem.rb b/omnibus/files/chef-gem/build-chef-gem.rb
new file mode 100644
index 0000000000..82c0ba08e2
--- /dev/null
+++ b/omnibus/files/chef-gem/build-chef-gem.rb
@@ -0,0 +1,123 @@
+require "shellwords"
+require "pathname"
+require "bundler"
+require_relative "../../../version_policy"
+
+# Common definitions and helpers (like compile environment and binary
+# locations) for all chef software definitions.
+module BuildChefGem
+  PLATFORM_FAMILY_FAMILIES = {
+    "linux" => %w{wrlinux debian fedora rhel suse gentoo slackware arch exherbo alpine},
+    "bsd" => %w{dragonflybsd freebsd netbsd openbsd},
+    "solaris" => %w{smartos omnios openindiana opensolaris solaris2 nextentacore},
+    "aix" => %w{aix},
+    "windows" => %w{windows},
+    "mac_os_x" => %w{mac_os_x},
+  }
+  def platform_family_families
+    PLATFORM_FAMILY_FAMILIES.keys
+  end
+
+  def platform_family_family
+    PLATFORM_FAMILY_FAMILIES.
+      select { |key, families| families.include?(Omnibus::Ohai["platform_family"]) }.
+      first[0]
+  end
+
+  def embedded_bin(binary)
+    windows_safe_path("#{install_dir}/embedded/bin/#{binary}")
+  end
+
+  def appbundler_bin
+    embedded_bin("appbundler")
+  end
+
+  def bundle_bin
+    embedded_bin("bundle")
+  end
+
+  def gem_bin
+    embedded_bin("gem")
+  end
+
+  def rake_bin
+    embedded_bin("rake")
+  end
+
+  def without_groups
+    # Add --without for every known OS except the one we're in.
+    exclude_os_groups = platform_family_families - [ platform_family_family ]
+    (INSTALL_WITHOUT_GROUPS + exclude_os_groups).map { |g| g.to_sym }
+  end
+
+  #
+  # Get the path to the top level shared Gemfile included by all individual
+  # Gemfiles
+  #
+  def shared_gemfile
+    File.join(install_dir, "Gemfile")
+  end
+
+  # A common env for building everything including nokogiri and dep-selector-libgecode
+  def env
+    env = with_standard_compiler_flags(with_embedded_path, bfd_flags: true)
+
+    # From dep-selector-libgecode
+    # On some RHEL-based systems, the default GCC that's installed is 4.1. We
+    # need to use 4.4, which is provided by the gcc44 and gcc44-c++ packages.
+    # These do not use the gcc binaries so we set the flags to point to the
+    # correct version here.
+    if File.exist?("/usr/bin/gcc44")
+      env["CC"]  = "gcc44"
+      env["CXX"] = "g++44"
+    end
+
+    # From dep-selector-libgecode
+    # Ruby DevKit ships with BSD Tar
+    env["PROG_TAR"] = "bsdtar" if windows?
+    env["ARFLAGS"] = "rv #{env["ARFLAGS"]}" if env["ARFLAGS"]
+
+    # Set up nokogiri environment and args
+    env["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true"
+    env
+  end
+
+  #
+  # Install arguments for various gems (to be passed to `gem install` or set in
+  # `bundle config build.<gemname>`).
+  #
+  def all_install_args
+    @all_install_args = {
+      "nokogiri" => %W{
+        --use-system-libraries
+        --with-xml2-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
+        --with-xml2-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxml2")}
+        --with-xslt-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")}
+        --with-xslt-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxslt")}
+        --with-iconv-dir=#{Shellwords.escape("#{install_dir}/embedded")}
+        --with-zlib-dir=#{Shellwords.escape("#{install_dir}/embedded")}
+      }.join(" "),
+    }
+  end
+
+  # gem install arguments for a particular gem. "" if no special args.
+  def install_args_for(gem_name)
+    all_install_args[gem_name] || ""
+  end
+
+  # Give block all the variables
+  def block(*args, &block)
+    super do
+      extend BuildChefGem
+      instance_eval(&block)
+    end
+  end
+
+  # Give build all the variables
+  def build(*args, &block)
+    super do
+      extend BuildChefGem
+      instance_eval(&block)
+    end
+  end
+end
diff --git a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
new file mode 100644
index 0000000000..5992ae8057
--- /dev/null
+++ b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb
@@ -0,0 +1,132 @@
+require "bundler"
+require "omnibus"
+require_relative "../build-chef-gem"
+
+module BuildChefGem
+  class GemInstallSoftwareDef
+    def self.define(software, software_filename)
+      new(software, software_filename).send(:define)
+    end
+
+    include BuildChefGem
+    include Omnibus::Logging
+
+    protected
+
+    def initialize(software, software_filename)
+      @software = software
+      @software_filename = software_filename
+    end
+
+    attr_reader :software, :software_filename
+
+    def define
+      software.name "#{File.basename(software_filename)[0..-4]}"
+      software.default_version gem_version
+
+      # If the source directory for building stuff changes, tell omnibus to
+      # de-cache us
+      software.source path: File.expand_path("../..", __FILE__)
+
+      # ruby and bundler and friends
+      software.dependency "ruby"
+      software.dependency "rubygems"
+
+      gem_name = self.gem_name
+      gem_version = self.gem_version
+      gemspec = self.gemspec
+      lockfile_path = self.lockfile_path
+
+      software.build do
+        extend BuildChefGem
+
+        if gem_version == "<skip>"
+          if gemspec
+            block do
+              log.info(log_key) { "#{gem_name} has source #{gemspec.source.name} in #{lockfile_path}. We only cache rubygems.org installs in omnibus to keep things simple. The chef step will build #{gem_name} ..." }
+            end
+          else
+            block do
+              log.info(log_key) { "#{gem_name} is not in the #{lockfile_path}. This can happen if your OS doesn't build it, or if chef no longer depends on it. Skipping ..." }
+            end
+          end
+        else
+          block do
+            log.info(log_key) { "Found version #{gem_version} of #{gem_name} in #{lockfile_path}. Building early to take advantage of omnibus caching ..." }
+          end
+          gem "install #{gem_name} -v #{gem_version} --no-doc --no-ri --ignore-dependencies --verbose -- #{install_args_for(gem_name)}", env: env
+        end
+      end
+    end
+
+    # Path above omnibus (where Gemfile is)
+    def root_path
+      File.expand_path("../../../../..", __FILE__)
+    end
+
+    def gemfile_path
+      # gemfile path could be relative to software filename (and often is)
+      @gemfile_path ||= begin
+        # Grab the version (and maybe source) from the lockfile so omnibus knows whether
+        # to toss the cache or not
+        gemfile_path = File.join(root_path, "Gemfile")
+        platform_gemfile_path = "#{gemfile_path}.#{Omnibus::Ohai["platform"]}"
+        if File.exist?(platform_gemfile_path)
+          gemfile_path = platform_gemfile_path
+        end
+        gemfile_path
+      end
+    end
+
+    def lockfile_path
+      @lockfile_path ||= "#{gemfile_path}.lock"
+    end
+
+    def gem_name
+      @gem_name ||= begin
+        # File must be named chef-<gemname>.rb
+        # Will look at chef/Gemfile.lock and install that version of the gem using "gem install"
+        # (and only that version)
+        if File.basename(software_filename) =~ /^chef-gem-(.+)\.rb$/
+          $1
+        else
+          raise "#{software_filename} must be named chef-<gemname>.rb to build a gem automatically"
+        end
+      end
+    end
+
+    def gemspec
+      @gemspec ||= begin
+        old_frozen = Bundler.settings[:frozen]
+        Bundler.settings[:frozen] = true
+        begin
+          bundle = Bundler::Definition.build(gemfile_path, lockfile_path, nil)
+          dependencies = bundle.dependencies.select { |d| (d.groups - without_groups).any? }
+          # This is sacrilege: figure out a way we can grab the list of dependencies *without*
+          # requiring everything to be installed or calling private methods ...
+          gemspec = bundle.resolve.for(bundle.send(:expand_dependencies, dependencies)).find { |s| s.name == gem_name }
+          if gemspec
+            log.info(software.name) { "Using #{gem_name} version #{gemspec.version} from #{gemfile_path}" }
+          elsif bundle.resolve.find { |s| s.name == gem_name }
+            log.info(software.name) { "#{gem_name} not loaded from #{gemfile_path}, skipping" }
+          else
+            raise "#{gem_name} not found in #{gemfile_path} or #{lockfile_path}"
+          end
+          gemspec
+        ensure
+          Bundler.settings[:frozen] = old_frozen
+        end
+      end
+    end
+
+    def gem_version
+      @gem_version ||= begin
+        if gemspec && gemspec.source.name == "rubygems repository https://rubygems.org/"
+          gemspec.version.to_s
+        else
+          "<skip>"
+        end
+      end
+    end
+  end
+end
diff --git a/omnibus/files/chef/build-chef.rb b/omnibus/files/chef/build-chef.rb
new file mode 100644
index 0000000000..d3e68d4e8a
--- /dev/null
+++ b/omnibus/files/chef/build-chef.rb
@@ -0,0 +1,146 @@
+require "shellwords"
+require "pathname"
+require "bundler"
+require_relative "../chef-gem/build-chef-gem"
+require_relative "../../../version_policy"
+
+# We use this to break up the `build` method into readable parts
+module BuildChef
+  include BuildChefGem
+
+  def create_bundle_config(gemfile, without: without_groups, retries: nil, jobs: nil, frozen: nil)
+    if without
+      without = without.dup
+      # no_aix, no_windows groups
+      without << "no_#{Omnibus::Ohai["platform"]}"
+    end
+
+    bundle_config = File.expand_path("../.bundle/config", gemfile)
+
+    block "Put build config into #{bundle_config}: #{ { without: without, retries: retries, jobs: jobs, frozen: frozen } }" do
+      # bundle config build.nokogiri #{nokogiri_build_config} messes up the line,
+      # so we write it directly ourselves.
+      new_bundle_config = "---\n"
+      new_bundle_config << "BUNDLE_WITHOUT: #{Array(without).join(":")}\n" if without
+      new_bundle_config << "BUNDLE_RETRY: #{retries}\n" if retries
+      new_bundle_config << "BUNDLE_JOBS: #{jobs}\n" if jobs
+      new_bundle_config << "BUNDLE_FROZEN: '1'\n" if frozen
+      all_install_args.each do |gem_name, install_args|
+        new_bundle_config << "BUNDLE_BUILD__#{gem_name.upcase}: #{install_args}\n"
+      end
+      create_file(bundle_config) { new_bundle_config }
+    end
+  end
+
+  #
+  # Get the (possibly platform-specific) path to the Gemfile.
+  # /var/omnibus/cache/src/chef/Gemfile or
+  # /var/omnibus/cache/src/chef/Gemfile.windows
+  #
+  def chef_gemfile
+    gemfile = File.join(project_dir, "Gemfile")
+    # Check for platform specific version
+    platform_gemfile = "#{gemfile}.#{Omnibus::Ohai["platform"]}"
+    if File.exist?(platform_gemfile)
+      gemfile = platform_gemfile
+    end
+    gemfile
+  end
+
+  #
+  # Some gems we installed don't end up in the `gem list` due to the fact that
+  # they have git sources (`gem 'chef', github: 'chef/chef'`) or paths (`gemspec`
+  # or `gem 'chef-config', path: 'chef-config'`). To get them in there, we need
+  # to go through these gems, run `rake install` from their top level, and
+  # then delete the git cached versions.
+  #
+  # Once we finish with all this, we update the Gemfile that will end up in the
+  # chef so that it doesn't have git or path references anymore.
+  #
+  def properly_reinstall_git_and_path_sourced_gems
+    # Emit blank line to separate different tasks
+    block { log.info(log_key) { "" } }
+    chef_env = env.dup.merge("BUNDLE_GEMFILE" => chef_gemfile)
+
+    # Reinstall git-sourced or path-sourced gems, and delete the originals
+    block "Reinstall git-sourced gems properly" do
+      # Grab info about the gem environment so we can make decisions
+      gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp
+      gem_install_dir = File.join(gemdir, "gems")
+
+      # bundle list --paths gets us the list of gem install paths. Get the ones
+      # that are installed local (git and path sources like `gem :x, github: 'chef/x'`
+      # or `gem :x, path: '.'` or `gemspec`). To do this, we just detect which ones
+      # have properly-installed paths (in the `gems` directory that shows up when
+      # you run `gem list`).
+      locally_installed_gems = shellout!("#{bundle_bin} list --paths", env: chef_env, cwd: project_dir).
+        stdout.lines.select { |gem_path| !gem_path.start_with?(gem_install_dir) }
+
+      # Install the gems for real using `rake install` in their directories
+      locally_installed_gems.each do |gem_path|
+        gem_path = gem_path.chomp
+        # We use the already-installed bundle to rake install, because (hopefully)
+        # just rake installing doesn't require anything special.
+        # Emit blank line to separate different tasks
+        log.info(log_key) { "" }
+        log.info(log_key) { "Properly installing git or path sourced gem #{gem_path} using rake install" }
+        shellout!("#{bundle_bin} exec #{rake_bin} install", env: chef_env, cwd: gem_path)
+      end
+    end
+  end
+
+  def install_shared_gemfile
+    # Emit blank line to separate different tasks
+    block { log.info(log_key) { "" } }
+
+    shared_gemfile = self.shared_gemfile
+    chef_env = env.dup.merge("BUNDLE_GEMFILE" => chef_gemfile)
+
+    # Show the config for good measure
+    bundle "config", env: chef_env
+
+    # Make `Gemfile` point to these by removing path and git sources and pinning versions.
+    block "Rewrite Gemfile using all properly-installed gems" do
+      gem_pins = ""
+      result = []
+      shellout!("#{bundle_bin} list", env: chef_env).stdout.lines.map do |line|
+        if line =~ /^\s*\*\s*(\S+)\s+\((\S+).*\)\s*$/
+          name, version = $1, $2
+          # rubocop is an exception, since different projects disagree
+          next if GEMS_ALLOWED_TO_FLOAT.include?(name)
+          gem_pins << "override_gem #{name.inspect}, #{version.inspect}\n"
+        end
+      end
+
+      create_file(shared_gemfile) { <<-EOM }
+        # Meant to be included in component Gemfiles at the end with:
+        #
+        #     instance_eval(IO.read("#{install_dir}/Gemfile"), "#{install_dir}/Gemfile")
+        #
+        # Override any existing gems with our own.
+        def override_gem(name, *args, &block)
+          # If the Gemfile re-specifies something in our lockfile, ignore it.
+          current = dependencies.find { |dep| dep.name == name }
+          dependencies.delete(current) if current
+          gem(name, *args, &block)
+        end
+        #{gem_pins}
+      EOM
+    end
+
+    shared_gemfile_env = env.dup.merge("BUNDLE_GEMFILE" => shared_gemfile)
+
+    # Create a `Gemfile.lock` at the final location
+    bundle "lock", env: shared_gemfile_env
+
+    # Freeze the location's Gemfile.lock.
+    create_bundle_config(shared_gemfile, frozen: true)
+
+    # Clear the now-unnecessary git caches, cached gems, and git-checked-out gems
+    block "Delete bundler git cache and git installs" do
+      gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp
+      remove_file "#{gemdir}/cache"
+      remove_file "#{gemdir}/bundler"
+    end
+  end
+end