Use locked dependencies to build chef

16 files changed, 234 insertions, 48 deletions

diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb
index aec14bfe0e..ce79c5cd1d 100644
--- a/omnibus/config/projects/chef.rb
+++ b/omnibus/config/projects/chef.rb
@@ -36,61 +36,20 @@ else
   install_dir "#{default_root}/#{name}"
 end
 
-override :ruby, version: "2.1.8"
-# Leave dev-kit pinned to 4.5 because 4.7 is 20MB larger and we don't want
-# to unnecessarily make the client any fatter.
-override :'ruby-windows-devkit', version: "4.5.2-20111229-1559" if windows? && windows_arch_i386?
-override :bundler,      version: "1.11.2"
-override :rubygems,     version: "2.5.2"
-
-# Chef Release version pinning
-override :chef, version: "local_source"
-override :ohai, version: "master"
-
 # Global FIPS override flag.
 if windows? || rhel?
   override :fips, enabled: true
 end
 
-dependency "preparation"
-dependency "rb-readline"
-dependency "nokogiri"
-dependency "pry"
-dependency "chef"
-dependency "shebang-cleanup"
-dependency "version-manifest"
-dependency "openssl-customization"
-
-if windows?
-  dependency "ruby-windows-devkit"
-  dependency "ruby-windows-devkit-bash"
-end
-
-# Lower level library pins
-override :xproto,             version: "7.0.28"
-override :"util-macros",      version: "1.19.0"
-override :makedepend,         version: "1.0.5"
-
-## We are currently on the latest of these:
-#override :"ncurses",          version: "5.9"
-#override :"zlib",             version: "1.2.8"
-#override :"pkg-config-lite",  version: "0.28-1"
-#override :"libffi",           version: "3.2.1"
-#override :"libyaml",          version: "0.1.6"
-#override :"libiconv",         version: "1.14"
-#override :"liblzma",          version: "5.2.2"
-#override :"libxml2",          version: "2.9.3"
-#override :"libxslt",          version: "1.1.28"
+# Load dynamically updated overrides
+overrides_path = File.expand_path("../../../../omnibus_overrides.rb", __FILE__)
+instance_eval(IO.read(overrides_path), overrides_path)
 
-## according to comment in omnibus-sw, latest versions don't work on solaris
-# https://github.com/chef/omnibus-software/blob/aefb7e79d29ca746c3f843673ef5e317fa3cba54/config/software/libtool.rb#L23
-#override :"libtool"
-
-## These can float as they are frequently updated in a way that works for us
-#override :"cacerts",                             # probably best to float?
-#override :"openssl"                              # leave this?
+dependency "preparation"
 
-dependency "clean-static-libs"
+# All actual dependencies are in chef-complete, so that the addition
+# or removal of a dependency doesn't dirty the entire project file
+dependency "chef-complete"
 
 package :rpm do
   signing_passphrase ENV["OMNIBUS_RPM_SIGNING_PASSPHRASE"]
diff --git a/omnibus/config/software/chef-appbundle.rb b/omnibus/config/software/chef-appbundle.rb
new file mode 100644
index 0000000000..19228738c3
--- /dev/null
+++ b/omnibus/config/software/chef-appbundle.rb
@@ -0,0 +1,14 @@
+name "chef-appbundle"
+default_version "local_source"
+source path: project.files_path
+
+dependency "chef"
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef-appbundle/build-chef-appbundle"
+  extend BuildChefAppbundle
+
+  appbundle_gem "chef"
+  appbundle_gem "ohai"
+end
diff --git a/omnibus/config/software/chef-complete.rb b/omnibus/config/software/chef-complete.rb
new file mode 100644
index 0000000000..46fc8046e4
--- /dev/null
+++ b/omnibus/config/software/chef-complete.rb
@@ -0,0 +1,20 @@
+name "chef-complete"
+
+license :project_license
+
+dependency "chef"
+dependency "chef-appbundle"
+dependency "chef-remove-docs"
+
+dependency "shebang-cleanup"
+dependency "version-manifest"
+dependency "openssl-customization"
+
+if windows?
+  # TODO can this be safely moved to before the chef?
+  # It would make caching better ...
+  dependency "ruby-windows-devkit"
+  dependency "ruby-windows-devkit-bash"
+end
+
+dependency "clean-static-libs"
diff --git a/omnibus/config/software/chef-gem-binding_of_caller.rb b/omnibus/config/software/chef-gem-binding_of_caller.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-binding_of_caller.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-byebug.rb b/omnibus/config/software/chef-gem-byebug.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-byebug.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-debug_inspector.rb b/omnibus/config/software/chef-gem-debug_inspector.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-debug_inspector.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-ffi-yajl.rb b/omnibus/config/software/chef-gem-ffi-yajl.rb
new file mode 100644
index 0000000000..e8279f7d81
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ffi-yajl.rb
@@ -0,0 +1,8 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
+
+dependency "chef-gem-libyajl2"
diff --git a/omnibus/config/software/chef-gem-ffi.rb b/omnibus/config/software/chef-gem-ffi.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ffi.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-json.rb b/omnibus/config/software/chef-gem-json.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-json.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-libyajl2.rb b/omnibus/config/software/chef-gem-libyajl2.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-libyajl2.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-mini_portile2.rb b/omnibus/config/software/chef-gem-mini_portile2.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-mini_portile2.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-nokogiri.rb b/omnibus/config/software/chef-gem-nokogiri.rb
new file mode 100644
index 0000000000..a18d156e11
--- /dev/null
+++ b/omnibus/config/software/chef-gem-nokogiri.rb
@@ -0,0 +1,8 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
+
+dependency "chef-gem-mini_portile2"
diff --git a/omnibus/config/software/chef-gem-ruby-prof.rb b/omnibus/config/software/chef-gem-ruby-prof.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ruby-prof.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-gem-ruby-shadow.rb b/omnibus/config/software/chef-gem-ruby-shadow.rb
new file mode 100644
index 0000000000..a9f8b758c9
--- /dev/null
+++ b/omnibus/config/software/chef-gem-ruby-shadow.rb
@@ -0,0 +1,6 @@
+# gem installs this gem from the version specified in chef's Gemfile.lock
+# so we can take advantage of omnibus's caching. Just duplicate this file and
+# add the new software def to chef software def if you want to separate
+# another gem's installation.
+require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def"
+BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__)
diff --git a/omnibus/config/software/chef-remove-docs.rb b/omnibus/config/software/chef-remove-docs.rb
new file mode 100644
index 0000000000..2e71e63792
--- /dev/null
+++ b/omnibus/config/software/chef-remove-docs.rb
@@ -0,0 +1,33 @@
+#
+# Copyright 2012-2014 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "chef-remove-docs"
+
+license :project_license
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef/build-chef"
+  extend BuildChef
+
+  delete "#{install_dir}/embedded/docs"
+  delete "#{install_dir}/embedded/share/man"
+  delete "#{install_dir}/embedded/share/doc"
+  delete "#{install_dir}/embedded/share/gtk-doc"
+  delete "#{install_dir}/embedded/ssl/man"
+  delete "#{install_dir}/embedded/man"
+  delete "#{install_dir}/embedded/info"
+end
diff --git a/omnibus/config/software/chef.rb b/omnibus/config/software/chef.rb
new file mode 100644
index 0000000000..02f4162313
--- /dev/null
+++ b/omnibus/config/software/chef.rb
@@ -0,0 +1,90 @@
+name "chef"
+default_version "local_source"
+
+license :project_license
+
+# For the specific super-special version "local_source", build the source from
+# the local git checkout. This is what you'd want to occur by default if you
+# just ran omnibus build locally.
+version("local_source") do
+  source path: File.expand_path("../..", project.files_path),
+         # Since we are using the local repo, we try to not copy any files
+         # that are generated in the process of bundle installing omnibus.
+         # If the install steps are well-behaved, this should not matter
+         # since we only perform bundle and gem installs from the
+         # omnibus cache source directory, but we do this regardless
+         # to maintain consistency between what a local build sees and
+         # what a github based build will see.
+         options: { exclude: [ "omnibus/vendor" ] }
+end
+
+# For any version other than "local_source", fetch from github.
+if version != "local_source"
+  source git: "git://github.com/chef/chef.git"
+end
+
+# For nokogiri
+dependency "libxml2"
+dependency "libxslt"
+dependency "libiconv"
+dependency "liblzma"
+dependency "zlib"
+
+# ruby and bundler and friends
+dependency "ruby"
+dependency "rubygems"
+dependency "bundler"
+
+# Install all the native gems separately
+# Worst offenders first to take best advantage of cache:
+dependency "chef-gem-ffi-yajl"
+dependency "chef-gem-nokogiri"
+dependency "chef-gem-libyajl2"
+dependency "chef-gem-ruby-prof"
+dependency "chef-gem-byebug"
+dependency "chef-gem-debug_inspector"
+dependency "chef-gem-binding_of_caller"
+
+# Now everyone else, in alphabetical order because we don't care THAT much
+Dir.entries(File.dirname(__FILE__)).sort.each do |gem_software|
+  if gem_software =~ /^(chef-gem-.+)\.rb$/
+    dependency $1
+  end
+end
+
+build do
+  # This is where we get the definitions below
+  require_relative "../../files/chef/build-chef"
+  extend BuildChef
+
+  chef_build_env = env.dup
+  chef_build_env["BUNDLE_GEMFILE"] = chef_gemfile
+
+  # Prepare to install: build config, retries, job, frozen=true
+  # TODO Windows install seems to sometimes install already-installed gems such
+  # as gherkin (and fail as a result) if you use jobs: 4.
+  create_bundle_config(chef_gemfile, retries: 4, jobs: 1, frozen: true)
+
+  # Install all the things. Arguments are specified in .bundle/config (see create_bundle_config)
+  block { log.info(log_key) { "" } }
+  bundle "install --verbose", env: chef_build_env
+
+  # For whatever reason, nokogiri software def deletes this (rather small) directory
+  block { log.info(log_key) { "" } }
+  block "Remove mini_portile test dir" do
+    mini_portile = shellout!("#{bundle_bin} show mini_portile").stdout.chomp
+    remove_directory File.join(mini_portile, "test")
+  end
+
+  # Check that it worked
+  block { log.info(log_key) { "" } }
+  bundle "check", env: chef_build_env
+
+  # fix up git-sourced gems
+  properly_reinstall_git_and_path_sourced_gems
+  install_shared_gemfile
+
+  # Check that the final gemfile worked
+  block { log.info(log_key) { "" } }
+  bundle "check", env: env, cwd: File.dirname(shared_gemfile)
+end