Merge pull request #4478 from chef/lcg/rfc-060-gem-metadata

RFC-060 gem metadata MVP

15 files changed, 245 insertions, 8 deletions

diff --git a/.travis.yml b/.travis.yml
index ea58b9b8d2..5504f4aa21 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,7 +68,6 @@ matrix:
     env: "GEMFILE_MOD=\"gem 'foodcritic', github: 'acrmp/foodcritic', branch: 'v5.0.0'\""
     script: bundle exec rake foodcritic_spec
   - rvm: 2.2
-    before_install:
     env: "GEMFILE_MOD=\"gem 'halite', github: 'poise/halite'\""
     script: bundle exec rake halite_spec
   - rvm: 2.2
@@ -78,6 +77,8 @@ matrix:
   - rvm: 2.2
     gemfile: kitchen-tests/Gemfile
     before_install:
+    - gem update --system
+    - gem install bundler
     - echo -n $DO_KEY_CHUNK_{0..30} >> ~/.ssh/id_aws.base64
     - cat ~/.ssh/id_aws.base64 | tr -d ' ' | base64 --decode > ~/.ssh/id_aws.pem
     before_script:
@@ -128,6 +129,7 @@ matrix:
     sudo: required
     dist: trusty
     before_install:
+      - gem update --system
       - gem install bundler
       - sudo apt-get update
       - sudo apt-get -y install squid3 git curl
@@ -145,6 +147,10 @@ matrix:
 
   allow_failures:
     - rvm: rbx
+    - rvm: 2.2
+      env: "GEMFILE_MOD=\"gem 'halite', github: 'poise/halite'\""
+      script: bundle exec rake halite_spec
+
 notifications:
   on_change: true
   on_failure: true
diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index 8161cd9ea7..3a20469397 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -900,6 +900,8 @@ module ChefConfig
     # break Chef community cookbooks and is very highly discouraged.
     default :ruby_encoding, Encoding::UTF_8
 
+    default :rubygems_url, "https://rubygems.org"
+
     # If installed via an omnibus installer, this gives the path to the
     # "embedded" directory which contains all of the software packaged with
     # omnibus. This is used to locate the cacert.pem file on windows.
diff --git a/chef.gemspec b/chef.gemspec
index 0e6f7e0234..2e6f635279 100644
--- a/chef.gemspec
+++ b/chef.gemspec
@@ -45,6 +45,10 @@ Gem::Specification.new do |s|
 
   s.add_dependency "proxifier", "~> 1.0"
 
+  # v1.10 is needed as a runtime dep now for 'bundler/inline'
+  # very deliberately avoiding putting a ceiling on this to avoid depsolver conflicts.
+  s.add_dependency "bundler", ">= 1.10"
+
   s.add_development_dependency "rack"
   s.add_development_dependency "cheffish", ">= 1.1", "< 3.0"
   s.add_development_dependency "github_changelog_generator", "1.11.3"
diff --git a/kitchen-tests/.kitchen.travis.yml b/kitchen-tests/.kitchen.travis.yml
index 3fcbcf6f78..100891bdf5 100644
--- a/kitchen-tests/.kitchen.travis.yml
+++ b/kitchen-tests/.kitchen.travis.yml
@@ -9,6 +9,8 @@ driver:
 
 provisioner:
   name: chef_github
+  chef_omnibus_url: "https://omnitruck.chef.io/current/install.sh"
+  chef_omnibus_install_options: "-n"
   github_owner: "chef"
   github_repo: "chef"
   refname: <%= ENV['TRAVIS_COMMIT'] %>
diff --git a/lib/chef/cookbook/cookbook_collection.rb b/lib/chef/cookbook/cookbook_collection.rb
index 81e7bb92b4..d06b8fd042 100644
--- a/lib/chef/cookbook/cookbook_collection.rb
+++ b/lib/chef/cookbook/cookbook_collection.rb
@@ -1,7 +1,7 @@
 #--
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
-# Copyright:: Copyright 2010-2016, Chef Software, Inc.
+# Copyright:: Copyright 2010-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +18,7 @@
 #
 
 require "chef/mash"
+require "chef/cookbook/gem_installer"
 
 class Chef
   # == Chef::CookbookCollection
@@ -54,5 +55,9 @@ class Chef
         cookbook_version.metadata.validate_ohai_version!
       end
     end
+
+    def install_gems(events)
+      Cookbook::GemInstaller.new(self, events).install
+    end
   end
 end
diff --git a/lib/chef/cookbook/gem_installer.rb b/lib/chef/cookbook/gem_installer.rb
new file mode 100644
index 0000000000..a85868ccfd
--- /dev/null
+++ b/lib/chef/cookbook/gem_installer.rb
@@ -0,0 +1,118 @@
+#--
+# Copyright:: Copyright (c) 2010-2016 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "bundler"
+require "bundler/inline"
+
+class Chef
+  class Cookbook
+    class GemInstaller
+
+      # @return [Chef::EventDispatch::Dispatcher] the client event dispatcher
+      attr_accessor :events
+      # @return [Chef::CookbookCollection] the cookbook collection
+      attr_accessor :cookbook_collection
+
+      def initialize(cookbook_collection, events)
+        @cookbook_collection = cookbook_collection
+        @events = events
+      end
+
+      # Installs the gems into the omnibus gemset.
+      #
+      def install
+        cookbook_gems = []
+
+        cookbook_collection.each do |cookbook_name, cookbook_version|
+          cookbook_gems += cookbook_version.metadata.gems
+        end
+
+        events.cookbook_gem_start(cookbook_gems)
+
+        unless cookbook_gems.empty?
+          begin
+            inline_gemfile do
+              source Chef::Config[:rubygems_url]
+              cookbook_gems.each do |args|
+                gem(*args)
+              end
+            end
+          rescue Exception => e
+            events.cookbook_gem_failed(e)
+            raise
+          end
+        end
+
+        events.cookbook_gem_finished
+      end
+
+      # Bundler::UI object so that we can intercept and log the output
+      # of the in-memory bundle install that we are going to do.
+      #
+      class ChefBundlerUI < Bundler::UI::Silent
+        attr_accessor :events
+
+        def initialize(events)
+          @events = events
+          super()
+        end
+
+        def confirm(msg, newline = nil)
+          # looks like "Installing time_ago_in_words 0.1.1" when installing
+          if msg =~ /Installing\s+(\S+)\s+(\S+)/
+            events.cookbook_gem_installing($1, $2)
+          end
+          Chef::Log.info(msg)
+        end
+
+        def error(msg, newline = nil)
+          Chef::Log.error(msg)
+        end
+
+        def debug(msg, newline = nil)
+          Chef::Log.debug(msg)
+        end
+
+        def info(msg, newline = nil)
+          # looks like "Using time_ago_in_words 0.1.1" when using, plus other misc output
+          if msg =~ /Using\s+(\S+)\s+(\S+)/
+            events.cookbook_gem_using($1, $2)
+          end
+          Chef::Log.info(msg)
+        end
+
+        def warn(msg, newline = nil)
+          Chef::Log.warn(msg)
+        end
+      end
+
+      private
+
+      # Helper to handle older bundler versions that do not support injecting the UI
+      # object.  On older bundler versions, we work, but you get no output other than
+      # on STDOUT.
+      #
+      def inline_gemfile(&block)
+        # requires https://github.com/bundler/bundler/pull/4245
+        gemfile(true, ui: ChefBundlerUI.new(events), &block)
+      rescue ArgumentError # Method#arity doesn't inspect optional arguments, so we rescue
+        # requires bundler 1.10.0
+        gemfile(true, &block)
+      end
+    end
+  end
+end
diff --git a/lib/chef/cookbook/metadata.rb b/lib/chef/cookbook/metadata.rb
index 1cad526b65..603f80748c 100644
--- a/lib/chef/cookbook/metadata.rb
+++ b/lib/chef/cookbook/metadata.rb
@@ -58,12 +58,14 @@ class Chef
       PRIVACY                = "privacy".freeze
       CHEF_VERSIONS          = "chef_versions".freeze
       OHAI_VERSIONS          = "ohai_versions".freeze
+      GEMS                   = "gems".freeze
 
       COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer,
                             :maintainer_email, :license, :platforms, :dependencies,
                             :recommendations, :suggestions, :conflicting, :providing,
                             :replacing, :attributes, :groupings, :recipes, :version,
-                            :source_url, :issues_url, :privacy, :chef_versions, :ohai_versions ]
+                            :source_url, :issues_url, :privacy, :chef_versions, :ohai_versions,
+                            :gems ]
 
       VERSION_CONSTRAINTS = { :depends      => DEPENDENCIES,
                               :recommends   => RECOMMENDATIONS,
@@ -93,6 +95,8 @@ class Chef
       attr_reader :chef_versions
       # @return [Array<Gem::Dependency>] Array of supported Ohai versions
       attr_reader :ohai_versions
+      # @return [Array<Array>] Array of gems to install with *args as an Array
+      attr_reader :gems
 
       # Builds a new Chef::Cookbook::Metadata object.
       #
@@ -130,6 +134,7 @@ class Chef
         @privacy = false
         @chef_versions = []
         @ohai_versions = []
+        @gems = []
 
         @errors = []
       end
@@ -420,6 +425,17 @@ class Chef
         @ohai_versions
       end
 
+      # Metadata DSL to set a gem to install from the cookbook metadata.  May be declared
+      # multiple times.  All the gems from all the cookbooks are combined into one Gemfile
+      # and depsolved together.  Uses Bundler's DSL for its implementation.
+      #
+      # @param args [Array<String>] Gem name and options to pass to Bundler's DSL
+      # @return [Array<Array>] Array of gem statements as args
+      def gem(*args)
+        @gems << args unless args.empty?
+        @gems
+      end
+
       # Adds a description for a recipe.
       #
       # === Parameters
@@ -573,6 +589,7 @@ class Chef
           PRIVACY                => self.privacy,
           CHEF_VERSIONS          => gem_requirements_to_array(*self.chef_versions),
           OHAI_VERSIONS          => gem_requirements_to_array(*self.ohai_versions),
+          GEMS                   => self.gems,
         }
       end
 
@@ -609,6 +626,7 @@ class Chef
         @privacy                      = o[PRIVACY] if o.has_key?(PRIVACY)
         @chef_versions                = gem_requirements_from_array("chef", o[CHEF_VERSIONS]) if o.has_key?(CHEF_VERSIONS)
         @ohai_versions                = gem_requirements_from_array("ohai", o[OHAI_VERSIONS]) if o.has_key?(OHAI_VERSIONS)
+        @gems                         = o[GEMS] if o.has_key?(GEMS)
         self
       end
 
diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index a6a18718c2..b3271a139a 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -139,6 +139,26 @@ class Chef
       def cookbook_sync_complete
       end
 
+      # Called when starting to collect gems from the cookbooks
+      def cookbook_gem_start(gems)
+      end
+
+      # Called when the result of installing the bundle is to install the gem
+      def cookbook_gem_installing(gem, version)
+      end
+
+      # Called when the result of installing the bundle is to use the gem
+      def cookbook_gem_using(gem, version)
+      end
+
+      # Called when finished installing cookbook gems
+      def cookbook_gem_finished
+      end
+
+      # Called when cookbook gem installation fails
+      def cookbook_gem_failed(exception)
+      end
+
       ## TODO: add cookbook name to the API for file load callbacks
 
       ## TODO: add callbacks for overall cookbook eval start and complete.
diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb
index 5462241049..3f832f1e92 100644
--- a/lib/chef/formatters/doc.rb
+++ b/lib/chef/formatters/doc.rb
@@ -180,6 +180,32 @@ class Chef
         unindent
       end
 
+      # Called when starting to collect gems from the cookbooks
+      def cookbook_gem_start(gems)
+        puts_line "Installing Cookbook Gems:"
+        indent
+      end
+
+      # Called when the result of installing the bundle is to install the gem
+      def cookbook_gem_installing(gem, version)
+        puts_line "- Installing #{gem} #{version}", :green
+      end
+
+      # Called when the result of installing the bundle is to use the gem
+      def cookbook_gem_using(gem, version)
+        puts_line "- Using #{gem} #{version}"
+      end
+
+      # Called when finished installing cookbook gems
+      def cookbook_gem_finished
+        unindent
+      end
+
+      # Called when cookbook gem installation fails
+      def cookbook_gem_failed(exception)
+        unindent
+      end
+
       # Called when cookbook loading starts.
       def library_load_start(file_count)
         puts_line "Compiling Cookbooks..."
diff --git a/lib/chef/policy_builder/expand_node_object.rb b/lib/chef/policy_builder/expand_node_object.rb
index 6a006ec992..980de60dd5 100644
--- a/lib/chef/policy_builder/expand_node_object.rb
+++ b/lib/chef/policy_builder/expand_node_object.rb
@@ -3,7 +3,7 @@
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
 # Author:: Daniel DeLeo (<dan@chef.io>)
-# Copyright:: Copyright 2008-2016, Chef Software, Inc.
+# Copyright:: Copyright 2008-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -75,12 +75,16 @@ class Chef
           cl.load_cookbooks
           cookbook_collection = Chef::CookbookCollection.new(cl)
           cookbook_collection.validate!
+          cookbook_collection.install_gems(events)
+
           run_context = Chef::RunContext.new(node, cookbook_collection, @events)
         else
           Chef::Cookbook::FileVendor.fetch_from_remote(api_service)
           cookbook_hash = sync_cookbooks
           cookbook_collection = Chef::CookbookCollection.new(cookbook_hash)
           cookbook_collection.validate!
+          cookbook_collection.install_gems(events)
+
           run_context = Chef::RunContext.new(node, cookbook_collection, @events)
         end
 
diff --git a/lib/chef/policy_builder/policyfile.rb b/lib/chef/policy_builder/policyfile.rb
index 679e3cfe47..8f35c66cab 100644
--- a/lib/chef/policy_builder/policyfile.rb
+++ b/lib/chef/policy_builder/policyfile.rb
@@ -3,7 +3,7 @@
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
 # Author:: Daniel DeLeo (<dan@chef.io>)
-# Copyright:: Copyright 2008-2016, Chef Software, Inc.
+# Copyright:: Copyright 2008-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -153,6 +153,8 @@ class Chef
         sync_cookbooks
         cookbook_collection = Chef::CookbookCollection.new(cookbooks_to_sync)
         cookbook_collection.validate!
+        cookbook_collection.install_gems(events)
+
         run_context = Chef::RunContext.new(node, cookbook_collection, events)
 
         setup_chef_class(run_context)
diff --git a/lib/chef/provider/package/rubygems.rb b/lib/chef/provider/package/rubygems.rb
index 7a2db6b32b..6b01927d50 100644
--- a/lib/chef/provider/package/rubygems.rb
+++ b/lib/chef/provider/package/rubygems.rb
@@ -535,7 +535,7 @@ class Chef
             src = " --clear-sources"
             src << (@new_resource.source && " --source=#{@new_resource.source}" || "")
           else
-            src = @new_resource.source && " --source=#{@new_resource.source} --source=https://rubygems.org"
+            src = @new_resource.source && " --source=#{@new_resource.source} --source=#{Chef::Config[:rubygems_url]}"
           end
           if !version.nil? && version.length > 0
             shell_out_with_timeout!("#{gem_binary_path} install #{name} -q --no-rdoc --no-ri -v \"#{version}\"#{src}#{opts}", :env => nil)
diff --git a/spec/integration/client/client_spec.rb b/spec/integration/client/client_spec.rb
index bfc88659e0..f6e50066bf 100644
--- a/spec/integration/client/client_spec.rb
+++ b/spec/integration/client/client_spec.rb
@@ -47,7 +47,7 @@ describe "chef-client" do
   # cf. CHEF-4914
   let(:chef_client) { "ruby '#{chef_dir}/chef-client' --minimal-ohai" }
 
-  let(:critical_env_vars) { %w{PATH RUBYOPT BUNDLE_GEMFILE GEM_PATH}.map { |o| "#{o}=#{ENV[o]}" } .join(" ") }
+  let(:critical_env_vars) { %w{_ORIGINAL_GEM_PATH GEM_PATH GEM_HOME GEM_ROOT BUNDLE_BIN_PATH BUNDLE_GEMFILE RUBYLIB RUBYOPT RUBY_ENGINE RUBY_ROOT RUBY_VERSION PATH}.map { |o| "#{o}=#{ENV[o]}" } .join(" ") }
 
   when_the_repository "has a cookbook with a no-op recipe" do
     before { file "cookbooks/x/recipes/default.rb", "" }
diff --git a/spec/unit/cookbook/metadata_spec.rb b/spec/unit/cookbook/metadata_spec.rb
index 0107667fcd..65cefa5ed5 100644
--- a/spec/unit/cookbook/metadata_spec.rb
+++ b/spec/unit/cookbook/metadata_spec.rb
@@ -30,7 +30,8 @@ describe Chef::Cookbook::Metadata do
                   :maintainer_email, :license, :platforms, :dependencies,
                   :recommendations, :suggestions, :conflicting, :providing,
                   :replacing, :attributes, :groupings, :recipes, :version,
-                  :source_url, :issues_url, :privacy, :ohai_versions, :chef_versions ]
+                  :source_url, :issues_url, :privacy, :ohai_versions, :chef_versions,
+                  :gems ]
     end
 
     it "does not depend on object identity for equality" do
@@ -428,6 +429,29 @@ describe Chef::Cookbook::Metadata do
     end
   end
 
+  describe "gem" do
+    def expect_gem_works(*args)
+      ret = []
+      args.each do |arg|
+        metadata.send(:gem, *arg)
+        ret << arg
+      end
+      expect(metadata.send(:gems)).to eql(ret)
+    end
+
+    it "works on a simple case" do
+      expect_gem_works(["foo", "~> 1.2"])
+    end
+
+    it "works if there's two gems" do
+      expect_gem_works(["foo", "~> 1.2"], ["bar", "~> 2.0"])
+    end
+
+    it "works if there's a more complicated constraint" do
+      expect_gem_works(["foo", "~> 1.2"], ["bar", ">= 2.4", "< 4.0"])
+    end
+  end
+
   describe "attribute groupings" do
     it "should allow you set a grouping" do
       group = {
@@ -786,6 +810,8 @@ describe Chef::Cookbook::Metadata do
       metadata.attribute "bizspark/has_login",
         :display_name => "You have nothing"
       metadata.version "1.2.3"
+      metadata.gem "foo", "~> 1.2"
+      metadata.gem "bar", ">= 2.2", "< 4.0"
       metadata.chef_version ">= 11.14.2", "< 11.18.10"
       metadata.chef_version ">= 12.2.1", "< 12.5.1"
       metadata.ohai_version ">= 7.1.0", "< 7.5.0"
@@ -825,6 +851,7 @@ describe Chef::Cookbook::Metadata do
         source_url
         issues_url
         privacy
+        gems
       }.each do |t|
         it "should include '#{t}'" do
           expect(deserialized_metadata[t]).to eq(metadata.send(t.to_sym))
@@ -871,6 +898,7 @@ describe Chef::Cookbook::Metadata do
         privacy
         chef_versions
         ohai_versions
+        gems
       }.each do |t|
         it "should match '#{t}'" do
           expect(deserialized_metadata.send(t.to_sym)).to eq(metadata.send(t.to_sym))
diff --git a/spec/unit/policy_builder/policyfile_spec.rb b/spec/unit/policy_builder/policyfile_spec.rb
index 6dab6d14b2..0f345ee344 100644
--- a/spec/unit/policy_builder/policyfile_spec.rb
+++ b/spec/unit/policy_builder/policyfile_spec.rb
@@ -658,6 +658,7 @@ describe Chef::PolicyBuilder::Policyfile do
               expect(cookbook_synchronizer).to receive(:sync_cookbooks)
               expect_any_instance_of(Chef::RunContext).to receive(:load).with(policy_builder.run_list_expansion_ish)
               expect_any_instance_of(Chef::CookbookCollection).to receive(:validate!)
+              expect_any_instance_of(Chef::CookbookCollection).to receive(:install_gems)
               run_context = policy_builder.setup_run_context
               expect(run_context.node).to eq(node)
               expect(run_context.cookbook_collection.keys).to match_array(%w{example1 example2})
@@ -667,6 +668,7 @@ describe Chef::PolicyBuilder::Policyfile do
               expect(cookbook_synchronizer).to receive(:sync_cookbooks)
               expect_any_instance_of(Chef::RunContext).to receive(:load).with(policy_builder.run_list_expansion_ish)
               expect_any_instance_of(Chef::CookbookCollection).to receive(:validate!)
+              expect_any_instance_of(Chef::CookbookCollection).to receive(:install_gems)
               run_context = policy_builder.setup_run_context
               expect(Chef.run_context).to eq(run_context)
             end