Merge pull request #4478 from chef/lcg/rfc-060-gem-metadata

RFC-060 gem metadata MVP

8 files changed, 198 insertions, 5 deletions

diff --git a/lib/chef/cookbook/cookbook_collection.rb b/lib/chef/cookbook/cookbook_collection.rb
index 81e7bb92b4..d06b8fd042 100644
--- a/lib/chef/cookbook/cookbook_collection.rb
+++ b/lib/chef/cookbook/cookbook_collection.rb
@@ -1,7 +1,7 @@
 #--
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
-# Copyright:: Copyright 2010-2016, Chef Software, Inc.
+# Copyright:: Copyright 2010-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +18,7 @@
 #
 
 require "chef/mash"
+require "chef/cookbook/gem_installer"
 
 class Chef
   # == Chef::CookbookCollection
@@ -54,5 +55,9 @@ class Chef
         cookbook_version.metadata.validate_ohai_version!
       end
     end
+
+    def install_gems(events)
+      Cookbook::GemInstaller.new(self, events).install
+    end
   end
 end
diff --git a/lib/chef/cookbook/gem_installer.rb b/lib/chef/cookbook/gem_installer.rb
new file mode 100644
index 0000000000..a85868ccfd
--- /dev/null
+++ b/lib/chef/cookbook/gem_installer.rb
@@ -0,0 +1,118 @@
+#--
+# Copyright:: Copyright (c) 2010-2016 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "bundler"
+require "bundler/inline"
+
+class Chef
+  class Cookbook
+    class GemInstaller
+
+      # @return [Chef::EventDispatch::Dispatcher] the client event dispatcher
+      attr_accessor :events
+      # @return [Chef::CookbookCollection] the cookbook collection
+      attr_accessor :cookbook_collection
+
+      def initialize(cookbook_collection, events)
+        @cookbook_collection = cookbook_collection
+        @events = events
+      end
+
+      # Installs the gems into the omnibus gemset.
+      #
+      def install
+        cookbook_gems = []
+
+        cookbook_collection.each do |cookbook_name, cookbook_version|
+          cookbook_gems += cookbook_version.metadata.gems
+        end
+
+        events.cookbook_gem_start(cookbook_gems)
+
+        unless cookbook_gems.empty?
+          begin
+            inline_gemfile do
+              source Chef::Config[:rubygems_url]
+              cookbook_gems.each do |args|
+                gem(*args)
+              end
+            end
+          rescue Exception => e
+            events.cookbook_gem_failed(e)
+            raise
+          end
+        end
+
+        events.cookbook_gem_finished
+      end
+
+      # Bundler::UI object so that we can intercept and log the output
+      # of the in-memory bundle install that we are going to do.
+      #
+      class ChefBundlerUI < Bundler::UI::Silent
+        attr_accessor :events
+
+        def initialize(events)
+          @events = events
+          super()
+        end
+
+        def confirm(msg, newline = nil)
+          # looks like "Installing time_ago_in_words 0.1.1" when installing
+          if msg =~ /Installing\s+(\S+)\s+(\S+)/
+            events.cookbook_gem_installing($1, $2)
+          end
+          Chef::Log.info(msg)
+        end
+
+        def error(msg, newline = nil)
+          Chef::Log.error(msg)
+        end
+
+        def debug(msg, newline = nil)
+          Chef::Log.debug(msg)
+        end
+
+        def info(msg, newline = nil)
+          # looks like "Using time_ago_in_words 0.1.1" when using, plus other misc output
+          if msg =~ /Using\s+(\S+)\s+(\S+)/
+            events.cookbook_gem_using($1, $2)
+          end
+          Chef::Log.info(msg)
+        end
+
+        def warn(msg, newline = nil)
+          Chef::Log.warn(msg)
+        end
+      end
+
+      private
+
+      # Helper to handle older bundler versions that do not support injecting the UI
+      # object.  On older bundler versions, we work, but you get no output other than
+      # on STDOUT.
+      #
+      def inline_gemfile(&block)
+        # requires https://github.com/bundler/bundler/pull/4245
+        gemfile(true, ui: ChefBundlerUI.new(events), &block)
+      rescue ArgumentError # Method#arity doesn't inspect optional arguments, so we rescue
+        # requires bundler 1.10.0
+        gemfile(true, &block)
+      end
+    end
+  end
+end
diff --git a/lib/chef/cookbook/metadata.rb b/lib/chef/cookbook/metadata.rb
index 1cad526b65..603f80748c 100644
--- a/lib/chef/cookbook/metadata.rb
+++ b/lib/chef/cookbook/metadata.rb
@@ -58,12 +58,14 @@ class Chef
       PRIVACY                = "privacy".freeze
       CHEF_VERSIONS          = "chef_versions".freeze
       OHAI_VERSIONS          = "ohai_versions".freeze
+      GEMS                   = "gems".freeze
 
       COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer,
                             :maintainer_email, :license, :platforms, :dependencies,
                             :recommendations, :suggestions, :conflicting, :providing,
                             :replacing, :attributes, :groupings, :recipes, :version,
-                            :source_url, :issues_url, :privacy, :chef_versions, :ohai_versions ]
+                            :source_url, :issues_url, :privacy, :chef_versions, :ohai_versions,
+                            :gems ]
 
       VERSION_CONSTRAINTS = { :depends      => DEPENDENCIES,
                               :recommends   => RECOMMENDATIONS,
@@ -93,6 +95,8 @@ class Chef
       attr_reader :chef_versions
       # @return [Array<Gem::Dependency>] Array of supported Ohai versions
       attr_reader :ohai_versions
+      # @return [Array<Array>] Array of gems to install with *args as an Array
+      attr_reader :gems
 
       # Builds a new Chef::Cookbook::Metadata object.
       #
@@ -130,6 +134,7 @@ class Chef
         @privacy = false
         @chef_versions = []
         @ohai_versions = []
+        @gems = []
 
         @errors = []
       end
@@ -420,6 +425,17 @@ class Chef
         @ohai_versions
       end
 
+      # Metadata DSL to set a gem to install from the cookbook metadata.  May be declared
+      # multiple times.  All the gems from all the cookbooks are combined into one Gemfile
+      # and depsolved together.  Uses Bundler's DSL for its implementation.
+      #
+      # @param args [Array<String>] Gem name and options to pass to Bundler's DSL
+      # @return [Array<Array>] Array of gem statements as args
+      def gem(*args)
+        @gems << args unless args.empty?
+        @gems
+      end
+
       # Adds a description for a recipe.
       #
       # === Parameters
@@ -573,6 +589,7 @@ class Chef
           PRIVACY                => self.privacy,
           CHEF_VERSIONS          => gem_requirements_to_array(*self.chef_versions),
           OHAI_VERSIONS          => gem_requirements_to_array(*self.ohai_versions),
+          GEMS                   => self.gems,
         }
       end
 
@@ -609,6 +626,7 @@ class Chef
         @privacy                      = o[PRIVACY] if o.has_key?(PRIVACY)
         @chef_versions                = gem_requirements_from_array("chef", o[CHEF_VERSIONS]) if o.has_key?(CHEF_VERSIONS)
         @ohai_versions                = gem_requirements_from_array("ohai", o[OHAI_VERSIONS]) if o.has_key?(OHAI_VERSIONS)
+        @gems                         = o[GEMS] if o.has_key?(GEMS)
         self
       end
 
diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index a6a18718c2..b3271a139a 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -139,6 +139,26 @@ class Chef
       def cookbook_sync_complete
       end
 
+      # Called when starting to collect gems from the cookbooks
+      def cookbook_gem_start(gems)
+      end
+
+      # Called when the result of installing the bundle is to install the gem
+      def cookbook_gem_installing(gem, version)
+      end
+
+      # Called when the result of installing the bundle is to use the gem
+      def cookbook_gem_using(gem, version)
+      end
+
+      # Called when finished installing cookbook gems
+      def cookbook_gem_finished
+      end
+
+      # Called when cookbook gem installation fails
+      def cookbook_gem_failed(exception)
+      end
+
       ## TODO: add cookbook name to the API for file load callbacks
 
       ## TODO: add callbacks for overall cookbook eval start and complete.
diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb
index 5462241049..3f832f1e92 100644
--- a/lib/chef/formatters/doc.rb
+++ b/lib/chef/formatters/doc.rb
@@ -180,6 +180,32 @@ class Chef
         unindent
       end
 
+      # Called when starting to collect gems from the cookbooks
+      def cookbook_gem_start(gems)
+        puts_line "Installing Cookbook Gems:"
+        indent
+      end
+
+      # Called when the result of installing the bundle is to install the gem
+      def cookbook_gem_installing(gem, version)
+        puts_line "- Installing #{gem} #{version}", :green
+      end
+
+      # Called when the result of installing the bundle is to use the gem
+      def cookbook_gem_using(gem, version)
+        puts_line "- Using #{gem} #{version}"
+      end
+
+      # Called when finished installing cookbook gems
+      def cookbook_gem_finished
+        unindent
+      end
+
+      # Called when cookbook gem installation fails
+      def cookbook_gem_failed(exception)
+        unindent
+      end
+
       # Called when cookbook loading starts.
       def library_load_start(file_count)
         puts_line "Compiling Cookbooks..."
diff --git a/lib/chef/policy_builder/expand_node_object.rb b/lib/chef/policy_builder/expand_node_object.rb
index 6a006ec992..980de60dd5 100644
--- a/lib/chef/policy_builder/expand_node_object.rb
+++ b/lib/chef/policy_builder/expand_node_object.rb
@@ -3,7 +3,7 @@
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
 # Author:: Daniel DeLeo (<dan@chef.io>)
-# Copyright:: Copyright 2008-2016, Chef Software, Inc.
+# Copyright:: Copyright 2008-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -75,12 +75,16 @@ class Chef
           cl.load_cookbooks
           cookbook_collection = Chef::CookbookCollection.new(cl)
           cookbook_collection.validate!
+          cookbook_collection.install_gems(events)
+
           run_context = Chef::RunContext.new(node, cookbook_collection, @events)
         else
           Chef::Cookbook::FileVendor.fetch_from_remote(api_service)
           cookbook_hash = sync_cookbooks
           cookbook_collection = Chef::CookbookCollection.new(cookbook_hash)
           cookbook_collection.validate!
+          cookbook_collection.install_gems(events)
+
           run_context = Chef::RunContext.new(node, cookbook_collection, @events)
         end
 
diff --git a/lib/chef/policy_builder/policyfile.rb b/lib/chef/policy_builder/policyfile.rb
index 679e3cfe47..8f35c66cab 100644
--- a/lib/chef/policy_builder/policyfile.rb
+++ b/lib/chef/policy_builder/policyfile.rb
@@ -3,7 +3,7 @@
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
 # Author:: Daniel DeLeo (<dan@chef.io>)
-# Copyright:: Copyright 2008-2016, Chef Software, Inc.
+# Copyright:: Copyright 2008-2016 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -153,6 +153,8 @@ class Chef
         sync_cookbooks
         cookbook_collection = Chef::CookbookCollection.new(cookbooks_to_sync)
         cookbook_collection.validate!
+        cookbook_collection.install_gems(events)
+
         run_context = Chef::RunContext.new(node, cookbook_collection, events)
 
         setup_chef_class(run_context)
diff --git a/lib/chef/provider/package/rubygems.rb b/lib/chef/provider/package/rubygems.rb
index 7a2db6b32b..6b01927d50 100644
--- a/lib/chef/provider/package/rubygems.rb
+++ b/lib/chef/provider/package/rubygems.rb
@@ -535,7 +535,7 @@ class Chef
             src = " --clear-sources"
             src << (@new_resource.source && " --source=#{@new_resource.source}" || "")
           else
-            src = @new_resource.source && " --source=#{@new_resource.source} --source=https://rubygems.org"
+            src = @new_resource.source && " --source=#{@new_resource.source} --source=#{Chef::Config[:rubygems_url]}"
           end
           if !version.nil? && version.length > 0
             shell_out_with_timeout!("#{gem_binary_path} install #{name} -q --no-rdoc --no-ri -v \"#{version}\"#{src}#{opts}", :env => nil)