Merge branch 'CHEF-2210'

4 files changed, 86 insertions, 43 deletions

diff --git a/chef/lib/chef/client.rb b/chef/lib/chef/client.rb
index 8384d41c22..fe4c9a46ed 100644
--- a/chef/lib/chef/client.rb
+++ b/chef/lib/chef/client.rb
@@ -22,6 +22,7 @@ require 'chef/config'
 require 'chef/mixin/params_validate'
 require 'chef/log'
 require 'chef/rest'
+require 'chef/api_client'
 require 'chef/platform'
 require 'chef/node'
 require 'chef/role'
diff --git a/chef/lib/chef/cookbook/cookbook_version_loader.rb b/chef/lib/chef/cookbook/cookbook_version_loader.rb
index ade4141fc0..48de17cc5a 100644
--- a/chef/lib/chef/cookbook/cookbook_version_loader.rb
+++ b/chef/lib/chef/cookbook/cookbook_version_loader.rb
@@ -56,7 +56,9 @@ class Chef
 
         remove_ignored_files
 
-        if File.exists?(File.join(@cookbook_path, "metadata.json"))
+        if File.exists?(File.join(@cookbook_path, "metadata.rb"))
+          @metadata_filenames << File.join(@cookbook_path, "metadata.rb")
+        elsif File.exists?(File.join(@cookbook_path, "metadata.json"))
           @metadata_filenames << File.join(@cookbook_path, "metadata.json")
         end
 
@@ -88,12 +90,14 @@ class Chef
       # Generates the Cookbook::Metadata object
       def metadata(cookbook_version)
         @metadata = Chef::Cookbook::Metadata.new(cookbook_version)
-        @metadata_filenames.each do |meta_json|
-          begin
-            @metadata.from_json(IO.read(meta_json))
-          rescue JSON::ParserError
-            Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + meta_json)
-            raise
+        @metadata_filenames.each do |metadata_file|
+          case metadata_file
+          when /\.rb$/
+            apply_ruby_metadata(metadata_file)
+          when /\.json$/
+            apply_json_metadata(metadata_file)
+          else
+            raise RuntimeError, "Invalid metadata file: #{metadata_file} for cookbook: #{cookbook_version}"
           end
         end
         @metadata
@@ -146,6 +150,24 @@ class Chef
         end
       end
 
+      def apply_ruby_metadata(file)
+        begin
+          @metadata.from_file(file)
+        rescue JSON::ParserError
+          Chef::Log.error("Error evaluating metadata.rb for #@cookbook_name in " + file)
+          raise
+        end
+      end
+
+      def apply_json_metadata(file)
+        begin
+          @metadata.from_json(IO.read(file))
+        rescue JSON::ParserError
+          Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + file)
+          raise
+        end
+      end
+
     end
   end
 end
diff --git a/chef/lib/chef/cookbook_uploader.rb b/chef/lib/chef/cookbook_uploader.rb
index 877f320022..6b7c6e72ad 100644
--- a/chef/lib/chef/cookbook_uploader.rb
+++ b/chef/lib/chef/cookbook_uploader.rb
@@ -1,3 +1,4 @@
+
 require 'rest_client'
 require 'chef/exceptions'
 require 'chef/knife/cookbook_metadata'
@@ -10,6 +11,23 @@ require 'chef/cookbook/file_system_file_vendor'
 class Chef
   class CookbookUploader
 
+    def self.work_queue
+      @work_queue ||= Queue.new
+    end
+
+    def self.setup_worker_threads
+      @worker_threads ||= begin
+        work_queue
+        (1...10).map do
+          Thread.new do
+            loop do
+              work_queue.pop.call
+            end
+          end
+        end
+      end
+    end
+
     attr_reader :cookbook
     attr_reader :path
     attr_reader :opts
@@ -35,51 +53,32 @@ class Chef
     end
 
     def upload_cookbook
+      Thread.abort_on_exception = true
       Chef::Log.info("Saving #{cookbook.name}")
 
       # Syntax Check
       validate_cookbook
-      # Generate metadata.json from metadata.rb
-      build_metadata
-
       # generate checksums of cookbook files and create a sandbox
       checksum_files = cookbook.checksums
       checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
       new_sandbox = rest.post_rest("sandboxes", { :checksums => checksums })
 
       Chef::Log.info("Uploading files")
+
+      self.class.setup_worker_threads
+
       # upload the new checksums and commit the sandbox
       new_sandbox['checksums'].each do |checksum, info|
         if info['needs_upload'] == true
           Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
-
-          # Checksum is the hexadecimal representation of the md5,
-          # but we need the base64 encoding for the content-md5
-          # header
-          checksum64 = Base64.encode64([checksum].pack("H*")).strip
-          timestamp = Time.now.utc.iso8601
-          file_contents = File.read(checksum_files[checksum])
-          # TODO - 5/28/2010, cw: make signing and sending the request streaming
-          sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
-                                                                             :http_method => :put,
-                                                                             :path        => URI.parse(info['url']).path,
-                                                                             :body        => file_contents,
-                                                                             :timestamp   => timestamp,
-                                                                             :user_id     => rest.client_name
-                                                                             )
-          headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
-          headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
-
-          begin
-            RestClient::Resource.new(info['url'], :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
-          rescue RestClient::Exception => e
-            Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
-            raise
-          end
+          self.class.work_queue << uploader_function_for(checksum_files[checksum], checksum, info['url'])
         else
           Chef::Log.debug("#{checksum_files[checksum]} has not changed")
         end
       end
+
+      sleep 0.1 until self.class.work_queue.empty?
+
       sandbox_url = new_sandbox['uri']
       Chef::Log.debug("Committing sandbox")
       # Retry if S3 is claims a checksum doesn't exist (the eventual
@@ -100,15 +99,35 @@ class Chef
       Chef::Log.info("Upload complete!")
     end
 
-    def build_metadata
-      Chef::Log.debug("Generating metadata")
-      # FIXME: This knife command should be factored out into a
-      # library for use here
-      kcm = Chef::Knife::CookbookMetadata.new
-      kcm.config[:cookbook_path] = path
-      kcm.name_args = [ cookbook.name.to_s ]
-      kcm.run
-      cookbook.reload_metadata!
+    def worker_thread(work_queue)
+    end
+
+    def uploader_function_for(file, checksum, url)
+      lambda do
+        # Checksum is the hexadecimal representation of the md5,
+        # but we need the base64 encoding for the content-md5
+        # header
+        checksum64 = Base64.encode64([checksum].pack("H*")).strip
+        timestamp = Time.now.utc.iso8601
+        file_contents = File.read(file)
+        # TODO - 5/28/2010, cw: make signing and sending the request streaming
+        sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                           :http_method => :put,
+                                                                           :path        => URI.parse(url).path,
+                                                                           :body        => file_contents,
+                                                                           :timestamp   => timestamp,
+                                                                           :user_id     => rest.client_name
+                                                                           )
+        headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+        headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
+
+        begin
+          RestClient::Resource.new(url, :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
+        rescue RestClient::Exception => e
+          Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
+          raise
+        end
+      end
     end
 
     def validate_cookbook
diff --git a/chef/lib/chef/knife/cookbook_show.rb b/chef/lib/chef/knife/cookbook_show.rb
index 074a4a7ea4..3545d20817 100644
--- a/chef/lib/chef/knife/cookbook_show.rb
+++ b/chef/lib/chef/knife/cookbook_show.rb
@@ -25,6 +25,7 @@ class Chef
       deps do
         require 'chef/json_compat'
         require 'uri'
+        require 'chef/cookbook_version'
       end
 
       banner "knife cookbook show COOKBOOK [VERSION] [PART] [FILENAME] (options)"