diff options
Diffstat (limited to 'chef/lib/chef')
27 files changed, 1179 insertions, 845 deletions
diff --git a/chef/lib/chef/application/indexer.rb b/chef/lib/chef/application/indexer.rb deleted file mode 100644 index 92bb9f08a7..0000000000 --- a/chef/lib/chef/application/indexer.rb +++ /dev/null @@ -1,141 +0,0 @@ -# -# Author:: AJ Christensen (<aj@opscode.com>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'chef/application' -require 'chef/queue' -require 'chef/search' -require 'chef/search_index' -require 'chef/config' -require 'chef/daemon' -require 'chef/log' - - -class Chef::Application::Indexer < Chef::Application - - option :config_file, - :short => "-c CONFIG", - :long => "--config CONFIG", - :default => "/etc/chef/server.rb", - :description => "The configuration file to use" - - option :log_level, - :short => "-l LEVEL", - :long => "--log_level LEVEL", - :description => "Set the log level (debug, info, warn, error, fatal)", - :proc => lambda { |l| l.to_sym } - - option :log_location, - :short => "-L LOGLOCATION", - :long => "--logfile LOGLOCATION", - :description => "Set the log file location, defaults to STDOUT - recommended for daemonizing", - :proc => nil - - option :help, - :short => "-h", - :long => "--help", - :description => "Show this message", - :on => :tail, - :boolean => true, - :show_options => true, - :exit => 0 - - option :user, - :short => "-u USER", - :long => "--user USER", - :description => "User to set privilege to", - :proc => nil - - option :group, - :short => "-g GROUP", - :long => "--group GROUP", - :description => "Group to set privilege to", - :proc => nil - - option :daemonize, - :short => "-d", - :long => "--daemonize", - :description => "Daemonize the process", - :proc => lambda { |p| true } - - option :version, - :short => "-v", - :long => "--version", - :description => "Show chef version", - :boolean => true, - :proc => lambda {|v| puts "Chef: #{::Chef::VERSION}"}, - :exit => 0 - - def initialize - super - - @chef_search_indexer = nil - end - - # Create a new search indexer and connect to the stomp queues - def setup_application - Chef::Daemon.change_privilege - - @chef_search_indexer = Chef::SearchIndex.new - Chef::Queue.connect - Chef::Queue.subscribe(:queue, "index") - Chef::Queue.subscribe(:queue, "remove") - end - - # Run the indexer, optionally daemonizing. - def run_application - if Chef::Config[:daemonize] - Chef::Daemon.daemonize("chef-indexer") - end - - if Chef::Config[:queue_prefix] - queue_prefix = Chef::Config[:queue_prefix] - queue_partial_url = "/queue/#{queue_prefix}/chef" - else - queue_partial_url = "/queue/chef" - end - - loop do - object, headers = Chef::Queue.receive_msg - Chef::Log.info("Headers #{headers.inspect}") - if headers["destination"] == "#{queue_partial_url}/index" - start_timer = Time.new - @chef_search_indexer.add(object) - @chef_search_indexer.commit - final_timer = Time.new - Chef::Log.info("Indexed object from #{headers['destination']} in #{final_timer - start_timer} seconds") - elsif headers["destination"] == "#{queue_partial_url}/remove" - start_timer = Time.new - @chef_search_indexer.delete(object) - @chef_search_indexer.commit - final_timer = Time.new - Chef::Log.info("Removed object from #{headers['destination']} in #{final_timer - start_timer} seconds") - end - end - rescue SystemExit => e - raise - rescue Exception => e - if Chef::Config[:interval] - Chef::Log.error("#{e.class}") - Chef::Log.fatal("#{e}\n#{e.backtrace.join("\n")}") - Chef::Log.fatal("Sleeping for #{Chef::Config[:delay]} seconds before trying again") - sleep Chef::Config[:delay] - retry - else - raise - end - end -end diff --git a/chef/lib/chef/client.rb b/chef/lib/chef/client.rb index c561fedb32..7f67c6bef0 100644 --- a/chef/lib/chef/client.rb +++ b/chef/lib/chef/client.rb @@ -46,9 +46,14 @@ class Chef @json_attribs = nil @node_name = nil @node_exists = true - Ohai::Log.logger = Chef::Log.logger + Mixlib::Auth::Log.logger = Ohai::Log.logger = Chef::Log.logger @ohai = Ohai::System.new - @rest = Chef::REST.new(Chef::Config[:registration_url]) + @ohai_has_run = false + if File.exists?(Chef::Config[:client_key]) + @rest = Chef::REST.new(Chef::Config[:chef_server_url]) + else + @rest = Chef::REST.new(Chef::Config[:chef_server_url], nil, nil) + end end # Do a full run for this Chef::Client. Calls: @@ -71,13 +76,9 @@ class Chef determine_node_name register - authenticate build_node(@node_name) save_node - sync_library_files - sync_attribute_files - sync_definitions - sync_recipes + sync_cookbooks save_node converge save_node @@ -116,9 +117,14 @@ class Chef end def determine_node_name - run_ohai unless @safe_name && @node_name - @node_name ||= @ohai[:fqdn] ? @ohai[:fqdn] : @ohai[:hostname] + run_ohai + if Chef::Config[:node_name] + @node_name = Chef::Config[:node_name] + else + @node_name ||= @ohai[:fqdn] ? @ohai[:fqdn] : @ohai[:hostname] + Chef::Config[:node_name] = @node_name + end @safe_name = @node_name.gsub(/\./, '_') end @node_name @@ -177,68 +183,22 @@ class Chef @node[:tags] = Array.new unless @node.attribute?(:tags) @node end - - # If this node has been registered before, this method will fetch the current registration - # data. - # - # If it has not, we register it by calling create_registration. - # + + # # === Returns # true:: Always returns true def register - determine_node_name unless @node_name - Chef::Log.debug("Registering #{@safe_name} for an openid") - - begin - if @rest.get_rest("registrations/#{@safe_name}") - @secret = Chef::FileCache.load(File.join("registration", @safe_name)) - end - rescue Net::HTTPServerException => e - case e.message - when /^404/ - create_registration - else - raise - end - rescue Chef::Exceptions::FileNotFound - Chef::Application.fatal! "A remote registration already exists for #{@safe_name}, however the local shared secret does not exist." + - " To remedy this, you could delete the registration via webUI/REST, change the node_name option in config.rb" + - " (or use the -N/--node-name option to the CLI) or" + - " copy the old shared secret to #{File.join(Chef::Config[:file_cache_path], 'registration', @safe_name)}", 3 + if File.exists?(Chef::Config[:validation_key]) + @vr = Chef::REST.new(Chef::Config[:client_url], Chef::Config[:validation_client_name], Chef::Config[:validation_key]) + @vr.register(@node_name, Chef::Config[:client_key]) + else + Chef::Log.debug("Validation key #{Chef::Config[:validation_key]} is not present - skipping registration") end - + # We now have the client key, and should use it from now on. + @rest = Chef::REST.new(Chef::Config[:chef_server_url]) true end - # Generates a random secret, stores it in the Chef::Filestore with the "registration" key, - # and posts our nodes registration information to the server. - # - # === Returns - # true:: Always returns true - def create_registration - @secret = random_password(500) - Chef::FileCache.store(File.join("registration", @safe_name), @secret) - @rest.post_rest("registrations", { :id => @safe_name, :password => @secret, :validation_token => @validation_token }) - true - end - - # Authenticates the node via OpenID. - # - # === Returns - # true:: Always returns true - def authenticate - determine_node_name unless @node_name - Chef::Log.debug("Authenticating #{@safe_name} via openid") - response = @rest.post_rest('openid/consumer/start', { - "openid_identifier" => "#{Chef::Config[:openid_url]}/openid/server/node/#{@safe_name}", - "submit" => "Verify" - }) - @rest.post_rest( - "#{Chef::Config[:openid_url]}#{response["action"]}", - { "password" => @secret } - ) - end - # Update the file caches for a given cache segment. Takes a segment name # and a hash that matches one of the cookbooks/_attribute_files style # remote file listings. @@ -246,103 +206,81 @@ class Chef # === Parameters # segment<String>:: The cache segment to update # remote_list<Hash>:: A cookbooks/_attribute_files style remote file listing - def update_file_cache(segment, remote_list) - # We need the list of known good attribute files, so we can delete any that are - # just laying about. + def update_file_cache(cookbook_name, parts) + Chef::Log.debug("Synchronizing cookbook #{cookbook_name}") + file_canonical = Hash.new - - remote_list.each do |rf| - cache_file = File.join("cookbooks", rf['cookbook'], segment, rf['name']) - file_canonical[cache_file] = true - # For back-compat between older clients and new chef servers - rf['checksum'] ||= nil - - current_checksum = nil - if Chef::FileCache.has_key?(cache_file) - current_checksum = checksum(Chef::FileCache.load(cache_file, false)) - end + parts.each do |segment, remote_list| + # segement = cookbook segment + # remote_list = list of file hashes + # + # We need the list of known good attribute files, so we can delete any that are + # just laying about. + + + remote_list.each do |rf| + cache_file = File.join("cookbooks", cookbook_name, segment, rf['name']) + file_canonical[cache_file] = true - rf_url = generate_cookbook_url( - rf['name'], - rf['cookbook'], - segment, - @node, - current_checksum ? { 'checksum' => current_checksum } : nil - ) - Chef::Log.debug(rf_url) + # For back-compat between older clients and new chef servers + rf['checksum'] ||= nil + + current_checksum = nil + if Chef::FileCache.has_key?(cache_file) + current_checksum = checksum(Chef::FileCache.load(cache_file, false)) + end - if current_checksum != rf['checksum'] - changed = true - begin - raw_file = @rest.get_rest(rf_url, true) - rescue Net::HTTPRetriableError => e - if e.response.kind_of?(Net::HTTPNotModified) - changed = false - Chef::Log.debug("Cache file #{cache_file} is unchanged") - else - raise e + rf_url = generate_cookbook_url( + rf['name'], + cookbook_name, + segment, + @node, + current_checksum ? { 'checksum' => current_checksum } : nil + ) + if current_checksum != rf['checksum'] + changed = true + begin + raw_file = @rest.get_rest(rf_url, true) + rescue Net::HTTPRetriableError => e + if e.response.kind_of?(Net::HTTPNotModified) + changed = false + Chef::Log.debug("Cache file #{cache_file} is unchanged") + else + raise e + end end - end - if changed - Chef::Log.info("Storing updated #{cache_file} in the cache.") - Chef::FileCache.move_to(raw_file.path, cache_file) + if changed + Chef::Log.info("Storing updated #{cache_file} in the cache.") + Chef::FileCache.move_to(raw_file.path, cache_file) + end end end - end - - Chef::FileCache.list.each do |cache_file| - if cache_file.match("cookbooks/.+?/#{segment}") - unless file_canonical[cache_file] - Chef::Log.info("Removing #{cache_file} from the cache; it is no longer on the server.") - Chef::FileCache.delete(cache_file) + + Chef::FileCache.list.each do |cache_file| + if cache_file =~ /^cookbooks\/(recipes|attributes|definitions|libraries)\// + unless file_canonical[cache_file] + Chef::Log.info("Removing #{cache_file} from the cache; it is no longer on the server.") + Chef::FileCache.delete(cache_file) + end end end + end end - - # Gets all the attribute files included in all the cookbooks available on the server, - # and executes them. - # - # === Returns - # true:: Always returns true - def sync_attribute_files - Chef::Log.debug("Synchronizing attributes") - update_file_cache("attributes", @rest.get_rest("cookbooks/_attribute_files?node=#{@node.name}")) - true - end - - # Gets all the library files included in all the cookbooks available on the server, - # and loads them. - # - # === Returns - # true:: Always returns true - def sync_library_files - Chef::Log.debug("Synchronizing libraries") - update_file_cache("libraries", @rest.get_rest("cookbooks/_library_files?node=#{@node.name}")) - true - end - - # Gets all the definition files included in all the cookbooks available on the server, - # and loads them. - # - # === Returns - # true:: Always returns true - def sync_definitions - Chef::Log.debug("Synchronizing definitions") - update_file_cache("definitions", @rest.get_rest("cookbooks/_definition_files?node=#{@node.name}")) - end - - # Gets all the recipe files included in all the cookbooks available on the server, - # and loads them. + + # Synchronizes all the cookbooks from the chef-server. # # === Returns # true:: Always returns true - def sync_recipes - Chef::Log.debug("Synchronizing recipes") - update_file_cache("recipes", @rest.get_rest("cookbooks/_recipe_files?node=#{@node.name}")) + def sync_cookbooks + Chef::Log.debug("Synchronizing cookbooks") + cookbook_hash = @rest.get_rest("nodes/#{@safe_name}/cookbooks") + cookbook_hash.each do |cookbook_name, parts| + update_file_cache(cookbook_name, parts) + end end # Updates the current node configuration on the server. @@ -382,15 +320,6 @@ class Chef cr.converge true end - - protected - # Generates a random password of "len" length. - def random_password(len) - chars = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a - newpass = "" - 1.upto(len) { |i| newpass << chars[rand(chars.size-1)] } - newpass - end end end diff --git a/chef/lib/chef/config.rb b/chef/lib/chef/config.rb index 977786a37d..11a4e4203d 100644 --- a/chef/lib/chef/config.rb +++ b/chef/lib/chef/config.rb @@ -17,6 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +require 'chef/log' require 'mixlib/config' class Chef @@ -54,11 +55,13 @@ class Chef :template_url, :remotefile_url, :search_url, + :chef_server_url, :role_url ].each do |u| c[u] = url end end end + # Override the config dispatch to set the value of log_location configuration option # # === Parameters @@ -79,7 +82,7 @@ class Chef authorized_openid_identifiers nil authorized_openid_providers nil - chef_server_url nil + chef_server_url "http://localhost:4000" cookbook_path [ "/var/chef/site-cookbooks", "/var/chef/cookbooks" ] couchdb_database "chef" couchdb_url "http://localhost:5984" @@ -105,14 +108,9 @@ class Chef openid_store_path "/var/chef/openid/db" openid_url "http://localhost:4001" pid_file nil - queue_host "localhost" - queue_password "" - queue_port 61613 - queue_retry_count 5 - queue_retry_delay 5 - queue_user "" - queue_prefix nil registration_url "http://localhost:4000" + certificate_url "http://localhost:4000" + client_url "http://localhost:4042" remotefile_url "http://localhost:4000" rest_timeout 60 run_command_stderr_timeout 120 @@ -130,5 +128,22 @@ class Chef role_path "/var/chef/roles" role_url "http://localhost:4000" recipe_url nil + solr_url "http://localhost:8983" + solr_jetty_path "/var/chef/solr-jetty" + solr_data_path "/var/chef/solr/data" + solr_home_path "/var/chef/solr" + solr_heap_size "256M" + solr_java_opts nil + nanite_host '0.0.0.0' + nanite_port '5672' + nanite_user 'nanite' + nanite_pass 'testing' + nanite_vhost '/nanite' + nanite_identity nil + + client_key "/etc/chef/client.pem" + validation_key "/etc/chef/validation.pem" + validation_client_name "chef-validator" + end end diff --git a/chef/lib/chef/cookbook_helper.rb b/chef/lib/chef/cookbook_helper.rb new file mode 100644 index 0000000000..520fa38fec --- /dev/null +++ b/chef/lib/chef/cookbook_helper.rb @@ -0,0 +1,90 @@ +module CookbookHelper + + require 'aws/s3' + + def put_in_couchdb_and_s3(cookbook_name, file, revision) + # TODO: set inital state of cookbook to something like 'pre-upload' + cookbook = Cookbook.on(database_from_orgname(params[:organization_id])).new(:display_name => cookbook_name, :revision => revision) + save cookbook + + id = cookbook['_id'] + Merb.logger.debug "Creating cookbook with id = #{id}" + + stream_to_s3(params[:file][:tempfile], id) + + # TODO: if upload successful, set cookbook state to something like 'active' + end + + def validate_tarball(filepath, cookbook_name) + raise "(try creating with 'tar czf cookbook.tgz cookbook/')" unless system("tar", "tzf", filepath) + + # TODO: modify/implement tests and uncomment the next lines + +# required_entry_roots = [cookbook_name] +# allowed_entry_roots = [cookbook_name, "ignore"] + +# entry_roots = `tar tzf #{filepath}`.split("\n").map{|e|e.split('/').first}.uniq + +# illegal_roots = entry_roots - allowed_entry_roots +# raise "tarball root may only contain #{allowed_entry_roots.join(', ')}" unless illegal_roots.empty? + +# missing_required_roots = required_entry_roots - entry_roots +# raise "tarball root must contain #{required_entry_roots.join(', ')}" unless missing_required_roots.empty? + end + + def get_all_cookbook_entries(cookbook_name) + rows = Cookbook.on(database_from_orgname(params[:organization_id])).by_display_name(:key => cookbook_name, :include_docs => true) + Merb.logger.debug "Cookbook has the following entries: #{rows.inspect}" + rows + end + + def cookbook_id(cookbook_name) + rows = get_all_cookbook_entries(cookbook_name) + return nil if rows.empty? + most_recent_record = rows.sort_by{|row| row['revision'].to_i}.last + Merb.logger.debug "Selected #{most_recent_record.inspect}" + [most_recent_record['_id'], most_recent_record['revision']] + end + + # TODO: should we do this once at start-up and test the connection before establishing it? + def establish_connection + AWS::S3::Base.establish_connection!( + :access_key_id => Merb::Config[:aws_secret_access_key_id], + :secret_access_key => Merb::Config[:aws_secret_access_key] + ) + end + + def stream_to_s3(path, object_id) + establish_connection + AWS::S3::S3Object.store("#{object_id}.tgz", open(path), Merb::Config[:aws_cookbook_tarball_s3_bucket]) + end + + def stream_from_s3(cookbook_name, id) + establish_connection + # TODO: if the cookbook is large and the user has a slow connection, will this cause the process's memory to bloat or will it just read from S3 slowly? + stream_file do |response| + AWS::S3::S3Object.stream("#{id}.tgz", Merb::Config[:aws_cookbook_tarball_s3_bucket]) do |chunk| + response.write chunk + end + end + end + + # TODO: the following methods were heisted from opscode-account. if this is how we want to do it, then do some hoisting + + def save(object) + if object.valid? + object.save + else + raise BadRequest, object.errors.full_messages + end + end + + def orgname_to_dbname(orgname) + "chef_#{orgname}" + end + + def database_from_orgname(orgname) + CouchRest::Database.new(CouchRest::Server.new,orgname_to_dbname(orgname)) + end + +end diff --git a/chef/lib/chef/couchdb.rb b/chef/lib/chef/couchdb.rb index 10f43b72c8..9f63ad6aaa 100644 --- a/chef/lib/chef/couchdb.rb +++ b/chef/lib/chef/couchdb.rb @@ -21,40 +21,77 @@ require 'chef/rest' require 'chef/log' require 'digest/sha2' require 'json' +require 'uuidtools' +require 'chef/nanite' class Chef class CouchDB include Chef::Mixin::ParamsValidate - def initialize(url=nil) + def initialize(url=nil, db=Chef::Config[:couchdb_database]) url ||= Chef::Config[:couchdb_url] - @rest = Chef::REST.new(url) + @db = db + @rest = Chef::REST.new(url, nil, nil) + end + + def couchdb_database(args=nil) + if args + @db = args + else + @db + end + end + + def create_id_map + create_design_document( + "id_map", + { + "version" => 1, + "language" => "javascript", + "views" => { + "name_to_id" => { + "map" => <<-EOJS + function(doc) { + emit([ doc.chef_type, doc.name], doc._id); + } + EOJS + }, + "id_to_name" => { + "map" => <<-EOJS + function(doc) { + emit(doc._id, [ doc.chef_type, doc.name ]); + } + EOJS + } + } + } + ) end def create_db @database_list = @rest.get_rest("_all_dbs") - unless @database_list.detect { |db| db == Chef::Config[:couchdb_database] } - response = @rest.put_rest(Chef::Config[:couchdb_database], Hash.new) + unless @database_list.detect { |db| db == couchdb_database } + response = @rest.put_rest(couchdb_database, Hash.new) end - Chef::Config[:couchdb_database] + couchdb_database end def create_design_document(name, data) create_db to_update = true begin - old_doc = @rest.get_rest("#{Chef::Config[:couchdb_database]}/_design%2F#{name}") + old_doc = @rest.get_rest("#{couchdb_database}/_design/#{name}") if data["version"] != old_doc["version"] data["_rev"] = old_doc["_rev"] Chef::Log.debug("Updating #{name} views") else to_update = false end - rescue - Chef::Log.debug("Creating #{name} views for the first time") + rescue + Chef::Log.debug("Creating #{name} views for the first time because: #{$!}") end if to_update - @rest.put_rest("#{Chef::Config[:couchdb_database]}/_design%2F#{name}", data) + @rest.put_rest("#{couchdb_database}/_design%2F#{name}", data) end true end @@ -70,7 +107,29 @@ class Chef :object => { :respond_to => :to_json }, } ) - @rest.put_rest("#{Chef::Config[:couchdb_database]}/#{obj_type}_#{safe_name(name)}", object) + r = get_view("id_map", "name_to_id", :key => [ obj_type, name ]) + uuid = nil + if r["rows"].length == 1 + uuid = r["rows"][0]["id"] + else + uuid = UUIDTools::UUID.random_create.to_s + end + + r = @rest.put_rest("#{couchdb_database}/#{uuid}", object) + Chef::Log.info("Sending #{uuid} to Nanite for indexing..") + n = Chef::Nanite.request( + "/index/add", + { + :id => uuid, + :database => couchdb_database, + :type => obj_type, + :item => object + }, + :persistent => true + ) do |response_full| + Chef::Log.debug("Finished indexing #{obj_type} #{uuid} in #{couchdb_database}"); + end + r end def load(obj_type, name) @@ -84,7 +143,9 @@ class Chef :name => { :kind_of => String }, } ) - @rest.get_rest("#{Chef::Config[:couchdb_database]}/#{obj_type}_#{safe_name(name)}") + doc = find_by_name(obj_type, name) + doc.couchdb = self if doc.respond_to?(:couchdb) + doc end def delete(obj_type, name, rev=nil) @@ -98,15 +159,30 @@ class Chef :name => { :kind_of => String }, } ) + del_id = nil + last_obj, obj_id = find_by_name(obj_type, name, true) unless rev - last_obj = @rest.get_rest("#{Chef::Config[:couchdb_database]}/#{obj_type}_#{safe_name(name)}") if last_obj.respond_to?(:couchdb_rev) rev = last_obj.couchdb_rev else rev = last_obj['_rev'] end end - @rest.delete_rest("#{Chef::Config[:couchdb_database]}/#{obj_type}_#{safe_name(name)}?rev=#{rev}") + r = @rest.delete_rest("#{couchdb_database}/#{obj_id}?rev=#{rev}") + r.couchdb = self if r.respond_to?(:couchdb) + Chef::Log.info("Sending #{obj_id} to Nanite for deletion..") + n = Chef::Nanite.request( + "/index/delete", + { + :id => obj_id, + :database => couchdb_database, + :type => obj_type + }, + :persistent => true + ) do |response_full| + Chef::Log.debug("Finished Deleting #{obj_type} #{obj_id} in #{couchdb_database}"); + end + r end def list(view, inflate=false) @@ -119,10 +195,13 @@ class Chef } ) if inflate - @rest.get_rest(view_uri(view, "all")) + r = @rest.get_rest(view_uri(view, "all")) + r["rows"].each { |i| i["value"].couchdb = self if i["value"].respond_to?(:couchdb=) } + r else - @rest.get_rest(view_uri(view, "all_id")) + r = @rest.get_rest(view_uri(view, "all_id")) end + r end def has_key?(obj_type, name) @@ -137,13 +216,25 @@ class Chef } ) begin - @rest.get_rest("#{Chef::Config[:couchdb_database]}/#{obj_type}_#{safe_name(name)}") + find_by_name(obj_type, name) true rescue false end end + def find_by_name(obj_type, name, with_id=false) + r = get_view("id_map", "name_to_id", :key => [ obj_type, name ], :include_docs => true) + if r["rows"].length == 0 + raise Chef::Exceptions::CouchDBNotFound, "Cannot find #{obj_type} #{name} in CouchDB!" + end + if with_id + [ r["rows"][0]["doc"], r["rows"][0]["id"] ] + else + r["rows"][0]["doc"] + end + end + def get_view(design, view, options={}) view_string = view_uri(design, view) view_string << "?" if options.length != 0 @@ -151,14 +242,21 @@ class Chef options.each { |k,v| view_string << "#{first ? '' : '&'}#{k}=#{URI.escape(v.to_json)}"; first = false } @rest.get_rest(view_string) end + + def bulk_get(*to_fetch) + response = @rest.post_rest("#{couchdb_database}/_all_docs?include_docs=true", { "keys" => to_fetch.flatten }) + response["rows"].collect { |r| r["doc"].couchdb = self; r["doc"] } + end def view_uri(design, view) - Chef::Config[:couchdb_version] ||= @rest.run_request(:GET, URI.parse(@rest.url + "/"), false, 10, false)["version"].gsub(/-.+/,"").to_f + Chef::Config[:couchdb_version] ||= @rest.run_request(:GET, URI.parse(@rest.url + "/"), {}, false, 10, false)["version"].gsub(/-.+/,"").to_f case Chef::Config[:couchdb_version] when 0.9 - "#{Chef::Config[:couchdb_database]}/_design/#{design}/_view/#{view}" + "#{couchdb_database}/_design/#{design}/_view/#{view}" when 0.8 - "#{Chef::Config[:couchdb_database]}/_view/#{design}/#{view}" + "#{couchdb_database}/_view/#{design}/#{view}" + else + "#{couchdb_database}/_design/#{design}/_view/#{view}" end end @@ -167,6 +265,6 @@ class Chef def safe_name(name) name.gsub(/\./, "_") end - + end end diff --git a/chef/lib/chef/data_bag.rb b/chef/lib/chef/data_bag.rb new file mode 100644 index 0000000000..e23ce04c83 --- /dev/null +++ b/chef/lib/chef/data_bag.rb @@ -0,0 +1,167 @@ +# +# Author:: Adam Jacob (<adam@opscode.com>) +# Copyright:: Copyright (c) 2009 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/config' +require 'chef/mixin/params_validate' +require 'chef/mixin/from_file' +require 'chef/couchdb' +require 'chef/data_bag_item' +require 'extlib' +require 'json' + +class Chef + class DataBag + + include Chef::Mixin::FromFile + include Chef::Mixin::ParamsValidate + + DESIGN_DOCUMENT = { + "version" => 2, + "language" => "javascript", + "views" => { + "all" => { + "map" => <<-EOJS + function(doc) { + if (doc.chef_type == "data_bag") { + emit(doc.name, doc); + } + } + EOJS + }, + "all_id" => { + "map" => <<-EOJS + function(doc) { + if (doc.chef_type == "data_bag") { + emit(doc.name, doc.name); + } + } + EOJS + }, + "entries" => { + "map" => <<-EOJS + function(doc) { + if (doc.chef_type == "data_bag_item") { + emit(doc.data_bag, doc.raw_data.id); + } + } + EOJS + } + } + } + + attr_accessor :couchdb_rev + + # Create a new Chef::DataBag + def initialize + @name = '' + @couchdb_rev = nil + @couchdb = Chef::CouchDB.new + end + + def name(arg=nil) + set_or_return( + :name, + arg, + :regex => /^[\-[:alnum:]_]+$/ + ) + end + + def to_hash + result = { + "name" => @name, + 'json_class' => self.class.name, + "chef_type" => "data_bag", + } + result["_rev"] = @couchdb_rev if @couchdb_rev + result + end + + # Serialize this object as a hash + def to_json(*a) + to_hash.to_json(*a) + end + + # Create a Chef::Role from JSON + def self.json_create(o) + bag = new + bag.name(o["name"]) + bag.couchdb_rev = o["_rev"] if o.has_key?("_rev") + bag + end + + # List all the Chef::DataBag objects in the CouchDB. If inflate is set to true, you will get + # the full list of all Roles, fully inflated. + def self.list(inflate=false) + couchdb = Chef::CouchDB.new + rs = couchdb.list("data_bags", inflate) + if inflate + rs["rows"].collect { |r| r["value"] } + else + rs["rows"].collect { |r| r["key"] } + end + end + + # Load a Data Bag by name from CouchDB + def self.load(name) + couchdb = Chef::CouchDB.new + couchdb.load("data_bag", name) + end + + # Remove this Data Bag from CouchDB + def destroy + removed = @couchdb.delete("data_bag", @name, @couchdb_rev) + rs = @couchdb.get_view("data_bags", "entries", :include_docs => true, :startkey => @name, :endkey => @name) + rs["rows"].each do |row| + row["doc"].couchdb = @couchdb + row["doc"].destroy + end + removed + end + + # Save this Data Bag to the CouchDB + def save + results = @couchdb.store("data_bag", @name, self) + @couchdb_rev = results["rev"] + end + + # List all the items in this Bag + def list(inflate=false) + rs = nil + if inflate + rs = @couchdb.get_view("data_bags", "entries", :include_docs => true, :startkey => @name, :endkey => @name) + rs["rows"].collect { |r| r["doc"].couchdb = @couchdb; r["doc"] } + else + rs = @couchdb.get_view("data_bags", "entries", :startkey => @name, :endkey => @name) + rs["rows"].collect { |r| r["value"] } + end + end + + # Set up our CouchDB design document + def self.create_design_document + couchdb = Chef::CouchDB.new + couchdb.create_design_document("data_bags", DESIGN_DOCUMENT) + end + + # As a string + def to_s + "data_bag[#{@name}]" + end + + end +end + diff --git a/chef/lib/chef/data_bag_item.rb b/chef/lib/chef/data_bag_item.rb new file mode 100644 index 0000000000..b59f4314da --- /dev/null +++ b/chef/lib/chef/data_bag_item.rb @@ -0,0 +1,188 @@ +# +# Author:: Adam Jacob (<adam@opscode.com>) +# Copyright:: Copyright (c) 2009 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/config' +require 'chef/mixin/params_validate' +require 'chef/mixin/from_file' +require 'chef/couchdb' +require 'extlib' +require 'json' + +class Chef + class DataBagItem + + include Chef::Mixin::FromFile + include Chef::Mixin::ParamsValidate + + DESIGN_DOCUMENT = { + "version" => 1, + "language" => "javascript", + "views" => { + "all" => { + "map" => <<-EOJS + function(doc) { + if (doc.chef_type == "data_bag_item") { + emit(doc.name, doc); + } + } + EOJS + }, + "all_id" => { + "map" => <<-EOJS + function(doc) { + if (doc.chef_type == "data_bag_item") { + emit(doc.name, doc.name); + } + } + EOJS + } + } + } + + attr_accessor :couchdb_rev, :raw_data + + # Create a new Chef::DataBagItem + def initialize(couchdb=nil) + @couchdb_rev = nil + @data_bag = nil + @raw_data = Hash.new + @couchdb = Chef::CouchDB.new + end + + def raw_data + @raw_data + end + + def raw_data=(new_data) + unless new_data.kind_of?(Hash) || new_data.kind_of?(Mash) + raise ArgumentError, "Data Bag Items must contain a Hash or Mash!" + end + unless new_data.has_key?("id") + raise ArgumentError, "Data Bag Items must have an id key in the hash! #{new_data.inspect}" + end + unless new_data["id"] =~ /^[\-[:alnum:]_]+$/ + raise ArgumentError, "Data Bag Item id does not match alphanumeric/-/_!" + end + @raw_data = new_data + end + + def data_bag(arg=nil) + set_or_return( + :data_bag, + arg, + :regex => /^[\-[:alnum:]_]+$/ + ) + end + + def name + object_name + end + + def object_name + if raw_data.has_key?('id') + id = raw_data['id'] + else + raise ArgumentError, "You must have an 'id' or :id key in the raw data" + end + + data_bag_name = self.data_bag + unless data_bag_name + raise ArgumentError, "You must have declared what bag this item belongs to!" + end + "data_bag_item_#{data_bag_name}_#{id}" + end + + def self.object_name(data_bag_name, id) + "data_bag_item_#{data_bag_name}_#{id}" + end + + def to_hash + result = self.raw_data + result["chef_type"] = "data_bag_item" + result["data_bag"] = self.data_bag + result["_rev"] = @couchdb_rev if @couchdb_rev + result + end + + # Serialize this object as a hash + def to_json(*a) + result = { + "name" => self.object_name, + "json_class" => self.class.name, + "chef_type" => "data_bag_item", + "data_bag" => self.data_bag, + "raw_data" => self.raw_data + } + result["_rev"] = @couchdb_rev if @couchdb_rev + result.to_json(*a) + end + + # Create a Chef::DataBagItem from JSON + def self.json_create(o) + bag_item = new + bag_item.data_bag(o["data_bag"]) + o.delete("data_bag") + o.delete("chef_type") + o.delete("json_class") + o.delete("name") + if o.has_key?("_rev") + bag_item.couchdb_rev = o["_rev"] + o.delete("_rev") + end + bag_item.raw_data = o["raw_data"] + bag_item + end + + # The Data Bag Item behaves like a hash - we pass all that stuff along to @raw_data. + def method_missing(method_symbol, *args, &block) + self.raw_data.send(method_symbol, *args, &block) + end + + # Load a Data Bag Item by name from CouchDB + def self.load(data_bag, name) + couchdb = Chef::CouchDB.new + couchdb.load("data_bag_item", object_name(data_bag, name)) + end + + # Remove this Data Bag Item from CouchDB + def destroy + removed = @couchdb.delete("data_bag_item", object_name, @couchdb_rev) + removed + end + + # Save this Data Bag Item to CouchDB + def save + results = @couchdb.store("data_bag_item", object_name, self) + @couchdb_rev = results["rev"] + end + + # Set up our CouchDB design document + def self.create_design_document + couchdb = Chef::CouchDB.new + couchdb.create_design_document("data_bag_items", DESIGN_DOCUMENT) + end + + # As a string + def to_s + "data_bag_item[#{@name}]" + end + + end +end + + diff --git a/chef/lib/chef/exceptions.rb b/chef/lib/chef/exceptions.rb index 7ab7fa5664..13b93502cc 100644 --- a/chef/lib/chef/exceptions.rb +++ b/chef/lib/chef/exceptions.rb @@ -32,5 +32,8 @@ class Chef class Group < RuntimeError; end class Link < RuntimeError; end class Mount < RuntimeError; end + class CouchDBNotFound < RuntimeError; end + class PrivateKeyMissing < RuntimeError; end + class CannotWritePrivateKey < RuntimeError; end end end diff --git a/chef/lib/chef/mixin/generate_url.rb b/chef/lib/chef/mixin/generate_url.rb index 0ea7cc7a31..6a3bfbb73e 100644 --- a/chef/lib/chef/mixin/generate_url.rb +++ b/chef/lib/chef/mixin/generate_url.rb @@ -29,19 +29,28 @@ class Chef else new_url = "cookbooks/#{cookbook}/#{type}?" new_url += "id=#{url}" - platform, version = Chef::Platform.find_platform_and_version(node) - if type == "files" || type == "templates" - new_url += "&platform=#{platform}&version=#{version}&fqdn=#{node[:fqdn]}&node_name=#{node.name}" - end - if args - args.each do |key, value| - new_url += "&#{key}=#{value}" - end - end + new_url = generate_cookbook_url_from_uri(new_url, node, args) end return new_url end + + def generate_cookbook_url_from_uri(uri, node, args=nil) + platform, version = Chef::Platform.find_platform_and_version(node) + uri =~ /cookbooks\/(.+?)\/(.+)\?/ + cookbook = $1 + type = $2 + if type == "files" || type == "templates" + uri += "&platform=#{platform}&version=#{version}&fqdn=#{node[:fqdn]}&node_name=#{node.name}" + end + if args + args.each do |key, value| + uri += "&#{key}=#{value}" + end + end + + uri + end end end diff --git a/chef/lib/chef/mixin/language.rb b/chef/lib/chef/mixin/language.rb index cf049926de..704dbf0c05 100644 --- a/chef/lib/chef/mixin/language.rb +++ b/chef/lib/chef/mixin/language.rb @@ -16,6 +16,8 @@ # limitations under the License. # +require 'chef/search/query' + class Chef module Mixin module Language @@ -73,6 +75,10 @@ class Chef has_platform end + + def search(*args, &block) + Chef::Search::Query.new.search(*args, &block) + end end end diff --git a/chef/lib/chef/nanite.rb b/chef/lib/chef/nanite.rb new file mode 100644 index 0000000000..a0f4827a68 --- /dev/null +++ b/chef/lib/chef/nanite.rb @@ -0,0 +1,84 @@ +# +# Author:: Adam Jacob (<adam@opscode.com>) +# Copyright:: Copyright (c) 2008 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef' +require 'chef/config' +require 'chef/mixin/params_validate' +require 'nanite' +require 'json' + +class Chef + class Nanite + + class << self + + def start_mapper(config={}) + Chef::Log.info("Running the Nanite Mapper") + ::Nanite::Log.logger = Chef::Log.logger + identity = Chef::Config[:nanite_identity] ? Chef::Config[:nanite_identity] : get_identity + ::Nanite.start_mapper( + :host => Chef::Config[:nanite_host], + :user => Chef::Config[:nanite_user], + :pass => Chef::Config[:nanite_pass], + :vhost => Chef::Config[:nanite_vhost], + :identity => identity, + :format => :json, + :log_level => Chef::Config[:log_level] + ) + end + + def get_identity(type="mapper") + id = nil + if Chef::FileCache.has_key?("nanite-#{type}-identity") + id = Chef::FileCache.load("nanite-#{type}-identity") + else + id = ::Nanite::Identity.generate + Chef::FileCache.store("nanite-#{type}-identity", id) + end + id + end + + def request(*args) + in_event do + ::Nanite.request(*args) + end + end + + def in_event(&block) + if EM.reactor_running? + begin + ::Nanite.ensure_mapper + rescue ::Nanite::MapperNotRunning + start_mapper + end + block.call + else + Chef::Log.warn("Starting Event Machine Loop") + Thread.new do + EM.run do + start_mapper + block.call + end + end + end + end + + end + + end +end diff --git a/chef/lib/chef/node.rb b/chef/lib/chef/node.rb index 824db99c2b..11ee5d471f 100644 --- a/chef/lib/chef/node.rb +++ b/chef/lib/chef/node.rb @@ -21,7 +21,6 @@ require 'chef/mixin/check_helper' require 'chef/mixin/params_validate' require 'chef/mixin/from_file' require 'chef/couchdb' -require 'chef/queue' require 'chef/run_list' require 'chef/node/attribute' require 'extlib' @@ -30,14 +29,14 @@ require 'json' class Chef class Node - attr_accessor :attribute, :recipe_list, :couchdb_rev, :run_state, :run_list, :override, :default + attr_accessor :attribute, :recipe_list, :couchdb_rev, :couchdb_id, :run_state, :run_list, :override, :default include Chef::Mixin::CheckHelper include Chef::Mixin::FromFile include Chef::Mixin::ParamsValidate DESIGN_DOCUMENT = { - "version" => 8, + "version" => 9, "language" => "javascript", "views" => { "all" => { @@ -120,7 +119,7 @@ class Chef } # Create a new Chef::Node object. - def initialize() + def initialize @name = nil @attribute = Mash.new @@ -129,6 +128,7 @@ class Chef @run_list = Chef::RunList.new @couchdb_rev = nil + @couchdb_id = nil @couchdb = Chef::CouchDB.new @run_state = { :template_cache => Hash.new, @@ -279,53 +279,32 @@ class Chef @run_list.detect { |r| r == item } ? true : false end - # Turns the node into an object that we can index. I apologize up front for the - # super confusion that is the recursive index_flatten hash, which comes up next. - # Faith, young one, faith. - # - # === Returns - # index_hash<Hash>:: A flattened hash of all the nodes attributes, suitable for indexing. - def to_index - index_hash = { - "index_name" => "node", - "id" => "node_#{@name}", - "name" => @name, - } - @attribute.each do |key, value| - if value.kind_of?(Hash) || value.kind_of?(Mash) - index_flatten_hash(key, value).each do |to_index| - to_index.each do |nk, nv| - index_hash[nk] = nv - end - end + # Set an attribute based on the missing method. If you pass an argument, we'll use that + # to set the attribute values. Otherwise, we'll wind up just returning the attributes + # value. + def method_missing(symbol, *args) + if args.length != 0 + @attribute[symbol] = args.length == 1 ? args[0] : args + else + if @attribute.has_key?(symbol) + @attribute[symbol] else - index_hash[key] = value + raise ArgumentError, "Attribute #{symbol.to_s} is not defined!" end end - index_hash["recipe"] = @run_list.recipes if @run_list.recipes.length > 0 + end + + # Transform the node to a Hash + def to_hash + index_hash = @attribute + index_hash["chef_type"] = "node" + index_hash["name"] = @name + index_hash["recipes"] = @run_list.recipes if @run_list.recipes.length > 0 index_hash["roles"] = @run_list.roles if @run_list.roles.length > 0 index_hash["run_list"] = @run_list.run_list if @run_list.run_list.length > 0 index_hash end - # Ah, song of my heart, index_flatten_hash. This method flattens a hash in preparation - # for indexing, by appending the name of it's parent to a current key with an _. Hence, - # node[:bar][:baz] = 'monkey' becomes bar_baz:monkey. - # - # === Returns - # results<Array>:: An array of hashes with one element. - def index_flatten_hash(parent_name, hash) - results = Array.new - hash.each do |k, v| - if v.kind_of?(Hash) || v.kind_of?(Mash) - results << index_flatten_hash("#{parent_name}_#{k}", v) - else - results << { "#{parent_name}_#{k}", v } - end - end - results.flatten - end - # Serialize this object as a hash def to_json(*a) result = { @@ -358,13 +337,15 @@ class Chef o["recipes"].each { |r| node.recipes << r } end node.couchdb_rev = o["_rev"] if o.has_key?("_rev") + node.couchdb_id = o["_id"] if o.has_key?("_id") node end # List all the Chef::Node objects in the CouchDB. If inflate is set to true, you will get # the full list of all Nodes, fully inflated. def self.list(inflate=false) - rs = Chef::CouchDB.new.list("nodes", inflate) + couchdb = Chef::CouchDB.new + rs = couchdb.list("nodes", inflate) if inflate rs["rows"].collect { |r| r["value"] } else @@ -374,25 +355,25 @@ class Chef # Load a node by name from CouchDB def self.load(name) - Chef::CouchDB.new.load("node", name) + couchdb = Chef::CouchDB.new + couchdb.load("node", name) end # Remove this node from the CouchDB def destroy - Chef::Queue.send_msg(:queue, :remove, self) @couchdb.delete("node", @name, @couchdb_rev) end # Save this node to the CouchDB def save - Chef::Queue.send_msg(:queue, :index, self) results = @couchdb.store("node", @name, self) @couchdb_rev = results["rev"] end - + # Set up our CouchDB design document def self.create_design_document - Chef::CouchDB.new.create_design_document("nodes", DESIGN_DOCUMENT) + couchdb = Chef::CouchDB.new + couchdb.create_design_document("nodes", DESIGN_DOCUMENT) end # As a string diff --git a/chef/lib/chef/openid_registration.rb b/chef/lib/chef/openid_registration.rb index 0939746494..dbe1b1d45c 100644 --- a/chef/lib/chef/openid_registration.rb +++ b/chef/lib/chef/openid_registration.rb @@ -178,4 +178,4 @@ class Chef end end -end
\ No newline at end of file +end diff --git a/chef/lib/chef/platform.rb b/chef/lib/chef/platform.rb index e5b0be0d1d..fa24d6a5c8 100644 --- a/chef/lib/chef/platform.rb +++ b/chef/lib/chef/platform.rb @@ -142,7 +142,6 @@ class Chef if pmap.has_key?(rtkey) pmap[rtkey] else - Chef::Log.error("#{rtkey.inspect} #{pmap.inspect}") raise( ArgumentError, "Cannot find a provider for #{resource_type} on #{platform} version #{version}" diff --git a/chef/lib/chef/provider/http_request.rb b/chef/lib/chef/provider/http_request.rb index a93c1c062a..447df10b0a 100644 --- a/chef/lib/chef/provider/http_request.rb +++ b/chef/lib/chef/provider/http_request.rb @@ -25,7 +25,7 @@ class Chef attr_accessor :rest def load_current_resource - @rest = Chef::REST.new(@new_resource.url) + @rest = Chef::REST.new(@new_resource.url, nil, nil) end # Send a GET request to @new_resource.url, with ?message=@new_resource.message @@ -99,4 +99,4 @@ class Chef end end -end
\ No newline at end of file +end diff --git a/chef/lib/chef/provider/remote_file.rb b/chef/lib/chef/provider/remote_file.rb index 3aff5650ff..9b5c9db04a 100644 --- a/chef/lib/chef/provider/remote_file.rb +++ b/chef/lib/chef/provider/remote_file.rb @@ -95,7 +95,7 @@ class Chef begin uri = URI.parse(source) if uri.absolute - r = Chef::REST.new(source) + r = Chef::REST.new(source, nil, nil) Chef::Log.debug("Downloading from absolute URI: #{source}") r.get_rest(source, true).open end diff --git a/chef/lib/chef/provider/template.rb b/chef/lib/chef/provider/template.rb index e3cbbb47ed..7f5b87346d 100644 --- a/chef/lib/chef/provider/template.rb +++ b/chef/lib/chef/provider/template.rb @@ -34,7 +34,6 @@ class Chef include Chef::Mixin::FindPreferredFile def action_create - Chef::Log.debug(@node.run_state.inspect) raw_template_file = nil cookbook_name = @new_resource.cookbook || @new_resource.cookbook_name @@ -137,4 +136,4 @@ class Chef end end -end
\ No newline at end of file +end diff --git a/chef/lib/chef/queue.rb b/chef/lib/chef/queue.rb deleted file mode 100644 index 72883b4bb2..0000000000 --- a/chef/lib/chef/queue.rb +++ /dev/null @@ -1,145 +0,0 @@ -# -# Author:: Adam Jacob (<adam@opscode.com>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/mixin/params_validate' -require 'json' -require 'stomp' - -class Chef - class Queue - - @client = nil - @queue_retry_delay = Chef::Config[:queue_retry_delay] - @queue_retry_count = Chef::Config[:queue_retry_count] - - class << self - include Chef::Mixin::ParamsValidate - - def connect - queue_user = Chef::Config[:queue_user] - queue_password = Chef::Config[:queue_password] - queue_host = Chef::Config[:queue_host] - queue_port = Chef::Config[:queue_port] - queue_retries = 1 unless queue_retries - - # Connection.open(login = "", passcode = "", host='localhost', port=61613, reliable=FALSE, reconnectDelay=5) - @client = Stomp::Connection.open(queue_user, queue_password, queue_host, queue_port, false) - - rescue Errno::ECONNREFUSED - Chef::Log.error("Connection refused connecting to stomp queue at #{queue_host}:#{queue_port}, retry #{queue_retries}/#{@queue_retry_count}") - sleep(@queue_retry_delay) - retry if (queue_retries += 1) < @queue_retry_count - raise Errno::ECONNREFUSED, "Connection refused connecting to stomp queue at #{queue_host}:#{queue_port}, giving up" - rescue Timeout::Error - Chef::Log.error("Timeout connecting to stomp queue at #{queue_host}:#{queue_port}, retry #{queue_retries}/#{@queue_retry_count}") - sleep(@queue_retry_delay) - retry if (queue_retries += 1) < @queue_retry_count - raise Timeout::Error, "Timeout connecting to stomp queue at #{queue_host}:#{queue_port}, giving up" - else - queue_retries = 1 # reset the number of retries on success - end - - def make_url(type, name) - validate( - { - :queue_type => type.to_sym, - :queue_name => name.to_sym, - }, - { - :queue_type => { - :equal_to => [ :topic, :queue ], - }, - :queue_name => { - :kind_of => [ String, Symbol ], - } - } - ) - if Chef::Config[:queue_prefix] - queue_prefix = Chef::Config[:queue_prefix] - queue_url = "/#{type}/#{queue_prefix}/chef/#{name}" - else - queue_url = "/#{type}/chef/#{name}" - end - queue_url - end - - def subscribe(type, name) - queue_url = make_url(type, name) - Chef::Log.debug("Subscribing to #{queue_url}") - connect if @client == nil - @client.subscribe(queue_url) - end - - def send_msg(type, name, msg) - queue_retries = 1 unless queue_retries - validate( - { - :message => msg, - }, - { - :message => { - :respond_to => :to_json - } - } - ) - queue_url = make_url(type, name) - json = msg.to_json - connect if @client == nil - Chef::Log.debug("Sending to #{queue_url}: #{json}") - begin - @client.send(queue_url, json) - rescue Errno::EPIPE - Chef::Log.debug("Lost connection to stomp queue, reconnecting") - connect - retry if (queue_retries += 1) < @queue_retry_count - raise Errno::EPIPE, "Lost connection to stomp queue, giving up" - else - queue_retries = 1 # reset the number of retries on success - end - end - - def receive_msg - connect if @client == nil - begin - raw_msg = @client.receive() - Chef::Log.debug("Received Message from #{raw_msg.headers["destination"]} containing: #{raw_msg.body}") - rescue - Chef::Log.debug("Received nil message from stomp, retrying") - retry - end - msg = JSON.parse(raw_msg.body) - return msg, raw_msg.headers - end - - def poll_msg - connect if @client == nil - raw_msg = @client.poll() - if raw_msg - msg = JSON.parse(raw_msg.body) - else - nil - end - end - - def disconnect - raise ArgumentError, "You must call connect before you can disconnect!" unless @client - @client.disconnect - end - end - end -end diff --git a/chef/lib/chef/recipe.rb b/chef/lib/chef/recipe.rb index 5421c08b57..d3e64d0737 100644 --- a/chef/lib/chef/recipe.rb +++ b/chef/lib/chef/recipe.rb @@ -23,7 +23,6 @@ require 'chef/mixin/language' require 'chef/resource_collection' require 'chef/cookbook_loader' require 'chef/rest' -require 'chef/search/result' class Chef class Recipe @@ -89,21 +88,6 @@ class Chef @collection.resources(*args) end - def search(type, query, attributes=[], &block) - Chef::Log.debug("Searching #{type} index with #{query}") - r = Chef::REST.new(Chef::Config[:search_url]) - - results = r.get_rest("search/#{type}?q=#{query}&a=#{attributes.join(',')}") - Chef::Log.debug("Searching #{type} index with #{query} returned #{results.length} entries") - if block - results.each do |sr| - block.call(sr) - end - else - results - end - end - # Sets a tag, or list of tags, for this node. Syntactic sugar for # @node[:tags]. # diff --git a/chef/lib/chef/rest.rb b/chef/lib/chef/rest.rb index 229fac2cdc..0e40068061 100644 --- a/chef/lib/chef/rest.rb +++ b/chef/lib/chef/rest.rb @@ -1,7 +1,8 @@ # # Author:: Adam Jacob (<adam@opscode.com>) # Author:: Thom May (<thom@clearairturbulence.org>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. +# Author:: Nuo Yan (<nuo@opscode.com>) +# Copyright:: Copyright (c) 2009 Opscode, Inc. # License:: Apache License, Version 2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +25,9 @@ require 'uri' require 'json' require 'tempfile' require 'singleton' +require 'mixlib/auth/signedheaderauth' + +include Mixlib::Auth::SignedHeaderAuth class Chef class REST @@ -32,47 +36,57 @@ class Chef include Singleton end - attr_accessor :url, :cookies + attr_accessor :url, :cookies, :signing_key - def initialize(url) + def initialize(url, client_name=Chef::Config[:node_name], signing_key=Chef::Config[:client_key]) @url = url @cookies = CookieJar.instance + @client_name = client_name + if signing_key + @signing_key = load_signing_key(signing_key) + else + @signing_key = nil + end + end + + def load_signing_key(key) + if File.exists?(key) && File.readable?(key) + IO.read(key) + else + raise Chef::Exceptions::PrivateKeyMissing, "I cannot find #{key}, which you told me to use to sign requests!" + end end - # Register for an OpenID - def register(user, pass, validation_token=nil) - Chef::Log.debug("Registering #{user} for an openid") - registration = nil + # Register the client + def register(name=Chef::Config[:node_name], destination=Chef::Config[:client_key]) + + if File.exists?(destination) + raise Chef::Exceptions::CannotWritePrivateKey, "I cannot write your private key to #{destination} - check permissions?" unless File.writable?(destination) + end + + # First, try and create a new registration begin - registration = get_rest("registrations/#{user}") - rescue Net::HTTPServerException => e - unless e.message =~ /^404/ - raise e - end + Chef::Log.info("Registering API Client #{name}") + response = post_rest("clients", {:clientname => name}) + rescue Net::HTTPServerException + # If that fails, go ahead and try and update it + response = put_rest("clients/#{name}", { :clientname => name, :private_key => true }) end - unless registration - post_rest( - "registrations", - { - :id => user, - :password => pass, - :validation_token => validation_token - } - ) + + Chef::Log.debug("Registration response: #{response.inspect}") + + raise Chef::Exceptions::CannotWritePrivateKey, "The response from the server did not include a private key!" unless response.has_key?("private_key") + + begin + # Write out the private key + file = File.open(destination, "w") + file.print(response["private_key"]) + file.close + rescue + raise Chef::Exceptions::CannotWritePrivateKey, "I cannot write your private key to #{destination}" end - end - - # Authenticate - def authenticate(user, pass) - Chef::Log.debug("Authenticating #{user} via openid") - response = post_rest('openid/consumer/start', { - "openid_identifier" => "#{Chef::Config[:openid_url]}/openid/server/node/#{user}", - "submit" => "Verify" - }) - post_rest( - "#{Chef::Config[:openid_url]}#{response["action"]}", - { "password" => pass } - ) + + true end # Send an HTTP GET request to the path @@ -81,23 +95,23 @@ class Chef # path:: The path to GET # raw:: Whether you want the raw body returned, or JSON inflated. Defaults # to JSON inflated. - def get_rest(path, raw=false) - run_request(:GET, create_url(path), false, 10, raw) + def get_rest(path, raw=false, headers={}) + run_request(:GET, create_url(path), headers, false, 10, raw) end # Send an HTTP DELETE request to the path - def delete_rest(path) - run_request(:DELETE, create_url(path)) + def delete_rest(path, headers={}) + run_request(:DELETE, create_url(path), headers) end # Send an HTTP POST request to the path - def post_rest(path, json) - run_request(:POST, create_url(path), json) + def post_rest(path, json, headers={}) + run_request(:POST, create_url(path), headers, json) end # Send an HTTP PUT request to the path - def put_rest(path, json) - run_request(:PUT, create_url(path), json) + def put_rest(path, json, headers={}) + run_request(:PUT, create_url(path), headers, json) end def create_url(path) @@ -108,6 +122,18 @@ class Chef end end + def sign_request(http_method, private_key, user_id, body = "", host="localhost") + #body = "" if body == false + timestamp = Time.now.utc.iso8601 + sign_obj = Mixlib::Auth::SignedHeaderAuth.signing_object( + :http_method=>http_method, + :body=>body, + :user_id=>user_id, + :timestamp=>timestamp) + signed = sign_obj.sign(private_key).merge({:host => host}) + signed.inject({}){|memo, kv| memo["#{kv[0].to_s.upcase}"] = kv[1];memo} + end + # Actually run an HTTP request. First argument is the HTTP method, # which should be one of :GET, :PUT, :POST or :DELETE. Next is the # URL, then an object to include in the body (which will be converted with @@ -117,7 +143,8 @@ class Chef # the helper methods (get_rest, post_rest, etc.) # # Will return the body of the response on success. - def run_request(method, url, data=false, limit=10, raw=false) + def run_request(method, url, headers={}, data=false, limit=10, raw=false) + http_retry_delay = Chef::Config[:http_retry_delay] http_retry_count = Chef::Config[:http_retry_count] @@ -129,21 +156,35 @@ class Chef if Chef::Config[:ssl_verify_mode] == :verify_none http.verify_mode = OpenSSL::SSL::VERIFY_NONE end - if File.exists?(Chef::Config[:ssl_client_cert]) + if Chef::Config[:ssl_client_cert] && File.exists?(Chef::Config[:ssl_client_cert]) http.cert = OpenSSL::X509::Certificate.new(File.read(Chef::Config[:ssl_client_cert])) http.key = OpenSSL::PKey::RSA.new(File.read(Chef::Config[:ssl_client_key])) end end + http.read_timeout = Chef::Config[:rest_timeout] - headers = Hash.new + unless raw - headers = { + headers = headers.merge({ 'Accept' => "application/json", - } + }) end + if @cookies.has_key?("#{url.host}:#{url.port}") headers['Cookie'] = @cookies["#{url.host}:#{url.port}"] end + + json_body = data ? data.to_json : nil + + if @signing_key + Chef::Log.debug("Signing the request as #{@client_name}") + if json_body + headers.merge!(sign_request(method, OpenSSL::PKey::RSA.new(@signing_key), @client_name, json_body, "#{url.host}:#{url.port}")) + else + headers.merge!(sign_request(method, OpenSSL::PKey::RSA.new(@signing_key), @client_name, "", "#{url.host}:#{url.port}")) + end + end + req = nil case method when :GET @@ -152,12 +193,16 @@ class Chef req = Net::HTTP::Get.new(req_path, headers) when :POST headers["Content-Type"] = 'application/json' if data - req = Net::HTTP::Post.new(url.path, headers) - req.body = data.to_json if data + req_path = "#{url.path}" + req_path << "?#{url.query}" if url.query + req = Net::HTTP::Post.new(req_path, headers) + req.body = json_body if json_body when :PUT headers["Content-Type"] = 'application/json' if data - req = Net::HTTP::Put.new(url.path, headers) - req.body = data.to_json if data + req_path = "#{url.path}" + req_path << "?#{url.query}" if url.query + req = Net::HTTP::Put.new(req_path, headers) + req.body = json_body if json_body when :DELETE req_path = "#{url.path}" req_path << "?#{url.query}" if url.query @@ -165,6 +210,8 @@ class Chef else raise ArgumentError, "You must provide :GET, :PUT, :POST or :DELETE as the method" end + + Chef::Log.debug("Sending HTTP Request via #{req.method} to #{url.host}:#{url.port}/#{req.path}") # Optionally handle HTTP Basic Authentication req.basic_auth(url.user, url.password) if url.user @@ -199,7 +246,8 @@ class Chef end response end - rescue Errno::ECONNREFUSED + + rescue Errno::ECONNREFUSED => e Chef::Log.error("Connection refused connecting to #{url.host}:#{url.port} for #{req.path} #{http_retries}/#{http_retry_count}") sleep(http_retry_delay) retry if (http_retries += 1) < http_retry_count @@ -216,7 +264,8 @@ class Chef @cookies["#{url.host}:#{url.port}"] = res['set-cookie'] end if res['content-type'] =~ /json/ - JSON.parse(res.body) + response_body = res.body.chomp + JSON.parse(response_body) else if raw tf @@ -228,11 +277,11 @@ class Chef if res['set-cookie'] @cookies["#{url.host}:#{url.port}"] = res['set-cookie'] end - run_request(:GET, create_url(res['location']), false, limit - 1, raw) + run_request(:GET, create_url(res['location']), {}, false, limit - 1, raw) else res.error! end end - + end end diff --git a/chef/lib/chef/role.rb b/chef/lib/chef/role.rb index 506a197416..461db69eaa 100644 --- a/chef/lib/chef/role.rb +++ b/chef/lib/chef/role.rb @@ -30,7 +30,7 @@ class Chef include Chef::Mixin::ParamsValidate DESIGN_DOCUMENT = { - "version" => 3, + "version" => 6, "language" => "javascript", "views" => { "all" => { @@ -50,21 +50,22 @@ class Chef } } EOJS - }, - }, + } + } } - attr_accessor :couchdb_rev + attr_accessor :couchdb_rev, :couchdb_id # Create a new Chef::Role object. - def initialize() + def initialize @name = '' @description = '' @default_attributes = Mash.new @override_attributes = Mash.new @recipes = Array.new @couchdb_rev = nil - @couchdb = Chef::CouchDB.new + @couchdb_id = nil + @couchdb = Chef::CouchDB.new end def name(arg=nil) @@ -139,13 +140,15 @@ class Chef role.override_attributes(o["override_attributes"]) role.recipes(o["recipes"]) role.couchdb_rev = o["_rev"] if o.has_key?("_rev") + role.couchdb_id = o["_id"] if o.has_key?("_id") role end # List all the Chef::Role objects in the CouchDB. If inflate is set to true, you will get # the full list of all Roles, fully inflated. def self.list(inflate=false) - rs = Chef::CouchDB.new.list("roles", inflate) + couchdb = Chef::CouchDB.new + rs = couchdb.list("roles", inflate) if inflate rs["rows"].collect { |r| r["value"] } else @@ -155,7 +158,8 @@ class Chef # Load a role by name from CouchDB def self.load(name) - Chef::CouchDB.new.load("role", name) + couchdb = Chef::CouchDB.new + couchdb.load("role", name) end # Remove this role from the CouchDB @@ -186,7 +190,8 @@ class Chef # Set up our CouchDB design document def self.create_design_document - Chef::CouchDB.new.create_design_document("roles", DESIGN_DOCUMENT) + couchdb = Chef::CouchDB.new + couchdb.create_design_document("roles", DESIGN_DOCUMENT) end # As a string @@ -212,6 +217,7 @@ class Chef # Sync all the json roles with couchdb from disk def self.sync_from_disk_to_couchdb + couchdb = Chef::CouchDB.new Dir[File.join(Chef::Config[:role_path], "*.json")].each do |role_file| short_name = File.basename(role_file, ".json") Chef::Log.warn("Loading #{short_name}") @@ -220,7 +226,7 @@ class Chef couch_role = Chef::Role.load(short_name) r.couchdb_rev = couch_role.couchdb_rev Chef::Log.debug("Replacing role #{short_name} with data from #{role_file}") - rescue Net::HTTPServerException + rescue Chef::Exceptions::CouchDBNotFound Chef::Log.debug("Creating role #{short_name} with data from #{role_file}") end r.save diff --git a/chef/lib/chef/run_list.rb b/chef/lib/chef/run_list.rb index 6eb0157580..f565185633 100644 --- a/chef/lib/chef/run_list.rb +++ b/chef/lib/chef/run_list.rb @@ -111,7 +111,8 @@ class Chef self end - def expand(from='server') + def expand(from='server', couchdb=nil) + couchdb = couchdb ? couchdb : Chef::CouchDB.new recipes = Array.new default_attrs = Mash.new override_attrs = Mash.new @@ -132,7 +133,7 @@ class Chef role = r.get_rest("roles/#{name}") elsif from == 'couchdb' # Load the role from couchdb - role = Chef::Role.load(name) + role = Chef::Role.load(name, couchdb) end role.recipes.each { |r| recipes << r unless recipes.include?(r) } default_attrs = Chef::Mixin::DeepMerge.merge(default_attrs, role.default_attributes) diff --git a/chef/lib/chef/search.rb b/chef/lib/chef/search.rb deleted file mode 100644 index 0d0c176921..0000000000 --- a/chef/lib/chef/search.rb +++ /dev/null @@ -1,88 +0,0 @@ -# -# Author:: Adam Jacob (<adam@opscode.com>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/search/result' -require 'ferret' - -class Chef - class Search - - attr_reader :index - - def initialize - @index = Ferret::Index::Index.new(:path => Chef::Config[:search_index_path]) - end - - def search(type, query="*", attributes=[], &block) - search_query = build_search_query(type, query) - start_time = Time.now - results = [] - block ||= lambda { |b| b } - - @index.search_each(search_query, :limit => :all) do |id, score| - results << block.call(build_hash(@index.doc(id))) - end - - Chef::Log.debug("Search #{search_query} complete in #{Time.now - start_time} seconds") - - attributes.empty? ? results : filter_by_attributes(results,attributes) - end - - def filter_by_attributes(results, attributes) - results.collect do |r| - nr = Hash.new - nr[:index_name] = r[:index_name] - nr[:id] = r[:id] - attributes.each do |attrib| - if r.has_key?(attrib) - nr[attrib] = r[attrib] - end - end - nr - end - end - - private :filter_by_attributes - - def list_indexes - indexes = Hash.new - @index.search_each("index_name:*", :limit => :all) do |id, score| - indexes[@index.doc(id)["index_name"]] = true - end - indexes.keys - end - - def has_index?(index) - list_indexes.detect { |i| i == index } - end - - private - def build_search_query(type, query) - query = "id:*" if query == '*' - "index_name:#{type} AND (#{query})" - end - - def build_hash(doc) - result = Chef::Search::Result.new - doc.fields.each do |f| - result[f] = doc[f] - end - result - end - end -end diff --git a/chef/lib/chef/search/query.rb b/chef/lib/chef/search/query.rb new file mode 100644 index 0000000000..be5e15269a --- /dev/null +++ b/chef/lib/chef/search/query.rb @@ -0,0 +1,63 @@ +# +# Author:: Adam Jacob (<adam@opscode.com>) +# Copyright:: Copyright (c) 2008 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/config' +require 'chef/node' +require 'chef/role' +require 'chef/data_bag' +require 'chef/data_bag_item' + +class Chef + class Search + class Query + def initialize(url=nil) + url ||= Chef::Config[:search_url] + @rest = Chef::REST.new(url) + end + + # Search Solr for objects of a given type, for a given query. If you give + # it a block, it will handle the paging for you dynamically. + def search(type, query="*:*", sort=nil, start=0, rows=20, &block) + unless type.kind_of?(String) || type.kind_of?(Symbol) + raise ArgumentError, "Type must be a string or a symbol!" + end + + response = @rest.get_rest("search/#{type}?q=#{escape(query)}&sort=#{escape(sort)}&start=#{escape(start)}&rows=#{escape(rows)}") + if block + response["rows"].each { |o| block.call(o) } + unless (response["start"] + response["rows"].length) >= response["total"] + nstart = response["start"] + rows + search(type, query, sort, nstart, rows, &block) + end + true + else + [ response["rows"], response["start"], response["total"] ] + end + end + + private + # escapes a query key/value for http + # Thanks to RSolr! + def escape(s) + s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) { + '%'+$1.unpack('H2'*$1.size).join('%').upcase + }.tr(' ', '+') + end + end + end +end diff --git a/chef/lib/chef/search/result.rb b/chef/lib/chef/search/result.rb deleted file mode 100644 index 97d35132a8..0000000000 --- a/chef/lib/chef/search/result.rb +++ /dev/null @@ -1,64 +0,0 @@ -# -# Author:: Adam Jacob (<adam@opscode.com>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'extlib' - -class Chef - class Search - class Result - - def initialize - proc = lambda do |h,k| - newhash = Mash.new(&proc) - h.each do |pk, pv| - rx = /^#{k.to_s}_/ - if pk =~ rx - newhash[ pk.gsub(rx,'') ] = pv - end - end - newhash - end - @internal = Mash.new(&proc) - end - - def method_missing(symbol, *args, &block) - @internal.send(symbol, *args, &block) - end - - # Serialize this object as a hash - def to_json(*a) - result = { - 'json_class' => self.class.name, - 'results' => @internal - } - result.to_json(*a) - end - - # Create a Chef::Search::Result from JSON - def self.json_create(o) - result = self.new - o['results'].each do |k,v| - result[k] = v - end - result - end - end - end -end - - diff --git a/chef/lib/chef/search_index.rb b/chef/lib/chef/search_index.rb deleted file mode 100644 index 6a76595aa1..0000000000 --- a/chef/lib/chef/search_index.rb +++ /dev/null @@ -1,77 +0,0 @@ -# -# Author:: Adam Jacob (<adam@opscode.com>) -# Copyright:: Copyright (c) 2008 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'ferret' - -class Chef - class SearchIndex - - attr_reader :index - - def initialize - @index = Ferret::Index::Index.new( - :path => Chef::Config[:search_index_path], - :key => [ :id ] - ) - end - - def add(new_object) - index_hash = create_index_object(new_object) - Chef::Log.debug("Indexing #{index_hash[:index_name]} with #{index_hash.inspect}") - @index.add_document(index_hash) - end - - def create_index_object(new_object) - index_hash = nil - - if new_object.respond_to?(:to_index) - index_hash = new_object.to_index - elsif new_object.kind_of?(Hash) - index_hash = new_object - else - raise Chef::Exceptions::SearchIndex, "Cannot transform argument to a Hash!" - end - - unless index_hash.has_key?(:index_name) || index_hash.has_key?("index_name") - raise Chef::Exceptions::SearchIndex, "Cannot index without an index_name key: #{index_hash.inspect}" - end - - unless index_hash.has_key?(:id) || index_hash.has_key?("id") - raise Chef::Exceptions::SearchIndex, "Cannot index without an id key: #{index_hash.inspect}" - end - - sanitized_hash = Hash.new - index_hash.each do |k,v| - sanitized_hash[k.to_sym] = v - end - - sanitized_hash - end - - def delete(index_obj) - to_delete = create_index_object(index_obj) - Chef::Log.debug("Removing #{to_delete.inspect} from the #{to_delete[:index_name]} index") - @index.delete(to_delete[:id]) - end - - def commit - @index.commit - end - - end -end diff --git a/chef/lib/chef/streaming_cookbook_uploader.rb b/chef/lib/chef/streaming_cookbook_uploader.rb new file mode 100644 index 0000000000..73c9034b42 --- /dev/null +++ b/chef/lib/chef/streaming_cookbook_uploader.rb @@ -0,0 +1,178 @@ +require 'net/http' +require 'mixlib/auth/signedheaderauth' +require 'openssl' + +# inspired by/cargo-culted from http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html +# TODO: confirm that code is public domain +class Chef + class StreamingCookbookUploader + + DefaultHeaders = { 'accept' => 'application/json' } + + class << self + + def post(to_url, user_id, secret_key_filename, params = {}, headers = {}) + make_request(:post, to_url, user_id, secret_key_filename, params, headers) + end + + def put(to_url, user_id, secret_key_filename, params = {}, headers = {}) + make_request(:put, to_url, user_id, secret_key_filename, params, headers) + end + + def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {}) + boundary = '----RubyMultipartClient' + rand(1000000).to_s + 'ZZZZZ' + parts = [] + content_file = nil + content_body = nil + + timestamp = Time.now.utc.iso8601 + secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename)) + + unless params.nil? || params.empty? + params.each do |key, value| + if value.kind_of?(File) + content_file = value + filepath = value.path + filename = File.basename(filepath) + parts << StringPart.new( "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" + + "Content-Type: application/octet-stream\r\n\r\n") + parts << StreamPart.new(value, File.size(filepath)) + parts << StringPart.new("\r\n") + else + content_body = value.to_s + parts << StringPart.new( "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n") + parts << StringPart.new(content_body + "\r\n") + end + end + parts << StringPart.new("--" + boundary + "--\r\n") + end + + body_stream = MultipartStream.new(parts) + + timestamp = Time.now.utc.iso8601 + + Chef::Log.logger.debug("Signing: method: #{http_verb}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}") + + signing_options = { + :http_method=>http_verb, + :user_id=>user_id, + :timestamp=>timestamp} + (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || "")) + + headers.merge!(Mixlib::Auth::SignedHeaderAuth.signing_object(signing_options).sign(secret_key)) + + content_file.rewind if content_file + + # net/http doesn't like symbols for header keys, so we'll to_s each one just in case + headers = DefaultHeaders.merge(Hash[*headers.map{ |k,v| [k.to_s, v] }.flatten]) + + url = URI.parse(to_url) + req = case http_verb + when :put + Net::HTTP::Put.new(url.path, headers) + when :post + Net::HTTP::Post.new(url.path, headers) + end + req.content_length = body_stream.size + req.content_type = 'multipart/form-data; boundary=' + boundary unless parts.empty? + req.body_stream = body_stream + res = Net::HTTP.new(url.host, url.port).start {|http| http.request(req) } + + # alias status to code and to_s to body for test purposes + # TODO: stop the following madness! + class << res + alias :to_s :body + + # BUGBUG this makes the response compatible with what respsonse_steps expects to test headers (response.headers[] -> response[]) + def headers + self + end + + def status + code.to_i + end + end + res + end + + end + + class StreamPart + def initialize(stream, size) + @stream, @size = stream, size + end + + def size + @size + end + + # read the specified amount from the stream + def read(offset, how_much) + @stream.read(how_much) + end + end + + class StringPart + def initialize(str) + @str = str + end + + def size + @str.length + end + + # read the specified amount from the string startiung at the offset + def read(offset, how_much) + @str[offset, how_much] + end + end + + class MultipartStream + def initialize(parts) + @parts = parts + @part_no = 0 + @part_offset = 0 + end + + def size + @parts.inject(0) {|size, part| size + part.size} + end + + def read(how_much) + return nil if @part_no >= @parts.size + + how_much_current_part = @parts[@part_no].size - @part_offset + + how_much_current_part = if how_much_current_part > how_much + how_much + else + how_much_current_part + end + + how_much_next_part = how_much - how_much_current_part + + current_part = @parts[@part_no].read(@part_offset, how_much_current_part) + + # recurse into the next part if the current one was not large enough + if how_much_next_part > 0 + @part_no += 1 + @part_offset = 0 + next_part = read(how_much_next_part) + current_part + if next_part + next_part + else + '' + end + else + @part_offset += how_much_current_part + current_part + end + end + end + + end + + +end |