diff options
39 files changed, 3518 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore index 3bc83b0ed8..6ff48eeb33 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ erl_crash.dump features/data/repo/checksums/ *.rbc .rvmrc +.bundle +chef-expander/conf/chef-expander.rb + diff --git a/chef-expander/.gitignore b/chef-expander/.gitignore new file mode 100644 index 0000000000..faeec99a25 --- /dev/null +++ b/chef-expander/.gitignore @@ -0,0 +1,12 @@ +.bundle +.autotest +coverage +doc +.DS_Store +*.swp +*.swo +erl_crash.dump +*.rake_tasks~ +.idea +*.rbc +conf/chef-expander.rb diff --git a/chef-expander/Gemfile b/chef-expander/Gemfile new file mode 100644 index 0000000000..1edc434e4d --- /dev/null +++ b/chef-expander/Gemfile @@ -0,0 +1,15 @@ +## Chef Expander ## +source "http://rubygems.org" + +gem "amqp", "~> 0.6.7" +gem "eventmachine", '~> 0.12.10' +gem "em-http-request", "~> 0.2.11" +gem 'yajl-ruby', "~> 0.7.7" +gem 'mixlib-log', "~> 1.1.0" +gem 'uuidtools', "~> 2.1.1" +gem 'bunny', '~> 0.6.0' +gem 'fast_xs', "~> 0.7.3" +gem 'highline', '~> 1.6.1' +gem 'rake', '~> 0.8.7' +gem 'rspec', '~> 1.3.0' +gem 'word-salad', '~> 1.0.0' diff --git a/chef-expander/Gemfile.lock b/chef-expander/Gemfile.lock new file mode 100644 index 0000000000..647b3b5f3d --- /dev/null +++ b/chef-expander/Gemfile.lock @@ -0,0 +1,36 @@ +GEM + remote: http://rubygems.org/ + specs: + addressable (2.1.1) + amqp (0.6.7) + eventmachine (>= 0.12.4) + bunny (0.6.0) + em-http-request (0.2.11) + addressable (>= 2.0.0) + eventmachine (>= 0.12.9) + eventmachine (0.12.10) + fast_xs (0.7.3) + highline (1.6.1) + mixlib-log (1.1.0) + rake (0.8.7) + rspec (1.3.0) + uuidtools (2.1.1) + word-salad (1.0.0) + yajl-ruby (0.7.7) + +PLATFORMS + ruby + +DEPENDENCIES + amqp (~> 0.6.7) + bunny (~> 0.6.0) + em-http-request (~> 0.2.11) + eventmachine (~> 0.12.10) + fast_xs (~> 0.7.3) + highline (~> 1.6.1) + mixlib-log (~> 1.1.0) + rake (~> 0.8.7) + rspec (~> 1.3.0) + uuidtools (~> 2.1.1) + word-salad (~> 1.0.0) + yajl-ruby (~> 0.7.7) diff --git a/chef-expander/LICENSE b/chef-expander/LICENSE new file mode 100644 index 0000000000..11069edd79 --- /dev/null +++ b/chef-expander/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/chef-expander/README.rdoc b/chef-expander/README.rdoc new file mode 100644 index 0000000000..c287d63fce --- /dev/null +++ b/chef-expander/README.rdoc @@ -0,0 +1,20 @@ += Chef Expander + +== What's This? +Chef Expander replaces the chef-solr-indexer daemon that was included with Chef 0.8 and 0.9 + +== Dependencies + +* bunny +* yajl +* eventmachine +* em-http-request +* amqp +* highline + +== Monitoring With Nagios +A Nagios plugin to monitor queue backlog is included in scripts/ directory as <tt>check_queue_size</tt> +To run it with the warning threshold at 250 messages and critical at 500 messages: + + check_queue_size -w 250 -c 500 + diff --git a/chef-expander/Rakefile b/chef-expander/Rakefile new file mode 100644 index 0000000000..01f2577669 --- /dev/null +++ b/chef-expander/Rakefile @@ -0,0 +1,68 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Copyright:: Copyright (c) 2010, 2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'rake/gempackagetask' +require 'rake/rdoctask' + +spec = eval(File.read("chef-expander.gemspec")) + +Rake::GemPackageTask.new(spec) do |pkg| + pkg.gem_spec = spec +end + +begin + require 'sdoc' + + Rake::RDocTask.new do |rdoc| + rdoc.title = "Chef Ruby API Documentation" + rdoc.main = "README.rdoc" + rdoc.options << '--fmt' << 'shtml' # explictly set shtml generator + rdoc.template = 'direct' # lighter template + rdoc.rdoc_files.include("README.rdoc", "LICENSE", "spec/tiny_server.rb", "lib/**/*.rb") + rdoc.rdoc_dir = "rdoc" + end +rescue LoadError + puts "sdoc is not available. (sudo) gem install sdoc to generate rdoc documentation." +end + +task :install => :package do + sh %{gem install pkg/chef-expander-#{Chef::Expander::VERSION} --no-rdoc --no-ri} +end + +begin + require 'spec/rake/spectask' + + Spec::Rake::SpecTask.new(:spec) do |spec| + spec.libs << 'lib' << 'spec' + spec.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""] + spec.spec_files = FileList['spec/**/*_spec.rb'] + end +rescue LoadError + desc "Rspec is not installed. `(sudo) gem install rspec` to run tests" + task :spec do + puts "Rspec is not installed. `(sudo) gem install rspec` to run tests" + exit 1 + end +end + +desc "install gem dependencies" +task :bundle do + sh("bundle install") +end + +task :default => :spec diff --git a/chef-expander/bin/chef-expander b/chef-expander/bin/chef-expander new file mode 100755 index 0000000000..8d2cdbba02 --- /dev/null +++ b/chef-expander/bin/chef-expander @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +require "rubygems" + +$:.unshift(File.dirname(__FILE__) + '/../lib/') + +require 'chef/expander' +require 'chef/expander/vnode_supervisor' + +Chef::Expander::VNodeSupervisor.start diff --git a/chef-expander/bin/chef-expander-cluster b/chef-expander/bin/chef-expander-cluster new file mode 100755 index 0000000000..789e6e81b8 --- /dev/null +++ b/chef-expander/bin/chef-expander-cluster @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "rubygems" + +$:.unshift(File.dirname(__FILE__) + '/../lib/') + +require 'chef/expander' +require 'chef/expander/cluster_supervisor' + +Chef::Expander::ClusterSupervisor.new.start diff --git a/chef-expander/bin/chef-expanderctl b/chef-expander/bin/chef-expanderctl new file mode 100755 index 0000000000..c1b201a7b2 --- /dev/null +++ b/chef-expander/bin/chef-expanderctl @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +require "rubygems" + +$:.unshift(File.dirname(__FILE__) + '/../lib/') + +require 'chef/expander' +require 'chef/expander/control' + +Chef::Expander::Control.run(ARGV) diff --git a/chef-expander/chef-expander.gemspec b/chef-expander/chef-expander.gemspec new file mode 100644 index 0000000000..158ce0e96e --- /dev/null +++ b/chef-expander/chef-expander.gemspec @@ -0,0 +1,33 @@ +$:.unshift(File.dirname(__FILE__) + '/lib') +require 'chef/expander/version' + +Gem::Specification.new do |s| + s.name = 'chef-expander' + s.version = Chef::Expander::VERSION + s.platform = Gem::Platform::RUBY + s.has_rdoc = true + s.extra_rdoc_files = ["README.rdoc", "LICENSE" ] + s.summary = "A systems integration framework, built to bring the benefits of configuration management to your entire infrastructure." + s.description = s.summary + s.author = "Adam Jacob" + s.email = "adam@opscode.com" + s.homepage = "http://wiki.opscode.com/display/chef" + + s.add_dependency "mixlib-log", ">= 1.2.0" + s.add_dependency "amqp", "~> 0.6.7" + s.add_dependency "eventmachine", '~> 0.12.10' + s.add_dependency "em-http-request", "~> 0.2.11" + s.add_dependency 'yajl-ruby', "~> 0.7.7" + s.add_dependency 'mixlib-log', "~> 1.1.0" + s.add_dependency 'uuidtools', "~> 2.1.1" + s.add_dependency 'bunny', '~> 0.6.0' + s.add_dependency 'fast_xs', "~> 0.7.3" + s.add_dependency 'highline', '~> 1.6.1' + s.add_dependency 'rake', '~> 0.8.7' + s.add_dependency "bunny", ">= 0.6.0" + + s.bindir = "bin" + s.executables = %w( chef-expander chef-expander-cluster chef-expanderctl ) + s.require_path = 'lib' + s.files = %w(LICENSE README.rdoc) + Dir.glob("{scripts,conf,lib}/**/*") +end diff --git a/chef-expander/conf/chef-expander.rb.example b/chef-expander/conf/chef-expander.rb.example new file mode 100644 index 0000000000..9b21daf65a --- /dev/null +++ b/chef-expander/conf/chef-expander.rb.example @@ -0,0 +1,9 @@ +# The URL is the only Solr config Chef Expander needs # +solr_url "http://localhost:8983" + +# Parameters for connecting to RabbitMQ +amqp_host 'localhost' +amqp_port '5672' +amqp_user 'chef' +amqp_pass 'testing' +amqp_vhost '/chef' diff --git a/chef-expander/data/sample_node.json b/chef-expander/data/sample_node.json new file mode 100644 index 0000000000..80dac4bace --- /dev/null +++ b/chef-expander/data/sample_node.json @@ -0,0 +1 @@ +{"name":"i-f09a939b","json_class":"Chef::Node","automatic":{"keys":{"ssh":{"host_dsa_public":"AAAAB3NzaC1kc3MAAACBAJ144IxziCQ3y9Q8erUviDO5hH0dxOEXYo5rTsUTB+eyvTbFPgIn1XmPyWBpqCpBEDHR7Mngzn9zIvXhf3YGNq4CaGKKE0JVcQXwveOXq2R7H32MUP4r0qP4wDrpjEtc/uQ4zJMJuSTLL+++HWClyD13QaVoWemlkpTkGE763/NpAAAAFQDuOq9biIZiQpcRnv4T9xQ22q9gUQAAAIAV55roDVnQfijL0RiHztUlRlTSq/57VJobKpn1gGaGltqpMi+k9NOecYSG98nqCibSubnjnGhotWAMsdd4QRfbwC3ofrv1fr/xBuqu4/yb8OjStQ7OrYe5Z5Qg8n6CT0QkL2GoWw3T3wg3np01dDakD2Yywv6zeoDbOeIGWizzQgAAAIEAmeWpAVdvxC7AnpgzOducU3j2zUzK5Uke3ZbX9TY8L5EI/cbogEQA6zp4QuKm68VcRe7Ca2Ljk+6vjqA8tsyQffNqnwru6B6xLUtCroF9BP3UktESBrGIaxlUGaSewdmaZ4JtoH40cdJtrAzI5GMDOUbtGTmVIWxjeKquk9NpWd0=","host_rsa_public":"AAAAB3NzaC1yc2EAAAABIwAAAQEArhqRYGssVycUxj8xPhDDmkveqg8mmMM6jyp1B4mWR0Nk2fuz5fzrVPsn+xXIwPPoF7PwoGH+VWqkqn0sIQyv/mUFhEi5hG+b+1+KZAOhFIryacRscY7BdVPCuEClVFSoJxVfDS40QFm1khsZW2u9/UIBLBBgxO3gtC7LCB0rpXmsU1YJrQ52EKRGkso01iOUpSqi9oIoFbBdNDOHw0+qwMf3R/zpqk/wA1loTOq9DbdKVYyrP5wvSbcfehuOocSAyfF1CwfsHPgZXrG9olyCI7LvfaMcdSNFoyAuS12Zu64knvrJ13kavabHzfVL/qwf1uDnXdTRUM5xIgUtKGLqDQ=="}},"dmi":{"bios":{},"version":"2.9"},"kernel":{"modules":{"ipv6":{"size":"243002","refcount":"12"}},"machine":"i686","name":"Linux","os":"GNU/Linux","version":"#9-Ubuntu SMP Thu Apr 15 04:14:01 UTC 2010","release":"2.6.32-305-ec2"},"platform_version":"10.04","fqdn":"lb-prod-i-f09a939b.opscode.us","filesystem":{"/dev/sda1":{"kb_size":"15481840","kb_available":"12433352","mount":"/","percent_used":"16%","kb_used":"2262056"},"proc":{"mount_options":["rw","noexec","nosuid","nodev"],"fs_type":"proc","mount":"/proc"},"/dev/sda2":{"kb_size":"350891748","kb_available":"332868096","mount_options":["rw"],"fs_type":"ext3","mount":"/mnt","percent_used":"1%","kb_used":"199372"},"none":{"kb_size":"890988","kb_available":"890988","mount_options":["rw","nosuid","mode=0755"],"fs_type":"tmpfs","mount":"/lib/init/rw","percent_used":"0%","kb_used":"0"},"devtmpfs":{"kb_size":"873976","kb_available":"873860","mount_options":["rw","mode=0755"],"fs_type":"devtmpfs","mount":"/dev","percent_used":"1%","kb_used":"116"}},"ipaddress":"10.192.26.6","command":{"ps":"ps -ef"},"memory":{"dirty":"164kB","vmalloc_used":"5732kB","page_tables":"0kB","buffers":"241844kB","slab_unreclaim":"6268kB","high_free":"207384kB","vmalloc_chunk":"116776kB","nfs_unstable":"0kB","slab":"48240kB","inactive":"191476kB","total":"1781976kB","vmalloc_total":"122880kB","low_free":"405380kB","low_total":"735440kB","free":"612764kB","commit_limit":"1808484kB","anon_pages":"64212kB","writeback":"0kB","cached":"768476kB","swap":{"total":"917496kB","free":"917496kB","cached":"0kB"},"high_total":"1046536kB","committed_as":"249708kB","bounce":"0kB","slab_reclaimable":"41972kB","mapped":"18684kB","active":"882952kB"},"idletime_seconds":6070168,"counters":{"network":{"interfaces":{"eql":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"5","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"ifb0":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"32","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"lo":{"tx":{"bytes":"32312142","packets":"307835","collisions":"0","queuelen":"0","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"32312142","packets":"307835","errors":"0","overrun":"0","drop":"0","frame":"0"}},"ifb1":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"32","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"eth0":{"tx":{"bytes":"1090545338","packets":"168873544","collisions":"0","queuelen":"1000","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"2226852623","packets":"286299389","errors":"0","overrun":"0","drop":"0","frame":"0"}},"dummy0":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"0","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}}}}},"domain":"opscode.us","os":"linux","idletime":"70 days 06 hours 09 minutes 28 seconds","lsb":{"codename":"lucid","id":"Ubuntu","description":"\"Ubuntu 10.04.1 LTS\"","release":"10.04"},"network":{"default_interface":"eth0","interfaces":{"eql":{"mtu":"576","encapsulation":"Serial"},"ifb0":{"flags":["BROADCAST","NOARP"],"number":"0","addresses":{"8a:7e:3d:8b:92:88":{"family":"lladdr"}},"mtu":"1500","type":"ifb","encapsulation":"Ethernet"},"lo":{"flags":["UP","LOOPBACK","RUNNING"],"addresses":{"::1":{"scope":"Node","prefixlen":"128","family":"inet6"},"127.0.0.1":{"netmask":"255.0.0.0","family":"inet"}},"mtu":"16436","encapsulation":"Loopback"},"ifb1":{"flags":["BROADCAST","NOARP"],"number":"1","addresses":{"d2:09:bf:06:5f:d1":{"family":"lladdr"}},"mtu":"1500","type":"ifb","encapsulation":"Ethernet"},"eth0":{"flags":["UP","BROADCAST","RUNNING","MULTICAST"],"number":"0","addresses":{"10.192.26.6":{"netmask":"255.255.254.0","broadcast":"10.192.27.255","family":"inet"},"12:31:39:0e:19:f4":{"family":"lladdr"},"fe80::1031:39ff:fe0e:19f4":{"scope":"Link","prefixlen":"64","family":"inet6"}},"mtu":"1500","type":"eth","arp":{"10.192.26.1":"fe:ff:ff:ff:ff:ff"},"encapsulation":"Ethernet"},"dummy0":{"flags":["BROADCAST","NOARP"],"number":"0","addresses":{"f2:77:75:3a:f4:71":{"family":"lladdr"}},"mtu":"1500","type":"dummy","encapsulation":"Ethernet"}}},"virtualization":{"role":"guest","emulator":"xen"},"current_user":"dan","ohai_time":1281135306.45911,"chef_packages":{"ohai":{"version":"0.5.4"},"chef":{"version":"0.9.0.a91"}},"os_version":"2.6.32-305-ec2","languages":{"perl":{"version":"5.10.1","archname":"i486-linux-gnu-thread-multi"},"python":{"version":"2.6.5","builddate":"Apr 16 2010, 13:09:56"},"ruby":{"target_os":"linux","bin_dir":"/usr/bin","host_vendor":"pc","target_vendor":"pc","ruby_bin":"/usr/bin/ruby1.8","target_cpu":"i486","version":"1.8.7","host_os":"linux-gnu","release_date":"2010-01-10","host_cpu":"i486","host":"i486-pc-linux-gnu","target":"i486-pc-linux-gnu","gems_dir":"/usr/lib/ruby/gems/1.8","platform":"i486-linux"},"erlang":{"version":"5.7.4","emulator":"BEAM","options":["SMP","ASYNC_THREADS"]}},"cpu":{"real":2,"total":2,"0":{"flags":["fpu","tsc","msr","pae","mce","cx8","apic","mca","cmov","pat","pse36","clflush","dts","acpi","mmx","fxsr","sse","sse2","ss","ht","tm","pbe","nx","lm","constant_tsc","arch_perfmon","pebs","bts","aperfmperf","pni","dtes64","monitor","ds_cpl","vmx","est","tm2","ssse3","cx16","xtpr","pdcm","dca","sse4_1","xsave","lahf_lm","tpr_shadow","vnmi","flexpriority"],"cores":"1","model":"23","core_id":"0","model_name":"Intel(R) Xeon(R) CPU E5410 @ 2.33GHz","family":"6","physical_id":"0","mhz":"2327.494","vendor_id":"GenuineIntel","cache_size":"6144 KB","stepping":"10"},"1":{"flags":["fpu","tsc","msr","pae","mce","cx8","apic","mca","cmov","pat","pse36","clflush","dts","acpi","mmx","fxsr","sse","sse2","ss","ht","tm","pbe","nx","lm","constant_tsc","arch_perfmon","pebs","bts","aperfmperf","pni","dtes64","monitor","ds_cpl","vmx","est","tm2","ssse3","cx16","xtpr","pdcm","dca","sse4_1","xsave","lahf_lm","tpr_shadow","vnmi","flexpriority"],"cores":"1","model":"23","core_id":"0","model_name":"Intel(R) Xeon(R) CPU E5410 @ 2.33GHz","family":"6","physical_id":"1","mhz":"2327.494","vendor_id":"GenuineIntel","cache_size":"6144 KB","stepping":"10"}},"uptime":"35 days 15 hours 06 minutes 20 seconds","hostname":"lb-prod-i-f09a939b","etc":{"group":{"cb":{"gid":7003,"members":[]},"sasl":{"gid":45,"members":[]},"kmem":{"gid":15,"members":[]},"jesse":{"gid":7009,"members":[]},"ssl-cert":{"gid":114,"members":[]},"lp":{"gid":7,"members":[]},"nagios":{"gid":113,"members":[]},"dip":{"gid":30,"members":["ubuntu"]},"www-data":{"gid":33,"members":[]},"opscode":{"gid":5049,"members":["opscode"]},"ubuntu":{"gid":1000,"members":[]},"cw":{"gid":7002,"members":[]},"news":{"gid":9,"members":[]},"ssh":{"gid":106,"members":[]},"adam":{"gid":7000,"members":[]},"mark":{"gid":7015,"members":[]},"dan":{"gid":7008,"members":[]},"sudo":{"gid":27,"members":[]},"nuo":{"gid":7006,"members":[]},"ronny":{"gid":7012,"members":[]},"cdrom":{"gid":24,"members":["ubuntu"]},"list":{"gid":38,"members":[]},"uucp":{"gid":10,"members":[]},"voice":{"gid":22,"members":[]},"netdev":{"gid":111,"members":[]},"shadow":{"gid":42,"members":[]},"root":{"gid":0,"members":[]},"games":{"gid":60,"members":[]},"adm":{"gid":4,"members":["ubuntu"]},"crontab":{"gid":102,"members":[]},"munin":{"gid":112,"members":[]},"joshua":{"gid":7004,"members":[]},"haldaemon":{"gid":108,"members":[]},"src":{"gid":40,"members":[]},"staff":{"gid":50,"members":[]},"gnats":{"gid":41,"members":[]},"haclient":{"gid":117,"members":[]},"seth":{"gid":7013,"members":[]},"operator":{"gid":37,"members":[]},"fuse":{"gid":104,"members":[]},"daemon":{"gid":1,"members":[]},"proxy":{"gid":13,"members":[]},"john":{"gid":7010,"members":[]},"disk":{"gid":6,"members":[]},"landscape":{"gid":109,"members":[]},"tty":{"gid":5,"members":[]},"sys":{"gid":3,"members":[]},"nathan":{"gid":7001,"members":[]},"postdrop":{"gid":116,"members":[]},"man":{"gid":12,"members":[]},"sysadmin":{"gid":2300,"members":["bryan","jesse","john","ronny","stephen","seth","joshua","opscode","nathan","mark","tim","dan","cw","adam","cb","nuo"]},"plugdev":{"gid":46,"members":["ubuntu"]},"tape":{"gid":26,"members":[]},"postfix":{"gid":115,"members":[]},"tim":{"gid":7005,"members":[]},"floppy":{"gid":25,"members":["ubuntu"]},"users":{"gid":100,"members":[]},"fax":{"gid":21,"members":[]},"syslog":{"gid":103,"members":[]},"irc":{"gid":39,"members":[]},"libuuid":{"gid":101,"members":[]},"mlocate":{"gid":105,"members":[]},"bin":{"gid":2,"members":[]},"video":{"gid":44,"members":["ubuntu"]},"messagebus":{"gid":107,"members":[]},"admin":{"gid":110,"members":["ubuntu"]},"bryan":{"gid":7011,"members":[]},"nogroup":{"gid":65534,"members":[]},"splunk":{"gid":7014,"members":[]},"utmp":{"gid":43,"members":[]},"dialout":{"gid":20,"members":["ubuntu"]},"stephen":{"gid":7007,"members":[]},"mail":{"gid":8,"members":[]},"backup":{"gid":34,"members":[]},"audio":{"gid":29,"members":["ubuntu"]}},"passwd":{"opscode":{"gecos":"Opscode deploy user","dir":"/home/opscode","gid":5049,"uid":5049,"shell":"/bin/bash"},"nathan":{"gecos":"Nathan Haneysmith","dir":"/home/nathan","gid":7001,"uid":7001,"shell":"/bin/bash"},"uucp":{"gecos":"uucp","dir":"/var/spool/uucp","gid":10,"uid":10,"shell":"/bin/sh"},"jesse":{"gecos":"Jesse Robbins","dir":"/home/jesse","gid":7009,"uid":7009,"shell":"/bin/bash"},"syslog":{"gecos":"","dir":"/home/syslog","gid":103,"uid":101,"shell":"/bin/false"},"list":{"gecos":"Mailing List Manager","dir":"/var/list","gid":38,"uid":38,"shell":"/bin/sh"},"games":{"gecos":"games","dir":"/usr/games","gid":60,"uid":5,"shell":"/bin/sh"},"munin":{"gecos":"","dir":"/var/lib/munin","gid":112,"uid":106,"shell":"/bin/false"},"tim":{"gecos":"","dir":"/home/tim","gid":7005,"uid":7005,"shell":"/bin/bash"},"john":{"gecos":"John Willis","dir":"/home/john","gid":7010,"uid":7010,"shell":"/bin/bash"},"sys":{"gecos":"sys","dir":"/dev","gid":3,"uid":3,"shell":"/bin/sh"},"nagios":{"gecos":"","dir":"/var/lib/nagios","gid":113,"uid":107,"shell":"/bin/false"},"nobody":{"gecos":"nobody","dir":"/nonexistent","gid":65534,"uid":65534,"shell":"/bin/sh"},"bryan":{"gecos":"Bryan Hale","dir":"/home/bryan","gid":7011,"uid":7011,"shell":"/bin/bash"},"libuuid":{"gecos":"","dir":"/var/lib/libuuid","gid":101,"uid":100,"shell":"/bin/sh"},"irc":{"gecos":"ircd","dir":"/var/run/ircd","gid":39,"uid":39,"shell":"/bin/sh"},"backup":{"gecos":"backup","dir":"/var/backups","gid":34,"uid":34,"shell":"/bin/sh"},"www-data":{"gecos":"www-data","dir":"/var/www","gid":33,"uid":33,"shell":"/bin/sh"},"lp":{"gecos":"lp","dir":"/var/spool/lpd","gid":7,"uid":7,"shell":"/bin/sh"},"man":{"gecos":"man","dir":"/var/cache/man","gid":12,"uid":6,"shell":"/bin/sh"},"splunk":{"gecos":"Splunk Server","dir":"/opt/splunk","gid":7014,"uid":7014,"shell":"/bin/bash"},"hacluster":{"gecos":"Heartbeat System Account,,,","dir":"/usr/lib/heartbeat","gid":117,"uid":109,"shell":"/bin/false"},"landscape":{"gecos":"","dir":"/var/lib/landscape","gid":109,"uid":105,"shell":"/bin/false"},"haldaemon":{"gecos":"Hardware abstraction layer,,,","dir":"/var/run/hald","gid":108,"uid":103,"shell":"/bin/false"},"messagebus":{"gecos":"","dir":"/var/run/dbus","gid":107,"uid":102,"shell":"/bin/false"},"proxy":{"gecos":"proxy","dir":"/bin","gid":13,"uid":13,"shell":"/bin/sh"},"mail":{"gecos":"mail","dir":"/var/mail","gid":8,"uid":8,"shell":"/bin/sh"},"sync":{"gecos":"sync","dir":"/bin","gid":65534,"uid":4,"shell":"/bin/sync"},"mark":{"gecos":"Mark Anderson","dir":"/home/mark","gid":7015,"uid":7015,"shell":"/usr/bin/zsh"},"seth":{"gecos":"Seth Falcon","dir":"/home/seth","gid":7013,"uid":7013,"shell":"/bin/bash"},"dan":{"gecos":"Daniel DeLeo","dir":"/home/dan","gid":7008,"uid":7008,"shell":"/bin/bash"},"sshd":{"gecos":"","dir":"/var/run/sshd","gid":65534,"uid":104,"shell":"/usr/sbin/nologin"},"root":{"gecos":"root","dir":"/root","gid":0,"uid":0,"shell":"/bin/bash"},"stephen":{"gecos":"Stephen Delano","dir":"/home/stephen","gid":7007,"uid":7007,"shell":"/bin/bash"},"bin":{"gecos":"bin","dir":"/bin","gid":2,"uid":2,"shell":"/bin/sh"},"ronny":{"gecos":"Ronny Jones","dir":"/home/ronny","gid":7012,"uid":7012,"shell":"/bin/bash"},"nuo":{"gecos":"Nuo Yan","dir":"/home/nuo","gid":7006,"uid":7006,"shell":"/bin/bash"},"joshua":{"gecos":"Joshua Timberman","dir":"/home/joshua","gid":7004,"uid":7004,"shell":"/usr/bin/zsh"},"adam":{"gecos":"Adam Jacob","dir":"/home/adam","gid":7000,"uid":7000,"shell":"/usr/bin/zsh"},"news":{"gecos":"news","dir":"/var/spool/news","gid":9,"uid":9,"shell":"/bin/sh"},"daemon":{"gecos":"daemon","dir":"/usr/sbin","gid":1,"uid":1,"shell":"/bin/sh"},"cw":{"gecos":"Chris Walters","dir":"/home/cw","gid":7002,"uid":7002,"shell":"/bin/bash"},"cb":{"gecos":"Chris Brown","dir":"/home/cb","gid":7003,"uid":7003,"shell":"/bin/bash"},"ubuntu":{"gecos":"Ubuntu,,,","dir":"/home/ubuntu","gid":1000,"uid":1000,"shell":"/bin/bash"},"gnats":{"gecos":"Gnats Bug-Reporting System (admin)","dir":"/var/lib/gnats","gid":41,"uid":41,"shell":"/bin/sh"},"postfix":{"gecos":"","dir":"/var/spool/postfix","gid":115,"uid":108,"shell":"/bin/false"}}},"ec2":{"public_hostname":"ec2-184-73-245-27.compute-1.amazonaws.com","placement_availability_zone":"us-east-1b","block_device_mapping_root":"/dev/sda1","public_keys_0_openssh_key":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPkRwNNzLbG4l8Ic3T7EOwuHpGzObCq+aHU0ewnbKEC1AQCS+ew9t/EYMzDt5N2PVxTfAP8if5dgRjeIBH13O0Xo+gJVArB+N2/0NHMEBxCzEPuT9j1+BJc6hpTbELtG1Y0eLCLS0oEwRNYSBa9GD/xkvSrhNA1+Jbf7qGtziVmFKpBKx/s6OUa1ofO+0aIqiR3e+YbCFFik1xHulWPuhwCM5fL8kkMdQBw8EC5lml5ysSuDjDJShLVu5882MMMZRfezCrmVjLoSszMo4CpSKqbzOIB9mGLlydT4yO+GzD8qx++ObpmOUFzfvRf4wQ0AeQTZVQnowqOkxys12yukUN opscode-platform-20100217\n","instance_id":"i-f09a939b","instance_type":"c1.medium","local_ipv4":"10.192.26.6","block_device_mapping_ephemeral0":"/dev/sda2","reservation_id":"r-56409f3d","public_ipv4":"184.73.245.27","local_hostname":"domU-12-31-39-0E-19-F4.compute-1.internal","kernel_id":"aki-754aa41c","hostname":"domU-12-31-39-0E-19-F4.compute-1.internal","block_device_mapping_swap":"sda3","ami_id":"ami-f8927a91","userdata":null,"security_groups":"platform-lb","block_device_mapping_ami":"/dev/sda1","ami_manifest_path":"(unknown)","ami_launch_index":"0"},"macaddress":"12:31:39:0e:19:f4","block_device":{"ram13":{"size":"131072","removable":"0"},"ram0":{"size":"131072","removable":"0"},"sda1":{"size":"31457280","removable":"0"},"loop0":{"size":"0","removable":"0"},"ram14":{"size":"131072","removable":"0"},"ram1":{"size":"131072","removable":"0"},"sda2":{"size":"712971264","removable":"0"},"loop1":{"size":"0","removable":"0"},"ram15":{"size":"131072","removable":"0"},"ram2":{"size":"131072","removable":"0"},"sda3":{"size":"1835008","removable":"0"},"loop2":{"size":"0","removable":"0"},"ram3":{"size":"131072","removable":"0"},"loop3":{"size":"0","removable":"0"},"ram4":{"size":"131072","removable":"0"},"loop4":{"size":"0","removable":"0"},"ram5":{"size":"131072","removable":"0"},"loop5":{"size":"0","removable":"0"},"ram6":{"size":"131072","removable":"0"},"loop6":{"size":"0","removable":"0"},"ram10":{"size":"131072","removable":"0"},"ram7":{"size":"131072","removable":"0"},"loop7":{"size":"0","removable":"0"},"ram11":{"size":"131072","removable":"0"},"ram8":{"size":"131072","removable":"0"},"ram12":{"size":"131072","removable":"0"},"ram9":{"size":"131072","removable":"0"}},"uptime_seconds":3078380,"platform":"ubuntu","cloud":{"public_ips":["184.73.245.27"],"private_ips":["10.192.26.6"],"provider":"ec2"}},"normal":{"ec2opts":{"lvm":{"ephemeral_mountpoint":"/mnt","ephemeral_logical_volume":"store","use_ephemeral":true,"ephemeral_devices":{"m1.small":["/dev/sda2"],"m1.xlarge":["/dev/sdb","/dev/sdc","/dev/sdd","/dev/sde"],"m1.large":["/dev/sdb","/dev/sdc"]},"ephemeral_volume_group":"ephemeral"}},"filesystem":{},"runit":{"sv_bin":"/usr/bin/sv","service_dir":"/etc/service","sv_dir":"/etc/sv"},"nagios":{"interval_length":1,"cache_dir":"/var/cache/nagios3","checks":{"memory":{"critical":150,"warning":250},"smtp_host":"","load":{"critical":"30,20,10","warning":"15,10,5"}},"sysadmin_email":"root@localhost","dir":"/etc/nagios3","default_host":{"max_check_attempts":1,"notification_interval":300,"retry_interval":15,"check_interval":15},"default_service":{"max_check_attempts":3,"notification_interval":1200,"retry_interval":15,"check_interval":60},"templates":{},"docroot":"/usr/share/nagios3/htdocs","sysadmin_sms_email":"root@localhost","default_contact_groups":["admins"],"check_external_commands":true,"state_dir":"/var/lib/nagios3","log_dir":"/var/log/nagios3","notifications_enabled":0,"config_subdir":"conf.d"},"nginx":{"worker_connections":4096,"gzip_types":["text/plain","text/html","text/css","application/x-javascript","text/xml","application/xml","application/xml+rss","text/javascript"],"dir":"/etc/nginx","binary":"/srv/nginx/0.8.33/sbin/nginx","server_names_hash_bucket_size":64,"gzip_comp_level":"2","gzip_proxied":"any","version":"0.8.33","worker_processes":2,"gzip_http_version":"1.0","gzip":"on","user":"www-data","log_dir":"/srv/nginx/log","keepalive_timeout":65,"keepalive":"on"},"authorization":{"sudo":{"groups":[],"users":[]}},"tags":[],"apache":{"allowed_openids":["http://adamhjk.myopenid.com/","http://bryanhale.myopenid.com/","http://jesserobbins.myopenid.com/","http://botchagalupe.myopenid.com/","http://jtimberman.myopenid.com/","http://tmonk42.myopenid.com/","http://ronny-jones.myopenid.com/","http://steviedu.myopenid.com/","http://seth.falcon.myopenid.com/","http://mark-a-anderson.myopenid.com/","http://timhinderliter.myopenid.com/","http://cwalters.myopenid.com","http://kallistec.myopenid.com/","http://cwalters.myopenid.com/","http://protoskeptomai.myopenid.com/","http://nuoyan.myopenid.com/"],"traceenable":"On","dir":"/etc/apache2","keepaliverequests":100,"timeout":300,"binary":"/usr/sbin/apache2","contact":"ops@example.com","prefork":{"maxspareservers":32,"minspareservers":16,"serverlimit":400,"startservers":16,"maxclients":400,"maxrequestsperchild":10000},"servertokens":"Prod","listen_ports":["80","443"],"keepalivetimeout":5,"serversignature":"On","icondir":"/usr/share/apache2/icons","user":"www-data","log_dir":"/var/log/apache2","worker":{"maxsparethreads":192,"startservers":4,"minsparethreads":64,"maxclients":1024,"maxrequestsperchild":0,"threadsperchild":64},"keepalive":"On"},"postfix":{"myorigin":"$myhostname","myhostname":"opscode.com","smtp_sasl_passwd":"Y3vDOkz8YDcxqL","smtp_sasl_security_options":"noanonymous","smtp_sasl_password_maps":"hash:/etc/postfix/sasl_passwd","smtp_sasl_auth_enable":"no","mail_type":"client","smtp_sasl_user_name":"community","mail_relay_networks":"127.0.0.0/8","relayhost":"lists.opscode.com","smtp_use_tls":"yes","smtp_tls_cafile":"/etc/postfix/cacert.pem","mydomain":"compute-1.internal"}},"chef_type":"node","default":{"dynect":{"zone":"opscode.us","domain":"opscode.us","username":"opscodeus","customer":"opscode","ec2":{"env":"na","type":"unknown"},"password":"nb7lO2hEPImEHtoTlYKq"},"opscode_lb":{"ha_role":"secondary"},"filesystem":{},"splunk":{"forward_server":"ec2-184-73-163-97.compute-1.amazonaws.com","auth":"admin:VYjsrtHdtieZD2QtX5za"},"monitor_group":"opscode-lb","ha_role":"secondary","opscode_lb_heartbeat":{"heartbeatsecret":"WIaVjtQurY0Q7sti"},"app_environment":"production"},"override":{"dynect":{"ec2":{"env":"prod","type":"lb"}},"filesystem":{},"opscode_lb_type":"external","postfix":{"myhostname":"opscode.com","smtp_sasl_passwd":"Y3vDOkz8YDcxqL","smtp_sasl_security_options":"noanonymous","smtp_sasl_auth_enable":"yes","relayhost":"lists.opscode.com","smtp_sasl_user_name":"community"},"rabbitmq":{"users":{"mapper":"EBjgOZGNsd4RpA3gSoom","chef":"Tat9THLuiOkdtyyH5VVf"}}},"run_list":["role[opscode-lb]","role[production]","role[ha-secondary]","recipe[aws]"],"_rev":"147-bb4f74122079d1feac675c1b2c57bfcc"} diff --git a/chef-expander/lib/chef/expander.rb b/chef-expander/lib/chef/expander.rb new file mode 100644 index 0000000000..9a58868a96 --- /dev/null +++ b/chef-expander/lib/chef/expander.rb @@ -0,0 +1,36 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Chef + module Expander + + # VNODES is the number of queues in rabbit that are available for subscribing. + # The name comes from riak, where the data ring (160bits) is chunked into + # many vnodes; vnodes outnumber physical nodes, so one node hosts several + # vnodes. That is the same design we use here. + # + # See the notes on topic queue benchmarking before adjusting this value. + VNODES = 1024 + + SHARED_CONTROL_QUEUE_NAME = "chef-search-control--shared" + BROADCAST_CONTROL_EXCHANGE_NAME = 'chef-search-control--broadcast' + + end +end diff --git a/chef-expander/lib/chef/expander/cluster_supervisor.rb b/chef-expander/lib/chef/expander/cluster_supervisor.rb new file mode 100644 index 0000000000..ccc5a9e730 --- /dev/null +++ b/chef-expander/lib/chef/expander/cluster_supervisor.rb @@ -0,0 +1,119 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/expander/loggable' +require 'chef/expander/version' +require 'chef/expander/configuration' +require 'chef/expander/vnode_supervisor' + +module Chef + module Expander + #==ClusterSupervisor + # Manages a cluster of chef-expander processes. Usually this class will + # be instantiated from the chef-expander-cluster executable. + # + # ClusterSupervisor works by forking the desired number of processes, then + # running VNodeSupervisor.start_cluster_worker within the forked process. + # ClusterSupervisor keeps track of the process ids of its children, and will + # periodically attempt to reap them in a non-blocking call. If they are + # reaped, ClusterSupervisor knows they died and need to be respawned. + # + # The child processes are responsible for checking on the master process and + # dying if the master has died (VNodeSupervisor does this when started in + # with start_cluster_worker). + # + #===TODO: + # * This implementation currently assumes there is only one cluster, so it + # will claim all of the vnodes. It may be advantageous to allow multiple + # clusters. + # * There is no heartbeat implementation at this time, so a zombified child + # process will not be automatically killed--This behavior is left to the + # meatcloud for now. + class ClusterSupervisor + include Loggable + + def initialize + @workers = {} + @running = true + @kill = :TERM + end + + def start + trap(:INT) { stop(:INT) } + trap(:TERM) { stop(:TERM)} + Expander.init_config(ARGV) + + log.info("Chef Expander #{Expander.version} starting cluster with #{Expander.config.node_count} nodes") + + start_workers + maintain_workers + end + + def start_workers + Expander.config.node_count.times do |i| + start_worker(i + 1) + end + end + + def start_worker(index) + log.info { "Starting cluster worker #{index}" } + worker_params = {:index => index} + child_pid = fork do + Expander.config.index = index + VNodeSupervisor.start_cluster_worker + end + @workers[child_pid] = worker_params + end + + def stop(signal) + log.info { "Stopping cluster on signal (#{signal})" } + @running = false + @kill = signal + end + + def maintain_workers + while @running + sleep 1 + workers_to_replace = {} + @workers.each do |process_id, worker_params| + if result = Process.waitpid2(process_id, Process::WNOHANG) + log.error { "worker #{worker_params[:index]} (PID: #{process_id}) died with status #{result[1].exitstatus || '(no status)'}"} + workers_to_replace[process_id] = worker_params + end + end + workers_to_replace.each do |dead_pid, worker_params| + @workers.delete(dead_pid) + start_worker(worker_params[:index]) + end + end + + @workers.each do |pid, worker_params| + log.info { "Stopping worker #{worker_params[:index]} (PID: #{pid})"} + Process.kill(@kill, pid) + end + @workers.each do |pid, worker_params| + Process.waitpid2(pid) + end + + end + + end + end +end diff --git a/chef-expander/lib/chef/expander/configuration.rb b/chef-expander/lib/chef/expander/configuration.rb new file mode 100644 index 0000000000..1888c56811 --- /dev/null +++ b/chef-expander/lib/chef/expander/configuration.rb @@ -0,0 +1,261 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'pp' +require 'optparse' +require 'singleton' + +require 'chef/expander/flattener' +require 'chef/expander/loggable' +require 'chef/expander/version' + +module Chef + module Expander + + def self.config + @config ||= Configuration::Base.new + end + + def self.init_config(argv) + config.apply_defaults + remaining_opts_after_parse = Configuration::CLI.parse_options(argv) + # Need to be able to override the default config file location on the command line + config_file_to_use = Configuration::CLI.config.config_file || config.config_file + config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use)) + # But for all other config options, the CLI config should win over config file + config.merge_config(Configuration::CLI.config) + remaining_opts_after_parse + end + + class ChefCompatibleConfig + + attr_reader :config_hash + + def initialize + @config_hash = {} + end + + def load(file) + file = File.expand_path(file) + instance_eval(IO.read(file), file, 1) + end + + def method_missing(method_name, *args, &block) + if args.size == 1 + @config_hash[method_name] = args.first + elsif args.empty? + @config_hash[method_name] or super + else + super + end + end + + end + + module Configuration + + class InvalidConfiguration < StandardError + end + + class Base + + include Loggable + + def self.from_chef_compat_config(file) + config = ChefCompatibleConfig.new + config.load(file) + from_hash(config.config_hash) + end + + def self.from_hash(config_hash) + config = new + config_hash.each do |setting, value| + setter = "#{setting}=".to_sym + if config.respond_to?(setter) + config.send(setter, value) + end + end + config + end + + def self.configurables + @configurables ||= [] + end + + def self.validations + @validations ||= [] + end + + def self.defaults + @defaults ||= {} + end + + def self.configurable(setting, default=nil, &validation) + attr_accessor(setting) + configurables << setting + defaults[setting] = default + validations << validation if block_given? + + setting + end + + configurable :config_file, File.expand_path(File.dirname(__FILE__) + '/../../../conf/chef-expander.rb') + + configurable :index do + invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer) + invalid("The index cannot be larger than the cluster size (node-count)") unless (index <= node_count.to_i) + end + + configurable :node_count do + invalid("You must specify the cluster size as an integer") unless node_count.kind_of?(Integer) + invalid("The cluster size (node-count) cannot be smaller than the index") unless node_count >= index.to_i + end + + configurable :ps_tag, "" + + configurable :solr_url, "http://localhost:8983" + + configurable :amqp_host, '0.0.0.0' + + configurable :amqp_port, '5672' + + configurable :amqp_user, 'chef' + + configurable :amqp_pass, 'testing' + + configurable :amqp_vhost, '/chef' + + configurable :log_level, :info + + # override the setter for log_level to also actually set the level + def log_level=(level) + if level #don't accept nil for an answer + level = level.to_sym + Loggable::LOGGER.level = level + @log_level = log_level + end + level + end + + def initialize + reset! + end + + def reset!(stdout=nil) + self.class.configurables.each do |setting| + send("#{setting}=".to_sym, nil) + end + @stdout = stdout || STDOUT + end + + def apply_defaults + self.class.defaults.each do |setting, value| + self.send("#{setting}=".to_sym, value) + end + end + + def merge_config(other) + self.class.configurables.each do |setting| + value = other.send(setting) + self.send("#{setting}=".to_sym, value) if value + end + end + + def fail_if_invalid + validate! + rescue InvalidConfiguration => e + @stdout.puts("Invalid configuration: #{e.message}") + exit(1) + end + + def invalid(message) + raise InvalidConfiguration, message + end + + def validate! + self.class.validations.each do |validation_proc| + instance_eval(&validation_proc) + end + end + + def vnode_numbers + vnodes_per_node = VNODES / node_count + lower_bound = (index - 1) * vnodes_per_node + upper_bound = lower_bound + vnodes_per_node + upper_bound += VNODES % vnodes_per_node if index == node_count + (lower_bound...upper_bound).to_a + end + + def amqp_config + {:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost} + end + + end + + module CLI + @config = Configuration::Base.new + + @option_parser = OptionParser.new do |o| + o.banner = "Usage: chef-expander [options]" + + o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf| + @config.config_file = File.expand_path(conf) + end + + o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i| + @config.index = i.to_i + end + + o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n| + @config.node_count = n.to_i + end + + o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l| + @config.log_level = l + end + + o.on_tail('-h', '--help', 'show this message') do + puts "chef-expander #{Expander.version}" + puts '' + puts o + exit 1 + end + + o.on_tail('-v', '--version', 'show the version and exit') do + puts "chef-expander #{Expander.version}" + exit 0 + end + + end + + def self.parse_options(argv) + @option_parser.parse!(argv.dup) + end + + def self.config + @config + end + + end + + end + + end +end diff --git a/chef-expander/lib/chef/expander/control.rb b/chef-expander/lib/chef/expander/control.rb new file mode 100644 index 0000000000..f8e3e99503 --- /dev/null +++ b/chef-expander/lib/chef/expander/control.rb @@ -0,0 +1,206 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'bunny' +require 'yajl' +require 'eventmachine' +require 'amqp' +require 'mq' +require 'highline' + +require 'chef/expander/node' +require 'chef/expander/configuration' + +require 'pp' + +module Chef + module Expander + class Control + + def self.run(argv) + remaining_args_after_opts = Expander.init_config(ARGV) + new(remaining_args_after_opts).run + end + + def self.desc(description) + @desc = description + end + + def self.option(*args) + #TODO + end + + def self.arg(*args) + #TODO + end + + def self.descriptions + @descriptions ||= [] + end + + def self.method_added(method_name) + if @desc + descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc] + @desc = nil + end + end + + #-- + # TODO: this is confusing and unneeded. Just whitelist the methods + # that map to commands and use +send+ + def self.compile + run_method = "def run; case @argv.first;" + descriptions.each do |method_name, command_name, desc| + run_method << "when '#{command_name}';#{method_name};" + end + run_method << "else; help; end; end;" + class_eval(run_method, __FILE__, __LINE__) + end + + def initialize(argv) + @argv = argv.dup + end + + desc "Show this message" + def help + puts "Chef Expander #{Expander.version}" + puts "Usage: chef-expanderctl COMMAND" + puts + puts "Commands:" + self.class.descriptions.each do |method_name, command_name, desc| + puts " #{command_name}".ljust(15) + desc + end + end + + desc "display the aggregate queue backlog" + def queue_depth + h = HighLine.new + message_counts = [] + + amqp_client = Bunny.new(Expander.config.amqp_config) + amqp_client.start + + 0.upto(VNODES - 1) do |vnode| + q = amqp_client.queue("vnode-#{vnode}", :durable => true) + message_counts << q.status[:message_count] + end + total_messages = message_counts.inject(0) { |sum, count| sum + count } + max = message_counts.max + min = message_counts.min + + avg = total_messages.to_f / message_counts.size.to_f + + puts " total messages: #{total_messages}" + puts " average queue depth: #{avg}" + puts " max queue depth: #{max}" + puts " min queue depth: #{min}" + ensure + amqp_client.stop if defined?(amqp_client) && amqp_client + end + + desc "show the backlog and consumer count for each vnode queue" + def queue_status + h = HighLine.new + queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)] + + total_messages = 0 + + amqp_client = Bunny.new(Expander.config.amqp_config) + amqp_client.start + + 0.upto(VNODES - 1) do |vnode| + q = amqp_client.queue("vnode-#{vnode}", :durable => true) + status = q.status + # returns {:message_count => method.message_count, :consumer_count => method.consumer_count} + queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s + total_messages += status[:message_count] + end + puts " total messages: #{total_messages}" + puts + puts h.list(queue_status, :columns_across, 3) + ensure + amqp_client.stop if defined?(amqp_client) && amqp_client + end + + desc "show the status of the nodes in the cluster" + def node_status + status_mutex = Mutex.new + h = ::HighLine.new + node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)] + + print("Collecting status info from the cluster...") + + AMQP.start(Expander.config.amqp_config) do + node = Expander::Node.local_node + node.exclusive_control_queue.subscribe do |header, message| + status = Yajl::Parser.parse(message) + status_mutex.synchronize do + node_status << status["hostname_f"] + node_status << status["pid"].to_s + node_status << status["guid"] + # BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges + # will not be true once vnode recovery is implemented + node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}" + end + end + node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name)) + EM.add_timer(2) { AMQP.stop;EM.stop } + end + + puts "done" + puts + puts h.list(node_status, :columns_across, 4) + puts + end + + desc "sets the log level of all nodes in the cluster" + def log_level + @argv.shift + level = @argv.first + acceptable_levels = %w{debug info warn error fatal} + unless acceptable_levels.include?(level) + puts "Log level must be one of #{acceptable_levels.join(', ')}" + exit 1 + end + + h = HighLine.new + response_mutex = Mutex.new + + responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)] + AMQP.start(Expander.config.amqp_config) do + node = Expander::Node.local_node + node.exclusive_control_queue.subscribe do |header, message| + reply = Yajl::Parser.parse(message) + n = reply['node'] + response_mutex.synchronize do + responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"] + end + end + node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name})) + EM.add_timer(2) { AMQP.stop; EM.stop } + end + puts h.list(responses, :columns_across, 4) + end + + + compile + end + end +end diff --git a/chef-expander/lib/chef/expander/flattener.rb b/chef-expander/lib/chef/expander/flattener.rb new file mode 100644 index 0000000000..90f7cd663e --- /dev/null +++ b/chef-expander/lib/chef/expander/flattener.rb @@ -0,0 +1,79 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/expander/configuration' + +module Chef + module Expander + # Flattens and expands nested Hashes representing Chef objects + # (e.g, Nodes, Roles, DataBagItems, etc.) into flat Hashes so the + # objects are suitable to be saved into Solr. This code is more or + # less copy-pasted from chef/solr/index which may or may not be a + # great idea, though that does minimize the dependencies and + # hopefully minimize the memory use of chef-expander. + class Flattener + UNDERSCORE = '_' + X = 'X' + + X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X' + X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X' + X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X' + + def initialize(item) + @item = item + end + + def flattened_item + @flattened_item || flatten_and_expand + end + + def flatten_and_expand + @flattened_item = Hash.new {|hash, key| hash[key] = []} + + @item.each do |key, value| + flatten_each([key.to_s], value) + end + + @flattened_item.each_value { |values| values.uniq! } + @flattened_item + end + + def flatten_each(keys, values) + case values + when Hash + values.each do |child_key, child_value| + add_field_value(keys, child_key) + flatten_each(keys + [child_key.to_s], child_value) + end + when Array + values.each { |child_value| flatten_each(keys, child_value) } + else + add_field_value(keys, values) + end + end + + def add_field_value(keys, value) + value = value.to_s + @flattened_item[keys.join(UNDERSCORE)] << value + @flattened_item[keys.last] << value + end + end + end +end diff --git a/chef-expander/lib/chef/expander/loggable.rb b/chef-expander/lib/chef/expander/loggable.rb new file mode 100644 index 0000000000..5bfb941ecb --- /dev/null +++ b/chef-expander/lib/chef/expander/loggable.rb @@ -0,0 +1,56 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'mixlib/log' + +module Chef + module Expander + module Loggable + class Logger + include Mixlib::Log + + def init(*args) + @logger = nil + super + end + + [:debug,:info,:warn,:error, :fatal].each do |level| + class_eval(<<-LOG_METHOD, __FILE__, __LINE__) + def #{level}(message=nil, &block) + @logger.#{level}(message, &block) + end + LOG_METHOD + end + end + + # TODO: it's admittedly janky to set up the default logging this way. + STDOUT.sync = true + LOGGER = Logger.new + LOGGER.init + LOGGER.level = :debug + + def log + LOGGER + end + + end + end +end + diff --git a/chef-expander/lib/chef/expander/node.rb b/chef-expander/lib/chef/expander/node.rb new file mode 100644 index 0000000000..23249e8685 --- /dev/null +++ b/chef-expander/lib/chef/expander/node.rb @@ -0,0 +1,177 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'uuidtools' +require 'amqp' +require 'mq' +require 'open3' + +require 'chef/expander/loggable' + +module Chef + module Expander + class Node + + include Loggable + + def self.from_hash(node_info) + new(node_info[:guid], node_info[:hostname_f], node_info[:pid]) + end + + def self.local_node + new(guid, hostname_f, Process.pid) + end + + def self.guid + return @guid if @guid + @guid = UUIDTools::UUID.random_create.to_s + end + + def self.hostname_f + @hostname ||= Open3.popen3("hostname -f") {|stdin, stdout, stderr| stdout.read }.strip + end + + attr_reader :guid + + attr_reader :hostname_f + + attr_reader :pid + + def initialize(guid, hostname_f, pid) + @guid, @hostname_f, @pid = guid, hostname_f, pid + end + + def start(&message_handler) + attach_to_queue(exclusive_control_queue, "exclusive control", &message_handler) + attach_to_queue(shared_control_queue, "shared_control", &message_handler) + attach_to_queue(broadcast_control_queue, "broadcast control", &message_handler) + end + + def attach_to_queue(queue, colloquial_name, &message_handler) + queue.subscribe(:ack => true) do |headers, payload| + log.debug { "received message on #{colloquial_name} queue: #{payload}" } + message_handler.call(payload) + headers.ack + end + end + + def stop + log.debug { "unsubscribing from broadcast control queue"} + broadcast_control_queue.unsubscribe(:nowait => false) + + log.debug { "unsubscribing from shared control queue" } + shared_control_queue.unsubscribe(:nowait => false) + + log.debug { "unsubscribing from exclusive control queue" } + exclusive_control_queue.unsubscribe(:nowait => false) + end + + def direct_message(message) + log.debug { "publishing direct message to node #{identifier}: #{message}" } + exclusive_control_queue.publish(message) + end + + def shared_message(message) + log.debug { "publishing shared message #{message}"} + shared_control_queue.publish(message) + end + + def broadcast_message(message) + log.debug { "publishing broadcast message #{message}" } + broadcast_control_exchange.publish(message) + end + + # The exclusive control queue is for point-to-point messaging, i.e., + # messages directly addressed to this node + def exclusive_control_queue + @exclusive_control_queue ||= begin + log.debug { "declaring exclusive control queue #{exclusive_control_queue_name}" } + MQ.queue(exclusive_control_queue_name) + end + end + + # The shared control queue is for 1 to (1 of N) messaging, i.e., + # messages that can go to any one node. + def shared_control_queue + @shared_control_queue ||= begin + log.debug { "declaring shared control queue #{shared_control_queue_name}" } + MQ.queue(shared_control_queue_name) + end + end + + # The broadcast control queue is for 1 to N messaging, i.e., + # messages that go to every node + def broadcast_control_queue + @broadcast_control_queue ||= begin + log.debug { "declaring broadcast control queue #{broadcast_control_queue_name}"} + q = MQ.queue(broadcast_control_queue_name) + log.debug { "binding broadcast control queue to broadcast control exchange"} + q.bind(broadcast_control_exchange) + q + end + end + + def broadcast_control_exchange + @broadcast_control_exchange ||= begin + log.debug { "declaring broadcast control exchange opscode-platfrom-control--broadcast" } + MQ.fanout(broadcast_control_exchange_name, :nowait => false) + end + end + + def shared_control_queue_name + SHARED_CONTROL_QUEUE_NAME + end + + def broadcast_control_queue_name + @broadcast_control_queue_name ||= "#{identifier}--broadcast" + end + + def broadcast_control_exchange_name + BROADCAST_CONTROL_EXCHANGE_NAME + end + + def exclusive_control_queue_name + @exclusive_control_queue_name ||= "#{identifier}--exclusive-control" + end + + def identifier + "#{hostname_f}--#{pid}--#{guid}" + end + + def ==(other) + other.respond_to?(:guid) && other.respond_to?(:hostname_f) && other.respond_to?(:pid) && + (other.guid == guid) && (other.hostname_f == hostname_f) && (other.pid == pid) + end + + def eql?(other) + (other.class == self.class) && (other.hash == hash) + end + + def hash + identifier.hash + end + + def to_hash + {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid} + end + + end + end +end diff --git a/chef-expander/lib/chef/expander/solrizer.rb b/chef-expander/lib/chef/expander/solrizer.rb new file mode 100644 index 0000000000..1a6fed9521 --- /dev/null +++ b/chef-expander/lib/chef/expander/solrizer.rb @@ -0,0 +1,275 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'set' +require 'yajl' +require 'fast_xs' +require 'em-http-request' +require 'chef/expander/loggable' +require 'chef/expander/flattener' + +module Chef + module Expander + class Solrizer + + @active_http_requests = Set.new + + def self.http_request_started(instance) + @active_http_requests << instance + end + + def self.http_request_completed(instance) + @active_http_requests.delete(instance) + end + + def self.http_requests_active? + !@active_http_requests.empty? + end + + def self.clear_http_requests + @active_http_requests.clear + end + + include Loggable + + ADD = "add" + DELETE = "delete" + SKIP = "skip" + + ITEM = "item" + ID = "id" + TYPE = "type" + DATABASE = "database" + ENQUEUED_AT = "enqueued_at" + + DATA_BAG_ITEM = "data_bag_item" + DATA_BAG = "data_bag" + + X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X' + X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X' + X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X' + + CONTENT_TYPE_XML = {"Content-Type" => "text/xml"} + + attr_reader :action + + attr_reader :indexer_payload + + attr_reader :chef_object + + attr_reader :obj_id + + attr_reader :obj_type + + attr_reader :database + + attr_reader :enqueued_at + + def initialize(object_command_json, &on_completion_block) + @start_time = Time.now.to_f + @on_completion_block = on_completion_block + if parsed_message = parse(object_command_json) + @action = parsed_message["action"] + @indexer_payload = parsed_message["payload"] + + extract_object_fields if @indexer_payload + else + @action = SKIP + end + end + + def extract_object_fields + @chef_object = @indexer_payload[ITEM] + @database = @indexer_payload[DATABASE] + @obj_id = @indexer_payload[ID] + @obj_type = @indexer_payload[TYPE] + @enqueued_at = @indexer_payload[ENQUEUED_AT] + @data_bag = @obj_type == DATA_BAG_ITEM ? @chef_object[DATA_BAG] : nil + end + + def parse(serialized_object) + Yajl::Parser.parse(serialized_object) + rescue Yajl::ParseError + log.error { "cannot index object because it is invalid JSON: #{serialized_object}" } + end + + def run + case @action + when ADD + add + when DELETE + delete + when SKIP + completed + log.info { "not indexing this item because of malformed JSON"} + else + completed + log.error { "cannot index object becuase it has an invalid action #{@action}" } + end + end + + def add + post_to_solr(pointyize_add) do + ["indexed #{indexed_object}", + "transit,xml,solr-post |", + [transit_time, @xml_time, @solr_post_time].join(","), + "|" + ].join(" ") + end + rescue Exception => e + log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"} + end + + def delete + post_to_solr(pointyize_delete) { "deleted #{indexed_object} transit-time[#{transit_time}s]"} + rescue Exception => e + log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"} + end + + def flattened_object + flattened_object = Flattener.new(@chef_object).flattened_item + + flattened_object[X_CHEF_id_CHEF_X] = [@obj_id] + flattened_object[X_CHEF_database_CHEF_X] = [@database] + flattened_object[X_CHEF_type_CHEF_X] = [@obj_type] + + log.debug {"adding flattened object to Solr: #{flattened_object.inspect}"} + + flattened_object + end + + START_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + ADD_DOC = "<add><doc>" + DELETE_DOC = "<delete>" + ID_OPEN = "<id>" + ID_CLOSE = "</id>" + END_ADD_DOC = "</doc></add>\n" + END_DELETE = "</delete>\n" + START_CONTENT = '<field name="content">' + CLOSE_FIELD = "</field>" + + FLD_CHEF_ID_FMT = '<field name="X_CHEF_id_CHEF_X">%s</field>' + FLD_CHEF_DB_FMT = '<field name="X_CHEF_database_CHEF_X">%s</field>' + FLD_CHEF_TY_FMT = '<field name="X_CHEF_type_CHEF_X">%s</field>' + FLD_DATA_BAG = '<field name="data_bag">%s</field>' + + KEYVAL_FMT = "%s__=__%s " + + # Takes a flattened hash where the values are arrays and converts it into + # a dignified XML document suitable for POST to Solr. + # The general structure of the output document is like this: + # <?xml version="1.0" encoding="UTF-8"?> + # <add> + # <doc> + # <field name="content"> + # key__=__value + # key__=__another_value + # other_key__=__yet another value + # </field> + # </doc> + # </add> + # The document as generated has minimal newlines and formatting, however. + def pointyize_add + xml = "" + xml << START_XML << ADD_DOC + xml << (FLD_CHEF_ID_FMT % @obj_id) + xml << (FLD_CHEF_DB_FMT % @database) + xml << (FLD_CHEF_TY_FMT % @obj_type) + xml << START_CONTENT + content = "" + flattened_object.each do |field, values| + values.each do |v| + content << (KEYVAL_FMT % [field, v]) + end + end + xml << content.fast_xs + xml << CLOSE_FIELD # ends content + xml << (FLD_DATA_BAG % @data_bag.fast_xs) if @data_bag + xml << END_ADD_DOC + @xml_time = Time.now.to_f - @start_time + xml + end + + # Takes a succinct document id, like 2342, and turns it into something + # even more compact, like + # "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n" + def pointyize_delete + xml = "" + xml << START_XML + xml << DELETE_DOC + xml << ID_OPEN + xml << @obj_id.to_s + xml << ID_CLOSE + xml << END_DELETE + xml + end + + def post_to_solr(document, &logger_block) + log.debug("POSTing document to SOLR:\n#{document}") + http_req = EventMachine::HttpRequest.new(solr_url).post(:body => document, :timeout => 1200, :head => CONTENT_TYPE_XML) + http_request_started + + http_req.callback do + completed + if http_req.response_header.status == 200 + log.info(&logger_block) + else + log.error { "Failed to post to solr: #{indexed_object}" } + end + end + http_req.errback do + completed + log.error { "Failed to post to solr (connection error): #{indexed_object}" } + end + end + + def completed + @solr_post_time = Time.now.to_f - @start_time + self.class.http_request_completed(self) + @on_completion_block.call + end + + def transit_time + Time.now.utc.to_i - @enqueued_at + end + + def solr_url + 'http://127.0.0.1:8983/solr/update' + end + + def indexed_object + "#{@obj_type}[#{@obj_id}] database[#{@database}]" + end + + def http_request_started + self.class.http_request_started(self) + end + + def eql?(other) + other.hash == hash + end + + def hash + "#{action}#{indexed_object}#@enqueued_at#{self.class.name}".hash + end + + end + end +end diff --git a/chef-expander/lib/chef/expander/version.rb b/chef-expander/lib/chef/expander/version.rb new file mode 100644 index 0000000000..9b0f814465 --- /dev/null +++ b/chef-expander/lib/chef/expander/version.rb @@ -0,0 +1,37 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'open3' + +module Chef + module Expander + + VERSION = "0.1.0" + + def self.version + @rev ||= begin + rev = Open3.popen3("git rev-parse HEAD") {|stdin, stdout, stderr| stdout.read }.strip + rev.empty? ? nil : " (#{rev})" + end + "#{VERSION}#@rev" + end + + end +end diff --git a/chef-expander/lib/chef/expander/vnode.rb b/chef-expander/lib/chef/expander/vnode.rb new file mode 100644 index 0000000000..32bb17da32 --- /dev/null +++ b/chef-expander/lib/chef/expander/vnode.rb @@ -0,0 +1,106 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'eventmachine' +require 'amqp' +require 'mq' + +require 'chef/expander/loggable' +require 'chef/expander/solrizer' + +module Chef + module Expander + class VNode + include Loggable + + attr_reader :vnode_number + + attr_reader :supervise_interval + + def initialize(vnode_number, supervisor, opts={}) + @vnode_number = vnode_number.to_i + @supervisor = supervisor + @queue = nil + @stopped = false + @supervise_interval = opts[:supervise_interval] || 30 + end + + def start + @supervisor.vnode_added(self) + + subscription_confirmed = Proc.new do + abort_on_multiple_subscribe + supervise_consumer_count + end + + queue.subscribe(:ack => true, :confirm => subscription_confirmed) do |headers, payload| + log.debug {"got #{payload} size(#{payload.size} bytes) on queue #{queue_name}"} + solrizer = Solrizer.new(payload) { headers.ack } + solrizer.run + end + + rescue MQ::Error => e + log.error {"Failed to start subscriber on #{queue_name} #{e.class.name}: #{e.message}"} + end + + def supervise_consumer_count + EM.add_periodic_timer(supervise_interval) do + abort_on_multiple_subscribe + end + end + + def abort_on_multiple_subscribe + queue.status do |message_count, subscriber_count| + if subscriber_count.to_i > 1 + log.error { "Detected extra consumers (#{subscriber_count} total) on queue #{queue_name}, cancelling subscription" } + stop + end + end + end + + def stop + log.debug {"Cancelling subscription on queue #{queue_name.inspect}"} + queue.unsubscribe if queue.subscribed? + @supervisor.vnode_removed(self) + @stopped = true + end + + def stopped? + @stopped + end + + def queue + @queue ||= begin + log.debug { "declaring queue #{queue_name}" } + MQ.queue(queue_name, :passive => false, :durable => true) + end + end + + def queue_name + "vnode-#{@vnode_number}" + end + + def control_queue_name + "#{queue_name}-control" + end + + end + end +end diff --git a/chef-expander/lib/chef/expander/vnode_supervisor.rb b/chef-expander/lib/chef/expander/vnode_supervisor.rb new file mode 100644 index 0000000000..40e9b62817 --- /dev/null +++ b/chef-expander/lib/chef/expander/vnode_supervisor.rb @@ -0,0 +1,265 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'yajl' +require 'eventmachine' +require 'amqp' +require 'mq' +require 'chef/expander/version' +require 'chef/expander/loggable' +require 'chef/expander/node' +require 'chef/expander/vnode' +require 'chef/expander/vnode_table' +require 'chef/expander/configuration' + +module ::AMQP + def self.hard_reset! + MQ.reset rescue nil + stop + EM.stop rescue nil + Thread.current[:mq], @conn = nil, nil + end +end + +module Chef + module Expander + class VNodeSupervisor + include Loggable + extend Loggable + + COULD_NOT_CONNECT = /Could not connect to server/.freeze + + def self.start_cluster_worker + @vnode_supervisor = new + @original_ppid = Process.ppid + trap_signals + + vnodes = Expander.config.vnode_numbers + + $0 = "chef-expander#{Expander.config.ps_tag} worker ##{Expander.config.index} (vnodes #{vnodes.min}-#{vnodes.max})" + + AMQP.start(Expander.config.amqp_config) do + start_consumers + await_parent_death + end + end + + def self.await_parent_death + @awaiting_parent_death = EM.add_periodic_timer(1) do + unless Process.ppid == @original_ppid + @awaiting_parent_death.cancel + stop("master process death") + end + end + end + + def self.start + @vnode_supervisor = new + trap_signals + + Expander.init_config(ARGV) + + log.info("Chef Search Expander #{Expander.version} starting up.") + + begin + AMQP.start(Expander.config.amqp_config) do + start_consumers + end + rescue AMQP::Error => e + if e.message =~ COULD_NOT_CONNECT + log.error { "Could not connect to rabbitmq. Make sure it is running and correctly configured." } + log.error { e.message } + + AMQP.hard_reset! + + sleep 5 + retry + else + raise + end + end + end + + def self.start_consumers + log.debug { "Setting prefetch count to 1"} + MQ.prefetch(1) + + vnodes = Expander.config.vnode_numbers + log.info("Starting Consumers for vnodes #{vnodes.min}-#{vnodes.max}") + @vnode_supervisor.start(vnodes) + end + + def self.trap_signals + Kernel.trap(:INT) { stop_immediately(:INT) } + Kernel.trap(:TERM) { stop_gracefully(:TERM) } + end + + def self.stop_immediately(signal) + log.info { "Initiating immediate shutdown on signal (#{signal})" } + @vnode_supervisor.stop + EM.add_timer(1) do + AMQP.stop + EM.stop + end + end + + def self.stop_gracefully(signal) + log.info { "Initiating graceful shutdown on signal (#{signal})" } + @vnode_supervisor.stop + wait_for_http_requests_to_complete + end + + def self.wait_for_http_requests_to_complete + if Expander::Solrizer.http_requests_active? + log.info { "waiting for in progress HTTP Requests to complete"} + EM.add_timer(1) do + wait_for_http_requests_to_complete + end + else + log.info { "HTTP requests completed, shutting down"} + AMQP.stop + EM.stop + end + end + + attr_reader :vnode_table + + attr_reader :local_node + + def initialize + @vnodes = {} + @vnode_table = VNodeTable.new(self) + @local_node = Node.local_node + @queue_name, @guid = nil, nil + end + + def start(vnode_ids) + @local_node.start do |message| + process_control_message(message) + end + + #start_vnode_table_publisher + + Array(vnode_ids).each { |vnode_id| spawn_vnode(vnode_id) } + end + + def stop + @local_node.stop + + #log.debug { "stopping vnode table updater" } + #@vnode_table_publisher.cancel + + log.info { "Stopping VNode queue subscribers"} + @vnodes.each do |vnode_number, vnode| + log.debug { "Stopping consumer on VNode #{vnode_number}"} + vnode.stop + end + + end + + def vnode_added(vnode) + log.debug { "vnode #{vnode.vnode_number} registered with supervisor" } + @vnodes[vnode.vnode_number.to_i] = vnode + end + + def vnode_removed(vnode) + log.debug { "vnode #{vnode.vnode_number} unregistered from supervisor" } + @vnodes.delete(vnode.vnode_number.to_i) + end + + def vnodes + @vnodes.keys.sort + end + + def spawn_vnode(vnode_number) + VNode.new(vnode_number, self).start + end + + def release_vnode + # TODO + end + + def process_control_message(message) + control_message = parse_symbolic(message) + case control_message[:action] + when "claim_vnode" + spawn_vnode(control_message[:vnode_id]) + when "recover_vnode" + recover_vnode(control_message[:vnode_id]) + when "release_vnodes" + raise "todo" + release_vnode() + when "update_vnode_table" + @vnode_table.update_table(control_message[:data]) + when "vnode_table_publish" + publish_vnode_table + when "status" + publish_status_to(control_message[:rsvp]) + when "set_log_level" + set_log_level(control_message[:level], control_message[:rsvp]) + else + log.error { "invalid control message #{control_message.inspect}" } + end + rescue Exception => e + log.error { "Error processing a control message."} + log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}" } + end + + + def start_vnode_table_publisher + @vnode_table_publisher = EM.add_periodic_timer(10) { publish_vnode_table } + end + + def publish_vnode_table + status_update = @local_node.to_hash + status_update[:vnodes] = vnodes + status_update[:update] = :add + @local_node.broadcast_message(Yajl::Encoder.encode({:action => :update_vnode_table, :data => status_update})) + end + + def publish_status_to(return_queue) + status_update = @local_node.to_hash + status_update[:vnodes] = vnodes + MQ.queue(return_queue).publish(Yajl::Encoder.encode(status_update)) + end + + def set_log_level(level, rsvp_to) + log.info { "setting log level to #{level} due to command from #{rsvp_to}" } + new_log_level = (Expander.config.log_level = level.to_sym) + reply = {:level => new_log_level, :node => @local_node.to_hash} + MQ.queue(rsvp_to).publish(Yajl::Encoder.encode(reply)) + end + + def recover_vnode(vnode_id) + if @vnode_table.local_node_is_leader? + log.debug { "Recovering vnode: #{vnode_id}" } + @local_node.shared_message(Yajl::Encoder.encode({:action => :claim_vnode, :vnode_id => vnode_id})) + else + log.debug { "Ignoring :recover_vnode message because this node is not the leader" } + end + end + + def parse_symbolic(message) + Yajl::Parser.new(:symbolize_keys => true).parse(message) + end + + end + end +end diff --git a/chef-expander/lib/chef/expander/vnode_table.rb b/chef-expander/lib/chef/expander/vnode_table.rb new file mode 100644 index 0000000000..025812b49c --- /dev/null +++ b/chef-expander/lib/chef/expander/vnode_table.rb @@ -0,0 +1,83 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'yajl' +require 'chef/expander/node' +require 'chef/expander/loggable' + +module Chef + module Expander + class VNodeTable + + include Loggable + + class InvalidVNodeTableUpdate < ArgumentError; end + + attr_reader :vnodes_by_node + + def initialize(vnode_supervisor) + @node_update_mutex = Mutex.new + @vnode_supervisor = vnode_supervisor + @vnodes_by_node = {} + end + + def nodes + @vnodes_by_node.keys + end + + def update_table(table_update) + case table_update[:update] + when "add", "update" + update_node(table_update) + when "remove" + remove_node(table_update) + else + raise InvalidVNodeTableUpdate, "no action or action not acceptable: #{table_update.inspect}" + end + log.debug { "current vnode table: #{@vnodes_by_node.inspect}" } + end + + def update_node(node_info) + @node_update_mutex.synchronize do + @vnodes_by_node[Node.from_hash(node_info)] = node_info[:vnodes] + end + end + + def remove_node(node_info) + @node_update_mutex.synchronize do + @vnodes_by_node.delete(Node.from_hash(node_info)) + end + end + + def leader_node + if @vnodes_by_node.empty? + nil + else + Array(@vnodes_by_node).reject { |node| node[1].empty? }.sort { |a,b| a[1].min <=> b[1].min }.first[0] + end + end + + def local_node_is_leader? + (Node.local_node == leader_node) || (@vnodes_by_node[Node.local_node].include?(0)) + end + + end + end +end diff --git a/chef-expander/notes/topic-queue-benchmarks.md b/chef-expander/notes/topic-queue-benchmarks.md new file mode 100644 index 0000000000..16dcbfca16 --- /dev/null +++ b/chef-expander/notes/topic-queue-benchmarks.md @@ -0,0 +1,48 @@ +# Time to process 10,000 messages # +* messages are (16 bytes), generated by bin/traffic-creator +* in all cases, the generator finishes well before the subscriber +* in all cases, the rabbitmq generates a load of ~80% - 90% on single core +* generator is run on the same physical machine, but see above caveat +* consumer is ruby amqp/eventmachine, prints a simple status message for each + message received. +* generator is ruby/bunny + +## 10 processes * 1,000 queues subscribed ## +10 consumer processes, each claiming 1/10th of 10,000 total queues. +start : 1282693162 +end : 1282694881 (incomplete, give up) +elapsed : 1771s (less than 6/s) + +## 10,000 queues subscribed ## +start : 1282690712 +end : 1282691371 (incomplete, gave up.) + +## 5,000 queues subscribed ## +start : 1282694987 +end : 1282695411 +total : 424s + +second run: +start : 1282695613 +end : 1282696041 +total : 428s (~23/s) + +## 1024 queues subscribed ## +start : 1282697018 +end : 1282697110 +total : 92s (~110/s) + +## 1000 queues subscribed ## +start : 1282689470 +end : 1282689560 +total : 90s + +## 100 queues subscribed ## +start : 1282689603 +end : 1282689698 +total : 95s + +## 10 queues subscribed ## +start : 1282689758 +end : 1282689850 +total : 92s
\ No newline at end of file diff --git a/chef-expander/scripts/check_queue_size b/chef-expander/scripts/check_queue_size new file mode 100755 index 0000000000..cc539f4683 --- /dev/null +++ b/chef-expander/scripts/check_queue_size @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +#-- +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################### +# check_queue_size +# A Nagios Check for Chef Server Queue Backlogs +############################################################################### + +require 'rubygems' + +Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do + + require 'bunny' + + $:.unshift(File.expand_path('./lib')) + require 'chef/expander' + require 'chef/expander/version' + require 'chef/expander/configuration' + + include Chef + + Expander.init_config([]) + + config = {:warn => 100, :crit => 200} + + option_parser = OptionParser.new do |o| + o.banner = "Usage: check_queue_size [options]" + + o.on('-w', '--warn WARN_THRESHOLD', 'number of messages to trigger a warning') do |i| + config[:warn] = i.to_i + end + + o.on('-c', '--critical CRITICAL_THRESHOLD', 'the number of messages to trigger a critical') do |n| + config[:crit] = n.to_i + end + + o.on_tail('-h', '--help', 'show this message') do + puts "chef-expander #{Expander.version}" + puts "queue size monitor" + puts '' + puts o + exit 127 + end + end + + option_parser.parse!(ARGV.dup) + + message_counts = [] + + begin + amqp_client = Bunny.new(Expander.config.amqp_config) + amqp_client.start + + 0.upto(Expander::VNODES - 1) do |vnode| + q = amqp_client.queue("vnode-#{vnode}", :durable => true) + message_counts << q.status[:message_count] + end + total_messages = message_counts.inject(:+) + + if total_messages >= config[:crit] + puts "Chef Expander Queue Size CRITICAL - messages: #{total_messages}" + exit(2) + elsif total_messages >= config[:warn] + puts "Chef Expander Queue Size WARNING - messages: #{total_messages}" + exit(1) + else + puts "Chef Expander Queue Size OK - messages: #{total_messages}" + exit(0) + end + + ensure + amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected? + end + +end diff --git a/chef-expander/scripts/make_solr_xml b/chef-expander/scripts/make_solr_xml new file mode 100755 index 0000000000..e44e045bf8 --- /dev/null +++ b/chef-expander/scripts/make_solr_xml @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "rubygems" +$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib')) + +require 'yajl' +require 'chef/expander/solrizer' + +USAGE = <<-EOH +#{$0} [file ...] + +Convert Chef object JSON files into XML documents of the same name but +with a ".xml" extension containing the XML used for Solr indexing. +EOH + +if ARGV.size == 0 + abort USAGE +end + +ARGV.each do |obj_file| + raw_json = open(obj_file, "r").read + item_json = Yajl::Parser.parse(raw_json) + payload = { + :item => item_json, + :type => item_json["chef_type"].to_s, + :database => "riak_search_test", + :id => item_json["name"], + :enqueued_at => Time.now.to_i + } + update_obj = {:action => "add", :payload => payload} + update_json = Yajl::Encoder.encode(update_obj) + solrizer = Chef::Expander::Solrizer.new(update_json) { :no_op } + solrizer.log.init(StringIO.new) + + out = File.basename(obj_file).sub(/\.json$/, "") + ".xml" + open(out, "w") do |f| + f.write(solrizer.pointyize_add) + end +end diff --git a/chef-expander/scripts/traffic-creator b/chef-expander/scripts/traffic-creator new file mode 100755 index 0000000000..a2c207cf36 --- /dev/null +++ b/chef-expander/scripts/traffic-creator @@ -0,0 +1,97 @@ +#!/usr/bin/env ruby +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "rubygems" + +$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib')) + +require 'pp' +require 'bunny' +require 'yajl' +require 'uuidtools' +require 'word_salad' + +require 'chef_search/expander/configuration' + +Chef::Expander.init_config(ARGV) + +MESSAGES_TO_SEND = 10_000 + +NUM_RAND_KEY_PAIRS = 50 +NUM_RAND_VALUE_PAIRS = 50 + +PERSISTENT_MESSAGES = true + +KEYS = NUM_RAND_VALUE_PAIRS.words + +SAMPLE_NODES = [] +Dir.glob(File.expand_path(File.dirname(__FILE__)) + '/../data/*_node.json') do |node_file| + SAMPLE_NODES << Yajl::Parser.parse(IO.read(node_file)) +end + +NUM_NODES = SAMPLE_NODES.size +puts "Read #{NUM_NODES} sample nodes" + +puts "Using rabbitmq config #{Chef::Expander.config.amqp_config.inspect}" + +puts "connecting to rabbitmq" +amqp_client = Bunny.new(Chef::Expander.config.amqp_config) +amqp_client.start + +puts 'declaring queues' +queues = {} +0.upto(1023) do |vnode| + queues[vnode] = amqp_client.queue("vnode-#{vnode}", :durable => true) +end + +def add_rand_keys(node) + rand_key_vals = Hash[*((2 * NUM_RAND_KEY_PAIRS).words)] + rand_vals = Hash[*(KEYS.zip(NUM_RAND_VALUE_PAIRS.words)).flatten] + node.merge(rand_key_vals.merge(rand_vals)) +end + +puts "sending #{MESSAGES_TO_SEND} messages" +start_time = Time.now +sent_messages = 0 +1.upto(MESSAGES_TO_SEND) do + node = SAMPLE_NODES[rand(NUM_NODES)] + node = add_rand_keys(node) + index_data = {:action => :add} + index_data[:payload] = {:item => node} + index_data[:payload][:type] = :node + index_data[:payload][:database] = :testdb + index_data[:payload][:enqueued_at] = Time.now.utc.to_i + + id = node["name"] + vnode = rand(1024) + index_data[:payload][:id] = id + + puts "queue: vnode-#{vnode} (#{sent_messages} / #{MESSAGES_TO_SEND})" + amqp_client.tx_select if PERSISTENT_MESSAGES + queues[vnode].publish(Yajl::Encoder.encode(index_data), :persistent => PERSISTENT_MESSAGES) + amqp_client.tx_commit if PERSISTENT_MESSAGES + sent_messages += 1 +end +end_time = Time.now + +total_time = end_time - start_time +rate = MESSAGES_TO_SEND.to_f / total_time +puts "done (#{total_time}s, #{rate} msg/s)" diff --git a/chef-expander/spec/fixtures/chef-expander.rb b/chef-expander/spec/fixtures/chef-expander.rb new file mode 100644 index 0000000000..32778a1a09 --- /dev/null +++ b/chef-expander/spec/fixtures/chef-expander.rb @@ -0,0 +1,42 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# CHEF EXPANDER CONFIGURATION ###### +# A Sample config file for spec tests # +####################################### + +## The Actual Config Settings for Chef Expander ## +# Solr +solr_url "http://localhost:8983" + +## Parameters for connecting to RabbitMQ ## +# Defaults: +#amqp_host 'localhost' +#amqp_port '5672' +#amqp_user 'guest' +amqp_pass 'config-file' # should override the defaults +amqp_vhost '/config-file' + +## Cluster Config, should be overridden by command line ## +node_count 42 +## Extraneous Crap (should be ignored and not raise an error) ## + +solr_ram_use "1024T" +another_setting "#{solr_ram_use} is an alot"
\ No newline at end of file diff --git a/chef-expander/spec/spec.opts b/chef-expander/spec/spec.opts new file mode 100644 index 0000000000..706088061b --- /dev/null +++ b/chef-expander/spec/spec.opts @@ -0,0 +1 @@ +-cbfs diff --git a/chef-expander/spec/spec_helper.rb b/chef-expander/spec/spec_helper.rb new file mode 100644 index 0000000000..a17bf4f939 --- /dev/null +++ b/chef-expander/spec/spec_helper.rb @@ -0,0 +1,75 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'pp' +require 'stringio' +require 'rubygems' +require 'bunny' +$:.unshift(File.expand_path('../../lib/', __FILE__)) +require 'chef/expander' + +include Chef + +OPSCODE_EXPANDER_MQ_CONFIG = {:user => "guest", :pass => "guest", :vhost => '/chef-expander-test'} + +HOW_TO_SETUP=<<-ERROR + +****************************** FAIL ******************************************* +* Running these tests requires a running instance of rabbitmq +* You also must configure a vhost "/chef-expander-test" +* and a user "guest" with password "guest" with full rights +* to that vhost +------------------------------------------------------------------------------- +> rabbitmq-server +> rabbitmqctl add_vhost /chef-expander-test +> rabbitmqctl set_permissions -p /chef-expander-test guest '.*' '.*' '.*' +> rabbitmqctl list_user_permissions guest +****************************** FAIL ******************************************* + +ERROR + +def debug_exception_to_stderr(e) + if ENV['DEBUG'] == "true" + STDERR.puts("#{e.class.name}: #{e.message}") + STDERR.puts("#{e.backtrace.join("\n")}") + end +end + +begin + b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) + b.start + b.stop +rescue Bunny::ProtocolError, Bunny::ServerDownError, Bunny::ConnectionError => e + STDERR.puts(HOW_TO_SETUP) + debug_exception_to_stderr(e) + exit(1) +rescue Exception => e + STDERR.puts(<<-EXPLAIN) +An unknown error occurred verifying prerequisites for unit tests. +This is most commonly caused by incorrect permissions settings in rabbitmq. + +Run with DEBUG=true to see the error. + +EXPLAIN + STDERR.puts(HOW_TO_SETUP) + debug_exception_to_stderr(e) + exit(2) +end + diff --git a/chef-expander/spec/unit/configuration_spec.rb b/chef-expander/spec/unit/configuration_spec.rb new file mode 100644 index 0000000000..2dddf686aa --- /dev/null +++ b/chef-expander/spec/unit/configuration_spec.rb @@ -0,0 +1,100 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'stringio' +require 'chef/expander/configuration' + +describe Expander::Configuration do + before do + @config = Expander::Configuration::Base.new + @config.reset! + @config.apply_defaults + @config.index = 1 + @config.node_count = 5 + end + + it "stores the number of nodes" do + @config.node_count.should == 5 + end + + it "stores the position in the node ring" do + @config.index.should == 1 + end + + it "computes the vnodes the node should claim" do + @config.vnode_numbers.should == (0..203).to_a + end + + it "assigns any remainder to the last node in the ring" do + @config.index = 5 + @config.vnode_numbers.should == (816..1023).to_a + end + + it "raises an invalid config error when then node index is not set" do + @config.index = nil + lambda { @config.validate! }.should raise_error(Expander::Configuration::InvalidConfiguration) + end + + it "raises an invalid config error when the node count is not set" do + @config.node_count = nil + lambda { @config.validate! }.should raise_error(Expander::Configuration::InvalidConfiguration) + end + + it "raises an invalid config error when the index is greater than the node count" do + @config.node_count = 5 + @config.index = 10 + lambda { @config.validate! }.should raise_error(Expander::Configuration::InvalidConfiguration) + end + + it "exits when the config is invalid" do + stdout = StringIO.new + @config.reset!(stdout) + @config.node_count = nil + lambda {@config.fail_if_invalid}.should raise_error(SystemExit) + stdout.string.should match(/You must specify this node's position in the ring as an integer/) + end + + it "has a setting for solr url defaulting to localhost:8983" do + @config.solr_url.should == "http://localhost:8983" + end + + it "has a setting for the amqp host to connect to, defaulting to 0.0.0.0" do + @config.amqp_host.should == '0.0.0.0' + end + + it "has a setting for the amqp port to use, defaulting to 5672" do + @config.amqp_port.should == '5672' + end + + it "has a setting for the amqp_user, defaulting to 'chef'" do + @config.amqp_user.should == 'chef' + end + + it "has a setting for the amqp password, defaulting to 'testing'" do + @config.amqp_pass.should == 'testing' + end + + it "has a setting for the amqp vhost, defaulting to /chef" do + @config.amqp_vhost.should == '/chef' + end + + it "generates an AMQP configuration hash suitable for passing to Bunny.new or AMQP.start" do + @config.amqp_config.should == {:host => '0.0.0.0', :port => '5672', :user => 'chef', :pass => 'testing', :vhost => '/chef'} + end + + it "merges another config on top of itself" do + other = Expander::Configuration::Base.new + other.solr_url = "somewhere with non-pitiful disk io" + @config.merge_config(other) + @config.solr_url.should == "somewhere with non-pitiful disk io" #if only it was that easy + end + + it "merges config settings so that defaults < config_file < command line " do + config_file = File.dirname(__FILE__) + '/../fixtures/chef-expander.rb' + argv = ["-c", config_file, '-n', '23'] + Expander.config.reset! + Expander.init_config(argv) + Expander.config.amqp_pass.should == 'config-file' + Expander.config.node_count.should == 23 + end + +end
\ No newline at end of file diff --git a/chef-expander/spec/unit/control_spec.rb b/chef-expander/spec/unit/control_spec.rb new file mode 100644 index 0000000000..7c456ec5e8 --- /dev/null +++ b/chef-expander/spec/unit/control_spec.rb @@ -0,0 +1,27 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'chef/expander/control' + +describe Expander::Control do + +end diff --git a/chef-expander/spec/unit/node_spec.rb b/chef-expander/spec/unit/node_spec.rb new file mode 100644 index 0000000000..2d9c1b207d --- /dev/null +++ b/chef-expander/spec/unit/node_spec.rb @@ -0,0 +1,185 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'ostruct' +require 'chef/expander/node' + +describe Expander::Node do + + it "can be created from a hash" do + node_info = { :foo => :blargh, + :guid => "93226974-6d0b-4ca6-8d42-124dd55e0076", + :hostname_f => "fermi.local", :pid => 12345} + unmodified_hash = node_info.dup + node_from_hash = Expander::Node.from_hash(node_info) + node_info.should == unmodified_hash + node_from_hash.guid.should == "93226974-6d0b-4ca6-8d42-124dd55e0076" + node_from_hash.hostname_f.should == "fermi.local" + node_from_hash.pid.should == 12345 + end + + describe "when first created" do + before do + @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" + @hostname_f = "fermi.local" + @pid = 12345 + @node = Expander::Node.new(@guid, @hostname_f, @pid) + end + + it "has the guid it was created with" do + @node.guid.should == @guid + end + + it "has the hostname it was created with" do + @node.hostname_f.should == @hostname_f + end + + it "has the pid it was created with" do + @node.pid.should == @pid + end + + it "names its shared control queue using a constant/consistent name" do + @node.shared_control_queue_name.should == "chef-search-control--shared" + end + + it "names its exclusive control queue after its hostname, pid, and guid" do + @node.exclusive_control_queue_name.should == "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076--exclusive-control" + end + + it "names its broadcast control queue after its hostname, pid, and guid" do + @node.broadcast_control_queue_name.should == "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076--broadcast" + end + + it "names the broadcast control exchange using a consistent name" do + @node.broadcast_control_exchange_name.should == 'chef-search-control--broadcast' + end + + it "generates its hash from a string concatenting the hostname, pid and guid" do + concat_string = "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076" + @node.hash.should == concat_string.hash + end + + it "is eql to another Node if it has the same guid, hostname, and pid" do + other = Expander::Node.new(@guid.dup, @hostname_f.dup, @pid) + @node.should eql(other) + end + + it "is == to another object if it has the same guid, hostname, and pid" do + other = Class.new(Expander::Node).new(@guid.dup, @hostname_f.dup, @pid) + other.should == @node + end + + it "converts to a hash" do + @node.to_hash.should == {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid} + end + + end + + describe "when describing the node it's running on" do + before do + hostname_f = OpenStruct.new(:stdout => "fermi.local\n") + Expander::Node.stub!(:shell_out!).and_return(hostname_f) + @node = Expander::Node.local_node + end + + it "uses the current machine's hostname -f for the hostname" do + @node.hostname_f.should == %x(hostname -f).strip + end + + it "uses the current process id for the pid" do + @node.pid.should == Process.pid + end + + it "generates a guid for the guid" do + @node.guid.should match /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + end + end + + describe "when sending and receiving messages" do + + before do + @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" + @hostname_f = "fermi.local" + @pid = rand(10000) + @node = Expander::Node.new(@guid, @hostname_f, @pid) + @log_stream = StringIO.new + @node.log.init(@log_stream) + end + + after do + b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) + b.start + b.exchange(@node.broadcast_control_exchange_name, :type => :fanout).delete + b.stop + end + + it "receives messages on the broadcast exchange" do + messages = [] + + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + @node.start do |message| + messages << message + end + + @node.broadcast_message("hello everybody") + + EM.add_timer(0.1) {AMQP.hard_reset!} + end + + messages.should == ["hello everybody"] + end + + it "receives messages on its exclusive queue" do + messages = [] + + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + @node.start do |message| + messages << message + end + + @node.direct_message("hello node") + + EM.add_timer(0.1) {AMQP.hard_reset!} + end + + messages.should == ["hello node"] + end + + it "receives messages on the shared queue" do + messages = [] + + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + @node.start do |message| + messages << message + end + + @node.shared_message("hello one of N") + + EM.add_timer(0.1) {AMQP.hard_reset!} + end + + messages.should == ["hello one of N"] + end + + end + +end diff --git a/chef-expander/spec/unit/solrizer_spec.rb b/chef-expander/spec/unit/solrizer_spec.rb new file mode 100644 index 0000000000..443e93a3fa --- /dev/null +++ b/chef-expander/spec/unit/solrizer_spec.rb @@ -0,0 +1,260 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'stringio' +require 'chef/expander/solrizer' +require 'yajl' +require 'rexml/document' + +describe Expander::Solrizer do + SEP = "__=__" + + describe "when created with an add request" do + before do + @now = Time.now.utc.to_i + @indexer_payload = {:item => {:foo => {:bar => :baz}}, + :type => :node, + :database => :testdb, + :id => "2342", + :enqueued_at => @now} + + @update_object = {:action => "add", :payload => @indexer_payload} + @update_json = Yajl::Encoder.encode(@update_object) + @solrizer = Expander::Solrizer.new(@update_json) { :no_op } + + @log_stream = StringIO.new + @solrizer.log.init(@log_stream) + @expected_fields = %w(X_CHEF_id_CHEF_X X_CHEF_database_CHEF_X X_CHEF_type_CHEF_X) + end + + it "extracts the indexing-specific payload from the update message" do + @solrizer.indexer_payload.should == { 'item' => {'foo' => {'bar' => "baz"}}, + 'type' => 'node', + 'database' => 'testdb', 'id' => "2342", + "enqueued_at"=>@now} + end + + it "extracts the action from the update message" do + @solrizer.action.should == "add" + end + + it "extracts the item to update from the update message" do + @solrizer.chef_object.should == {"foo" => {"bar" => "baz"}} + end + + it "extracts the database name from the update message" do + @solrizer.database.should == "testdb" + end + + it "extracts the object id from the update message" do + @solrizer.obj_id.should == "2342" + end + + it "extracts the object type from the update message" do + @solrizer.obj_type.should == "node" + end + + it "extracts the time the object was enqueued from the message" do + @solrizer.enqueued_at.should == @now + end + + it "is eql to another Solrizer object that has the same object type, id, database, action, and enqueued_at time" do + eql_solrizer = Expander::Solrizer.new(@update_json) + @solrizer.should eql eql_solrizer + end + + it "is not eql to another Solrizer if the enqueued_at time is different" do + update_hash = @update_object.dup + update_hash[:payload] = update_hash[:payload].merge({:enqueued_at => (Time.now.utc.to_i + 10000)}) + update_json = Yajl::Encoder.encode(update_hash) + uneql_solrizer = Expander::Solrizer.new(update_json) + @solrizer.should_not eql(uneql_solrizer) + end + + it "is not eql to another Solrizer if the object id is different" do + update_hash = @update_object.dup + update_hash[:payload] = update_hash[:payload].merge({:id => 12345}) + update_json = Yajl::Encoder.encode(update_hash) + uneql_solrizer = Expander::Solrizer.new(update_json) + @solrizer.should_not eql(uneql_solrizer) + end + + it "is not eql to another Solrizer if the database is different" do + update_hash = @update_object.dup + update_hash[:payload] = update_hash[:payload].merge({:database => "nononono"}) + update_json = Yajl::Encoder.encode(update_hash) + uneql_solrizer = Expander::Solrizer.new(update_json) + @solrizer.should_not eql(uneql_solrizer) + end + + it "is not eql to another Solrizer if the action is different" do + update_hash = @update_object.dup + update_hash[:action] = :delete + update_json = Yajl::Encoder.encode(update_hash) + uneql_solrizer = Expander::Solrizer.new(update_json) + @solrizer.should_not eql(uneql_solrizer) + end + + describe "when flattening to XML" do + before do + @expected_object = {"foo" => ["bar"], + "foo_bar" => ["baz"], + "bar" => ["baz"], + "X_CHEF_id_CHEF_X" => ["2342"], + "X_CHEF_database_CHEF_X" => ["testdb"], + "X_CHEF_type_CHEF_X" => ["node"]} + @expected_fields = %w(X_CHEF_id_CHEF_X X_CHEF_database_CHEF_X X_CHEF_type_CHEF_X) + end + + it "generates the flattened and expanded representation of the object" do + @solrizer.flattened_object.should == @expected_object + end + + it "has the expected fields in the document" do + doc = REXML::Document.new(@solrizer.pointyize_add) + flds = doc.elements.to_a("add/doc/field").map {|f| f.attributes["name"] } + @expected_fields.each do |field| + flds.should include(field) + end + end + + it "the content field contains key value pairs delimited with the right separator" do + doc = REXML::Document.new(@solrizer.pointyize_add) + doc.elements.each("add/doc/field[@name='content']") do |content| + raw = content.text + @expected_object.each do |k, v| + s = "#{k}#{SEP}#{v.first}" + raw.index(s).should_not be_nil + end + end + end + end + + describe "when flattening data to XML that needs XML escaping" do + before do + @indexer_payload[:type] = :role + @indexer_payload[:item] = { "a&w" => "<rootbeer/>" } + update_object = {:action => "add", :payload => @indexer_payload} + update_json = Yajl::Encoder.encode(update_object) + @solrizer = Expander::Solrizer.new(update_json) { :no_op } + @solrizer.log.init(@log_stream) + end + + it "the content field contains escaped keys and values" do + raw = @solrizer.pointyize_add + raw.should match("a&w#{SEP}<rootbeer/>") + end + end + + describe "when flattening data bag XML" do + before do + @indexer_payload[:type] = :data_bag_item + @indexer_payload[:item] = {:k1 => "v1", "data_bag" => "stuff"} + update_object = {:action => "add", :payload => @indexer_payload} + update_json = Yajl::Encoder.encode(update_object) + @solrizer = Expander::Solrizer.new(update_json) { :no_op } + @solrizer.log.init(@log_stream) + @expected_fields << "data_bag" + end + + it "contains a data_bag field with the right name" do + doc = REXML::Document.new(@solrizer.pointyize_add) + flds = doc.elements.to_a("add/doc/field[@name='data_bag']") + flds.size.should == 1 + flds.first.text.should == "stuff" + end + + it "has the expected fields in the document" do + doc = REXML::Document.new(@solrizer.pointyize_add) + flds = doc.elements.to_a("add/doc/field").map {|f| f.attributes["name"] } + @expected_fields.each do |field| + flds.should include(field) + end + end + describe "and data bag name needs escaping" do + before do + @indexer_payload[:item] = {:k1 => "v1", "data_bag" => "a&w>"} + update_object = {:action => "add", :payload => @indexer_payload} + update_json = Yajl::Encoder.encode(update_object) + @solrizer = Expander::Solrizer.new(update_json) { :no_op } + @solrizer.log.init(@log_stream) + end + + it "contains a data_bag field with an escaped name" do + raw = @solrizer.pointyize_add + raw.should match("data_bag#{SEP}a&w>") + end + end + end + + describe "when no HTTP request is in progress" do + + it "does not report that an HTTP request is in progress" do + Expander::Solrizer.http_requests_active?.should be_false + end + + end + + describe "when an HTTP request is in progress" do + before do + Expander::Solrizer.clear_http_requests + @solrizer.http_request_started + end + + it "registers the in-progress HTTP request" do + Expander::Solrizer.http_requests_active?.should be_true + end + + it "removes itself from the list of active http requests when the request completes" do + @solrizer.completed + Expander::Solrizer.http_requests_active?.should be_false + end + + end + + + end + + describe "when created with a delete request" do + before do + @indexer_payload = {:id => "2342"} + @update_object = {:action => "add", :payload => @indexer_payload} + @update_json = Yajl::Encoder.encode(@update_object) + @solrizer = Expander::Solrizer.new(@update_json) + end + + it "extracts the indexer payload" do + @solrizer.indexer_payload.should == {"id" => "2342"} + end + + it "extracts the object id" do + @solrizer.obj_id.should == "2342" + end + + it "converts the delete request to XML" do + expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n" + @solrizer.pointyize_delete.should == expected + end + + end + +end diff --git a/chef-expander/spec/unit/vnode_spec.rb b/chef-expander/spec/unit/vnode_spec.rb new file mode 100644 index 0000000000..9321c03ef3 --- /dev/null +++ b/chef-expander/spec/unit/vnode_spec.rb @@ -0,0 +1,79 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'chef/expander/vnode_supervisor' +require 'chef/expander/vnode' + +describe Expander::VNode do + before do + @supervisor = Expander::VNodeSupervisor.new + @vnode = Expander::VNode.new("2342", @supervisor, :supervise_interval => 0.1) + @log_stream = StringIO.new + @vnode.log.init(@log_stream) + end + + it "has the vnode number it was created with" do + @vnode.vnode_number.should == 2342 + end + + it "has a queue named after its vnode number" do + @vnode.queue_name.should == "vnode-2342" + end + + it "has a control queue name" do + @vnode.control_queue_name.should == "vnode-2342-control" + end + + describe "when connecting to rabbitmq" do + it "disconnects if there is another subscriber" do + begin + q = nil + b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) + b.start + q = b.queue(@vnode.queue_name, :passive => false, :durable => true, :exclusive => false, :auto_delete => false) + t = Thread.new { q.subscribe { |message| nil }} + + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + EM.add_timer(0.5) do + AMQP.stop + EM.stop + end + @vnode.start + end + t.kill + + @vnode.should be_stopped + @log_stream.string.should match(/Detected extra consumers/) + ensure + q && q.delete + b.stop + end + end + + it "calls back to the supervisor when it subscribes to the queue" do + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + MQ.topic('foo') + EM.add_timer(0.1) do + AMQP.stop + EM.stop + end + @vnode.start + end + @supervisor.vnodes.should == [2342] + end + + it "calls back to the supervisor when it stops subscribing" do + @supervisor.vnode_added(@vnode) + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + MQ.topic('foo') + EM.add_timer(0.1) do + @vnode.stop + AMQP.stop + EM.stop + end + end + @supervisor.vnodes.should be_empty + end + + end + +end diff --git a/chef-expander/spec/unit/vnode_supervisor_spec.rb b/chef-expander/spec/unit/vnode_supervisor_spec.rb new file mode 100644 index 0000000000..59622675e8 --- /dev/null +++ b/chef-expander/spec/unit/vnode_supervisor_spec.rb @@ -0,0 +1,152 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'chef/expander/vnode_supervisor' + +describe Expander::VNodeSupervisor do + before do + @log_stream = StringIO.new + @local_node = Expander::Node.new("1101d02d-1547-45ab-b2f6-f0153d0abb34", "fermi.local", 12342) + @vnode_supervisor = Expander::VNodeSupervisor.new + @vnode_supervisor.instance_variable_set(:@local_node, @local_node) + @vnode_supervisor.log.init(@log_stream) + @vnode = Expander::VNode.new("42", @vnode_supervisor) + end + + after do + b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) + b.start + b.exchange(@vnode_supervisor.local_node.broadcast_control_exchange_name, :type => :fanout).delete + b.queue(@vnode_supervisor.local_node.broadcast_control_queue_name).purge + b.stop + end + + it "keeps a list of vnodes" do + @vnode_supervisor.vnodes.should be_empty + @vnode_supervisor.vnode_added(@vnode) + @vnode_supervisor.vnodes.should == [42] + end + + it "has a callback for vnode removal" do + @vnode_supervisor.vnode_added(@vnode) + @vnode_supervisor.vnodes.should == [42] + @vnode_supervisor.vnode_removed(@vnode) + @vnode_supervisor.vnodes.should be_empty + end + + it "spawns a vnode" do + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + @vnode_supervisor.spawn_vnode(42) + MQ.topic('foo') + EM.add_timer(0.1) do + AMQP.hard_reset! + end + end + @vnode_supervisor.vnodes.should == [42] + end + + it "subscribes to the control queue" do + AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do + @vnode_supervisor.start([]) + @vnode_supervisor.should_receive(:process_control_message).with("hello_robot_overlord") + Expander::Node.local_node.broadcast_message("hello_robot_overlord") + EM.add_timer(0.1) do + AMQP.hard_reset! + end + end + end + + it "periodically publishes its list of vnodes to the gossip queue" do + pending("disabled until cluster healing is implemented") + end + + describe "when responding to control messages" do + it "passes vnode table updates to its vnode table" do + vnode_table_update = Expander::Node.local_node.to_hash + vnode_table_update[:vnodes] = (0...16).to_a + vnode_table_update[:update] = :add + update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) + @vnode_supervisor.process_control_message(update_message) + @vnode_supervisor.vnode_table.vnodes_by_node[Expander::Node.local_node].should == (0...16).to_a + end + + it "publishes the vnode table when it receives a :vnode_table_publish message" do + pending "disabled until cluster healing is implemented" + update_message = Yajl::Encoder.encode({:action => :vnode_table_publish}) + @vnode_supervisor.process_control_message(update_message) + end + + describe "and it is the leader" do + before do + vnode_table_update = Expander::Node.local_node.to_hash + vnode_table_update[:vnodes] = (0...16).to_a + vnode_table_update[:update] = :add + update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) + @vnode_supervisor.process_control_message(update_message) + end + + it "distributes the vnode when it receives a recover_vnode message and it is the leader" do + control_msg = {:action => :recover_vnode, :vnode_id => 23} + + @vnode_supervisor.local_node.should_receive(:shared_message) + @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) + end + + it "waits before re-advertising a vnode as available" do + pending("not yet implemented") + vnode_table_update = Expander::Node.local_node.to_hash + vnode_table_update[:vnodes] = (0...16).to_a + vnode_table_update[:update] = :add + update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) + @vnode_supervisor.process_control_message(update_message) + + control_msg = {:action => :recover_vnode, :vnode_id => 23} + + @vnode_supervisor.local_node.should_receive(:shared_message).once + @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) + @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) + end + end + + + it "doesn't distribute a vnode when it is not the leader" do + vnode_table_update = Expander::Node.local_node.to_hash + vnode_table_update[:vnodes] = (16...32).to_a + vnode_table_update[:update] = :add + update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) + @vnode_supervisor.process_control_message(update_message) + + vnode_table_update = Expander::Node.new("1c53daf0-34a1-4e4f-8069-332665453b44", 'fermi.local', 2342).to_hash + vnode_table_update[:vnodes] = (0...16).to_a + vnode_table_update[:update] = :add + update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) + @vnode_supervisor.process_control_message(update_message) + + control_msg = {:action => :recover_vnode, :vnode_id => 42} + + @vnode_supervisor.local_node.should_not_receive(:shared_message) + @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) + end + + end + +end diff --git a/chef-expander/spec/unit/vnode_table_spec.rb b/chef-expander/spec/unit/vnode_table_spec.rb new file mode 100644 index 0000000000..faa76b84c1 --- /dev/null +++ b/chef-expander/spec/unit/vnode_table_spec.rb @@ -0,0 +1,114 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Author:: Chris Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +require 'chef/expander/vnode_table' +require 'chef/expander/vnode_supervisor' + +describe Expander::VNodeTable do + before do + @vnode_supervisor = Expander::VNodeSupervisor.new + @vnode_table = Expander::VNodeTable.new(@vnode_supervisor) + + @log_stream = StringIO.new + @vnode_table.log.init(@log_stream) + end + + describe "when first created" do + it "has no nodes" do + @vnode_table.nodes.should be_empty + end + end + + describe "when one node's vnode info has been added" do + before do + @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" + @hostname_f = "fermi.localhost" + @pid = 12345 + @vnodes = (0..511).to_a + @update = {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid, :vnodes => @vnodes, :update => 'update'} + @vnode_table.update_table(@update) + end + + it "has one vnode" do + @vnode_table.should have(1).nodes + @vnode_table.nodes.first.should == Expander::Node.from_hash(@update) + end + + it "removes the node from the table when it exits the cluster" do + update = @update + update[:update] = 'remove' + @vnode_table.update_table(update) + @vnode_table.should have(0).nodes + end + + end + + describe "when several nodes are in the table" do + before do + @node_1 = Expander::Node.new("93226974-6d0b-4ca6-8d42-124dd55e0076", "fermi.local", 12345) + @node_1_hash = @node_1.to_hash + @node_1_hash[:vnodes] = (0..511).to_a + @node_1_hash[:update] = "update" + @node_2 = Expander::Node.new("ad265988-f650-4a31-a97b-5dbf4db8e1b0", "fermi.local", 23425) + @node_2_hash = @node_2.to_hash + @node_2_hash[:vnodes] = (512..767).to_a + @node_2_hash[:update] = "update" + @vnode_table.update_table(@node_1_hash) + @vnode_table.update_table(@node_2_hash) + end + + it "determines the node with the lowest numbered vnode is the leader node" do + @vnode_table.leader_node.should == @node_1 + end + + it "determines the local node is the leader when the local node matches the leader node" do + Expander::Node.stub!(:local_node).and_return(@node_1) + @vnode_table.local_node_is_leader?.should be_true + end + + it "determines the local node is not the leader when the local node doesn't match the leader node" do + Expander::Node.stub!(:local_node).and_return(@node_2) + @vnode_table.local_node_is_leader?.should be_false + end + end + + describe "when only one node has claimed any vnodes" do + before do + @node_1 = Expander::Node.new("93226974-6d0b-4ca6-8d42-124dd55e0076", "fermi.local", 12345) + @node_1_hash = @node_1.to_hash + @node_1_hash[:vnodes] = (0..511).to_a + @node_1_hash[:update] = "update" + @node_2 = Expander::Node.new("ad265988-f650-4a31-a97b-5dbf4db8e1b0", "fermi.local", 23425) + @node_2_hash = @node_2.to_hash + @node_2_hash[:vnodes] = [] + @node_2_hash[:update] = "update" + @vnode_table.update_table(@node_1_hash) + @vnode_table.update_table(@node_2_hash) + end + + it "still reliably determines who the leader is" do + pending + end + + end + +end |