diff options
| author | Seth Falcon <seth@opscode.com> | 2012-10-08 15:41:22 -0700 |
|---|---|---|
| committer | Seth Falcon <seth@opscode.com> | 2012-10-08 15:41:22 -0700 |
| commit | ded6dda0bfec60c482d0cbf08f15c5086af78d6c (patch) | |
| tree | c1c5c5b6d89d3dc76ea806522ab52ff73267dd10 | |
| parent | 7253b47582c2560ab15390d98a4ec3ae32bdfbf5 (diff) | |
| download | chef-ded6dda0bfec60c482d0cbf08f15c5086af78d6c.tar.gz | |
Remove chef-expander
The chef-expander code has moved to its own git repository located at:
https://github.com/opscode/chef-expander
41 files changed, 1 insertions, 3911 deletions
diff --git a/README.rdoc b/README.rdoc index 726c99ca97..fdff0a5c78 100644 --- a/README.rdoc +++ b/README.rdoc @@ -76,7 +76,7 @@ These are available in most common distribution packaging systems, including Deb Make sure the RubyGems for the Chef Server are installed (should be from the +rake install+ earlier): -* chef-expander +* chef-expander (now in its own repository at https://github.com/opscode/chef-expander) * chef-server * chef-server-api * chef-server-webui (now in its own repository at https://github.com/opscode/chef-server-webui) diff --git a/chef-expander/.gitignore b/chef-expander/.gitignore deleted file mode 100644 index 681b81f181..0000000000 --- a/chef-expander/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -.bundle -.autotest -coverage -doc -.DS_Store -*.swp -*.swo -erl_crash.dump -*.rake_tasks~ -.idea -*.rbc -conf/chef-expander.rb -Gemfile.lock diff --git a/chef-expander/Gemfile b/chef-expander/Gemfile deleted file mode 100644 index 1eb1987930..0000000000 --- a/chef-expander/Gemfile +++ /dev/null @@ -1,4 +0,0 @@ -## Chef Expander ## -source :rubygems - -gemspec diff --git a/chef-expander/LICENSE b/chef-expander/LICENSE deleted file mode 100644 index 11069edd79..0000000000 --- a/chef-expander/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/chef-expander/README.rdoc b/chef-expander/README.rdoc deleted file mode 100644 index c287d63fce..0000000000 --- a/chef-expander/README.rdoc +++ /dev/null @@ -1,20 +0,0 @@ -= Chef Expander - -== What's This? -Chef Expander replaces the chef-solr-indexer daemon that was included with Chef 0.8 and 0.9 - -== Dependencies - -* bunny -* yajl -* eventmachine -* em-http-request -* amqp -* highline - -== Monitoring With Nagios -A Nagios plugin to monitor queue backlog is included in scripts/ directory as <tt>check_queue_size</tt> -To run it with the warning threshold at 250 messages and critical at 500 messages: - - check_queue_size -w 250 -c 500 - diff --git a/chef-expander/Rakefile b/chef-expander/Rakefile deleted file mode 100644 index c8c772ed39..0000000000 --- a/chef-expander/Rakefile +++ /dev/null @@ -1,55 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Copyright:: Copyright (c) 2010, 2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -$: << File.dirname(__FILE__) -require 'lib/chef/expander/version' -require 'rake/gempackagetask' -require 'rake/rdoctask' -require '../chef/tasks/rspec.rb' - -spec = eval(File.read("chef-expander.gemspec")) - -Rake::GemPackageTask.new(spec) do |pkg| - pkg.gem_spec = spec -end - -begin - require 'sdoc' - - Rake::RDocTask.new do |rdoc| - rdoc.title = "Chef Ruby API Documentation" - rdoc.main = "README.rdoc" - rdoc.options << '--fmt' << 'shtml' # explictly set shtml generator - rdoc.template = 'direct' # lighter template - rdoc.rdoc_files.include("README.rdoc", "LICENSE", "spec/tiny_server.rb", "lib/**/*.rb") - rdoc.rdoc_dir = "rdoc" - end -rescue LoadError - puts "sdoc is not available. (sudo) gem install sdoc to generate rdoc documentation." -end - -task :install => :package do - sh %{gem install pkg/chef-expander-#{Chef::Expander::VERSION} --no-rdoc --no-ri} -end - -desc "install gem dependencies" -task :bundle do - sh("bundle install") -end - -task :default => :spec diff --git a/chef-expander/bin/chef-expander b/chef-expander/bin/chef-expander deleted file mode 100755 index 789e6e81b8..0000000000 --- a/chef-expander/bin/chef-expander +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env ruby -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require "rubygems" - -$:.unshift(File.dirname(__FILE__) + '/../lib/') - -require 'chef/expander' -require 'chef/expander/cluster_supervisor' - -Chef::Expander::ClusterSupervisor.new.start diff --git a/chef-expander/bin/chef-expander-vnode b/chef-expander/bin/chef-expander-vnode deleted file mode 100755 index 8d2cdbba02..0000000000 --- a/chef-expander/bin/chef-expander-vnode +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env ruby -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -require "rubygems" - -$:.unshift(File.dirname(__FILE__) + '/../lib/') - -require 'chef/expander' -require 'chef/expander/vnode_supervisor' - -Chef::Expander::VNodeSupervisor.start diff --git a/chef-expander/bin/chef-expanderctl b/chef-expander/bin/chef-expanderctl deleted file mode 100755 index c1b201a7b2..0000000000 --- a/chef-expander/bin/chef-expanderctl +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env ruby -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -require "rubygems" - -$:.unshift(File.dirname(__FILE__) + '/../lib/') - -require 'chef/expander' -require 'chef/expander/control' - -Chef::Expander::Control.run(ARGV) diff --git a/chef-expander/chef-expander.gemspec b/chef-expander/chef-expander.gemspec deleted file mode 100644 index 9fc6bda8ae..0000000000 --- a/chef-expander/chef-expander.gemspec +++ /dev/null @@ -1,34 +0,0 @@ -$:.unshift(File.dirname(__FILE__) + '/lib') - -sandbox = Module.new -sandbox.module_eval(IO.read(File.expand_path('../lib/chef/expander/version.rb', __FILE__))) - -Gem::Specification.new do |s| - s.name = 'chef-expander' - s.version = sandbox::Chef::Expander::VERSION - s.platform = Gem::Platform::RUBY - s.has_rdoc = true - s.extra_rdoc_files = ["README.rdoc", "LICENSE" ] - s.summary = "A systems integration framework, built to bring the benefits of configuration management to your entire infrastructure." - s.description = s.summary - s.author = "Adam Jacob" - s.email = "adam@opscode.com" - s.homepage = "http://wiki.opscode.com/display/chef" - - s.add_dependency "mixlib-log", ">= 1.2.0" - s.add_dependency "amqp", "~> 0.6.7" - s.add_dependency "eventmachine", '~> 0.12.10' - s.add_dependency "em-http-request", "~> 0.2.11" - s.add_dependency 'yajl-ruby', "~> 1.0" - s.add_dependency 'uuidtools', "~> 2.1.1" - s.add_dependency 'bunny', '~> 0.6.0' - s.add_dependency 'fast_xs', "~> 0.7.3" - s.add_dependency 'highline', '~> 1.6.1' - - %w(rake rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem } - - s.bindir = "bin" - s.executables = %w( chef-expander chef-expander-vnode chef-expanderctl ) - s.require_path = 'lib' - s.files = %w(LICENSE README.rdoc) + Dir.glob("{scripts,conf,lib}/**/*") -end diff --git a/chef-expander/conf/chef-expander.rb.example b/chef-expander/conf/chef-expander.rb.example deleted file mode 100644 index 69fb972798..0000000000 --- a/chef-expander/conf/chef-expander.rb.example +++ /dev/null @@ -1,9 +0,0 @@ -# The URL is the only Solr config Chef Expander needs # -solr_url "http://localhost:8983/solr" - -# Parameters for connecting to RabbitMQ -amqp_host 'localhost' -amqp_port '5672' -amqp_user 'chef' -amqp_pass 'testing' -amqp_vhost '/chef' diff --git a/chef-expander/data/sample_node.json b/chef-expander/data/sample_node.json deleted file mode 100644 index 80dac4bace..0000000000 --- a/chef-expander/data/sample_node.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"i-f09a939b","json_class":"Chef::Node","automatic":{"keys":{"ssh":{"host_dsa_public":"AAAAB3NzaC1kc3MAAACBAJ144IxziCQ3y9Q8erUviDO5hH0dxOEXYo5rTsUTB+eyvTbFPgIn1XmPyWBpqCpBEDHR7Mngzn9zIvXhf3YGNq4CaGKKE0JVcQXwveOXq2R7H32MUP4r0qP4wDrpjEtc/uQ4zJMJuSTLL+++HWClyD13QaVoWemlkpTkGE763/NpAAAAFQDuOq9biIZiQpcRnv4T9xQ22q9gUQAAAIAV55roDVnQfijL0RiHztUlRlTSq/57VJobKpn1gGaGltqpMi+k9NOecYSG98nqCibSubnjnGhotWAMsdd4QRfbwC3ofrv1fr/xBuqu4/yb8OjStQ7OrYe5Z5Qg8n6CT0QkL2GoWw3T3wg3np01dDakD2Yywv6zeoDbOeIGWizzQgAAAIEAmeWpAVdvxC7AnpgzOducU3j2zUzK5Uke3ZbX9TY8L5EI/cbogEQA6zp4QuKm68VcRe7Ca2Ljk+6vjqA8tsyQffNqnwru6B6xLUtCroF9BP3UktESBrGIaxlUGaSewdmaZ4JtoH40cdJtrAzI5GMDOUbtGTmVIWxjeKquk9NpWd0=","host_rsa_public":"AAAAB3NzaC1yc2EAAAABIwAAAQEArhqRYGssVycUxj8xPhDDmkveqg8mmMM6jyp1B4mWR0Nk2fuz5fzrVPsn+xXIwPPoF7PwoGH+VWqkqn0sIQyv/mUFhEi5hG+b+1+KZAOhFIryacRscY7BdVPCuEClVFSoJxVfDS40QFm1khsZW2u9/UIBLBBgxO3gtC7LCB0rpXmsU1YJrQ52EKRGkso01iOUpSqi9oIoFbBdNDOHw0+qwMf3R/zpqk/wA1loTOq9DbdKVYyrP5wvSbcfehuOocSAyfF1CwfsHPgZXrG9olyCI7LvfaMcdSNFoyAuS12Zu64knvrJ13kavabHzfVL/qwf1uDnXdTRUM5xIgUtKGLqDQ=="}},"dmi":{"bios":{},"version":"2.9"},"kernel":{"modules":{"ipv6":{"size":"243002","refcount":"12"}},"machine":"i686","name":"Linux","os":"GNU/Linux","version":"#9-Ubuntu SMP Thu Apr 15 04:14:01 UTC 2010","release":"2.6.32-305-ec2"},"platform_version":"10.04","fqdn":"lb-prod-i-f09a939b.opscode.us","filesystem":{"/dev/sda1":{"kb_size":"15481840","kb_available":"12433352","mount":"/","percent_used":"16%","kb_used":"2262056"},"proc":{"mount_options":["rw","noexec","nosuid","nodev"],"fs_type":"proc","mount":"/proc"},"/dev/sda2":{"kb_size":"350891748","kb_available":"332868096","mount_options":["rw"],"fs_type":"ext3","mount":"/mnt","percent_used":"1%","kb_used":"199372"},"none":{"kb_size":"890988","kb_available":"890988","mount_options":["rw","nosuid","mode=0755"],"fs_type":"tmpfs","mount":"/lib/init/rw","percent_used":"0%","kb_used":"0"},"devtmpfs":{"kb_size":"873976","kb_available":"873860","mount_options":["rw","mode=0755"],"fs_type":"devtmpfs","mount":"/dev","percent_used":"1%","kb_used":"116"}},"ipaddress":"10.192.26.6","command":{"ps":"ps -ef"},"memory":{"dirty":"164kB","vmalloc_used":"5732kB","page_tables":"0kB","buffers":"241844kB","slab_unreclaim":"6268kB","high_free":"207384kB","vmalloc_chunk":"116776kB","nfs_unstable":"0kB","slab":"48240kB","inactive":"191476kB","total":"1781976kB","vmalloc_total":"122880kB","low_free":"405380kB","low_total":"735440kB","free":"612764kB","commit_limit":"1808484kB","anon_pages":"64212kB","writeback":"0kB","cached":"768476kB","swap":{"total":"917496kB","free":"917496kB","cached":"0kB"},"high_total":"1046536kB","committed_as":"249708kB","bounce":"0kB","slab_reclaimable":"41972kB","mapped":"18684kB","active":"882952kB"},"idletime_seconds":6070168,"counters":{"network":{"interfaces":{"eql":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"5","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"ifb0":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"32","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"lo":{"tx":{"bytes":"32312142","packets":"307835","collisions":"0","queuelen":"0","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"32312142","packets":"307835","errors":"0","overrun":"0","drop":"0","frame":"0"}},"ifb1":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"32","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}},"eth0":{"tx":{"bytes":"1090545338","packets":"168873544","collisions":"0","queuelen":"1000","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"2226852623","packets":"286299389","errors":"0","overrun":"0","drop":"0","frame":"0"}},"dummy0":{"tx":{"bytes":"0","packets":"0","collisions":"0","queuelen":"0","errors":"0","carrier":"0","overrun":"0","drop":"0"},"rx":{"bytes":"0","packets":"0","errors":"0","overrun":"0","drop":"0","frame":"0"}}}}},"domain":"opscode.us","os":"linux","idletime":"70 days 06 hours 09 minutes 28 seconds","lsb":{"codename":"lucid","id":"Ubuntu","description":"\"Ubuntu 10.04.1 LTS\"","release":"10.04"},"network":{"default_interface":"eth0","interfaces":{"eql":{"mtu":"576","encapsulation":"Serial"},"ifb0":{"flags":["BROADCAST","NOARP"],"number":"0","addresses":{"8a:7e:3d:8b:92:88":{"family":"lladdr"}},"mtu":"1500","type":"ifb","encapsulation":"Ethernet"},"lo":{"flags":["UP","LOOPBACK","RUNNING"],"addresses":{"::1":{"scope":"Node","prefixlen":"128","family":"inet6"},"127.0.0.1":{"netmask":"255.0.0.0","family":"inet"}},"mtu":"16436","encapsulation":"Loopback"},"ifb1":{"flags":["BROADCAST","NOARP"],"number":"1","addresses":{"d2:09:bf:06:5f:d1":{"family":"lladdr"}},"mtu":"1500","type":"ifb","encapsulation":"Ethernet"},"eth0":{"flags":["UP","BROADCAST","RUNNING","MULTICAST"],"number":"0","addresses":{"10.192.26.6":{"netmask":"255.255.254.0","broadcast":"10.192.27.255","family":"inet"},"12:31:39:0e:19:f4":{"family":"lladdr"},"fe80::1031:39ff:fe0e:19f4":{"scope":"Link","prefixlen":"64","family":"inet6"}},"mtu":"1500","type":"eth","arp":{"10.192.26.1":"fe:ff:ff:ff:ff:ff"},"encapsulation":"Ethernet"},"dummy0":{"flags":["BROADCAST","NOARP"],"number":"0","addresses":{"f2:77:75:3a:f4:71":{"family":"lladdr"}},"mtu":"1500","type":"dummy","encapsulation":"Ethernet"}}},"virtualization":{"role":"guest","emulator":"xen"},"current_user":"dan","ohai_time":1281135306.45911,"chef_packages":{"ohai":{"version":"0.5.4"},"chef":{"version":"0.9.0.a91"}},"os_version":"2.6.32-305-ec2","languages":{"perl":{"version":"5.10.1","archname":"i486-linux-gnu-thread-multi"},"python":{"version":"2.6.5","builddate":"Apr 16 2010, 13:09:56"},"ruby":{"target_os":"linux","bin_dir":"/usr/bin","host_vendor":"pc","target_vendor":"pc","ruby_bin":"/usr/bin/ruby1.8","target_cpu":"i486","version":"1.8.7","host_os":"linux-gnu","release_date":"2010-01-10","host_cpu":"i486","host":"i486-pc-linux-gnu","target":"i486-pc-linux-gnu","gems_dir":"/usr/lib/ruby/gems/1.8","platform":"i486-linux"},"erlang":{"version":"5.7.4","emulator":"BEAM","options":["SMP","ASYNC_THREADS"]}},"cpu":{"real":2,"total":2,"0":{"flags":["fpu","tsc","msr","pae","mce","cx8","apic","mca","cmov","pat","pse36","clflush","dts","acpi","mmx","fxsr","sse","sse2","ss","ht","tm","pbe","nx","lm","constant_tsc","arch_perfmon","pebs","bts","aperfmperf","pni","dtes64","monitor","ds_cpl","vmx","est","tm2","ssse3","cx16","xtpr","pdcm","dca","sse4_1","xsave","lahf_lm","tpr_shadow","vnmi","flexpriority"],"cores":"1","model":"23","core_id":"0","model_name":"Intel(R) Xeon(R) CPU E5410 @ 2.33GHz","family":"6","physical_id":"0","mhz":"2327.494","vendor_id":"GenuineIntel","cache_size":"6144 KB","stepping":"10"},"1":{"flags":["fpu","tsc","msr","pae","mce","cx8","apic","mca","cmov","pat","pse36","clflush","dts","acpi","mmx","fxsr","sse","sse2","ss","ht","tm","pbe","nx","lm","constant_tsc","arch_perfmon","pebs","bts","aperfmperf","pni","dtes64","monitor","ds_cpl","vmx","est","tm2","ssse3","cx16","xtpr","pdcm","dca","sse4_1","xsave","lahf_lm","tpr_shadow","vnmi","flexpriority"],"cores":"1","model":"23","core_id":"0","model_name":"Intel(R) Xeon(R) CPU E5410 @ 2.33GHz","family":"6","physical_id":"1","mhz":"2327.494","vendor_id":"GenuineIntel","cache_size":"6144 KB","stepping":"10"}},"uptime":"35 days 15 hours 06 minutes 20 seconds","hostname":"lb-prod-i-f09a939b","etc":{"group":{"cb":{"gid":7003,"members":[]},"sasl":{"gid":45,"members":[]},"kmem":{"gid":15,"members":[]},"jesse":{"gid":7009,"members":[]},"ssl-cert":{"gid":114,"members":[]},"lp":{"gid":7,"members":[]},"nagios":{"gid":113,"members":[]},"dip":{"gid":30,"members":["ubuntu"]},"www-data":{"gid":33,"members":[]},"opscode":{"gid":5049,"members":["opscode"]},"ubuntu":{"gid":1000,"members":[]},"cw":{"gid":7002,"members":[]},"news":{"gid":9,"members":[]},"ssh":{"gid":106,"members":[]},"adam":{"gid":7000,"members":[]},"mark":{"gid":7015,"members":[]},"dan":{"gid":7008,"members":[]},"sudo":{"gid":27,"members":[]},"nuo":{"gid":7006,"members":[]},"ronny":{"gid":7012,"members":[]},"cdrom":{"gid":24,"members":["ubuntu"]},"list":{"gid":38,"members":[]},"uucp":{"gid":10,"members":[]},"voice":{"gid":22,"members":[]},"netdev":{"gid":111,"members":[]},"shadow":{"gid":42,"members":[]},"root":{"gid":0,"members":[]},"games":{"gid":60,"members":[]},"adm":{"gid":4,"members":["ubuntu"]},"crontab":{"gid":102,"members":[]},"munin":{"gid":112,"members":[]},"joshua":{"gid":7004,"members":[]},"haldaemon":{"gid":108,"members":[]},"src":{"gid":40,"members":[]},"staff":{"gid":50,"members":[]},"gnats":{"gid":41,"members":[]},"haclient":{"gid":117,"members":[]},"seth":{"gid":7013,"members":[]},"operator":{"gid":37,"members":[]},"fuse":{"gid":104,"members":[]},"daemon":{"gid":1,"members":[]},"proxy":{"gid":13,"members":[]},"john":{"gid":7010,"members":[]},"disk":{"gid":6,"members":[]},"landscape":{"gid":109,"members":[]},"tty":{"gid":5,"members":[]},"sys":{"gid":3,"members":[]},"nathan":{"gid":7001,"members":[]},"postdrop":{"gid":116,"members":[]},"man":{"gid":12,"members":[]},"sysadmin":{"gid":2300,"members":["bryan","jesse","john","ronny","stephen","seth","joshua","opscode","nathan","mark","tim","dan","cw","adam","cb","nuo"]},"plugdev":{"gid":46,"members":["ubuntu"]},"tape":{"gid":26,"members":[]},"postfix":{"gid":115,"members":[]},"tim":{"gid":7005,"members":[]},"floppy":{"gid":25,"members":["ubuntu"]},"users":{"gid":100,"members":[]},"fax":{"gid":21,"members":[]},"syslog":{"gid":103,"members":[]},"irc":{"gid":39,"members":[]},"libuuid":{"gid":101,"members":[]},"mlocate":{"gid":105,"members":[]},"bin":{"gid":2,"members":[]},"video":{"gid":44,"members":["ubuntu"]},"messagebus":{"gid":107,"members":[]},"admin":{"gid":110,"members":["ubuntu"]},"bryan":{"gid":7011,"members":[]},"nogroup":{"gid":65534,"members":[]},"splunk":{"gid":7014,"members":[]},"utmp":{"gid":43,"members":[]},"dialout":{"gid":20,"members":["ubuntu"]},"stephen":{"gid":7007,"members":[]},"mail":{"gid":8,"members":[]},"backup":{"gid":34,"members":[]},"audio":{"gid":29,"members":["ubuntu"]}},"passwd":{"opscode":{"gecos":"Opscode deploy user","dir":"/home/opscode","gid":5049,"uid":5049,"shell":"/bin/bash"},"nathan":{"gecos":"Nathan Haneysmith","dir":"/home/nathan","gid":7001,"uid":7001,"shell":"/bin/bash"},"uucp":{"gecos":"uucp","dir":"/var/spool/uucp","gid":10,"uid":10,"shell":"/bin/sh"},"jesse":{"gecos":"Jesse Robbins","dir":"/home/jesse","gid":7009,"uid":7009,"shell":"/bin/bash"},"syslog":{"gecos":"","dir":"/home/syslog","gid":103,"uid":101,"shell":"/bin/false"},"list":{"gecos":"Mailing List Manager","dir":"/var/list","gid":38,"uid":38,"shell":"/bin/sh"},"games":{"gecos":"games","dir":"/usr/games","gid":60,"uid":5,"shell":"/bin/sh"},"munin":{"gecos":"","dir":"/var/lib/munin","gid":112,"uid":106,"shell":"/bin/false"},"tim":{"gecos":"","dir":"/home/tim","gid":7005,"uid":7005,"shell":"/bin/bash"},"john":{"gecos":"John Willis","dir":"/home/john","gid":7010,"uid":7010,"shell":"/bin/bash"},"sys":{"gecos":"sys","dir":"/dev","gid":3,"uid":3,"shell":"/bin/sh"},"nagios":{"gecos":"","dir":"/var/lib/nagios","gid":113,"uid":107,"shell":"/bin/false"},"nobody":{"gecos":"nobody","dir":"/nonexistent","gid":65534,"uid":65534,"shell":"/bin/sh"},"bryan":{"gecos":"Bryan Hale","dir":"/home/bryan","gid":7011,"uid":7011,"shell":"/bin/bash"},"libuuid":{"gecos":"","dir":"/var/lib/libuuid","gid":101,"uid":100,"shell":"/bin/sh"},"irc":{"gecos":"ircd","dir":"/var/run/ircd","gid":39,"uid":39,"shell":"/bin/sh"},"backup":{"gecos":"backup","dir":"/var/backups","gid":34,"uid":34,"shell":"/bin/sh"},"www-data":{"gecos":"www-data","dir":"/var/www","gid":33,"uid":33,"shell":"/bin/sh"},"lp":{"gecos":"lp","dir":"/var/spool/lpd","gid":7,"uid":7,"shell":"/bin/sh"},"man":{"gecos":"man","dir":"/var/cache/man","gid":12,"uid":6,"shell":"/bin/sh"},"splunk":{"gecos":"Splunk Server","dir":"/opt/splunk","gid":7014,"uid":7014,"shell":"/bin/bash"},"hacluster":{"gecos":"Heartbeat System Account,,,","dir":"/usr/lib/heartbeat","gid":117,"uid":109,"shell":"/bin/false"},"landscape":{"gecos":"","dir":"/var/lib/landscape","gid":109,"uid":105,"shell":"/bin/false"},"haldaemon":{"gecos":"Hardware abstraction layer,,,","dir":"/var/run/hald","gid":108,"uid":103,"shell":"/bin/false"},"messagebus":{"gecos":"","dir":"/var/run/dbus","gid":107,"uid":102,"shell":"/bin/false"},"proxy":{"gecos":"proxy","dir":"/bin","gid":13,"uid":13,"shell":"/bin/sh"},"mail":{"gecos":"mail","dir":"/var/mail","gid":8,"uid":8,"shell":"/bin/sh"},"sync":{"gecos":"sync","dir":"/bin","gid":65534,"uid":4,"shell":"/bin/sync"},"mark":{"gecos":"Mark Anderson","dir":"/home/mark","gid":7015,"uid":7015,"shell":"/usr/bin/zsh"},"seth":{"gecos":"Seth Falcon","dir":"/home/seth","gid":7013,"uid":7013,"shell":"/bin/bash"},"dan":{"gecos":"Daniel DeLeo","dir":"/home/dan","gid":7008,"uid":7008,"shell":"/bin/bash"},"sshd":{"gecos":"","dir":"/var/run/sshd","gid":65534,"uid":104,"shell":"/usr/sbin/nologin"},"root":{"gecos":"root","dir":"/root","gid":0,"uid":0,"shell":"/bin/bash"},"stephen":{"gecos":"Stephen Delano","dir":"/home/stephen","gid":7007,"uid":7007,"shell":"/bin/bash"},"bin":{"gecos":"bin","dir":"/bin","gid":2,"uid":2,"shell":"/bin/sh"},"ronny":{"gecos":"Ronny Jones","dir":"/home/ronny","gid":7012,"uid":7012,"shell":"/bin/bash"},"nuo":{"gecos":"Nuo Yan","dir":"/home/nuo","gid":7006,"uid":7006,"shell":"/bin/bash"},"joshua":{"gecos":"Joshua Timberman","dir":"/home/joshua","gid":7004,"uid":7004,"shell":"/usr/bin/zsh"},"adam":{"gecos":"Adam Jacob","dir":"/home/adam","gid":7000,"uid":7000,"shell":"/usr/bin/zsh"},"news":{"gecos":"news","dir":"/var/spool/news","gid":9,"uid":9,"shell":"/bin/sh"},"daemon":{"gecos":"daemon","dir":"/usr/sbin","gid":1,"uid":1,"shell":"/bin/sh"},"cw":{"gecos":"Chris Walters","dir":"/home/cw","gid":7002,"uid":7002,"shell":"/bin/bash"},"cb":{"gecos":"Chris Brown","dir":"/home/cb","gid":7003,"uid":7003,"shell":"/bin/bash"},"ubuntu":{"gecos":"Ubuntu,,,","dir":"/home/ubuntu","gid":1000,"uid":1000,"shell":"/bin/bash"},"gnats":{"gecos":"Gnats Bug-Reporting System (admin)","dir":"/var/lib/gnats","gid":41,"uid":41,"shell":"/bin/sh"},"postfix":{"gecos":"","dir":"/var/spool/postfix","gid":115,"uid":108,"shell":"/bin/false"}}},"ec2":{"public_hostname":"ec2-184-73-245-27.compute-1.amazonaws.com","placement_availability_zone":"us-east-1b","block_device_mapping_root":"/dev/sda1","public_keys_0_openssh_key":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPkRwNNzLbG4l8Ic3T7EOwuHpGzObCq+aHU0ewnbKEC1AQCS+ew9t/EYMzDt5N2PVxTfAP8if5dgRjeIBH13O0Xo+gJVArB+N2/0NHMEBxCzEPuT9j1+BJc6hpTbELtG1Y0eLCLS0oEwRNYSBa9GD/xkvSrhNA1+Jbf7qGtziVmFKpBKx/s6OUa1ofO+0aIqiR3e+YbCFFik1xHulWPuhwCM5fL8kkMdQBw8EC5lml5ysSuDjDJShLVu5882MMMZRfezCrmVjLoSszMo4CpSKqbzOIB9mGLlydT4yO+GzD8qx++ObpmOUFzfvRf4wQ0AeQTZVQnowqOkxys12yukUN opscode-platform-20100217\n","instance_id":"i-f09a939b","instance_type":"c1.medium","local_ipv4":"10.192.26.6","block_device_mapping_ephemeral0":"/dev/sda2","reservation_id":"r-56409f3d","public_ipv4":"184.73.245.27","local_hostname":"domU-12-31-39-0E-19-F4.compute-1.internal","kernel_id":"aki-754aa41c","hostname":"domU-12-31-39-0E-19-F4.compute-1.internal","block_device_mapping_swap":"sda3","ami_id":"ami-f8927a91","userdata":null,"security_groups":"platform-lb","block_device_mapping_ami":"/dev/sda1","ami_manifest_path":"(unknown)","ami_launch_index":"0"},"macaddress":"12:31:39:0e:19:f4","block_device":{"ram13":{"size":"131072","removable":"0"},"ram0":{"size":"131072","removable":"0"},"sda1":{"size":"31457280","removable":"0"},"loop0":{"size":"0","removable":"0"},"ram14":{"size":"131072","removable":"0"},"ram1":{"size":"131072","removable":"0"},"sda2":{"size":"712971264","removable":"0"},"loop1":{"size":"0","removable":"0"},"ram15":{"size":"131072","removable":"0"},"ram2":{"size":"131072","removable":"0"},"sda3":{"size":"1835008","removable":"0"},"loop2":{"size":"0","removable":"0"},"ram3":{"size":"131072","removable":"0"},"loop3":{"size":"0","removable":"0"},"ram4":{"size":"131072","removable":"0"},"loop4":{"size":"0","removable":"0"},"ram5":{"size":"131072","removable":"0"},"loop5":{"size":"0","removable":"0"},"ram6":{"size":"131072","removable":"0"},"loop6":{"size":"0","removable":"0"},"ram10":{"size":"131072","removable":"0"},"ram7":{"size":"131072","removable":"0"},"loop7":{"size":"0","removable":"0"},"ram11":{"size":"131072","removable":"0"},"ram8":{"size":"131072","removable":"0"},"ram12":{"size":"131072","removable":"0"},"ram9":{"size":"131072","removable":"0"}},"uptime_seconds":3078380,"platform":"ubuntu","cloud":{"public_ips":["184.73.245.27"],"private_ips":["10.192.26.6"],"provider":"ec2"}},"normal":{"ec2opts":{"lvm":{"ephemeral_mountpoint":"/mnt","ephemeral_logical_volume":"store","use_ephemeral":true,"ephemeral_devices":{"m1.small":["/dev/sda2"],"m1.xlarge":["/dev/sdb","/dev/sdc","/dev/sdd","/dev/sde"],"m1.large":["/dev/sdb","/dev/sdc"]},"ephemeral_volume_group":"ephemeral"}},"filesystem":{},"runit":{"sv_bin":"/usr/bin/sv","service_dir":"/etc/service","sv_dir":"/etc/sv"},"nagios":{"interval_length":1,"cache_dir":"/var/cache/nagios3","checks":{"memory":{"critical":150,"warning":250},"smtp_host":"","load":{"critical":"30,20,10","warning":"15,10,5"}},"sysadmin_email":"root@localhost","dir":"/etc/nagios3","default_host":{"max_check_attempts":1,"notification_interval":300,"retry_interval":15,"check_interval":15},"default_service":{"max_check_attempts":3,"notification_interval":1200,"retry_interval":15,"check_interval":60},"templates":{},"docroot":"/usr/share/nagios3/htdocs","sysadmin_sms_email":"root@localhost","default_contact_groups":["admins"],"check_external_commands":true,"state_dir":"/var/lib/nagios3","log_dir":"/var/log/nagios3","notifications_enabled":0,"config_subdir":"conf.d"},"nginx":{"worker_connections":4096,"gzip_types":["text/plain","text/html","text/css","application/x-javascript","text/xml","application/xml","application/xml+rss","text/javascript"],"dir":"/etc/nginx","binary":"/srv/nginx/0.8.33/sbin/nginx","server_names_hash_bucket_size":64,"gzip_comp_level":"2","gzip_proxied":"any","version":"0.8.33","worker_processes":2,"gzip_http_version":"1.0","gzip":"on","user":"www-data","log_dir":"/srv/nginx/log","keepalive_timeout":65,"keepalive":"on"},"authorization":{"sudo":{"groups":[],"users":[]}},"tags":[],"apache":{"allowed_openids":["http://adamhjk.myopenid.com/","http://bryanhale.myopenid.com/","http://jesserobbins.myopenid.com/","http://botchagalupe.myopenid.com/","http://jtimberman.myopenid.com/","http://tmonk42.myopenid.com/","http://ronny-jones.myopenid.com/","http://steviedu.myopenid.com/","http://seth.falcon.myopenid.com/","http://mark-a-anderson.myopenid.com/","http://timhinderliter.myopenid.com/","http://cwalters.myopenid.com","http://kallistec.myopenid.com/","http://cwalters.myopenid.com/","http://protoskeptomai.myopenid.com/","http://nuoyan.myopenid.com/"],"traceenable":"On","dir":"/etc/apache2","keepaliverequests":100,"timeout":300,"binary":"/usr/sbin/apache2","contact":"ops@example.com","prefork":{"maxspareservers":32,"minspareservers":16,"serverlimit":400,"startservers":16,"maxclients":400,"maxrequestsperchild":10000},"servertokens":"Prod","listen_ports":["80","443"],"keepalivetimeout":5,"serversignature":"On","icondir":"/usr/share/apache2/icons","user":"www-data","log_dir":"/var/log/apache2","worker":{"maxsparethreads":192,"startservers":4,"minsparethreads":64,"maxclients":1024,"maxrequestsperchild":0,"threadsperchild":64},"keepalive":"On"},"postfix":{"myorigin":"$myhostname","myhostname":"opscode.com","smtp_sasl_passwd":"Y3vDOkz8YDcxqL","smtp_sasl_security_options":"noanonymous","smtp_sasl_password_maps":"hash:/etc/postfix/sasl_passwd","smtp_sasl_auth_enable":"no","mail_type":"client","smtp_sasl_user_name":"community","mail_relay_networks":"127.0.0.0/8","relayhost":"lists.opscode.com","smtp_use_tls":"yes","smtp_tls_cafile":"/etc/postfix/cacert.pem","mydomain":"compute-1.internal"}},"chef_type":"node","default":{"dynect":{"zone":"opscode.us","domain":"opscode.us","username":"opscodeus","customer":"opscode","ec2":{"env":"na","type":"unknown"},"password":"nb7lO2hEPImEHtoTlYKq"},"opscode_lb":{"ha_role":"secondary"},"filesystem":{},"splunk":{"forward_server":"ec2-184-73-163-97.compute-1.amazonaws.com","auth":"admin:VYjsrtHdtieZD2QtX5za"},"monitor_group":"opscode-lb","ha_role":"secondary","opscode_lb_heartbeat":{"heartbeatsecret":"WIaVjtQurY0Q7sti"},"app_environment":"production"},"override":{"dynect":{"ec2":{"env":"prod","type":"lb"}},"filesystem":{},"opscode_lb_type":"external","postfix":{"myhostname":"opscode.com","smtp_sasl_passwd":"Y3vDOkz8YDcxqL","smtp_sasl_security_options":"noanonymous","smtp_sasl_auth_enable":"yes","relayhost":"lists.opscode.com","smtp_sasl_user_name":"community"},"rabbitmq":{"users":{"mapper":"EBjgOZGNsd4RpA3gSoom","chef":"Tat9THLuiOkdtyyH5VVf"}}},"run_list":["role[opscode-lb]","role[production]","role[ha-secondary]","recipe[aws]"],"_rev":"147-bb4f74122079d1feac675c1b2c57bfcc"} diff --git a/chef-expander/lib/chef/expander.rb b/chef-expander/lib/chef/expander.rb deleted file mode 100644 index 9a58868a96..0000000000 --- a/chef-expander/lib/chef/expander.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Chef - module Expander - - # VNODES is the number of queues in rabbit that are available for subscribing. - # The name comes from riak, where the data ring (160bits) is chunked into - # many vnodes; vnodes outnumber physical nodes, so one node hosts several - # vnodes. That is the same design we use here. - # - # See the notes on topic queue benchmarking before adjusting this value. - VNODES = 1024 - - SHARED_CONTROL_QUEUE_NAME = "chef-search-control--shared" - BROADCAST_CONTROL_EXCHANGE_NAME = 'chef-search-control--broadcast' - - end -end diff --git a/chef-expander/lib/chef/expander/cluster_supervisor.rb b/chef-expander/lib/chef/expander/cluster_supervisor.rb deleted file mode 100644 index 56d6e70d5f..0000000000 --- a/chef-expander/lib/chef/expander/cluster_supervisor.rb +++ /dev/null @@ -1,130 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/expander/loggable' -require 'chef/expander/daemonizable' -require 'chef/expander/version' -require 'chef/expander/configuration' -require 'chef/expander/vnode_supervisor' - -module Chef - module Expander - #==ClusterSupervisor - # Manages a cluster of chef-expander processes. Usually this class will - # be instantiated from the chef-expander-cluster executable. - # - # ClusterSupervisor works by forking the desired number of processes, then - # running VNodeSupervisor.start_cluster_worker within the forked process. - # ClusterSupervisor keeps track of the process ids of its children, and will - # periodically attempt to reap them in a non-blocking call. If they are - # reaped, ClusterSupervisor knows they died and need to be respawned. - # - # The child processes are responsible for checking on the master process and - # dying if the master has died (VNodeSupervisor does this when started in - # with start_cluster_worker). - # - #===TODO: - # * This implementation currently assumes there is only one cluster, so it - # will claim all of the vnodes. It may be advantageous to allow multiple - # clusters. - # * There is no heartbeat implementation at this time, so a zombified child - # process will not be automatically killed--This behavior is left to the - # meatcloud for now. - class ClusterSupervisor - include Loggable - include Daemonizable - - def initialize - @workers = {} - @running = true - @kill = :TERM - end - - def start - trap(:INT) { stop(:INT) } - trap(:TERM) { stop(:TERM)} - Expander.init_config(ARGV) - - log.info("Chef Expander #{Expander.version} starting cluster with #{Expander.config.node_count} nodes") - configure_process - start_workers - maintain_workers - release_locks - rescue Configuration::InvalidConfiguration => e - log.fatal {"Configuration Error: " + e.message} - exit(2) - rescue Exception => e - raise if SystemExit === e - - log.fatal {e} - exit(1) - end - - def start_workers - Expander.config.node_count.times do |i| - start_worker(i + 1) - end - end - - def start_worker(index) - log.info { "Starting cluster worker #{index}" } - worker_params = {:index => index} - child_pid = fork do - Expander.config.index = index - VNodeSupervisor.start_cluster_worker - end - @workers[child_pid] = worker_params - end - - def stop(signal) - log.info { "Stopping cluster on signal (#{signal})" } - @running = false - @kill = signal - end - - def maintain_workers - while @running - sleep 1 - workers_to_replace = {} - @workers.each do |process_id, worker_params| - if result = Process.waitpid2(process_id, Process::WNOHANG) - log.error { "worker #{worker_params[:index]} (PID: #{process_id}) died with status #{result[1].exitstatus || '(no status)'}"} - workers_to_replace[process_id] = worker_params - end - end - workers_to_replace.each do |dead_pid, worker_params| - @workers.delete(dead_pid) - start_worker(worker_params[:index]) - end - end - - @workers.each do |pid, worker_params| - log.info { "Stopping worker #{worker_params[:index]} (PID: #{pid})"} - Process.kill(@kill, pid) - end - @workers.each do |pid, worker_params| - Process.waitpid2(pid) - end - - end - - end - end -end diff --git a/chef-expander/lib/chef/expander/configuration.rb b/chef-expander/lib/chef/expander/configuration.rb deleted file mode 100644 index 8fbaba762a..0000000000 --- a/chef-expander/lib/chef/expander/configuration.rb +++ /dev/null @@ -1,320 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'pp' -require 'optparse' -require 'singleton' - -require 'chef/expander/flattener' -require 'chef/expander/loggable' -require 'chef/expander/version' - -module Chef - module Expander - - def self.config - @config ||= Configuration::Base.new - end - - def self.init_config(argv) - config.apply_defaults - remaining_opts_after_parse = Configuration::CLI.parse_options(argv) - # Need to be able to override the default config file location on the command line - config_file_to_use = Configuration::CLI.config.config_file || config.config_file - config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use)) - # But for all other config options, the CLI config should win over config file - config.merge_config(Configuration::CLI.config) - config.validate! - remaining_opts_after_parse - end - - class ChefCompatibleConfig - - attr_reader :config_hash - - def initialize - @config_hash = {} - end - - def load(file) - file = File.expand_path(file) - instance_eval(IO.read(file), file, 1) if File.readable?(file) - end - - def method_missing(method_name, *args, &block) - if args.size == 1 - @config_hash[method_name] = args.first - elsif args.empty? - @config_hash[method_name] or super - else - super - end - end - - end - - module Configuration - - class InvalidConfiguration < StandardError - end - - class Base - - DEFAULT_PIDFILE = Object.new - - include Loggable - - def self.from_chef_compat_config(file) - config = ChefCompatibleConfig.new - config.load(file) - from_hash(config.config_hash) - end - - def self.from_hash(config_hash) - config = new - config_hash.each do |setting, value| - setter = "#{setting}=".to_sym - if config.respond_to?(setter) - config.send(setter, value) - end - end - config - end - - def self.configurables - @configurables ||= [] - end - - def self.validations - @validations ||= [] - end - - def self.defaults - @defaults ||= {} - end - - def self.configurable(setting, default=nil, &validation) - attr_accessor(setting) - configurables << setting - defaults[setting] = default - validations << validation if block_given? - - setting - end - - configurable :config_file, "/etc/chef/solr.rb" do - unless (config_file && File.exist?(config_file) && File.readable?(config_file)) - log.warn {"* " * 40} - log.warn {"Config file #{config_file} does not exist or cannot be read by user (#{Process.euid})"} - log.warn {"Default configuration settings will be used"} - log.warn {"* " * 40} - end - end - - configurable :index do - unless index.nil? # in single-cluster mode, this setting is not required. - invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer) - invalid("The index cannot be larger than the cluster size (node-count)") unless (index.to_i <= node_count.to_i) - end - end - - configurable :node_count, 1 do - invalid("You must specify the node_count as an integer") unless node_count.kind_of?(Integer) - invalid("The node_count must be 1 or greater") unless node_count >= 1 - invalid("The node_count cannot be smaller than the index") unless node_count >= index.to_i - end - - configurable :ps_tag, "" - - configurable :solr_url, "http://localhost:8983/solr" - - # override the setter for solr_url for backward compatibilty - def solr_url=(url) - if url && url == "http://localhost:8983" - log.warn {"You seem to have a legacy setting for solr_url: did you mean #{url}/solr ?"} - url = "#{url}/solr" - end - @solr_url = url - end - - configurable :amqp_host, '0.0.0.0' - - configurable :amqp_port, 5672 - - configurable :amqp_user, 'chef' - - configurable :amqp_pass, 'testing' - - configurable :amqp_vhost, '/chef' - - configurable :user, nil - - configurable :group, nil - - configurable :daemonize, false - - alias :daemonize? :daemonize - - configurable :pidfile, DEFAULT_PIDFILE - - def pidfile - if @pidfile.equal?(DEFAULT_PIDFILE) - Process.euid == 0 ? '/var/run/chef-expander.pid' : '/tmp/chef-expander.pid' - else - @pidfile - end - end - - configurable :log_level, :info - - # override the setter for log_level to also actually set the level - def log_level=(level) - if level #don't accept nil for an answer - level = level.to_sym - Loggable::LOGGER.level = level - @log_level = log_level - end - level - end - - configurable :log_location, STDOUT - - # override the setter for log_location to re-init the logger - def log_location=(location) - Loggable::LOGGER.init(location) unless location.nil? - end - - def initialize - reset! - end - - def reset!(stdout=nil) - self.class.configurables.each do |setting| - send("#{setting}=".to_sym, nil) - end - @stdout = stdout || STDOUT - end - - def apply_defaults - self.class.defaults.each do |setting, value| - self.send("#{setting}=".to_sym, value) - end - end - - def merge_config(other) - self.class.configurables.each do |setting| - value = other.send(setting) - self.send("#{setting}=".to_sym, value) if value - end - end - - def fail_if_invalid - validate! - rescue InvalidConfiguration => e - @stdout.puts("Invalid configuration: #{e.message}") - exit(1) - end - - def invalid(message) - raise InvalidConfiguration, message - end - - def validate! - self.class.validations.each do |validation_proc| - instance_eval(&validation_proc) - end - end - - def vnode_numbers - vnodes_per_node = VNODES / node_count - lower_bound = (index - 1) * vnodes_per_node - upper_bound = lower_bound + vnodes_per_node - upper_bound += VNODES % vnodes_per_node if index == node_count - (lower_bound...upper_bound).to_a - end - - def amqp_config - {:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost} - end - - end - - module CLI - @config = Configuration::Base.new - - @option_parser = OptionParser.new do |o| - o.banner = "Usage: chef-expander [options]" - - o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf| - @config.config_file = File.expand_path(conf) - end - - o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i| - @config.index = i.to_i - end - - o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n| - @config.node_count = n.to_i - end - - o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l| - @config.log_level = l - end - - o.on('-L', '--logfile LOG_LOCATION', 'Logfile to use') do |l| - @config.log_location = l - end - - o.on('-d', '--daemonize', 'fork into the background') do - @config.daemonize = true - end - - o.on('-P', '--pid PIDFILE') do |p| - @config.pidfile = p - end - - o.on_tail('-h', '--help', 'show this message') do - puts "chef-expander #{Expander.version}" - puts '' - puts o - exit 1 - end - - o.on_tail('-v', '--version', 'show the version and exit') do - puts "chef-expander #{Expander.version}" - exit 0 - end - - end - - def self.parse_options(argv) - @option_parser.parse!(argv.dup) - end - - def self.config - @config - end - - end - - end - - end -end diff --git a/chef-expander/lib/chef/expander/control.rb b/chef-expander/lib/chef/expander/control.rb deleted file mode 100644 index f8e3e99503..0000000000 --- a/chef-expander/lib/chef/expander/control.rb +++ /dev/null @@ -1,206 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'bunny' -require 'yajl' -require 'eventmachine' -require 'amqp' -require 'mq' -require 'highline' - -require 'chef/expander/node' -require 'chef/expander/configuration' - -require 'pp' - -module Chef - module Expander - class Control - - def self.run(argv) - remaining_args_after_opts = Expander.init_config(ARGV) - new(remaining_args_after_opts).run - end - - def self.desc(description) - @desc = description - end - - def self.option(*args) - #TODO - end - - def self.arg(*args) - #TODO - end - - def self.descriptions - @descriptions ||= [] - end - - def self.method_added(method_name) - if @desc - descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc] - @desc = nil - end - end - - #-- - # TODO: this is confusing and unneeded. Just whitelist the methods - # that map to commands and use +send+ - def self.compile - run_method = "def run; case @argv.first;" - descriptions.each do |method_name, command_name, desc| - run_method << "when '#{command_name}';#{method_name};" - end - run_method << "else; help; end; end;" - class_eval(run_method, __FILE__, __LINE__) - end - - def initialize(argv) - @argv = argv.dup - end - - desc "Show this message" - def help - puts "Chef Expander #{Expander.version}" - puts "Usage: chef-expanderctl COMMAND" - puts - puts "Commands:" - self.class.descriptions.each do |method_name, command_name, desc| - puts " #{command_name}".ljust(15) + desc - end - end - - desc "display the aggregate queue backlog" - def queue_depth - h = HighLine.new - message_counts = [] - - amqp_client = Bunny.new(Expander.config.amqp_config) - amqp_client.start - - 0.upto(VNODES - 1) do |vnode| - q = amqp_client.queue("vnode-#{vnode}", :durable => true) - message_counts << q.status[:message_count] - end - total_messages = message_counts.inject(0) { |sum, count| sum + count } - max = message_counts.max - min = message_counts.min - - avg = total_messages.to_f / message_counts.size.to_f - - puts " total messages: #{total_messages}" - puts " average queue depth: #{avg}" - puts " max queue depth: #{max}" - puts " min queue depth: #{min}" - ensure - amqp_client.stop if defined?(amqp_client) && amqp_client - end - - desc "show the backlog and consumer count for each vnode queue" - def queue_status - h = HighLine.new - queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)] - - total_messages = 0 - - amqp_client = Bunny.new(Expander.config.amqp_config) - amqp_client.start - - 0.upto(VNODES - 1) do |vnode| - q = amqp_client.queue("vnode-#{vnode}", :durable => true) - status = q.status - # returns {:message_count => method.message_count, :consumer_count => method.consumer_count} - queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s - total_messages += status[:message_count] - end - puts " total messages: #{total_messages}" - puts - puts h.list(queue_status, :columns_across, 3) - ensure - amqp_client.stop if defined?(amqp_client) && amqp_client - end - - desc "show the status of the nodes in the cluster" - def node_status - status_mutex = Mutex.new - h = ::HighLine.new - node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)] - - print("Collecting status info from the cluster...") - - AMQP.start(Expander.config.amqp_config) do - node = Expander::Node.local_node - node.exclusive_control_queue.subscribe do |header, message| - status = Yajl::Parser.parse(message) - status_mutex.synchronize do - node_status << status["hostname_f"] - node_status << status["pid"].to_s - node_status << status["guid"] - # BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges - # will not be true once vnode recovery is implemented - node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}" - end - end - node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name)) - EM.add_timer(2) { AMQP.stop;EM.stop } - end - - puts "done" - puts - puts h.list(node_status, :columns_across, 4) - puts - end - - desc "sets the log level of all nodes in the cluster" - def log_level - @argv.shift - level = @argv.first - acceptable_levels = %w{debug info warn error fatal} - unless acceptable_levels.include?(level) - puts "Log level must be one of #{acceptable_levels.join(', ')}" - exit 1 - end - - h = HighLine.new - response_mutex = Mutex.new - - responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)] - AMQP.start(Expander.config.amqp_config) do - node = Expander::Node.local_node - node.exclusive_control_queue.subscribe do |header, message| - reply = Yajl::Parser.parse(message) - n = reply['node'] - response_mutex.synchronize do - responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"] - end - end - node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name})) - EM.add_timer(2) { AMQP.stop; EM.stop } - end - puts h.list(responses, :columns_across, 4) - end - - - compile - end - end -end diff --git a/chef-expander/lib/chef/expander/daemonizable.rb b/chef-expander/lib/chef/expander/daemonizable.rb deleted file mode 100644 index 0d76bd6b6e..0000000000 --- a/chef-expander/lib/chef/expander/daemonizable.rb +++ /dev/null @@ -1,150 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Copyright:: Copyright (c) 2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'etc' -require 'chef/expander/loggable' - -module Chef - module Expander - - class AlreadyRunning < RuntimeError - end - - class NoSuchUser < ArgumentError - end - - class NoSuchGroup < ArgumentError - end - - module Daemonizable - include Loggable - - # Daemonizes the process if configured to do so, and ensures that only one - # copy of the process is running with a given config by obtaining an - # exclusive lock on the pidfile. Also sets process user and group if so - # configured. - # ===Raises - # * AlreadyRunning::: when another process has the exclusive lock on the pidfile - # * NoSuchUser::: when a user is configured that doesn't exist - # * NoSuchGroup::: when a group is configured that doesn't exist - # * SystemCallError::: if there is an error creating the pidfile - def configure_process - Expander.config.daemonize? ? daemonize : ensure_exclusive - set_user_and_group - end - - def daemonize - acquire_locks - exit if fork - Process.setsid - exit if fork - write_pid - Dir.chdir('/') - STDIN.reopen("/dev/null") - STDOUT.reopen("/dev/null", "a") - STDERR.reopen("/dev/null", "a") - end - - # When not forking into the background, this ensures only one chef-expander - # is running with a given config and writes the process id to the pidfile. - def ensure_exclusive - acquire_locks - write_pid - end - - def set_user_and_group - return nil if Expander.config.user.nil? - - if Expander.config.group.nil? - log.info {"Changing user to #{Expander.config.user}"} - else - log.info {"Changing user to #{Expander.config.user} and group to #{Expander.config.group}"} - end - - unless (set_group && set_user) - log.error {"Unable to change user to #{Expander.config.user} - Are you root?"} - end - end - - # Deletes the pidfile, releasing the exclusive lock on it in the process. - def release_locks - File.unlink(@pidfile.path) if File.exist?(@pidfile.path) - @pidfile.close unless @pidfile.closed? - end - - private - - def set_user - Process::Sys.setuid(target_uid) - true - rescue Errno::EPERM => e - log.debug {e} - false - end - - def set_group - if gid = target_uid - Process::Sys.setgid(gid) - end - true - rescue Errno::EPERM - log.debug {e} - false - end - - def target_uid - user = Expander.config.user - user.kind_of?(Fixnum) ? user : Etc.getpwnam(user).uid - rescue ArgumentError => e - log.debug {e} - raise NoSuchUser, "Cannot change user to #{user} - failed to find the uid" - end - - def target_gid - if group = Expander.config.group - group.kind_of?(Fixnum) ? group : Etc.getgrnam(group).gid - else - nil - end - rescue ArgumentError => e - log.debug {e} - raise NoSuchGroup, "Cannot change group to #{group} - failed to find the gid" - end - - def acquire_locks - @pidfile = File.open(Expander.config.pidfile, File::RDWR|File::CREAT, 0644) - unless @pidfile.flock(File::LOCK_EX | File::LOCK_NB) - pid = @pidfile.read.strip - msg = "Another instance of chef-expander (pid: #{pid}) has a lock on the pidfile (#{Expander.config.pidfile}). \n"\ - "Configure a different pidfile to run multiple instances of chef-expander at once." - raise AlreadyRunning, msg - end - rescue Exception - @pidfile.close if @pidfile && !@pidfile.closed? - raise - end - - def write_pid - @pidfile.truncate(0) - @pidfile.print("#{Process.pid}\n") - @pidfile.flush - end - - end - end -end
\ No newline at end of file diff --git a/chef-expander/lib/chef/expander/flattener.rb b/chef-expander/lib/chef/expander/flattener.rb deleted file mode 100644 index 90f7cd663e..0000000000 --- a/chef-expander/lib/chef/expander/flattener.rb +++ /dev/null @@ -1,79 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/expander/configuration' - -module Chef - module Expander - # Flattens and expands nested Hashes representing Chef objects - # (e.g, Nodes, Roles, DataBagItems, etc.) into flat Hashes so the - # objects are suitable to be saved into Solr. This code is more or - # less copy-pasted from chef/solr/index which may or may not be a - # great idea, though that does minimize the dependencies and - # hopefully minimize the memory use of chef-expander. - class Flattener - UNDERSCORE = '_' - X = 'X' - - X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X' - X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X' - X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X' - - def initialize(item) - @item = item - end - - def flattened_item - @flattened_item || flatten_and_expand - end - - def flatten_and_expand - @flattened_item = Hash.new {|hash, key| hash[key] = []} - - @item.each do |key, value| - flatten_each([key.to_s], value) - end - - @flattened_item.each_value { |values| values.uniq! } - @flattened_item - end - - def flatten_each(keys, values) - case values - when Hash - values.each do |child_key, child_value| - add_field_value(keys, child_key) - flatten_each(keys + [child_key.to_s], child_value) - end - when Array - values.each { |child_value| flatten_each(keys, child_value) } - else - add_field_value(keys, values) - end - end - - def add_field_value(keys, value) - value = value.to_s - @flattened_item[keys.join(UNDERSCORE)] << value - @flattened_item[keys.last] << value - end - end - end -end diff --git a/chef-expander/lib/chef/expander/loggable.rb b/chef-expander/lib/chef/expander/loggable.rb deleted file mode 100644 index 5dd534dab5..0000000000 --- a/chef-expander/lib/chef/expander/loggable.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/expander/logger' -require 'mixlib/log' - -module Chef - module Expander - module Loggable - - # TODO: it's admittedly janky to set up the default logging this way. - STDOUT.sync = true - LOGGER = Logger.new(STDOUT) - LOGGER.level = :debug - - def log - LOGGER - end - - end - end -end - diff --git a/chef-expander/lib/chef/expander/logger.rb b/chef-expander/lib/chef/expander/logger.rb deleted file mode 100644 index 234344e0cb..0000000000 --- a/chef-expander/lib/chef/expander/logger.rb +++ /dev/null @@ -1,135 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Copyright:: Copyright (c) 2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'logger' - -module Chef - module Expander - - class InvalidLogDevice < ArgumentError - end - - class InvalidLogLevel < ArgumentError - end - - # Customized Logger class that dispenses with the unnecessary mutexing. - # As long as you write one line at a time, the OS will take care of keeping - # your output in order. Expander commonly runs as a cluster of worker - # processes so the mutexing wasn't actually helping us anyway. - # - # We don't use the program name field in the logger, so support for that - # has been removed. The log format is also hardcoded since we don't ever - # change the format. - class Logger < ::Logger - - LEVELS = { :debug=>DEBUG, :info=>INFO, :warn=>WARN, :error=>ERROR, :fatal=>FATAL} - LEVEL_INTEGERS = LEVELS.invert - LEVEL_TO_STR = Hash[LEVEL_INTEGERS.map {|i,sym| [i,sym.to_s.upcase]}] - - LOG_DEVICES = [] - - at_exit do - LOG_DEVICES.each {|io| io.close if io.respond_to?(:closed?) && !io.closed?} - end - - attr_reader :log_device - - # (re-)initialize the Logger with a new IO object or file to log to. - def init(log_device) - @log_device = initialize_log_device(log_device) - end - - def initialize(log_device) - @level = DEBUG - init(log_device) - end - - def level=(new_level) - @level = if new_level.kind_of?(Fixnum) && LEVEL_INTEGERS.key?(new_level) - new - elsif LEVELS.key?(new_level) - LEVELS[new_level] - else - raise InvalidLogLevel, "#{new_level} is not a valid log level. Valid log levels are [#{LEVEL_INTEGERS.keys.join(',')}] and [#{LEVELS.join(',')}]" - end - end - - def <<(msg) - @log_device.print(msg) - end - - def add(severity=UNKNOWN, message = nil, progname = nil, &block) - return true unless severity >= @level - - message ||= progname # level methods (e.g, #debug) pass explicit message as progname - - if message.nil? && block_given? - message = yield - end - - self << sprintf("[%s] %s: %s\n", Time.new.rfc2822(), LEVEL_TO_STR[severity], msg2str(message)) - true - end - - alias :log :add - - private - - def msg2str(msg) - case msg - when ::String - msg - when ::Exception - "#{ msg.message } (#{ msg.class })\n" << - (msg.backtrace || []).join("\n") - else - msg.inspect - end - end - - def logging_at_severity?(severity=nil) - end - - def initialize_log_device(dev) - unless dev.respond_to? :sync= - assert_valid_path!(dev) - dev = File.open(dev.to_str, "a") - LOG_DEVICES << dev - end - - dev.sync = true - dev - end - - def assert_valid_path!(path) - enclosing_directory = File.dirname(path) - unless File.directory?(enclosing_directory) - raise InvalidLogDevice, "You must create the enclosing directory #{enclosing_directory} before the log file #{path} can be created." - end - if File.exist?(path) - unless File.writable?(path) - raise InvalidLogDevice, "The log file you specified (#{path}) is not writable by user #{Process.euid}" - end - elsif !File.writable?(enclosing_directory) - raise InvalidLogDevice, "You specified a log file #{path} but user #{Process.euid} is not permitted to create files there." - end - end - - end - end -end
\ No newline at end of file diff --git a/chef-expander/lib/chef/expander/node.rb b/chef-expander/lib/chef/expander/node.rb deleted file mode 100644 index 23249e8685..0000000000 --- a/chef-expander/lib/chef/expander/node.rb +++ /dev/null @@ -1,177 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'uuidtools' -require 'amqp' -require 'mq' -require 'open3' - -require 'chef/expander/loggable' - -module Chef - module Expander - class Node - - include Loggable - - def self.from_hash(node_info) - new(node_info[:guid], node_info[:hostname_f], node_info[:pid]) - end - - def self.local_node - new(guid, hostname_f, Process.pid) - end - - def self.guid - return @guid if @guid - @guid = UUIDTools::UUID.random_create.to_s - end - - def self.hostname_f - @hostname ||= Open3.popen3("hostname -f") {|stdin, stdout, stderr| stdout.read }.strip - end - - attr_reader :guid - - attr_reader :hostname_f - - attr_reader :pid - - def initialize(guid, hostname_f, pid) - @guid, @hostname_f, @pid = guid, hostname_f, pid - end - - def start(&message_handler) - attach_to_queue(exclusive_control_queue, "exclusive control", &message_handler) - attach_to_queue(shared_control_queue, "shared_control", &message_handler) - attach_to_queue(broadcast_control_queue, "broadcast control", &message_handler) - end - - def attach_to_queue(queue, colloquial_name, &message_handler) - queue.subscribe(:ack => true) do |headers, payload| - log.debug { "received message on #{colloquial_name} queue: #{payload}" } - message_handler.call(payload) - headers.ack - end - end - - def stop - log.debug { "unsubscribing from broadcast control queue"} - broadcast_control_queue.unsubscribe(:nowait => false) - - log.debug { "unsubscribing from shared control queue" } - shared_control_queue.unsubscribe(:nowait => false) - - log.debug { "unsubscribing from exclusive control queue" } - exclusive_control_queue.unsubscribe(:nowait => false) - end - - def direct_message(message) - log.debug { "publishing direct message to node #{identifier}: #{message}" } - exclusive_control_queue.publish(message) - end - - def shared_message(message) - log.debug { "publishing shared message #{message}"} - shared_control_queue.publish(message) - end - - def broadcast_message(message) - log.debug { "publishing broadcast message #{message}" } - broadcast_control_exchange.publish(message) - end - - # The exclusive control queue is for point-to-point messaging, i.e., - # messages directly addressed to this node - def exclusive_control_queue - @exclusive_control_queue ||= begin - log.debug { "declaring exclusive control queue #{exclusive_control_queue_name}" } - MQ.queue(exclusive_control_queue_name) - end - end - - # The shared control queue is for 1 to (1 of N) messaging, i.e., - # messages that can go to any one node. - def shared_control_queue - @shared_control_queue ||= begin - log.debug { "declaring shared control queue #{shared_control_queue_name}" } - MQ.queue(shared_control_queue_name) - end - end - - # The broadcast control queue is for 1 to N messaging, i.e., - # messages that go to every node - def broadcast_control_queue - @broadcast_control_queue ||= begin - log.debug { "declaring broadcast control queue #{broadcast_control_queue_name}"} - q = MQ.queue(broadcast_control_queue_name) - log.debug { "binding broadcast control queue to broadcast control exchange"} - q.bind(broadcast_control_exchange) - q - end - end - - def broadcast_control_exchange - @broadcast_control_exchange ||= begin - log.debug { "declaring broadcast control exchange opscode-platfrom-control--broadcast" } - MQ.fanout(broadcast_control_exchange_name, :nowait => false) - end - end - - def shared_control_queue_name - SHARED_CONTROL_QUEUE_NAME - end - - def broadcast_control_queue_name - @broadcast_control_queue_name ||= "#{identifier}--broadcast" - end - - def broadcast_control_exchange_name - BROADCAST_CONTROL_EXCHANGE_NAME - end - - def exclusive_control_queue_name - @exclusive_control_queue_name ||= "#{identifier}--exclusive-control" - end - - def identifier - "#{hostname_f}--#{pid}--#{guid}" - end - - def ==(other) - other.respond_to?(:guid) && other.respond_to?(:hostname_f) && other.respond_to?(:pid) && - (other.guid == guid) && (other.hostname_f == hostname_f) && (other.pid == pid) - end - - def eql?(other) - (other.class == self.class) && (other.hash == hash) - end - - def hash - identifier.hash - end - - def to_hash - {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid} - end - - end - end -end diff --git a/chef-expander/lib/chef/expander/solrizer.rb b/chef-expander/lib/chef/expander/solrizer.rb deleted file mode 100644 index 1faf288674..0000000000 --- a/chef-expander/lib/chef/expander/solrizer.rb +++ /dev/null @@ -1,275 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'set' -require 'yajl' -require 'fast_xs' -require 'em-http-request' -require 'chef/expander/loggable' -require 'chef/expander/flattener' - -module Chef - module Expander - class Solrizer - - @active_http_requests = Set.new - - def self.http_request_started(instance) - @active_http_requests << instance - end - - def self.http_request_completed(instance) - @active_http_requests.delete(instance) - end - - def self.http_requests_active? - !@active_http_requests.empty? - end - - def self.clear_http_requests - @active_http_requests.clear - end - - include Loggable - - ADD = "add" - DELETE = "delete" - SKIP = "skip" - - ITEM = "item" - ID = "id" - TYPE = "type" - DATABASE = "database" - ENQUEUED_AT = "enqueued_at" - - DATA_BAG_ITEM = "data_bag_item" - DATA_BAG = "data_bag" - - X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X' - X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X' - X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X' - - CONTENT_TYPE_XML = {"Content-Type" => "text/xml"} - - attr_reader :action - - attr_reader :indexer_payload - - attr_reader :chef_object - - attr_reader :obj_id - - attr_reader :obj_type - - attr_reader :database - - attr_reader :enqueued_at - - def initialize(object_command_json, &on_completion_block) - @start_time = Time.now.to_f - @on_completion_block = on_completion_block - if parsed_message = parse(object_command_json) - @action = parsed_message["action"] - @indexer_payload = parsed_message["payload"] - - extract_object_fields if @indexer_payload - else - @action = SKIP - end - end - - def extract_object_fields - @chef_object = @indexer_payload[ITEM] - @database = @indexer_payload[DATABASE] - @obj_id = @indexer_payload[ID] - @obj_type = @indexer_payload[TYPE] - @enqueued_at = @indexer_payload[ENQUEUED_AT] - @data_bag = @obj_type == DATA_BAG_ITEM ? @chef_object[DATA_BAG] : nil - end - - def parse(serialized_object) - Yajl::Parser.parse(serialized_object) - rescue Yajl::ParseError - log.error { "cannot index object because it is invalid JSON: #{serialized_object}" } - end - - def run - case @action - when ADD - add - when DELETE - delete - when SKIP - completed - log.info { "not indexing this item because of malformed JSON"} - else - completed - log.error { "cannot index object becuase it has an invalid action #{@action}" } - end - end - - def add - post_to_solr(pointyize_add) do - ["indexed #{indexed_object}", - "transit,xml,solr-post |", - [transit_time, @xml_time, @solr_post_time].join(","), - "|" - ].join(" ") - end - rescue Exception => e - log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"} - end - - def delete - post_to_solr(pointyize_delete) { "deleted #{indexed_object} transit-time[#{transit_time}s]"} - rescue Exception => e - log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"} - end - - def flattened_object - flattened_object = Flattener.new(@chef_object).flattened_item - - flattened_object[X_CHEF_id_CHEF_X] = [@obj_id] - flattened_object[X_CHEF_database_CHEF_X] = [@database] - flattened_object[X_CHEF_type_CHEF_X] = [@obj_type] - - log.debug {"adding flattened object to Solr: #{flattened_object.inspect}"} - - flattened_object - end - - START_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" - ADD_DOC = "<add><doc>" - DELETE_DOC = "<delete>" - ID_OPEN = "<id>" - ID_CLOSE = "</id>" - END_ADD_DOC = "</doc></add>\n" - END_DELETE = "</delete>\n" - START_CONTENT = '<field name="content">' - CLOSE_FIELD = "</field>" - - FLD_CHEF_ID_FMT = '<field name="X_CHEF_id_CHEF_X">%s</field>' - FLD_CHEF_DB_FMT = '<field name="X_CHEF_database_CHEF_X">%s</field>' - FLD_CHEF_TY_FMT = '<field name="X_CHEF_type_CHEF_X">%s</field>' - FLD_DATA_BAG = '<field name="data_bag">%s</field>' - - KEYVAL_FMT = "%s__=__%s " - - # Takes a flattened hash where the values are arrays and converts it into - # a dignified XML document suitable for POST to Solr. - # The general structure of the output document is like this: - # <?xml version="1.0" encoding="UTF-8"?> - # <add> - # <doc> - # <field name="content"> - # key__=__value - # key__=__another_value - # other_key__=__yet another value - # </field> - # </doc> - # </add> - # The document as generated has minimal newlines and formatting, however. - def pointyize_add - xml = "" - xml << START_XML << ADD_DOC - xml << (FLD_CHEF_ID_FMT % @obj_id) - xml << (FLD_CHEF_DB_FMT % @database) - xml << (FLD_CHEF_TY_FMT % @obj_type) - xml << START_CONTENT - content = "" - flattened_object.each do |field, values| - values.each do |v| - content << (KEYVAL_FMT % [field, v]) - end - end - xml << content.fast_xs - xml << CLOSE_FIELD # ends content - xml << (FLD_DATA_BAG % @data_bag.fast_xs) if @data_bag - xml << END_ADD_DOC - @xml_time = Time.now.to_f - @start_time - xml - end - - # Takes a succinct document id, like 2342, and turns it into something - # even more compact, like - # "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n" - def pointyize_delete - xml = "" - xml << START_XML - xml << DELETE_DOC - xml << ID_OPEN - xml << @obj_id.to_s - xml << ID_CLOSE - xml << END_DELETE - xml - end - - def post_to_solr(document, &logger_block) - log.debug("POSTing document to SOLR:\n#{document}") - http_req = EventMachine::HttpRequest.new(solr_url).post(:body => document, :timeout => 1200, :head => CONTENT_TYPE_XML) - http_request_started - - http_req.callback do - completed - if http_req.response_header.status == 200 - log.info(&logger_block) - else - log.error { "Failed to post to solr: #{indexed_object}" } - end - end - http_req.errback do - completed - log.error { "Failed to post to solr (connection error): #{indexed_object}" } - end - end - - def completed - @solr_post_time = Time.now.to_f - @start_time - self.class.http_request_completed(self) - @on_completion_block.call - end - - def transit_time - Time.now.utc.to_i - @enqueued_at - end - - def solr_url - "#{Expander.config.solr_url}/update" - end - - def indexed_object - "#{@obj_type}[#{@obj_id}] database[#{@database}]" - end - - def http_request_started - self.class.http_request_started(self) - end - - def eql?(other) - other.hash == hash - end - - def hash - "#{action}#{indexed_object}#@enqueued_at#{self.class.name}".hash - end - - end - end -end diff --git a/chef-expander/lib/chef/expander/version.rb b/chef-expander/lib/chef/expander/version.rb deleted file mode 100644 index 5a0bac588d..0000000000 --- a/chef-expander/lib/chef/expander/version.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'open3' - -module Chef - module Expander - - VERSION = "11.0.0.alpha" - - def self.version - @rev ||= begin - begin - rev = Open3.popen3("git rev-parse HEAD") {|stdin, stdout, stderr| stdout.read }.strip - rescue Errno::ENOENT - rev = "" - end - rev.empty? ? nil : " (#{rev})" - end - "#{VERSION}#@rev" - end - - end -end diff --git a/chef-expander/lib/chef/expander/vnode.rb b/chef-expander/lib/chef/expander/vnode.rb deleted file mode 100644 index 32bb17da32..0000000000 --- a/chef-expander/lib/chef/expander/vnode.rb +++ /dev/null @@ -1,106 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'eventmachine' -require 'amqp' -require 'mq' - -require 'chef/expander/loggable' -require 'chef/expander/solrizer' - -module Chef - module Expander - class VNode - include Loggable - - attr_reader :vnode_number - - attr_reader :supervise_interval - - def initialize(vnode_number, supervisor, opts={}) - @vnode_number = vnode_number.to_i - @supervisor = supervisor - @queue = nil - @stopped = false - @supervise_interval = opts[:supervise_interval] || 30 - end - - def start - @supervisor.vnode_added(self) - - subscription_confirmed = Proc.new do - abort_on_multiple_subscribe - supervise_consumer_count - end - - queue.subscribe(:ack => true, :confirm => subscription_confirmed) do |headers, payload| - log.debug {"got #{payload} size(#{payload.size} bytes) on queue #{queue_name}"} - solrizer = Solrizer.new(payload) { headers.ack } - solrizer.run - end - - rescue MQ::Error => e - log.error {"Failed to start subscriber on #{queue_name} #{e.class.name}: #{e.message}"} - end - - def supervise_consumer_count - EM.add_periodic_timer(supervise_interval) do - abort_on_multiple_subscribe - end - end - - def abort_on_multiple_subscribe - queue.status do |message_count, subscriber_count| - if subscriber_count.to_i > 1 - log.error { "Detected extra consumers (#{subscriber_count} total) on queue #{queue_name}, cancelling subscription" } - stop - end - end - end - - def stop - log.debug {"Cancelling subscription on queue #{queue_name.inspect}"} - queue.unsubscribe if queue.subscribed? - @supervisor.vnode_removed(self) - @stopped = true - end - - def stopped? - @stopped - end - - def queue - @queue ||= begin - log.debug { "declaring queue #{queue_name}" } - MQ.queue(queue_name, :passive => false, :durable => true) - end - end - - def queue_name - "vnode-#{@vnode_number}" - end - - def control_queue_name - "#{queue_name}-control" - end - - end - end -end diff --git a/chef-expander/lib/chef/expander/vnode_supervisor.rb b/chef-expander/lib/chef/expander/vnode_supervisor.rb deleted file mode 100644 index e2e1dcbf8d..0000000000 --- a/chef-expander/lib/chef/expander/vnode_supervisor.rb +++ /dev/null @@ -1,265 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'yajl' -require 'eventmachine' -require 'amqp' -require 'mq' -require 'chef/expander/version' -require 'chef/expander/loggable' -require 'chef/expander/node' -require 'chef/expander/vnode' -require 'chef/expander/vnode_table' -require 'chef/expander/configuration' - -module ::AMQP - def self.hard_reset! - MQ.reset rescue nil - stop - EM.stop rescue nil - Thread.current[:mq], @conn = nil, nil - end -end - -module Chef - module Expander - class VNodeSupervisor - include Loggable - extend Loggable - - COULD_NOT_CONNECT = /Could not connect to server/.freeze - - def self.start_cluster_worker - @vnode_supervisor = new - @original_ppid = Process.ppid - trap_signals - - vnodes = Expander.config.vnode_numbers - - $0 = "chef-expander#{Expander.config.ps_tag} worker ##{Expander.config.index} (vnodes #{vnodes.min}-#{vnodes.max})" - - AMQP.start(Expander.config.amqp_config) do - start_consumers - await_parent_death - end - end - - def self.await_parent_death - @awaiting_parent_death = EM.add_periodic_timer(1) do - unless Process.ppid == @original_ppid - @awaiting_parent_death.cancel - stop_immediately("master process death") - end - end - end - - def self.start - @vnode_supervisor = new - trap_signals - - Expander.init_config(ARGV) - - log.info("Chef Search Expander #{Expander.version} starting up.") - - begin - AMQP.start(Expander.config.amqp_config) do - start_consumers - end - rescue AMQP::Error => e - if e.message =~ COULD_NOT_CONNECT - log.error { "Could not connect to rabbitmq. Make sure it is running and correctly configured." } - log.error { e.message } - - AMQP.hard_reset! - - sleep 5 - retry - else - raise - end - end - end - - def self.start_consumers - log.debug { "Setting prefetch count to 1"} - MQ.prefetch(1) - - vnodes = Expander.config.vnode_numbers - log.info("Starting Consumers for vnodes #{vnodes.min}-#{vnodes.max}") - @vnode_supervisor.start(vnodes) - end - - def self.trap_signals - Kernel.trap(:INT) { stop_immediately(:INT) } - Kernel.trap(:TERM) { stop_gracefully(:TERM) } - end - - def self.stop_immediately(signal) - log.info { "Initiating immediate shutdown on signal (#{signal})" } - @vnode_supervisor.stop - EM.add_timer(1) do - AMQP.stop - EM.stop - end - end - - def self.stop_gracefully(signal) - log.info { "Initiating graceful shutdown on signal (#{signal})" } - @vnode_supervisor.stop - wait_for_http_requests_to_complete - end - - def self.wait_for_http_requests_to_complete - if Expander::Solrizer.http_requests_active? - log.info { "waiting for in progress HTTP Requests to complete"} - EM.add_timer(1) do - wait_for_http_requests_to_complete - end - else - log.info { "HTTP requests completed, shutting down"} - AMQP.stop - EM.stop - end - end - - attr_reader :vnode_table - - attr_reader :local_node - - def initialize - @vnodes = {} - @vnode_table = VNodeTable.new(self) - @local_node = Node.local_node - @queue_name, @guid = nil, nil - end - - def start(vnode_ids) - @local_node.start do |message| - process_control_message(message) - end - - #start_vnode_table_publisher - - Array(vnode_ids).each { |vnode_id| spawn_vnode(vnode_id) } - end - - def stop - @local_node.stop - - #log.debug { "stopping vnode table updater" } - #@vnode_table_publisher.cancel - - log.info { "Stopping VNode queue subscribers"} - @vnodes.each do |vnode_number, vnode| - log.debug { "Stopping consumer on VNode #{vnode_number}"} - vnode.stop - end - - end - - def vnode_added(vnode) - log.debug { "vnode #{vnode.vnode_number} registered with supervisor" } - @vnodes[vnode.vnode_number.to_i] = vnode - end - - def vnode_removed(vnode) - log.debug { "vnode #{vnode.vnode_number} unregistered from supervisor" } - @vnodes.delete(vnode.vnode_number.to_i) - end - - def vnodes - @vnodes.keys.sort - end - - def spawn_vnode(vnode_number) - VNode.new(vnode_number, self).start - end - - def release_vnode - # TODO - end - - def process_control_message(message) - control_message = parse_symbolic(message) - case control_message[:action] - when "claim_vnode" - spawn_vnode(control_message[:vnode_id]) - when "recover_vnode" - recover_vnode(control_message[:vnode_id]) - when "release_vnodes" - raise "todo" - release_vnode() - when "update_vnode_table" - @vnode_table.update_table(control_message[:data]) - when "vnode_table_publish" - publish_vnode_table - when "status" - publish_status_to(control_message[:rsvp]) - when "set_log_level" - set_log_level(control_message[:level], control_message[:rsvp]) - else - log.error { "invalid control message #{control_message.inspect}" } - end - rescue Exception => e - log.error { "Error processing a control message."} - log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}" } - end - - - def start_vnode_table_publisher - @vnode_table_publisher = EM.add_periodic_timer(10) { publish_vnode_table } - end - - def publish_vnode_table - status_update = @local_node.to_hash - status_update[:vnodes] = vnodes - status_update[:update] = :add - @local_node.broadcast_message(Yajl::Encoder.encode({:action => :update_vnode_table, :data => status_update})) - end - - def publish_status_to(return_queue) - status_update = @local_node.to_hash - status_update[:vnodes] = vnodes - MQ.queue(return_queue).publish(Yajl::Encoder.encode(status_update)) - end - - def set_log_level(level, rsvp_to) - log.info { "setting log level to #{level} due to command from #{rsvp_to}" } - new_log_level = (Expander.config.log_level = level.to_sym) - reply = {:level => new_log_level, :node => @local_node.to_hash} - MQ.queue(rsvp_to).publish(Yajl::Encoder.encode(reply)) - end - - def recover_vnode(vnode_id) - if @vnode_table.local_node_is_leader? - log.debug { "Recovering vnode: #{vnode_id}" } - @local_node.shared_message(Yajl::Encoder.encode({:action => :claim_vnode, :vnode_id => vnode_id})) - else - log.debug { "Ignoring :recover_vnode message because this node is not the leader" } - end - end - - def parse_symbolic(message) - Yajl::Parser.new(:symbolize_keys => true).parse(message) - end - - end - end -end diff --git a/chef-expander/lib/chef/expander/vnode_table.rb b/chef-expander/lib/chef/expander/vnode_table.rb deleted file mode 100644 index 025812b49c..0000000000 --- a/chef-expander/lib/chef/expander/vnode_table.rb +++ /dev/null @@ -1,83 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'yajl' -require 'chef/expander/node' -require 'chef/expander/loggable' - -module Chef - module Expander - class VNodeTable - - include Loggable - - class InvalidVNodeTableUpdate < ArgumentError; end - - attr_reader :vnodes_by_node - - def initialize(vnode_supervisor) - @node_update_mutex = Mutex.new - @vnode_supervisor = vnode_supervisor - @vnodes_by_node = {} - end - - def nodes - @vnodes_by_node.keys - end - - def update_table(table_update) - case table_update[:update] - when "add", "update" - update_node(table_update) - when "remove" - remove_node(table_update) - else - raise InvalidVNodeTableUpdate, "no action or action not acceptable: #{table_update.inspect}" - end - log.debug { "current vnode table: #{@vnodes_by_node.inspect}" } - end - - def update_node(node_info) - @node_update_mutex.synchronize do - @vnodes_by_node[Node.from_hash(node_info)] = node_info[:vnodes] - end - end - - def remove_node(node_info) - @node_update_mutex.synchronize do - @vnodes_by_node.delete(Node.from_hash(node_info)) - end - end - - def leader_node - if @vnodes_by_node.empty? - nil - else - Array(@vnodes_by_node).reject { |node| node[1].empty? }.sort { |a,b| a[1].min <=> b[1].min }.first[0] - end - end - - def local_node_is_leader? - (Node.local_node == leader_node) || (@vnodes_by_node[Node.local_node].include?(0)) - end - - end - end -end diff --git a/chef-expander/notes/topic-queue-benchmarks.md b/chef-expander/notes/topic-queue-benchmarks.md deleted file mode 100644 index 16dcbfca16..0000000000 --- a/chef-expander/notes/topic-queue-benchmarks.md +++ /dev/null @@ -1,48 +0,0 @@ -# Time to process 10,000 messages # -* messages are (16 bytes), generated by bin/traffic-creator -* in all cases, the generator finishes well before the subscriber -* in all cases, the rabbitmq generates a load of ~80% - 90% on single core -* generator is run on the same physical machine, but see above caveat -* consumer is ruby amqp/eventmachine, prints a simple status message for each - message received. -* generator is ruby/bunny - -## 10 processes * 1,000 queues subscribed ## -10 consumer processes, each claiming 1/10th of 10,000 total queues. -start : 1282693162 -end : 1282694881 (incomplete, give up) -elapsed : 1771s (less than 6/s) - -## 10,000 queues subscribed ## -start : 1282690712 -end : 1282691371 (incomplete, gave up.) - -## 5,000 queues subscribed ## -start : 1282694987 -end : 1282695411 -total : 424s - -second run: -start : 1282695613 -end : 1282696041 -total : 428s (~23/s) - -## 1024 queues subscribed ## -start : 1282697018 -end : 1282697110 -total : 92s (~110/s) - -## 1000 queues subscribed ## -start : 1282689470 -end : 1282689560 -total : 90s - -## 100 queues subscribed ## -start : 1282689603 -end : 1282689698 -total : 95s - -## 10 queues subscribed ## -start : 1282689758 -end : 1282689850 -total : 92s
\ No newline at end of file diff --git a/chef-expander/scripts/check_queue_size b/chef-expander/scripts/check_queue_size deleted file mode 100755 index cc539f4683..0000000000 --- a/chef-expander/scripts/check_queue_size +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env ruby -#-- -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################### -# check_queue_size -# A Nagios Check for Chef Server Queue Backlogs -############################################################################### - -require 'rubygems' - -Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do - - require 'bunny' - - $:.unshift(File.expand_path('./lib')) - require 'chef/expander' - require 'chef/expander/version' - require 'chef/expander/configuration' - - include Chef - - Expander.init_config([]) - - config = {:warn => 100, :crit => 200} - - option_parser = OptionParser.new do |o| - o.banner = "Usage: check_queue_size [options]" - - o.on('-w', '--warn WARN_THRESHOLD', 'number of messages to trigger a warning') do |i| - config[:warn] = i.to_i - end - - o.on('-c', '--critical CRITICAL_THRESHOLD', 'the number of messages to trigger a critical') do |n| - config[:crit] = n.to_i - end - - o.on_tail('-h', '--help', 'show this message') do - puts "chef-expander #{Expander.version}" - puts "queue size monitor" - puts '' - puts o - exit 127 - end - end - - option_parser.parse!(ARGV.dup) - - message_counts = [] - - begin - amqp_client = Bunny.new(Expander.config.amqp_config) - amqp_client.start - - 0.upto(Expander::VNODES - 1) do |vnode| - q = amqp_client.queue("vnode-#{vnode}", :durable => true) - message_counts << q.status[:message_count] - end - total_messages = message_counts.inject(:+) - - if total_messages >= config[:crit] - puts "Chef Expander Queue Size CRITICAL - messages: #{total_messages}" - exit(2) - elsif total_messages >= config[:warn] - puts "Chef Expander Queue Size WARNING - messages: #{total_messages}" - exit(1) - else - puts "Chef Expander Queue Size OK - messages: #{total_messages}" - exit(0) - end - - ensure - amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected? - end - -end diff --git a/chef-expander/scripts/check_queue_size_munin b/chef-expander/scripts/check_queue_size_munin deleted file mode 100755 index c0725196db..0000000000 --- a/chef-expander/scripts/check_queue_size_munin +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env ruby - -require 'rubygems' -require 'munin_plugin' -Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do - require 'bunny' - - $:.unshift(File.expand_path('../../lib', __FILE__)) - require 'chef/expander' - require 'chef/expander/version' - require 'chef/expander/configuration' - - include Chef - -munin_plugin do - graph_title "Expander Queue Size" - graph_vlabel "Events" - graph_category "Opscode" - graph_info "Events in the Expander Queue waiting to be consumed by Solr." - queuesize.label "events" - queuesize.draw "LINE" - queuesize.warning "100" - queuesize.critical "200" - - collect do - - Expander.init_config([]) - - message_counts = [] - - begin - amqp_client = Bunny.new(Expander.config.amqp_config) - amqp_client.start - - 0.upto(Expander::VNODES - 1) do |vnode| - q = amqp_client.queue("vnode-#{vnode}", :durable => true) - message_counts << q.status[:message_count] - end - total_messages = message_counts.inject(:+) - - ensure - amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected? - end - - queuesize.value total_messages - - end - - - end -end diff --git a/chef-expander/scripts/make_solr_xml b/chef-expander/scripts/make_solr_xml deleted file mode 100755 index e44e045bf8..0000000000 --- a/chef-expander/scripts/make_solr_xml +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env ruby -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require "rubygems" -$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib')) - -require 'yajl' -require 'chef/expander/solrizer' - -USAGE = <<-EOH -#{$0} [file ...] - -Convert Chef object JSON files into XML documents of the same name but -with a ".xml" extension containing the XML used for Solr indexing. -EOH - -if ARGV.size == 0 - abort USAGE -end - -ARGV.each do |obj_file| - raw_json = open(obj_file, "r").read - item_json = Yajl::Parser.parse(raw_json) - payload = { - :item => item_json, - :type => item_json["chef_type"].to_s, - :database => "riak_search_test", - :id => item_json["name"], - :enqueued_at => Time.now.to_i - } - update_obj = {:action => "add", :payload => payload} - update_json = Yajl::Encoder.encode(update_obj) - solrizer = Chef::Expander::Solrizer.new(update_json) { :no_op } - solrizer.log.init(StringIO.new) - - out = File.basename(obj_file).sub(/\.json$/, "") + ".xml" - open(out, "w") do |f| - f.write(solrizer.pointyize_add) - end -end diff --git a/chef-expander/scripts/traffic-creator b/chef-expander/scripts/traffic-creator deleted file mode 100755 index a2c207cf36..0000000000 --- a/chef-expander/scripts/traffic-creator +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env ruby -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require "rubygems" - -$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib')) - -require 'pp' -require 'bunny' -require 'yajl' -require 'uuidtools' -require 'word_salad' - -require 'chef_search/expander/configuration' - -Chef::Expander.init_config(ARGV) - -MESSAGES_TO_SEND = 10_000 - -NUM_RAND_KEY_PAIRS = 50 -NUM_RAND_VALUE_PAIRS = 50 - -PERSISTENT_MESSAGES = true - -KEYS = NUM_RAND_VALUE_PAIRS.words - -SAMPLE_NODES = [] -Dir.glob(File.expand_path(File.dirname(__FILE__)) + '/../data/*_node.json') do |node_file| - SAMPLE_NODES << Yajl::Parser.parse(IO.read(node_file)) -end - -NUM_NODES = SAMPLE_NODES.size -puts "Read #{NUM_NODES} sample nodes" - -puts "Using rabbitmq config #{Chef::Expander.config.amqp_config.inspect}" - -puts "connecting to rabbitmq" -amqp_client = Bunny.new(Chef::Expander.config.amqp_config) -amqp_client.start - -puts 'declaring queues' -queues = {} -0.upto(1023) do |vnode| - queues[vnode] = amqp_client.queue("vnode-#{vnode}", :durable => true) -end - -def add_rand_keys(node) - rand_key_vals = Hash[*((2 * NUM_RAND_KEY_PAIRS).words)] - rand_vals = Hash[*(KEYS.zip(NUM_RAND_VALUE_PAIRS.words)).flatten] - node.merge(rand_key_vals.merge(rand_vals)) -end - -puts "sending #{MESSAGES_TO_SEND} messages" -start_time = Time.now -sent_messages = 0 -1.upto(MESSAGES_TO_SEND) do - node = SAMPLE_NODES[rand(NUM_NODES)] - node = add_rand_keys(node) - index_data = {:action => :add} - index_data[:payload] = {:item => node} - index_data[:payload][:type] = :node - index_data[:payload][:database] = :testdb - index_data[:payload][:enqueued_at] = Time.now.utc.to_i - - id = node["name"] - vnode = rand(1024) - index_data[:payload][:id] = id - - puts "queue: vnode-#{vnode} (#{sent_messages} / #{MESSAGES_TO_SEND})" - amqp_client.tx_select if PERSISTENT_MESSAGES - queues[vnode].publish(Yajl::Encoder.encode(index_data), :persistent => PERSISTENT_MESSAGES) - amqp_client.tx_commit if PERSISTENT_MESSAGES - sent_messages += 1 -end -end_time = Time.now - -total_time = end_time - start_time -rate = MESSAGES_TO_SEND.to_f / total_time -puts "done (#{total_time}s, #{rate} msg/s)" diff --git a/chef-expander/spec/fixtures/chef-expander.rb b/chef-expander/spec/fixtures/chef-expander.rb deleted file mode 100644 index 84883abe2c..0000000000 --- a/chef-expander/spec/fixtures/chef-expander.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# CHEF EXPANDER CONFIGURATION ###### -# A Sample config file for spec tests # -####################################### - -## The Actual Config Settings for Chef Expander ## -# Solr -solr_url "http://localhost:8983/solr" - -## Parameters for connecting to RabbitMQ ## -# Defaults: -#amqp_host 'localhost' -#amqp_port '5672' -#amqp_user 'guest' -amqp_pass 'config-file' # should override the defaults -amqp_vhost '/config-file' - -## Cluster Config, should be overridden by command line ## -node_count 42 -## Extraneous Config (should be ignored and not raise an error) ## - -solr_ram_use "1024T" -another_setting "#{solr_ram_use} is an alot"
\ No newline at end of file diff --git a/chef-expander/spec/spec.opts b/chef-expander/spec/spec.opts deleted file mode 100644 index 706088061b..0000000000 --- a/chef-expander/spec/spec.opts +++ /dev/null @@ -1 +0,0 @@ --cbfs diff --git a/chef-expander/spec/spec_helper.rb b/chef-expander/spec/spec_helper.rb deleted file mode 100644 index 578143fe92..0000000000 --- a/chef-expander/spec/spec_helper.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'pp' -require 'stringio' -require 'rubygems' -require 'bunny' -$:.unshift(File.expand_path('../../lib/', __FILE__)) -require 'chef/expander' - -include Chef - -OPSCODE_EXPANDER_MQ_CONFIG = {:user => "guest", :pass => "guest", :vhost => '/chef-expander-test'} - -HOW_TO_SETUP=<<-ERROR - -****************************** FAIL ******************************************* -* Running these tests requires a running instance of rabbitmq -* You also must configure a vhost "/chef-expander-test" -* and a user "guest" with password "guest" with full rights -* to that vhost -------------------------------------------------------------------------------- -> rabbitmq-server -> rabbitmqctl add_vhost /chef-expander-test -> rabbitmqctl set_permissions -p /chef-expander-test guest '.*' '.*' '.*' -> rabbitmqctl list_user_permissions guest -****************************** FAIL ******************************************* - -ERROR - -def debug_exception_to_stderr(e) - if ENV['DEBUG'] == "true" - STDERR.puts("#{e.class.name}: #{e.message}") - STDERR.puts("#{e.backtrace.join("\n")}") - end -end - -begin - b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) - b.start - b.stop -rescue Bunny::ProtocolError, Bunny::ServerDownError, Bunny::ConnectionError => e - STDERR.puts(HOW_TO_SETUP) - debug_exception_to_stderr(e) - exit(1) -rescue Exception => e - STDERR.puts(<<-EXPLAIN) -An unknown error occurred verifying prerequisites for unit tests. -This is most commonly caused by incorrect permissions settings in rabbitmq. - -Run with DEBUG=true to see the error. - -EXPLAIN - STDERR.puts(HOW_TO_SETUP) - debug_exception_to_stderr(e) - exit(2) -end - -FIXTURE_PATH = File.expand_path('../fixtures', __FILE__) diff --git a/chef-expander/spec/unit/configuration_spec.rb b/chef-expander/spec/unit/configuration_spec.rb deleted file mode 100644 index eed19c2b67..0000000000 --- a/chef-expander/spec/unit/configuration_spec.rb +++ /dev/null @@ -1,141 +0,0 @@ -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'stringio' -require 'chef/expander/configuration' - -describe Expander::Configuration do - before do - @config = Expander::Configuration::Base.new - @config.reset! - @config.apply_defaults - @config.index = 1 - @config.node_count = 5 - end - - it "stores the number of nodes" do - @config.node_count.should == 5 - end - - it "stores the position in the node ring" do - @config.index.should == 1 - end - - it "computes the vnodes the node should claim" do - @config.vnode_numbers.should == (0..203).to_a - end - - it "assigns any remainder to the last node in the ring" do - @config.index = 5 - @config.vnode_numbers.should == (816..1023).to_a - end - - it "does not raise an invalid config error when then node index is not set" do - @config.index = nil - @config.index.should == nil - end - - it "raises an invalid config error when the node count is not set" do - @config.node_count = nil - lambda { @config.validate! }.should raise_error(Expander::Configuration::InvalidConfiguration) - end - - it "raises an invalid config error when the index is greater than the node count" do - @config.node_count = 5 - @config.index = 10 - lambda { @config.validate! }.should raise_error(Expander::Configuration::InvalidConfiguration) - end - - it "exits when the config is invalid" do - stdout = StringIO.new - @config.reset!(stdout) - @config.node_count = nil - @config.log.stub!(:warn) - lambda {@config.fail_if_invalid}.should raise_error(SystemExit) - stdout.string.should match(/You must specify the node_count as an integer/) - end - - it "has a setting for solr url defaulting to localhost:8983" do - @config.solr_url.should == "http://localhost:8983/solr" - end - - it "has a setting for the amqp host to connect to, defaulting to 0.0.0.0" do - @config.amqp_host.should == '0.0.0.0' - end - - it "has a setting for the amqp port to use, defaulting to 5672" do - @config.amqp_port.should == 5672 - end - - it "has a setting for the amqp_user, defaulting to 'chef'" do - @config.amqp_user.should == 'chef' - end - - it "has a setting for the amqp password, defaulting to 'testing'" do - @config.amqp_pass.should == 'testing' - end - - it "has a setting for the amqp vhost, defaulting to /chef" do - @config.amqp_vhost.should == '/chef' - end - - it "uses /etc/chef/solr.rb as the default configuration file" do - @config.config_file.should == '/etc/chef/solr.rb' - end - - it "has a pidfile, using /var/run/chef-expander.pid as the default when running as root" do - Process.stub!(:euid).and_return(0) - @config.pidfile.should == "/var/run/chef-expander.pid" - end - - it "has a pidfile, using /tmp/chef-expander.pid as the default when running as non-root" do - Process.stub!(:euid).and_return(1000) - @config.pidfile.should == "/tmp/chef-expander.pid" - end - - it "has a user setting, defaulting to nil" do - @config.user.should be_nil - end - - it "has a group setting, defaulting to nil" do - @config.group.should be_nil - end - - it "configures whether the process should daemonize" do - @config.daemonize?.should be_false - end - - it "sets the log location to an IO object" do - @config.log_location = STDERR - @config.log.log_device.should == STDERR - end - - it "sets the log location to a File" do - @config.log_location = File.join(FIXTURE_PATH, 'expander.log') - @config.log.log_device.path.should == File.join(FIXTURE_PATH, 'expander.log') - end - - it "generates an AMQP configuration hash suitable for passing to Bunny.new or AMQP.start" do - @config.amqp_config.should == {:host => '0.0.0.0', :port => 5672, :user => 'chef', :pass => 'testing', :vhost => '/chef'} - end - - it "merges another config on top of itself" do - other = Expander::Configuration::Base.new - other.solr_url = "somewhere with non-pitiful disk io" - @config.merge_config(other) - @config.solr_url.should == "somewhere with non-pitiful disk io" #if only it was that easy - end - - it "merges config settings so that defaults < config_file < command line " do - config_file = File.dirname(__FILE__) + '/../fixtures/chef-expander.rb' - argv = ["-c", config_file, '-n', '23'] - Expander.config.reset! - Expander.init_config(argv) - Expander.config.amqp_pass.should == 'config-file' - Expander.config.node_count.should == 23 - end - - it "appends /solr to the legacy solr_url" do - Expander.config.solr_url = "http://localhost:8983" - @config.solr_url.should == "http://localhost:8983/solr" - end -end
\ No newline at end of file diff --git a/chef-expander/spec/unit/control_spec.rb b/chef-expander/spec/unit/control_spec.rb deleted file mode 100644 index 7c456ec5e8..0000000000 --- a/chef-expander/spec/unit/control_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'chef/expander/control' - -describe Expander::Control do - -end diff --git a/chef-expander/spec/unit/node_spec.rb b/chef-expander/spec/unit/node_spec.rb deleted file mode 100644 index 2d9c1b207d..0000000000 --- a/chef-expander/spec/unit/node_spec.rb +++ /dev/null @@ -1,185 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'ostruct' -require 'chef/expander/node' - -describe Expander::Node do - - it "can be created from a hash" do - node_info = { :foo => :blargh, - :guid => "93226974-6d0b-4ca6-8d42-124dd55e0076", - :hostname_f => "fermi.local", :pid => 12345} - unmodified_hash = node_info.dup - node_from_hash = Expander::Node.from_hash(node_info) - node_info.should == unmodified_hash - node_from_hash.guid.should == "93226974-6d0b-4ca6-8d42-124dd55e0076" - node_from_hash.hostname_f.should == "fermi.local" - node_from_hash.pid.should == 12345 - end - - describe "when first created" do - before do - @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" - @hostname_f = "fermi.local" - @pid = 12345 - @node = Expander::Node.new(@guid, @hostname_f, @pid) - end - - it "has the guid it was created with" do - @node.guid.should == @guid - end - - it "has the hostname it was created with" do - @node.hostname_f.should == @hostname_f - end - - it "has the pid it was created with" do - @node.pid.should == @pid - end - - it "names its shared control queue using a constant/consistent name" do - @node.shared_control_queue_name.should == "chef-search-control--shared" - end - - it "names its exclusive control queue after its hostname, pid, and guid" do - @node.exclusive_control_queue_name.should == "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076--exclusive-control" - end - - it "names its broadcast control queue after its hostname, pid, and guid" do - @node.broadcast_control_queue_name.should == "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076--broadcast" - end - - it "names the broadcast control exchange using a consistent name" do - @node.broadcast_control_exchange_name.should == 'chef-search-control--broadcast' - end - - it "generates its hash from a string concatenting the hostname, pid and guid" do - concat_string = "fermi.local--12345--93226974-6d0b-4ca6-8d42-124dd55e0076" - @node.hash.should == concat_string.hash - end - - it "is eql to another Node if it has the same guid, hostname, and pid" do - other = Expander::Node.new(@guid.dup, @hostname_f.dup, @pid) - @node.should eql(other) - end - - it "is == to another object if it has the same guid, hostname, and pid" do - other = Class.new(Expander::Node).new(@guid.dup, @hostname_f.dup, @pid) - other.should == @node - end - - it "converts to a hash" do - @node.to_hash.should == {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid} - end - - end - - describe "when describing the node it's running on" do - before do - hostname_f = OpenStruct.new(:stdout => "fermi.local\n") - Expander::Node.stub!(:shell_out!).and_return(hostname_f) - @node = Expander::Node.local_node - end - - it "uses the current machine's hostname -f for the hostname" do - @node.hostname_f.should == %x(hostname -f).strip - end - - it "uses the current process id for the pid" do - @node.pid.should == Process.pid - end - - it "generates a guid for the guid" do - @node.guid.should match /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ - end - end - - describe "when sending and receiving messages" do - - before do - @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" - @hostname_f = "fermi.local" - @pid = rand(10000) - @node = Expander::Node.new(@guid, @hostname_f, @pid) - @log_stream = StringIO.new - @node.log.init(@log_stream) - end - - after do - b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) - b.start - b.exchange(@node.broadcast_control_exchange_name, :type => :fanout).delete - b.stop - end - - it "receives messages on the broadcast exchange" do - messages = [] - - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - @node.start do |message| - messages << message - end - - @node.broadcast_message("hello everybody") - - EM.add_timer(0.1) {AMQP.hard_reset!} - end - - messages.should == ["hello everybody"] - end - - it "receives messages on its exclusive queue" do - messages = [] - - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - @node.start do |message| - messages << message - end - - @node.direct_message("hello node") - - EM.add_timer(0.1) {AMQP.hard_reset!} - end - - messages.should == ["hello node"] - end - - it "receives messages on the shared queue" do - messages = [] - - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - @node.start do |message| - messages << message - end - - @node.shared_message("hello one of N") - - EM.add_timer(0.1) {AMQP.hard_reset!} - end - - messages.should == ["hello one of N"] - end - - end - -end diff --git a/chef-expander/spec/unit/solrizer_spec.rb b/chef-expander/spec/unit/solrizer_spec.rb deleted file mode 100644 index ca5f92ed75..0000000000 --- a/chef-expander/spec/unit/solrizer_spec.rb +++ /dev/null @@ -1,277 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'stringio' -require 'chef/expander/solrizer' -require 'yajl' -require 'rexml/document' - -describe Expander::Solrizer do - SEP = "__=__" - - describe "when created with an add request" do - before do - @now = Time.now.utc.to_i - @indexer_payload = {:item => {:foo => {:bar => :baz}}, - :type => :node, - :database => :testdb, - :id => "2342", - :enqueued_at => @now} - - @update_object = {:action => "add", :payload => @indexer_payload} - @update_json = Yajl::Encoder.encode(@update_object) - @solrizer = Expander::Solrizer.new(@update_json) { :no_op } - - @log_stream = StringIO.new - @solrizer.log.init(@log_stream) - @expected_fields = %w(X_CHEF_id_CHEF_X X_CHEF_database_CHEF_X X_CHEF_type_CHEF_X) - end - - it "extracts the indexing-specific payload from the update message" do - @solrizer.indexer_payload.should == { 'item' => {'foo' => {'bar' => "baz"}}, - 'type' => 'node', - 'database' => 'testdb', 'id' => "2342", - "enqueued_at"=>@now} - end - - it "extracts the action from the update message" do - @solrizer.action.should == "add" - end - - it "extracts the item to update from the update message" do - @solrizer.chef_object.should == {"foo" => {"bar" => "baz"}} - end - - it "extracts the database name from the update message" do - @solrizer.database.should == "testdb" - end - - it "extracts the object id from the update message" do - @solrizer.obj_id.should == "2342" - end - - it "extracts the object type from the update message" do - @solrizer.obj_type.should == "node" - end - - it "extracts the time the object was enqueued from the message" do - @solrizer.enqueued_at.should == @now - end - - it "is eql to another Solrizer object that has the same object type, id, database, action, and enqueued_at time" do - eql_solrizer = Expander::Solrizer.new(@update_json) - @solrizer.should eql eql_solrizer - end - - it "is not eql to another Solrizer if the enqueued_at time is different" do - update_hash = @update_object.dup - update_hash[:payload] = update_hash[:payload].merge({:enqueued_at => (Time.now.utc.to_i + 10000)}) - update_json = Yajl::Encoder.encode(update_hash) - uneql_solrizer = Expander::Solrizer.new(update_json) - @solrizer.should_not eql(uneql_solrizer) - end - - it "is not eql to another Solrizer if the object id is different" do - update_hash = @update_object.dup - update_hash[:payload] = update_hash[:payload].merge({:id => 12345}) - update_json = Yajl::Encoder.encode(update_hash) - uneql_solrizer = Expander::Solrizer.new(update_json) - @solrizer.should_not eql(uneql_solrizer) - end - - it "is not eql to another Solrizer if the database is different" do - update_hash = @update_object.dup - update_hash[:payload] = update_hash[:payload].merge({:database => "nononono"}) - update_json = Yajl::Encoder.encode(update_hash) - uneql_solrizer = Expander::Solrizer.new(update_json) - @solrizer.should_not eql(uneql_solrizer) - end - - it "is not eql to another Solrizer if the action is different" do - update_hash = @update_object.dup - update_hash[:action] = :delete - update_json = Yajl::Encoder.encode(update_hash) - uneql_solrizer = Expander::Solrizer.new(update_json) - @solrizer.should_not eql(uneql_solrizer) - end - - describe "when flattening to XML" do - before do - @expected_object = {"foo" => ["bar"], - "foo_bar" => ["baz"], - "bar" => ["baz"], - "X_CHEF_id_CHEF_X" => ["2342"], - "X_CHEF_database_CHEF_X" => ["testdb"], - "X_CHEF_type_CHEF_X" => ["node"]} - @expected_fields = %w(X_CHEF_id_CHEF_X X_CHEF_database_CHEF_X X_CHEF_type_CHEF_X) - end - - it "generates the flattened and expanded representation of the object" do - @solrizer.flattened_object.should == @expected_object - end - - it "has the expected fields in the document" do - doc = REXML::Document.new(@solrizer.pointyize_add) - flds = doc.elements.to_a("add/doc/field").map {|f| f.attributes["name"] } - @expected_fields.each do |field| - flds.should include(field) - end - end - - it "the content field contains key value pairs delimited with the right separator" do - doc = REXML::Document.new(@solrizer.pointyize_add) - doc.elements.each("add/doc/field[@name='content']") do |content| - raw = content.text - @expected_object.each do |k, v| - s = "#{k}#{SEP}#{v.first}" - raw.index(s).should_not be_nil - end - end - end - end - - describe "when flattening data to XML that needs XML escaping" do - before do - @indexer_payload[:type] = :role - @indexer_payload[:item] = { "a&w" => "<rootbeer/>" } - update_object = {:action => "add", :payload => @indexer_payload} - update_json = Yajl::Encoder.encode(update_object) - @solrizer = Expander::Solrizer.new(update_json) { :no_op } - @solrizer.log.init(@log_stream) - end - - it "the content field contains escaped keys and values" do - raw = @solrizer.pointyize_add - raw.should match("a&w#{SEP}<rootbeer/>") - end - end - - describe "when flattening data bag XML" do - before do - @indexer_payload[:type] = :data_bag_item - @indexer_payload[:item] = {:k1 => "v1", "data_bag" => "stuff"} - update_object = {:action => "add", :payload => @indexer_payload} - update_json = Yajl::Encoder.encode(update_object) - @solrizer = Expander::Solrizer.new(update_json) { :no_op } - @solrizer.log.init(@log_stream) - @expected_fields << "data_bag" - end - - it "contains a data_bag field with the right name" do - doc = REXML::Document.new(@solrizer.pointyize_add) - flds = doc.elements.to_a("add/doc/field[@name='data_bag']") - flds.size.should == 1 - flds.first.text.should == "stuff" - end - - it "has the expected fields in the document" do - doc = REXML::Document.new(@solrizer.pointyize_add) - flds = doc.elements.to_a("add/doc/field").map {|f| f.attributes["name"] } - @expected_fields.each do |field| - flds.should include(field) - end - end - describe "and data bag name needs escaping" do - before do - @indexer_payload[:item] = {:k1 => "v1", "data_bag" => "a&w>"} - update_object = {:action => "add", :payload => @indexer_payload} - update_json = Yajl::Encoder.encode(update_object) - @solrizer = Expander::Solrizer.new(update_json) { :no_op } - @solrizer.log.init(@log_stream) - end - - it "contains a data_bag field with an escaped name" do - raw = @solrizer.pointyize_add - raw.should match("data_bag#{SEP}a&w>") - end - end - end - - describe "when no HTTP request is in progress" do - - it "does not report that an HTTP request is in progress" do - Expander::Solrizer.http_requests_active?.should be_false - end - - end - - describe "when an HTTP request is in progress" do - before do - Expander::Solrizer.clear_http_requests - @solrizer.http_request_started - end - - it "registers the in-progress HTTP request" do - Expander::Solrizer.http_requests_active?.should be_true - end - - it "removes itself from the list of active http requests when the request completes" do - @solrizer.completed - Expander::Solrizer.http_requests_active?.should be_false - end - - end - - - end - - describe "when created with a delete request" do - before do - @indexer_payload = {:id => "2342"} - @update_object = {:action => "add", :payload => @indexer_payload} - @update_json = Yajl::Encoder.encode(@update_object) - @solrizer = Expander::Solrizer.new(@update_json) - end - - it "extracts the indexer payload" do - @solrizer.indexer_payload.should == {"id" => "2342"} - end - - it "extracts the object id" do - @solrizer.obj_id.should == "2342" - end - - it "converts the delete request to XML" do - expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n" - @solrizer.pointyize_delete.should == expected - end - - end - - describe "solr_url" do - before do - @indexer_payload = {:id => "2342"} - @update_object = {:action => "add", :payload => @indexer_payload} - @update_json = Yajl::Encoder.encode(@update_object) - @solrizer = Expander::Solrizer.new(@update_json) - end - - it "appends /update to the default solr_url" do - @solrizer.solr_url.should == "http://localhost:8983/solr/update" - end - - it "appends /update to a configured solr_url" do - Expander.config.solr_url = "https://vhost/abcdef/solr/collection" - @solrizer.solr_url.should == "https://vhost/abcdef/solr/collection/update" - end - end -end diff --git a/chef-expander/spec/unit/vnode_spec.rb b/chef-expander/spec/unit/vnode_spec.rb deleted file mode 100644 index 9321c03ef3..0000000000 --- a/chef-expander/spec/unit/vnode_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'chef/expander/vnode_supervisor' -require 'chef/expander/vnode' - -describe Expander::VNode do - before do - @supervisor = Expander::VNodeSupervisor.new - @vnode = Expander::VNode.new("2342", @supervisor, :supervise_interval => 0.1) - @log_stream = StringIO.new - @vnode.log.init(@log_stream) - end - - it "has the vnode number it was created with" do - @vnode.vnode_number.should == 2342 - end - - it "has a queue named after its vnode number" do - @vnode.queue_name.should == "vnode-2342" - end - - it "has a control queue name" do - @vnode.control_queue_name.should == "vnode-2342-control" - end - - describe "when connecting to rabbitmq" do - it "disconnects if there is another subscriber" do - begin - q = nil - b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) - b.start - q = b.queue(@vnode.queue_name, :passive => false, :durable => true, :exclusive => false, :auto_delete => false) - t = Thread.new { q.subscribe { |message| nil }} - - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - EM.add_timer(0.5) do - AMQP.stop - EM.stop - end - @vnode.start - end - t.kill - - @vnode.should be_stopped - @log_stream.string.should match(/Detected extra consumers/) - ensure - q && q.delete - b.stop - end - end - - it "calls back to the supervisor when it subscribes to the queue" do - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - MQ.topic('foo') - EM.add_timer(0.1) do - AMQP.stop - EM.stop - end - @vnode.start - end - @supervisor.vnodes.should == [2342] - end - - it "calls back to the supervisor when it stops subscribing" do - @supervisor.vnode_added(@vnode) - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - MQ.topic('foo') - EM.add_timer(0.1) do - @vnode.stop - AMQP.stop - EM.stop - end - end - @supervisor.vnodes.should be_empty - end - - end - -end diff --git a/chef-expander/spec/unit/vnode_supervisor_spec.rb b/chef-expander/spec/unit/vnode_supervisor_spec.rb deleted file mode 100644 index 59622675e8..0000000000 --- a/chef-expander/spec/unit/vnode_supervisor_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'chef/expander/vnode_supervisor' - -describe Expander::VNodeSupervisor do - before do - @log_stream = StringIO.new - @local_node = Expander::Node.new("1101d02d-1547-45ab-b2f6-f0153d0abb34", "fermi.local", 12342) - @vnode_supervisor = Expander::VNodeSupervisor.new - @vnode_supervisor.instance_variable_set(:@local_node, @local_node) - @vnode_supervisor.log.init(@log_stream) - @vnode = Expander::VNode.new("42", @vnode_supervisor) - end - - after do - b = Bunny.new(OPSCODE_EXPANDER_MQ_CONFIG) - b.start - b.exchange(@vnode_supervisor.local_node.broadcast_control_exchange_name, :type => :fanout).delete - b.queue(@vnode_supervisor.local_node.broadcast_control_queue_name).purge - b.stop - end - - it "keeps a list of vnodes" do - @vnode_supervisor.vnodes.should be_empty - @vnode_supervisor.vnode_added(@vnode) - @vnode_supervisor.vnodes.should == [42] - end - - it "has a callback for vnode removal" do - @vnode_supervisor.vnode_added(@vnode) - @vnode_supervisor.vnodes.should == [42] - @vnode_supervisor.vnode_removed(@vnode) - @vnode_supervisor.vnodes.should be_empty - end - - it "spawns a vnode" do - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - @vnode_supervisor.spawn_vnode(42) - MQ.topic('foo') - EM.add_timer(0.1) do - AMQP.hard_reset! - end - end - @vnode_supervisor.vnodes.should == [42] - end - - it "subscribes to the control queue" do - AMQP.start(OPSCODE_EXPANDER_MQ_CONFIG) do - @vnode_supervisor.start([]) - @vnode_supervisor.should_receive(:process_control_message).with("hello_robot_overlord") - Expander::Node.local_node.broadcast_message("hello_robot_overlord") - EM.add_timer(0.1) do - AMQP.hard_reset! - end - end - end - - it "periodically publishes its list of vnodes to the gossip queue" do - pending("disabled until cluster healing is implemented") - end - - describe "when responding to control messages" do - it "passes vnode table updates to its vnode table" do - vnode_table_update = Expander::Node.local_node.to_hash - vnode_table_update[:vnodes] = (0...16).to_a - vnode_table_update[:update] = :add - update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) - @vnode_supervisor.process_control_message(update_message) - @vnode_supervisor.vnode_table.vnodes_by_node[Expander::Node.local_node].should == (0...16).to_a - end - - it "publishes the vnode table when it receives a :vnode_table_publish message" do - pending "disabled until cluster healing is implemented" - update_message = Yajl::Encoder.encode({:action => :vnode_table_publish}) - @vnode_supervisor.process_control_message(update_message) - end - - describe "and it is the leader" do - before do - vnode_table_update = Expander::Node.local_node.to_hash - vnode_table_update[:vnodes] = (0...16).to_a - vnode_table_update[:update] = :add - update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) - @vnode_supervisor.process_control_message(update_message) - end - - it "distributes the vnode when it receives a recover_vnode message and it is the leader" do - control_msg = {:action => :recover_vnode, :vnode_id => 23} - - @vnode_supervisor.local_node.should_receive(:shared_message) - @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) - end - - it "waits before re-advertising a vnode as available" do - pending("not yet implemented") - vnode_table_update = Expander::Node.local_node.to_hash - vnode_table_update[:vnodes] = (0...16).to_a - vnode_table_update[:update] = :add - update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) - @vnode_supervisor.process_control_message(update_message) - - control_msg = {:action => :recover_vnode, :vnode_id => 23} - - @vnode_supervisor.local_node.should_receive(:shared_message).once - @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) - @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) - end - end - - - it "doesn't distribute a vnode when it is not the leader" do - vnode_table_update = Expander::Node.local_node.to_hash - vnode_table_update[:vnodes] = (16...32).to_a - vnode_table_update[:update] = :add - update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) - @vnode_supervisor.process_control_message(update_message) - - vnode_table_update = Expander::Node.new("1c53daf0-34a1-4e4f-8069-332665453b44", 'fermi.local', 2342).to_hash - vnode_table_update[:vnodes] = (0...16).to_a - vnode_table_update[:update] = :add - update_message = Yajl::Encoder.encode({:action => :update_vnode_table, :data => vnode_table_update}) - @vnode_supervisor.process_control_message(update_message) - - control_msg = {:action => :recover_vnode, :vnode_id => 42} - - @vnode_supervisor.local_node.should_not_receive(:shared_message) - @vnode_supervisor.process_control_message(Yajl::Encoder.encode(control_msg)) - end - - end - -end diff --git a/chef-expander/spec/unit/vnode_table_spec.rb b/chef-expander/spec/unit/vnode_table_spec.rb deleted file mode 100644 index faa76b84c1..0000000000 --- a/chef-expander/spec/unit/vnode_table_spec.rb +++ /dev/null @@ -1,114 +0,0 @@ -# -# Author:: Daniel DeLeo (<dan@opscode.com>) -# Author:: Seth Falcon (<seth@opscode.com>) -# Author:: Chris Walters (<cw@opscode.com>) -# Copyright:: Copyright (c) 2010-2011 Opscode, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') - -require 'chef/expander/vnode_table' -require 'chef/expander/vnode_supervisor' - -describe Expander::VNodeTable do - before do - @vnode_supervisor = Expander::VNodeSupervisor.new - @vnode_table = Expander::VNodeTable.new(@vnode_supervisor) - - @log_stream = StringIO.new - @vnode_table.log.init(@log_stream) - end - - describe "when first created" do - it "has no nodes" do - @vnode_table.nodes.should be_empty - end - end - - describe "when one node's vnode info has been added" do - before do - @guid = "93226974-6d0b-4ca6-8d42-124dd55e0076" - @hostname_f = "fermi.localhost" - @pid = 12345 - @vnodes = (0..511).to_a - @update = {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid, :vnodes => @vnodes, :update => 'update'} - @vnode_table.update_table(@update) - end - - it "has one vnode" do - @vnode_table.should have(1).nodes - @vnode_table.nodes.first.should == Expander::Node.from_hash(@update) - end - - it "removes the node from the table when it exits the cluster" do - update = @update - update[:update] = 'remove' - @vnode_table.update_table(update) - @vnode_table.should have(0).nodes - end - - end - - describe "when several nodes are in the table" do - before do - @node_1 = Expander::Node.new("93226974-6d0b-4ca6-8d42-124dd55e0076", "fermi.local", 12345) - @node_1_hash = @node_1.to_hash - @node_1_hash[:vnodes] = (0..511).to_a - @node_1_hash[:update] = "update" - @node_2 = Expander::Node.new("ad265988-f650-4a31-a97b-5dbf4db8e1b0", "fermi.local", 23425) - @node_2_hash = @node_2.to_hash - @node_2_hash[:vnodes] = (512..767).to_a - @node_2_hash[:update] = "update" - @vnode_table.update_table(@node_1_hash) - @vnode_table.update_table(@node_2_hash) - end - - it "determines the node with the lowest numbered vnode is the leader node" do - @vnode_table.leader_node.should == @node_1 - end - - it "determines the local node is the leader when the local node matches the leader node" do - Expander::Node.stub!(:local_node).and_return(@node_1) - @vnode_table.local_node_is_leader?.should be_true - end - - it "determines the local node is not the leader when the local node doesn't match the leader node" do - Expander::Node.stub!(:local_node).and_return(@node_2) - @vnode_table.local_node_is_leader?.should be_false - end - end - - describe "when only one node has claimed any vnodes" do - before do - @node_1 = Expander::Node.new("93226974-6d0b-4ca6-8d42-124dd55e0076", "fermi.local", 12345) - @node_1_hash = @node_1.to_hash - @node_1_hash[:vnodes] = (0..511).to_a - @node_1_hash[:update] = "update" - @node_2 = Expander::Node.new("ad265988-f650-4a31-a97b-5dbf4db8e1b0", "fermi.local", 23425) - @node_2_hash = @node_2.to_hash - @node_2_hash[:vnodes] = [] - @node_2_hash[:update] = "update" - @vnode_table.update_table(@node_1_hash) - @vnode_table.update_table(@node_2_hash) - end - - it "still reliably determines who the leader is" do - pending - end - - end - -end |