diff options
33 files changed, 1979 insertions, 0 deletions
diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore b/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore new file mode 100644 index 0000000000..041413b040 --- /dev/null +++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore @@ -0,0 +1,2 @@ +nodes/ +tmp/ diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb new file mode 100644 index 0000000000..68fc3af2dd --- /dev/null +++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb @@ -0,0 +1,3 @@ +name 'acceptance-cookbook' +depends "kitchen_acceptance" +depends "data-collector-test" diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb new file mode 100644 index 0000000000..7f4be2c7f7 --- /dev/null +++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb @@ -0,0 +1,2 @@ +log "Running 'destroy' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'" +kitchen "destroy" diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb new file mode 100644 index 0000000000..c707e874f0 --- /dev/null +++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb @@ -0,0 +1,2 @@ +log "Running 'provision' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'" +kitchen "converge" diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb new file mode 100644 index 0000000000..e4a547272b --- /dev/null +++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb @@ -0,0 +1,2 @@ +log "Running 'verify' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'" +kitchen "verify" diff --git a/acceptance/data-collector/.acceptance/data-collector-test/.gitignore b/acceptance/data-collector/.acceptance/data-collector-test/.gitignore new file mode 100644 index 0000000000..ec2a890bd3 --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/.gitignore @@ -0,0 +1,16 @@ +.vagrant +Berksfile.lock +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +bin/* +.bundle/* + +.kitchen/ +.kitchen.local.yml diff --git a/acceptance/data-collector/.acceptance/data-collector-test/Berksfile b/acceptance/data-collector/.acceptance/data-collector-test/Berksfile new file mode 100644 index 0000000000..34fea2166b --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/Berksfile @@ -0,0 +1,3 @@ +source 'https://supermarket.chef.io' + +metadata diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb new file mode 100644 index 0000000000..3fb2c730b0 --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb @@ -0,0 +1,85 @@ +require "json" +require "sinatra" + +class Chef + class Node + # dummy class for JSON parsing + end +end + +module ApiHelpers + def self.payload_type(payload) + message_type = payload["message_type"] + status = payload["status"] + + message_type == "run_converge" ? "#{message_type}.#{status}" : message_type + end +end + +class Counter + def self.reset + @@counters = Hash.new { |h, k| h[k] = 0 } + end + + def self.increment(payload) + counter_name = ApiHelpers.payload_type(payload) + @@counters[counter_name] += 1 + end + + def self.to_json + @@counters.to_json + end +end + +class MessageCache + include ApiHelpers + + def self.reset + @@message_cache = {} + end + + def self.store(payload) + cache_key = ApiHelpers.payload_type(payload) + + @@message_cache[cache_key] = payload + end + + def self.fetch(cache_key) + @@message_cache[cache_key].to_json + end +end + +Counter.reset + +get "/" do + "Data Collector API server" +end + +get "/reset-counters" do + Counter.reset + "counters reset" +end + +get "/counters" do + Counter.to_json +end + +get "/cache/:key" do |cache_key| + MessageCache.fetch(cache_key) +end + +get "/reset-cache" do + MessageCache.reset + "cache reset" +end + +post "/data-collector/v0" do + body = request.body.read + payload = JSON.load(body) + + Counter.increment(payload) + MessageCache.store(payload) + + status 201 + "message received" +end diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile b/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile new file mode 100644 index 0000000000..94fc334d88 --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gem "sinatra" diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb new file mode 100644 index 0000000000..705bbdf603 --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb @@ -0,0 +1,4 @@ +chef_server_url "http://localhost:8889" +node_name "data-collector-test" +data_collector_server_url "http://localhost:9292/data-collector/v0" +data_collector_mode :both diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb new file mode 100644 index 0000000000..5c11f9169d --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb @@ -0,0 +1,4 @@ +chef_server_url "http://localhost:8889" +node_name "data-collector-test" +data_collector_server_url "http://localhost:9292/data-collector/v0" +data_collector_mode :client diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb new file mode 100644 index 0000000000..f8374107ea --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb @@ -0,0 +1,2 @@ +chef_server_url "http://localhost:8889" +node_name "data-collector-test" diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb new file mode 100644 index 0000000000..fa6db7de2c --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb @@ -0,0 +1,4 @@ +chef_server_url "http://localhost:8889" +node_name "data-collector-test" +data_collector_server_url "http://localhost:9292/data-collector/v0" +data_collector_mode :solo diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru b/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru new file mode 100644 index 0000000000..81cf29d9fb --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru @@ -0,0 +1,2 @@ +require_relative "./api" +run Sinatra::Application diff --git a/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb b/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb new file mode 100644 index 0000000000..dbd376aa83 --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb @@ -0,0 +1,7 @@ +name 'data-collector-test' +maintainer 'Adam Leff' +maintainer_email 'adamleff@chef.io' +license 'Apache 2.0' +description 'Installs/Configures data-collector-test' +long_description 'Installs/Configures data-collector-test' +version '0.1.0' diff --git a/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb b/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb new file mode 100644 index 0000000000..8c3fd063ce --- /dev/null +++ b/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb @@ -0,0 +1,38 @@ +api_root_dir = "/var/opt/data_collector_api" + +directory api_root_dir do + recursive true +end + +cookbook_file ::File.join(api_root_dir, "Gemfile") do + source "apigemfile" +end + +cookbook_file ::File.join(api_root_dir, "config.ru") + +cookbook_file ::File.join(api_root_dir, "api.rb") + +execute "bundle install --binstubs" do + cwd api_root_dir +end + +pid_file = "/var/run/api.pid" +running_pid = ::File.exist?(pid_file) ? ::File.read(pid_file).strip : nil + +execute "kill existing API process" do + command "kill #{running_pid}" + not_if { running_pid.nil? } +end + +execute "start API" do + command "bundle exec rackup -D -P #{pid_file}" + cwd api_root_dir +end + +directory "/etc/chef" + +["both-mode", "client-mode", "no-endpoint", "solo-mode"].each do |config_file| + cookbook_file "/etc/chef/#{config_file}.rb" do + source "client-rb-#{config_file}.rb" + end +end diff --git a/acceptance/data-collector/.kitchen.yml b/acceptance/data-collector/.kitchen.yml new file mode 100644 index 0000000000..f719e3ea69 --- /dev/null +++ b/acceptance/data-collector/.kitchen.yml @@ -0,0 +1,9 @@ +verifier: + name: busser + +suites: + - name: default + includes: + - ubuntu-14.04 + run_list: + - recipe[data-collector-test::default] diff --git a/acceptance/data-collector/Berksfile b/acceptance/data-collector/Berksfile new file mode 100644 index 0000000000..b8f003071b --- /dev/null +++ b/acceptance/data-collector/Berksfile @@ -0,0 +1,3 @@ +source "https://supermarket.chef.io" + +cookbook "data-collector-test", path: File.join(File.expand_path(File.dirname(__FILE__)), ".acceptance", "data-collector-test") diff --git a/acceptance/data-collector/Berksfile.lock b/acceptance/data-collector/Berksfile.lock new file mode 100644 index 0000000000..39f4ce30dc --- /dev/null +++ b/acceptance/data-collector/Berksfile.lock @@ -0,0 +1,6 @@ +DEPENDENCIES + data-collector-test + path: .acceptance/data-collector-test + +GRAPH + data-collector-test (0.1.0) diff --git a/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb b/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb new file mode 100644 index 0000000000..8bc68b45c6 --- /dev/null +++ b/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb @@ -0,0 +1,88 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'json' +require 'serverspec' + +set :backend, :exec + +shared_examples_for "reset counters" do + it "resets the counters" do + expect(command("curl http://localhost:9292/reset-counters").exit_status).to eq(0) + end +end + +shared_examples_for "reset cache" do + it "resets the message cache" do + expect(command("curl http://localhost:9292/reset-cache").exit_status).to eq(0) + end +end + +shared_examples_for "successful chef run" do |cmd| + include_examples "reset counters" + include_examples "reset cache" + + it "runs chef and expects a zero exit status" do + expect(command(cmd).exit_status).to eq(0) + end +end + +shared_examples_for "unsuccessful chef run" do |cmd| + include_examples "reset counters" + include_examples "reset cache" + + it "runs chef and expects a non-zero exit status" do + expect(command(cmd).exit_status).not_to eq(0) + end +end + +shared_examples_for "counter checks" do |counters_to_check| + counters_to_check.each do |counter, value| + it "counter #{counter} should equal #{value}" do + counter_values = JSON.load(command("curl http://localhost:9292/counters").stdout) + expect(counter_values[counter]).to eq(value) + end + end +end + + +describe "CCR with no data collector URL configured" do + include_examples "successful chef run", "chef-client -z -c /etc/chef/no-endpoint.rb" + include_examples "counter checks", {"run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil} +end + +describe "CCR, local mode, config in solo mode" do + include_examples "successful chef run", "chef-client -z -c /etc/chef/solo-mode.rb" + include_examples "counter checks", {"run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil} +end + +describe "CCR, local mode, config in client mode" do + include_examples "successful chef run", "chef-client -z -c /etc/chef/client-mode.rb" + include_examples "counter checks", {"run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil} +end + +describe "CCR, local mode, config in both mode" do + include_examples "successful chef run", "chef-client -z -c /etc/chef/both-mode.rb" + include_examples "counter checks", {"run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil} +end + +describe "CCR, local mode, config in solo mode, failed run" do + include_examples "unsuccessful chef run", "chef-client -z -c /etc/chef/solo-mode.rb -r 'recipe[cookbook-that-does-not-exist::default]'" + include_examples "counter checks", {"run_start" => 1, "run_converge.success" => nil, "run_converge.failure" => 1} +end diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb index 502852b648..da70b11b88 100644 --- a/chef-config/lib/chef-config/config.rb +++ b/chef-config/lib/chef-config/config.rb @@ -796,6 +796,15 @@ module ChefConfig config_context :chefdk do end + # Configuration options for Data Collector reporting. These settings allow + # the user to configure where to send their Data Collector data, what token + # to send, and whether Data Collector should report its findings in client + # mode vs. solo mode. + default :data_collector_server_url, nil + default :data_collector_token, nil + default :data_collector_mode, :both + default :data_collector_raise_on_failure, false + configurable(:http_proxy) configurable(:http_proxy_user) configurable(:http_proxy_pass) diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 054b284bd5..2df5b7271f 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -45,6 +45,7 @@ require "chef/formatters/doc" require "chef/formatters/minimal" require "chef/version" require "chef/resource_reporter" +require "chef/data_collector" require "chef/audit/audit_reporter" require "chef/run_lock" require "chef/policy_builder" @@ -263,6 +264,7 @@ class Chef run_ohai register unless Chef::Config[:solo_legacy_mode] + register_data_collector_reporter load_node @@ -415,6 +417,13 @@ class Chef end end + # Register the data collector reporter to send event information to the + # data collector server + # @api private + def register_data_collector_reporter + events.register(Chef::DataCollector::Reporter.new) if Chef::DataCollector.register_reporter? + end + # # Callback to fire notifications that the Chef run is starting # diff --git a/lib/chef/data_collector.rb b/lib/chef/data_collector.rb new file mode 100644 index 0000000000..a71dea60c0 --- /dev/null +++ b/lib/chef/data_collector.rb @@ -0,0 +1,277 @@ +# +# Author:: Adam Leff (<adamleff@chef.io>) +# Auther:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "uri" +require "chef/event_dispatch/base" +require "chef/data_collector/resource_report" +require "chef/data_collector/serializers/node_update" +require "chef/data_collector/serializers/run_end" +require "chef/data_collector/serializers/run_start" + +class Chef + class DataCollector + def self.register_reporter? + Chef::Config[:data_collector_server_url] && + !Chef::Config[:why_run] && + self.reporter_enabled_for_current_mode? + end + + def self.reporter_enabled_for_current_mode? + if Chef::Config[:solo] || Chef::Config[:local_mode] + acceptable_modes = [:solo, :both] + else + acceptable_modes = [:client, :both] + end + + acceptable_modes.include?(Chef::Config[:data_collector_mode]) + end + + class Reporter < EventDispatch::Base + attr_reader :updated_resources, :status, :exception, :error_descriptions, + :expanded_run_list, :run_status, :http, :resource_count, + :current_resource_report, :enabled + + def initialize + @updated_resources = [] + @resource_count = 0 + @current_resource_loaded = nil + @status = "success" + @exception = nil + @error_descriptions = {} + @expanded_run_list = {} + @http = Chef::HTTP.new(data_collector_server_url) + @enabled = true + end + + def run_started(current_run_status) + update_run_status(current_run_status) + + disable_reporter_on_error do + send_to_data_collector(Serializers::RunStart.new(run_status)) + end + end + + def run_completed(node) + send_run_completion + end + + def run_failed(exception) + update_exception(exception) + send_run_completion + end + + def resource_current_state_loaded(new_resource, action, current_resource) + return if nested_resource?(new_resource) + update_current_resource_report( + Chef::DataCollector::ResourceReport.new( + new_resource, + action, + current_resource + ) + ) + end + + def resource_up_to_date(new_resource, action) + current_resource_report.up_to_date unless nested_resource?(new_resource) + increment_resource_count + end + + def resource_skipped(new_resource, action, conditional) + increment_resource_count + return if nested_resource?(new_resource) + + update_current_resource_report( + Chef::DataCollector::ResourceReport.new( + new_resource, + action + ) + ) + current_resource_report.skipped(conditional) + end + + def resource_updated(new_resource, action) + current_resource_report.updated unless nested_resource?(new_resource) + increment_resource_count + end + + def resource_failed(new_resource, action, exception) + current_resource_report.failed(exception) unless nested_resource?(new_resource) + increment_resource_count + update_error_description( + Formatters::ErrorMapper.resource_failed( + new_resource, + action, + exception + ).for_json + ) + end + + def resource_completed(new_resource) + if current_resource_report && !nested_resource?(new_resource) + current_resource_report.finish + add_updated_resource(current_resource_report) + update_current_resource_report(nil) + end + end + + def run_list_expanded(run_list_expansion) + @expanded_run_list = run_list_expansion + end + + def run_list_expand_failed(node, exception) + update_error_description( + Formatters::ErrorMapper.run_list_expand_failed( + node, + exception + ).for_json + ) + end + + def cookbook_resolution_failed(expanded_run_list, exception) + update_error_description( + Formatters::ErrorMapper.cookbook_resolution_failed( + expanded_run_list, + exception + ).for_json + ) + end + + def cookbook_sync_failed(cookbooks, exception) + update_error_description( + Formatters::ErrorMapper.cookbook_sync_failed( + cookbooks, + exception + ).for_json + ) + end + + private + + def disable_reporter_on_error + yield + rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e + disable_data_collector_reporter + code = if e.respond_to?(:response) && e.response.code + e.response.code.to_s + else + "Exception Code Empty" + end + + msg = "Error while reporting run start to Data Collector. " \ + "URL: #{data_collector_server_url} " \ + "Exception: #{code} -- #{e.message} " + + if Chef::Config[:data_collector_raise_on_failure] + Chef::Log.error(msg) + raise + else + Chef::Log.warn(msg) + end + end + + def send_to_data_collector(message) + return unless data_collector_accessible? + + Chef::Log.debug("data_collector_reporter: POSTing the following message to #{data_collector_server_url}: #{message.to_json}") + http.post(nil, message.to_json, headers) + end + + def send_run_completion + # If run_status is nil we probably failed before the client triggered + # the run_started callback. In this case we'll skip updating because + # we have nothing to report. + return unless run_status + + send_to_data_collector(Serializers::NodeUpdate.new(run_status)) + send_to_data_collector( + Serializers::RunEnd.new( + run_status: run_status, + expanded_run_list: expanded_run_list, + total_resource_count: resource_count, + updated_resources: updated_resources, + status: status, + error_descriptions: error_descriptions + ) + ) + end + + def headers(additional_headers = {}) + headers = { "Content-Type" => "application/json" } + headers["x-data-collector-token"] = data_collector_token unless data_collector_token.nil? + + headers.merge(additional_headers) + end + + def data_collector_server_url + Chef::Config[:data_collector_server_url] + end + + def data_collector_token + Chef::Config[:data_collector_token] + end + + def increment_resource_count + @resource_count += 1 + end + + def add_updated_resource(resource_report) + @updated_resources << resource_report + end + + def disable_data_collector_reporter + # TODO: should the event dispatcher support de-registering? + @enabled = false + end + + def data_collector_accessible? + @enabled + end + + def update_status(status) + @status = status + end + + def update_run_status(run_status) + @run_status = run_status + end + + def update_exception(ex) + @exception = ex + update_status("failure") + end + + def update_current_resource_report(resource_report) + @current_resource_report = resource_report + end + + def update_error_description(discription_hash) + @error_descriptions = discription_hash + end + + # If we are getting messages about a resource while we are in the middle of + # another resource's update, we assume that the nested resource is just the + # implementation of a provider, and we want to hide it from the reporting + # output. + def nested_resource?(new_resource) + @current_resource_report && @current_resource_report.new_resource != new_resource + end + end + end +end diff --git a/lib/chef/data_collector/resource_report.rb b/lib/chef/data_collector/resource_report.rb new file mode 100644 index 0000000000..a9f9a31f4a --- /dev/null +++ b/lib/chef/data_collector/resource_report.rb @@ -0,0 +1,85 @@ +# +# Author:: Adam Leff (<adamleff@chef.io>) +# Auther:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Chef + class DataCollector + class ResourceReport + + attr_reader :action, :current_resource, :elapsed_time, :new_resource, :status + attr_accessor :conditional, :exception + + def initialize(new_resource, action, current_resource=nil) + @new_resource = new_resource + @action = action + @current_resource = current_resource + end + + def skipped(conditional) + @status = "skipped" + @conditional = conditional + end + + def updated + @status = "updated" + end + + def failed(exception) + @current_resource = nil + @status = "failed" + @exception = exception + end + + def up_to_date + @status = "up-to-date" + end + + def finish + @elapsed_time = new_resource.elapsed_time + end + + def to_hash + hash = { + "type" => new_resource.resource_name.to_sym, + "name" => new_resource.name.to_s, + "id" => new_resource.identity.to_s, + "after" => new_resource.state_for_resource_reporter, + "before" => current_resource ? current_resource.state_for_resource_reporter : {}, + "duration" => (elapsed_time * 1000).to_i.to_s, + "delta" => new_resource.respond_to?(:diff) ? new_resource.diff : "", + "result" => action.to_s, + "status" => status + } + + if new_resource.cookbook_name + hash["cookbook_name"] = new_resource.cookbook_name + hash["cookbook_version"] = new_resource.cookbook_version.version + end + + hash["conditional"] = conditional.to_text if status == "skipped" + hash["error_message"] = exception.message unless exception.nil? + + hash + end + alias :to_h :to_hash + alias :for_json :to_hash + end + end +end + diff --git a/lib/chef/data_collector/serializers/base.rb b/lib/chef/data_collector/serializers/base.rb new file mode 100644 index 0000000000..793ca34fb4 --- /dev/null +++ b/lib/chef/data_collector/serializers/base.rb @@ -0,0 +1,101 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# Author:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'json' +require 'securerandom' + +class Chef + class DataCollector + class Serializers + class Base + def document + raise "#{self.class} does not implement the #document method, which should return a hash of data to send" + end + + def message_type + raise "#{self.class} does not implement the #message_type method, which return a string containing the message type" + end + + def to_json + document.to_json + end + + def chef_server_fqdn + return URI(Chef::Config[:chef_server_url]).host unless Chef::Config[:chef_server_url].nil? + return "localhost" unless defined?(run_status) + + run_status.node["fqdn"] + end + + def organization + solo_run? ? data_collector_organization : chef_server_organization + end + + def data_collector_organization + Chef::Config[:data_collector_organization] || "chef_solo" + end + + def chef_server_organization + return nil unless Chef::Config[:chef_server_url] + + Chef::Config[:chef_server_url].match(%r(/organizations/(\w+))).nil? ? "unknown_organization" : $1 + end + + def collector_source + solo_run? ? "chef_solo" : "chef_client" + end + + def solo_run? + Chef::Config[:solo] || Chef::Config[:local_mode] + end + + def node_uuid + read_node_uuid || generate_node_uuid + end + + def generate_node_uuid + uuid = SecureRandom.uuid + update_metadata("node_uuid", uuid) + + uuid + end + + def read_node_uuid + metadata["node_uuid"] + end + + def metadata + @metadata ||= JSON.load(Chef::FileCache.load(metadata_filename)) + rescue Chef::Exceptions::FileNotFound + @metadata = {} + end + + def update_metadata(key, value) + metadata[key] = value + Chef::FileCache.store(metadata_filename, metadata.to_json, 0644) + end + + def metadata_filename + "data_collector_metadata.json" + end + end + end + end +end diff --git a/lib/chef/data_collector/serializers/node_update.rb b/lib/chef/data_collector/serializers/node_update.rb new file mode 100644 index 0000000000..b12e4d81da --- /dev/null +++ b/lib/chef/data_collector/serializers/node_update.rb @@ -0,0 +1,66 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# Author:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "securerandom" +require "chef/data_collector/serializers/base" + +class Chef + class DataCollector + class Serializers + class NodeUpdate < Base + + attr_reader :run_status + + def initialize(run_status) + @run_status = run_status + end + + def message_type + "action" + end + + def node + run_status.node + end + + def document + { + "entity_name" => node.name, + "entity_type" => "node", + "entity_uuid" => node_uuid, + "id" => SecureRandom.uuid, + "message_version" => "1.0.0", + "message_type" => message_type, + "organization_name" => organization, + "recorded_at" => Time.now.utc.iso8601, + "remote_hostname" => node["fqdn"], + "requestor_name" => node.name, + "requestor_type" => "client", + "service_hostname" => chef_server_fqdn, + "source" => collector_source, + "task" => "update", + "user_agent" => Chef::HTTP::HTTPRequest::DEFAULT_UA, + "data" => node + } + end + end + end + end +end diff --git a/lib/chef/data_collector/serializers/run_end.rb b/lib/chef/data_collector/serializers/run_end.rb new file mode 100644 index 0000000000..d7c2c08746 --- /dev/null +++ b/lib/chef/data_collector/serializers/run_end.rb @@ -0,0 +1,85 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# Author:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "securerandom" +require "chef/data_collector/serializers/base" + +class Chef + class DataCollector + class Serializers + class RunEnd < Base + attr_reader :error_descriptions + attr_reader :expanded_run_list + attr_reader :run_status + attr_reader :status + attr_reader :total_resource_count + attr_reader :updated_resources + + def initialize(opts) + @error_descriptions = opts[:error_descriptions] + @expanded_run_list = opts[:expanded_run_list] + @run_status = opts[:run_status] + @total_resource_count = opts[:total_resource_count] + @updated_resources = opts[:updated_resources] + @status = opts[:status] + end + + def message_type + "run_converge" + end + + def document + document = { + "chef_server_fqdn" => chef_server_fqdn, + "entity_uuid" => node_uuid, + "expanded_run_list" => expanded_run_list, + "id" => run_status.run_id, + "message_version" => "1.0.0", + "message_type" => message_type, + "node_name" => run_status.node.name, + "organization_name" => organization, + "resources" => updated_resources.map(&:for_json), + "run_id" => run_status.run_id, + "run_list" => run_status.node.run_list.for_json, + "start_time" => run_status.start_time.utc.iso8601, + "end_time" => run_status.end_time.utc.iso8601, + "source" => collector_source, + "status" => status, + "total_resource_count" => total_resource_count, + "updated_resource_count" => updated_resources.count + } + + document["error"] = formatted_exception if run_status.exception + + document + end + + def formatted_exception + { + "class" => run_status.exception.class, + "message" => run_status.exception.message, + "backtrace" => run_status.exception.backtrace, + "description" => error_descriptions + } + end + end + end + end +end diff --git a/lib/chef/data_collector/serializers/run_start.rb b/lib/chef/data_collector/serializers/run_start.rb new file mode 100644 index 0000000000..e128065226 --- /dev/null +++ b/lib/chef/data_collector/serializers/run_start.rb @@ -0,0 +1,55 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# Author:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "chef/data_collector/serializers/base" + +class Chef + class DataCollector + class Serializers + class RunStart < Base + + attr_reader :run_status + + def initialize(run_status) + @run_status = run_status + end + + def message_type + "run_start" + end + + def document + { + "chef_server_fqdn" => chef_server_fqdn, + "entity_uuid" => node_uuid, + "id" => run_status.run_id, + "message_version" => "1.0.0", + "message_type" => message_type, + "node_name" => run_status.node.name, + "organization_name" => organization, + "run_id" => run_status.run_id, + "source" => collector_source, + "start_time" => run_status.start_time.utc.iso8601 + } + end + end + end + end +end diff --git a/spec/unit/data_collector/serializers/base_spec.rb b/spec/unit/data_collector/serializers/base_spec.rb new file mode 100644 index 0000000000..5a86d7919f --- /dev/null +++ b/spec/unit/data_collector/serializers/base_spec.rb @@ -0,0 +1,210 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "spec_helper" +require "chef/data_collector/serializers/base" + +describe Chef::DataCollector::Serializers::Base do + let(:serializer) { described_class.new } + + describe '#document' do + it "should raise an exception" do + expect { serializer.document }.to raise_error(RuntimeError) + end + end + + describe '#message_type' do + it "should raise an exception" do + expect { serializer.message_type }.to raise_error(RuntimeError) + end + end + + describe '#to_json' do + let(:document) { double("document") } + + it "should convert the document to JSON" do + allow(serializer).to receive(:document).and_return(document) + expect(document).to receive(:to_json) + serializer.to_json + end + end + + describe '#organization' do + context "when the run is a solo run" do + it "returns the data collector organization" do + allow(serializer).to receive(:solo_run?).and_return(true) + expect(serializer).to receive(:data_collector_organization).and_return("org1") + expect(serializer.organization).to eq("org1") + end + end + + context "when the run is not a solo run" do + it "returns the data collector organization" do + allow(serializer).to receive(:solo_run?).and_return(false) + expect(serializer).to receive(:chef_server_organization).and_return("org2") + expect(serializer.organization).to eq("org2") + end + end + end + + describe '#data_collector_organization' do + context "when the org is specified in the config" do + it "returns the org from the config" do + Chef::Config[:data_collector_organization] = "org1" + expect(serializer.data_collector_organization).to eq("org1") + end + end + + context "when the org is not specified in the config" do + it "returns the default chef_solo org" do + expect(serializer.data_collector_organization).to eq("chef_solo") + end + end + end + + describe '#chef_server_organization' do + context "when the URL is properly formatted" do + it "returns the org from the parsed URL" do + Chef::Config[:chef_server_url] = "http://mycompany.com/organizations/myorg" + expect(serializer.chef_server_organization).to eq("myorg") + end + end + + context "when the URL is not properly formatted" do + it "returns unknown_organization" do + Chef::Config[:chef_server_url] = "http://mycompany.com/what/url/is/this" + expect(serializer.chef_server_organization).to eq("unknown_organization") + end + end + end + + describe '#collector_source' do + context "when the run is a solo run" do + it "returns chef_solo" do + allow(serializer).to receive(:solo_run?).and_return(true) + expect(serializer.collector_source).to eq("chef_solo") + end + end + + context "when the run is not a solo run" do + it "returns chef_client" do + allow(serializer).to receive(:solo_run?).and_return(false) + expect(serializer.collector_source).to eq("chef_client") + end + end + end + + describe '#solo_run?' do + context "when :solo is set in Chef::Config" do + it "returns true" do + Chef::Config[:solo] = true + Chef::Config[:local_mode] = nil + expect(serializer.solo_run?).to be_truthy + end + end + + context "when :local_mode is set in Chef::Config" do + it "returns true" do + Chef::Config[:solo] = nil + Chef::Config[:local_mode] = true + expect(serializer.solo_run?).to be_truthy + end + end + + context "when neither :solo or :local_mode is set in Chef::Config" do + it "returns false" do + Chef::Config[:solo] = nil + Chef::Config[:local_mode] = nil + expect(serializer.solo_run?).to be_falsey + end + end + end + + describe '#node_uuid' do + context "when the node UUID can be read" do + it "returns the read-in node UUID" do + allow(serializer).to receive(:read_node_uuid).and_return("read_uuid") + expect(serializer.node_uuid).to eq("read_uuid") + end + end + + context "when the node UUID cannot be read" do + it "generated a new node UUID" do + allow(serializer).to receive(:read_node_uuid).and_return(nil) + allow(serializer).to receive(:generate_node_uuid).and_return("generated_uuid") + expect(serializer.node_uuid).to eq("generated_uuid") + end + end + end + + describe '#generate_node_uuid' do + it "generates a new UUID, stores it, and returns it" do + expect(SecureRandom).to receive(:uuid).and_return("generated_uuid") + expect(serializer).to receive(:update_metadata).with("node_uuid", "generated_uuid") + expect(serializer.generate_node_uuid).to eq("generated_uuid") + end + end + + describe '#read_node_uuid' do + it "reads the node UUID from metadata" do + expect(serializer).to receive(:metadata).and_return({ "node_uuid" => "read_uuid" }) + expect(serializer.read_node_uuid).to eq("read_uuid") + end + end + + describe "metadata" do + let(:metadata_filename) { "fake_metadata_file.json" } + + before do + allow(serializer).to receive(:metadata_filename).and_return(metadata_filename) + end + + context "when the metadata file exists" do + it "returns the contents of the metadata file" do + expect(Chef::FileCache).to receive(:load).with(metadata_filename).and_return('{"foo":"bar"}') + expect(serializer.metadata["foo"]).to eq("bar") + end + end + + context "when the metadata file does not exist" do + it "returns an empty hash" do + expect(Chef::FileCache).to receive(:load).with(metadata_filename).and_raise(Chef::Exceptions::FileNotFound) + expect(serializer.metadata).to eq({}) + end + end + end + + describe '#update_metadata' do + let(:metadata) { double("metadata") } + + it "updates the file" do + allow(serializer).to receive(:metadata_filename).and_return("fake_metadata_file.json") + allow(serializer).to receive(:metadata).and_return(metadata) + expect(metadata).to receive(:[]=).with("new_key", "new_value") + expect(metadata).to receive(:to_json).and_return("metadata_json") + expect(Chef::FileCache).to receive(:store).with( + "fake_metadata_file.json", + "metadata_json", + 0644 + ) + + serializer.update_metadata("new_key", "new_value") + end + end +end diff --git a/spec/unit/data_collector/serializers/node_update_spec.rb b/spec/unit/data_collector/serializers/node_update_spec.rb new file mode 100644 index 0000000000..cbd3d73af0 --- /dev/null +++ b/spec/unit/data_collector/serializers/node_update_spec.rb @@ -0,0 +1,74 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "spec_helper" +require "chef/data_collector/serializers/base" +require "chef/data_collector/serializers/node_update" + +describe Chef::DataCollector::Serializers::NodeUpdate do + let(:node) { double("node") } + let(:run_status) { double("run_status") } + let(:serializer) { described_class.new(run_status) } + let(:required_fields) do + %w( + entity_name + entity_type + entity_uuid + id + message_type + message_version + organization_name + recorded_at + remote_hostname + requestor_name + requestor_type + service_hostname + source + task + user_agent + ) + end + let(:optional_fields) { %w(data) } + + before do + allow(serializer).to receive(:chef_server_fqdn).and_return("chef_server_fqdn") + allow(serializer).to receive(:node_uuid).and_return("node_uuid") + allow(serializer).to receive(:organization).and_return("organization") + allow(serializer).to receive(:collector_source).and_return("collector_source") + allow(serializer).to receive(:node).and_return(node) + allow(node).to receive(:name).and_return("node_name") + allow(node).to receive(:[]).and_return({ "fqdn" => "node_fqdn" }) + end + + describe '#message_type' do + it "has a message_type of run_start" do + expect(serializer.message_type).to eq("action") + end + + it "is not missing any required fields" do + missing_fields = required_fields.select { |key| !serializer.document.key?(key) } + expect(missing_fields).to eq([]) + end + + it "does not have any extra fields" do + extra_fields = serializer.document.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) } + expect(extra_fields).to eq([]) + end + end +end diff --git a/spec/unit/data_collector/serializers/run_end_spec.rb b/spec/unit/data_collector/serializers/run_end_spec.rb new file mode 100644 index 0000000000..272a210c1e --- /dev/null +++ b/spec/unit/data_collector/serializers/run_end_spec.rb @@ -0,0 +1,136 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "spec_helper" +require "chef/data_collector/serializers/base" +require "chef/data_collector/serializers/run_end" + +describe Chef::DataCollector::Serializers::RunEnd do + let(:node) { double("node") } + let(:run_list) { double("run_list")} + let(:run_status) { double("run_status") } + let(:serializer) { described_class.new({}) } + let(:resource) { double("resource")} + + before do + allow(serializer).to receive(:chef_server_fqdn).and_return("chef_server_fqdn") + allow(serializer).to receive(:node_uuid).and_return("node_uuid") + allow(serializer).to receive(:organization).and_return("organization") + allow(serializer).to receive(:collector_source).and_return("collector_source") + allow(serializer).to receive(:expanded_run_list).and_return("expanded_run_list") + allow(serializer).to receive(:updated_resources).and_return([resource]) + allow(serializer).to receive(:formatted_exception).and_return("formatted_exception") + allow(serializer).to receive(:status).and_return("status") + allow(serializer).to receive(:total_resource_count).and_return("total_resource_count") + allow(serializer).to receive(:run_status).and_return(run_status) + allow(node).to receive(:name).and_return("node_name") + allow(node).to receive(:run_list).and_return(run_list) + allow(resource).to receive(:for_json).and_return("resource_json") + allow(run_list).to receive(:for_json).and_return("run_list_json") + allow(run_status).to receive(:run_id).and_return("run_id") + allow(run_status).to receive(:node).and_return(node) + allow(run_status).to receive(:start_time).and_return(Time.now) + allow(run_status).to receive(:end_time).and_return(Time.now) + end + + describe '#message_type' do + it "has a message_type of run_converge" do + expect(serializer.message_type).to eq("run_converge") + end + + context "when the run was successful" do + let(:required_fields) do + %w( + chef_server_fqdn + entity_uuid + id + end_time + expanded_run_list + message_type + message_version + node_name + organization_name + resources + run_id + run_list + source + start_time + status + total_resource_count + updated_resource_count + ) + end + let(:optional_fields) { %w(error) } + + before do + allow(run_status).to receive(:exception).and_return(nil) + end + + it "is not missing any required fields" do + missing_fields = required_fields.select { |key| !serializer.document.key?(key) } + expect(missing_fields).to eq([]) + end + + it "does not have any extra fields" do + extra_fields = serializer.document.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) } + expect(extra_fields).to eq([]) + end + end + + context "when the run was not successful" do + let(:required_fields) do + %w( + chef_server_fqdn + entity_uuid + id + end_time + error + expanded_run_list + message_type + message_version + node_name + organization_name + resources + run_id + run_list + source + start_time + status + total_resource_count + updated_resource_count + ) + end + let(:optional_fields) { [] } + + before do + allow(run_status).to receive(:exception).and_return("exception") + end + + it "is not missing any required fields" do + missing_fields = required_fields.select { |key| !serializer.document.key?(key) } + expect(missing_fields).to eq([]) + end + + it "does not have any extra fields" do + extra_fields = serializer.document.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) } + expect(extra_fields).to eq([]) + end + end + end +end diff --git a/spec/unit/data_collector/serializers/run_start_spec.rb b/spec/unit/data_collector/serializers/run_start_spec.rb new file mode 100644 index 0000000000..ddcba3b2dd --- /dev/null +++ b/spec/unit/data_collector/serializers/run_start_spec.rb @@ -0,0 +1,69 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "spec_helper" +require "chef/data_collector/serializers/base" +require "chef/data_collector/serializers/run_start" + +describe Chef::DataCollector::Serializers::RunStart do + let(:node) { double("node", name: "node_name") } + let(:run_status) { double("run_status") } + let(:serializer) { described_class.new(run_status) } + let(:required_fields) do + %w( + chef_server_fqdn + entity_uuid + id + message_version + message_type + node_name + organization_name + run_id + source + start_time + ) + end + let(:optional_fields) { [] } + + before do + allow(serializer).to receive(:chef_server_fqdn).and_return("chef_server_fqdn") + allow(serializer).to receive(:node_uuid).and_return("node_uuid") + allow(serializer).to receive(:organization).and_return("organization") + allow(serializer).to receive(:collector_source).and_return("collector_source") + allow(run_status).to receive(:run_id).and_return("run_id") + allow(run_status).to receive(:node).and_return(node) + allow(run_status).to receive(:start_time).and_return(Time.now) + end + + describe '#message_type' do + it "has a message_type of run_start" do + expect(serializer.message_type).to eq("run_start") + end + + it "is not missing any required fields" do + missing_fields = required_fields.select { |key| !serializer.document.key?(key) } + expect(missing_fields).to eq([]) + end + + it "does not have any extra fields" do + extra_fields = serializer.document.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) } + expect(extra_fields).to eq([]) + end + end +end diff --git a/spec/unit/data_collector_spec.rb b/spec/unit/data_collector_spec.rb new file mode 100644 index 0000000000..e4f3feb503 --- /dev/null +++ b/spec/unit/data_collector_spec.rb @@ -0,0 +1,518 @@ +# +# Author:: Adam Leff (<adamleff@chef.io) +# Author:: Ryan Cragun (<ryan@chef.io>) +# +# Copyright:: Copyright 2012-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "spec_helper" +require "chef/data_collector" + +describe Chef::DataCollector do + describe ".register_reporter?" do + context "when no data collector URL is configured" do + it "returns false" do + Chef::Config[:data_collector_server_url] = nil + expect(Chef::DataCollector.register_reporter?).to be_falsey + end + end + + context "when a data collector URL is configured" do + before do + Chef::Config[:data_collector_server_url] = "http://data_collector" + end + + context "when operating in why_run mode" do + it "returns false" do + Chef::Config[:why_run] = true + expect(Chef::DataCollector.register_reporter?).to be_falsey + end + end + + context "when not operating in why_run mode" do + before do + Chef::Config[:why_run] = false + end + + context "when report is enabled for current mode" do + it "returns true" do + allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(true) + expect(Chef::DataCollector.register_reporter?).to be_truthy + end + end + + context "when report is disabled for current mode" do + it "returns false" do + allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(false) + expect(Chef::DataCollector.register_reporter?).to be_falsey + end + end + end + end + end + + describe ".reporter_enabled_for_current_mode?" do + context "when running in solo mode" do + before do + Chef::Config[:solo] = true + Chef::Config[:local_mode] = false + end + + context "when data_collector_mode is :solo" do + it "returns true" do + Chef::Config[:data_collector_mode] = :solo + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + + context "when data_collector_mode is :client" do + it "returns false" do + Chef::Config[:data_collector_mode] = :client + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false) + end + end + + context "when data_collector_mode is :both" do + it "returns true" do + Chef::Config[:data_collector_mode] = :both + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + end + + context "when running in local mode" do + before do + Chef::Config[:solo] = false + Chef::Config[:local_mode] = true + end + + context "when data_collector_mode is :solo" do + it "returns true" do + Chef::Config[:data_collector_mode] = :solo + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + + context "when data_collector_mode is :client" do + it "returns false" do + Chef::Config[:data_collector_mode] = :client + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false) + end + end + + context "when data_collector_mode is :both" do + it "returns true" do + Chef::Config[:data_collector_mode] = :both + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + end + + context "when running in client mode" do + before do + Chef::Config[:solo] = false + Chef::Config[:local_mode] = false + end + + context "when data_collector_mode is :solo" do + it "returns false" do + Chef::Config[:data_collector_mode] = :solo + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false) + end + end + + context "when data_collector_mode is :client" do + it "returns true" do + Chef::Config[:data_collector_mode] = :client + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + + context "when data_collector_mode is :both" do + it "returns true" do + Chef::Config[:data_collector_mode] = :both + expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true) + end + end + end + end +end + +describe Chef::DataCollector::Reporter do + let(:reporter) { described_class.new } + + describe '#run_started' do + before do + allow(reporter).to receive(:update_run_status) + allow(reporter).to receive(:send_to_data_collector) + end + + it "updates the run status" do + expect(reporter).to receive(:update_run_status).with("test_run_status") + reporter.run_started("test_run_status") + end + + it "sends the RunStart serializer output to the Data Collector server" do + expect(Chef::DataCollector::Serializers::RunStart).to receive(:new).and_return("run_start_data") + expect(reporter).to receive(:send_to_data_collector).with("run_start_data") + reporter.run_started("test_run_status") + end + end + + describe '#run_completed' do + it 'sends the run completion' do + expect(reporter).to receive(:send_run_completion) + reporter.run_completed("fake_node") + end + end + + describe '#run_failed' do + it "updates the exception and sends the run completion" do + expect(reporter).to receive(:update_exception).with("test_exception") + expect(reporter).to receive(:send_run_completion) + reporter.run_failed("test_exception") + end + end + + describe '#resource_current_state_loaded' do + let(:new_resource) { double("new_resource") } + let(:action) { double("action") } + let(:current_resource) { double("current_resource") } + + context "when resource is a nested resource" do + it "does not update the resource report" do + allow(reporter).to receive(:nested_resource?).and_return(true) + expect(reporter).not_to receive(:update_current_resource_report) + reporter.resource_current_state_loaded(new_resource, action, current_resource) + end + end + + context "when resource is not a nested resource" do + it "updates the resource report" do + allow(reporter).to receive(:nested_resource?).and_return(false) + expect(Chef::DataCollector::ResourceReport).to receive(:new).with( + new_resource, + action, + current_resource) + .and_return("resource_report") + expect(reporter).to receive(:update_current_resource_report).with("resource_report") + reporter.resource_current_state_loaded(new_resource, action, current_resource) + end + end + end + + describe '#resource_up_to_date' do + let(:new_resource) { double("new_resource") } + let(:action) { double("action") } + let(:resource_report) { double("resource_report")} + + before do + allow(reporter).to receive(:increment_resource_count) + allow(reporter).to receive(:nested_resource?) + allow(reporter).to receive(:current_resource_report).and_return(resource_report) + allow(resource_report).to receive(:up_to_date) + end + + it "increments the resource count" do + expect(reporter).to receive(:increment_resource_count) + reporter.resource_up_to_date(new_resource, action) + end + + context "when the resource is a nested resource" do + it "does not mark the resource report as up-to-date" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(true) + expect(resource_report).not_to receive(:up_to_date) + reporter.resource_up_to_date(new_resource, action) + end + end + + context "when the resource is not a nested resource" do + it "marks the resource report as up-to-date" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(false) + expect(resource_report).to receive(:up_to_date) + reporter.resource_up_to_date(new_resource, action) + end + end + end + + describe '#resource_skipped' do + let(:new_resource) { double("new_resource") } + let(:action) { double("action") } + let(:conditional) { double("conditional") } + let(:resource_report) { double("resource_report")} + + before do + allow(reporter).to receive(:increment_resource_count) + allow(reporter).to receive(:nested_resource?) + allow(reporter).to receive(:current_resource_report).and_return(resource_report) + allow(resource_report).to receive(:skipped) + end + + it "increments the resource count" do + expect(reporter).to receive(:increment_resource_count) + reporter.resource_skipped(new_resource, action, conditional) + end + + context "when the resource is a nested resource" do + it "does not mark the resource report as skipped" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(true) + expect(resource_report).not_to receive(:skipped).with(conditional) + reporter.resource_skipped(new_resource, action, conditional) + end + end + + context "when the resource is not a nested resource" do + it "updates the resource report" do + allow(reporter).to receive(:nested_resource?).and_return(false) + expect(Chef::DataCollector::ResourceReport).to receive(:new).with( + new_resource, + action) + .and_return("resource_report") + expect(reporter).to receive(:update_current_resource_report).with("resource_report") + reporter.resource_skipped(new_resource, action, conditional) + end + + it "marks the resource report as skipped" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(false) + expect(resource_report).to receive(:skipped).with(conditional) + reporter.resource_skipped(new_resource, action, conditional) + end + end + end + + describe '#resource_updated' do + let(:resource_report) { double("resource_report")} + + before do + allow(reporter).to receive(:current_resource_report).and_return(resource_report) + allow(resource_report).to receive(:updated) + end + + it "increments the resource count" do + expect(reporter).to receive(:increment_resource_count) + reporter.resource_updated("new_resource", "action") + end + + it "marks the resource report as updated" do + expect(resource_report).to receive(:updated) + reporter.resource_updated("new_resource", "action") + end + end + + describe '#resource_failed' do + let(:new_resource) { double("new_resource") } + let(:action) { double("action") } + let(:exception) { double("exception") } + let(:error_mapper) { double("error_mapper")} + let(:resource_report) { double("resource_report")} + + + before do + allow(reporter).to receive(:increment_resource_count) + allow(reporter).to receive(:update_error_description) + allow(reporter).to receive(:current_resource_report).and_return(resource_report) + allow(resource_report).to receive(:failed) + allow(Chef::Formatters::ErrorMapper).to receive(:resource_failed).and_return(error_mapper) + allow(error_mapper).to receive(:for_json) + end + + it "increments the resource count" do + expect(reporter).to receive(:increment_resource_count) + reporter.resource_failed(new_resource, action, exception) + end + + it "updates the error description" do + expect(Chef::Formatters::ErrorMapper).to receive(:resource_failed).with( + new_resource, + action, + exception + ).and_return(error_mapper) + expect(error_mapper).to receive(:for_json).and_return("error_description") + expect(reporter).to receive(:update_error_description).with("error_description") + reporter.resource_failed(new_resource, action, exception) + end + + context "when the resource is not a nested resource" do + it "marks the resource report as failed" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(false) + expect(resource_report).to receive(:failed).with(exception) + reporter.resource_failed(new_resource, action, exception) + end + end + + context "when the resource is a nested resource" do + it "does not mark the resource report as failed" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(true) + expect(resource_report).not_to receive(:failed).with(exception) + reporter.resource_failed(new_resource, action, exception) + end + end + end + + describe '#resource_completed' do + let(:new_resource) { double("new_resource") } + let(:resource_report) { double("resource_report") } + + before do + allow(reporter).to receive(:add_updated_resource) + allow(reporter).to receive(:update_current_resource_report) + allow(resource_report).to receive(:finish) + end + + context "when there is no current resource report" do + it "does not add the updated resource" do + allow(reporter).to receive(:current_resource_report).and_return(nil) + expect(reporter).not_to receive(:add_updated_resource) + reporter.resource_completed(new_resource) + end + end + + context "when there is a current resource report" do + before do + allow(reporter).to receive(:current_resource_report).and_return(resource_report) + end + + context "when the resource is a nested resource" do + it "does not add the updated resource" do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(true) + expect(reporter).not_to receive(:add_updated_resource) + reporter.resource_completed(new_resource) + end + end + + context "when the resource is not a nested resource" do + before do + allow(reporter).to receive(:nested_resource?).with(new_resource).and_return(false) + end + + it "marks the current resource report as finished" do + expect(resource_report).to receive(:finish) + reporter.resource_completed(new_resource) + end + + it "adds the resource to the updated resource list" do + expect(reporter).to receive(:add_updated_resource).with(resource_report) + reporter.resource_completed(new_resource) + end + + it "nils out the current resource report" do + expect(reporter).to receive(:update_current_resource_report).with(nil) + reporter.resource_completed(new_resource) + end + end + end + end + + describe '#run_list_expanded' do + it "sets the expanded run list" do + reporter.run_list_expanded("test_run_list") + expect(reporter.expanded_run_list).to eq("test_run_list") + end + end + + describe '#run_list_expand_failed' do + let(:node) { double("node") } + let(:error_mapper) { double("error_mapper") } + let(:exception) { double("exception") } + + it "updates the error description" do + expect(Chef::Formatters::ErrorMapper).to receive(:run_list_expand_failed).with( + node, + exception + ).and_return(error_mapper) + expect(error_mapper).to receive(:for_json).and_return("error_description") + expect(reporter).to receive(:update_error_description).with("error_description") + reporter.run_list_expand_failed(node, exception) + end + end + + describe '#cookbook_resolution_failed' do + let(:error_mapper) { double("error_mapper") } + let(:exception) { double("exception") } + let(:expanded_run_list) { double("expanded_run_list") } + + it "updates the error description" do + expect(Chef::Formatters::ErrorMapper).to receive(:cookbook_resolution_failed).with( + expanded_run_list, + exception + ).and_return(error_mapper) + expect(error_mapper).to receive(:for_json).and_return("error_description") + expect(reporter).to receive(:update_error_description).with("error_description") + reporter.cookbook_resolution_failed(expanded_run_list, exception) + end + + end + + describe '#cookbook_sync_failed' do + let(:cookbooks) { double("cookbooks") } + let(:error_mapper) { double("error_mapper") } + let(:exception) { double("exception") } + + it "updates the error description" do + expect(Chef::Formatters::ErrorMapper).to receive(:cookbook_sync_failed).with( + cookbooks, + exception + ).and_return(error_mapper) + expect(error_mapper).to receive(:for_json).and_return("error_description") + expect(reporter).to receive(:update_error_description).with("error_description") + reporter.cookbook_sync_failed(cookbooks, exception) + end + end + + describe '#disable_reporter_on_error' do + context "when no exception is raise by the block" do + it "does not disable the reporter" do + expect(reporter).not_to receive(:disable_data_collector_reporter) + reporter.send(:disable_reporter_on_error) { true } + end + + it "does not raise an exception" do + expect { reporter.send(:disable_reporter_on_error) { true } }.not_to raise_error + end + end + + context "when an unexpected exception is raised by the block" do + it "re-raises the exception" do + expect { reporter.send(:disable_reporter_on_error) { raise RuntimeError, "bummer" } }.to raise_error(RuntimeError) + end + end + + [ Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError ].each do |exception_class| + context "when the block raises a #{exception_class} exception" do + it "disables the reporter" do + expect(reporter).to receive(:disable_data_collector_reporter) + reporter.send(:disable_reporter_on_error) { raise exception_class.new("bummer") } + end + + context "when raise-on-failure is enabled" do + it "logs an error and raises" do + Chef::Config[:data_collector_raise_on_failure] = true + expect(Chef::Log).to receive(:error) + expect { reporter.send(:disable_reporter_on_error) { raise exception_class.new("bummer") } }.to raise_error(exception_class) + end + end + + context "when raise-on-failure is disabled" do + it "logs a warning and does not raise an exception" do + Chef::Config[:data_collector_raise_on_failure] = false + expect(Chef::Log).to receive(:warn) + expect { reporter.send(:disable_reporter_on_error) { raise exception_class.new("bummer") } }.not_to raise_error + end + end + end + end + end +end |