From dfa54f1756bc740a12759f742d426288b62f8304 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 29 Oct 2014 09:24:05 -0700 Subject: First pass at DSL additions --- chef.gemspec | 2 +- lib/chef/dsl/audit.rb | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/chef/recipe.rb | 2 ++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 lib/chef/dsl/audit.rb diff --git a/chef.gemspec b/chef.gemspec index f623f8bb82..0461872668 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -43,7 +43,7 @@ Gem::Specification.new do |s| # rspec_junit_formatter 0.2.0 drops ruby 1.8.7 support s.add_development_dependency "rspec_junit_formatter", "~> 0.2.0" - %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem, "~> 3.0" } + %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem, "~> 3.1" } s.bindir = "bin" s.executables = %w( chef-client chef-solo knife chef-shell chef-apply ) diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb new file mode 100644 index 0000000000..a90f7f66e0 --- /dev/null +++ b/lib/chef/dsl/audit.rb @@ -0,0 +1,61 @@ +# +# Author:: Tyler Ball () +# Copyright:: Copyright (c) 2014 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'rspec/core' + +class Chef + module DSL + module Audit + + # List of `controls` example groups to be executed + @example_groups = nil + + # Adds the control_group and block (containing controls to execute) to the runner's list of pending examples + def control_group(group_name, &group_block) + puts "entered group named #{group_name}" + @example_groups = [] + + if group_block + yield + end + + # TODO add the @example_groups list to the runner for later execution + p @example_groups + + # Reset this to nil so we can tell if a `controls` message is sent outside a `control_group` block + # Prevents defining then un-defining the `controls` singleton method + @example_groups = nil + end + + def controls(*args, &control_block) + if @example_groups.nil? + raise "Cannot define a `controls` unless inside a `control_group`" + end + + example_name = args[0] + puts "entered control block named #{example_name}" + # TODO is this the correct way to define one? + # https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/example_group.rb#L197 + # https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/example_group.rb#L323 + @example_groups << ::RSpec::Core::ExampleGroup.describe(args, &control_block) + end + + end + end +end + + diff --git a/lib/chef/recipe.rb b/lib/chef/recipe.rb index e54a1d98e3..621d93099b 100644 --- a/lib/chef/recipe.rb +++ b/lib/chef/recipe.rb @@ -24,6 +24,7 @@ require 'chef/dsl/platform_introspection' require 'chef/dsl/include_recipe' require 'chef/dsl/registry_helper' require 'chef/dsl/reboot_pending' +require 'chef/dsl/audit' require 'chef/mixin/from_file' @@ -40,6 +41,7 @@ class Chef include Chef::DSL::Recipe include Chef::DSL::RegistryHelper include Chef::DSL::RebootPending + include Chef::DSL::Audit include Chef::Mixin::FromFile include Chef::Mixin::Deprecation -- cgit v1.2.1 From 7a49ae038a148c137d72cb5a60a3581b4db264ab Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Wed, 29 Oct 2014 10:50:29 -0700 Subject: Add setup phase to audit-mode. --- lib/chef/audit.rb | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/chef/dsl/audit.rb | 15 ++++++------ 2 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 lib/chef/audit.rb diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb new file mode 100644 index 0000000000..6fd067448b --- /dev/null +++ b/lib/chef/audit.rb @@ -0,0 +1,64 @@ +# +# Author:: Claire McQuin () +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'rspec/core' + +require 'chef/config' + +class Chef + class Audit + + def initialize + @configuration = RSpec::Core::Configuration.new + @world = RSpec::Core::World.new(@configuration) + @runner = RSpec::Core::Runner.new(nil, @configuration, @world) + end + + def setup + @configuration.output_stream = Chef::Config[:log_location] + @configuration.error_stream = Chef::Config[:log_location] + + configure_formatters + configure_expectation_frameworks + end + + private + # Adds formatters to RSpec. + # By default, two formatters are added: one for outputting readable text + # of audits run and one for sending JSON data back to reporting. + def configure_formatters + # TODO (future): We should allow for an audit-mode formatter config option + # and use this formatter as default/fallback if none is specified. + @configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) + # TODO: Add JSON formatter for audit reporting to analytics. + end + + def configure_expectation_frameworks + @configuration.expect_with(:rspec) do |config| + # :should is deprecated in RSpec 3+ and we have chosen to explicitly disable + # it in audits. If :should is used in an audit, this will cause the audit to + # fail with message "undefined method `should`" rather than print a deprecation + # message. + config.syntax = :expect + end + + #TODO: serverspec? + end + + end +end diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index a90f7f66e0..24c1bb0464 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -21,11 +21,11 @@ class Chef module DSL module Audit - # List of `controls` example groups to be executed + # List of `control` example groups to be executed @example_groups = nil - # Adds the control_group and block (containing controls to execute) to the runner's list of pending examples - def control_group(group_name, &group_block) + # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples + def controls(group_name, &group_block) puts "entered group named #{group_name}" @example_groups = [] @@ -36,14 +36,15 @@ class Chef # TODO add the @example_groups list to the runner for later execution p @example_groups - # Reset this to nil so we can tell if a `controls` message is sent outside a `control_group` block - # Prevents defining then un-defining the `controls` singleton method + # Reset this to nil so we can tell if a `control` message is sent outside a `controls` block + # Prevents defining then un-defining the `control` singleton method + # TODO this does not prevent calling `control` inside `control` @example_groups = nil end - def controls(*args, &control_block) + def control(*args, &control_block) if @example_groups.nil? - raise "Cannot define a `controls` unless inside a `control_group`" + raise "Cannot define a `control` unless inside a `controls` block" end example_name = args[0] -- cgit v1.2.1 From 4cfb1e47aa8e9501f4f2a01f1d8cc0deb2cfa13b Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 30 Oct 2014 17:34:37 -0700 Subject: Creating our own example group class to simplify adding examples to the spec runner --- lib/chef/audit/chef_example_group.rb | 10 +++++++ lib/chef/client.rb | 56 +++++++++++++++++++++++++++--------- lib/chef/dsl/audit.rb | 34 ++++------------------ lib/chef/event_dispatch/base.rb | 16 +++++++++++ lib/chef/exceptions.rb | 2 ++ 5 files changed, 75 insertions(+), 43 deletions(-) create mode 100644 lib/chef/audit/chef_example_group.rb diff --git a/lib/chef/audit/chef_example_group.rb b/lib/chef/audit/chef_example_group.rb new file mode 100644 index 0000000000..cd874d57b7 --- /dev/null +++ b/lib/chef/audit/chef_example_group.rb @@ -0,0 +1,10 @@ +require 'rspec/core' + +class Chef + class Audit + class ChefExampleGroup < ::RSpec::Core::ExampleGroup + # Can encompass tests in a `control` block or `describe` block + define_example_group_method :control + end + end +end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 4f37bd0ee3..7fde1eb12e 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -295,6 +295,7 @@ class Chef end # We now have the client key, and should use it from now on. @rest = Chef::REST.new(config[:chef_server_url], client_name, config[:client_key]) + # TODO register this where we register all other event listeners @resource_reporter = Chef::ResourceReporter.new(@rest) @events.register(@resource_reporter) rescue Exception => e @@ -307,18 +308,35 @@ class Chef # Converges the node. # # === Returns - # true:: Always returns true + # The thrown exception, if there was one. If this returns nil the converge was successful. def converge(run_context) - @events.converge_start(run_context) - Chef::Log.debug("Converging node #{node_name}") - @runner = Chef::Runner.new(run_context) - runner.converge - @events.converge_complete - true - rescue Exception - # TODO: should this be a separate #converge_failed(exception) method? - @events.converge_complete - raise + converge_exception = nil + catch(:end_client_run_early) do + begin + @events.converge_start(run_context) + Chef::Log.debug("Converging node #{node_name}") + @runner = Chef::Runner.new(run_context) + runner.converge + @events.converge_complete + rescue Exception => e + @events.converge_failed(e) + converge_exception = e + end + end + converge_exception + end + + def run_audits(run_context) + audit_exception = nil + begin + @events.audit_start(run_context) + # TODO + @events.audit_complete + rescue Exception => e + @events.audit_failed(e) + audit_exception = e + end + audit_exception end # Expands the run list. Delegates to the policy_builder. @@ -396,11 +414,17 @@ class Chef run_context = setup_run_context - catch(:end_client_run_early) do - converge(run_context) + converge_exception = converge(run_context) + if converge_exception + + else + save_updated_node end - save_updated_node + audit_exception = run_audits(run_context) + if audit_exception + + end run_status.stop_clock Chef::Log.info("Chef Run complete in #{run_status.elapsed_time} seconds") @@ -411,6 +435,10 @@ class Chef Chef::Platform::Rebooter.reboot_if_needed!(node) true + + # TODO get rid of resuce here, push down into sub-methods, clean up this method, return exceptions and raise new + # exception wrapping all known exceptions. sub-method should do all their own event reporting. + rescue Exception => e # CHEF-3336: Send the error first in case something goes wrong below and we don't know why Chef::Log.debug("Re-raising exception: #{e.class} - #{e.message}\n#{e.backtrace.join("\n ")}") diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 24c1bb0464..fee89c02ca 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -15,44 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -require 'rspec/core' +#require 'chef/audit' +require 'chef/audit/chef_example_group' class Chef module DSL module Audit - # List of `control` example groups to be executed - @example_groups = nil - # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples def controls(group_name, &group_block) - puts "entered group named #{group_name}" - @example_groups = [] - - if group_block - yield - end + raise ::Chef::Exceptions::NoAuditsProvided("You must provide a block with audits") unless group_block # TODO add the @example_groups list to the runner for later execution - p @example_groups - - # Reset this to nil so we can tell if a `control` message is sent outside a `controls` block - # Prevents defining then un-defining the `control` singleton method - # TODO this does not prevent calling `control` inside `control` - @example_groups = nil - end - - def control(*args, &control_block) - if @example_groups.nil? - raise "Cannot define a `control` unless inside a `controls` block" - end - - example_name = args[0] - puts "entered control block named #{example_name}" - # TODO is this the correct way to define one? - # https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/example_group.rb#L197 - # https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/example_group.rb#L323 - @example_groups << ::RSpec::Core::ExampleGroup.describe(args, &control_block) + # run_context.audit_runner.register + ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block) end end diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb index 50d261cecd..555998ced4 100644 --- a/lib/chef/event_dispatch/base.rb +++ b/lib/chef/event_dispatch/base.rb @@ -225,6 +225,22 @@ class Chef def converge_complete end + # Called if the converge phase fails + def converge_failed(exception) + end + + # Called before audit phase starts + def audit_start(run_context) + end + + # Called when the audit phase is finished. + def audit_complete + end + + # Called if the audit phase fails + def audit_failed(exception) + end + # TODO: need events for notification resolve? # def notifications_resolved # end diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index f5d91c24a6..55d6fcc893 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -384,5 +384,7 @@ class Chef super "Found more than one provider for #{resource.resource_name} resource: #{classes}" end end + + class NoAuditsProvided < RuntimeError; end end end -- cgit v1.2.1 From 66665a7f699a592b77249a31af229f1412d90458 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 30 Oct 2014 20:44:36 -0700 Subject: Adding logic for exceptions from converge phase not interfering with audit phase and vice-versa --- lib/chef/client.rb | 25 ++++++++++--------------- lib/chef/exceptions.rb | 22 ++++++++++++++++++++++ lib/chef/formatters/doc.rb | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 7fde1eb12e..b41db70689 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -246,7 +246,6 @@ class Chef @policy_builder ||= Chef::PolicyBuilder.strategy.new(node_name, ohai.data, json_attribs, @override_runlist, events) end - def save_updated_node if Chef::Config[:solo] # nothing to do @@ -260,6 +259,7 @@ class Chef def run_ohai ohai.all_plugins + @events.ohai_completed(node) end def node_name @@ -326,6 +326,7 @@ class Chef converge_exception end + # TODO are failed audits going to raise exceptions, or only be handled by the reporters? def run_audits(run_context) audit_exception = nil begin @@ -351,7 +352,6 @@ class Chef policy_builder.expand_run_list end - def do_windows_admin_check if Chef::Platform.windows? Chef::Log.debug("Checking for administrator privileges....") @@ -398,7 +398,7 @@ class Chef Chef::Log.debug("Chef-client request_id: #{request_id}") enforce_path_sanity run_ohai - @events.ohai_completed(node) + register unless Chef::Config[:solo] load_node @@ -414,16 +414,14 @@ class Chef run_context = setup_run_context - converge_exception = converge(run_context) - if converge_exception - - else - save_updated_node - end - - audit_exception = run_audits(run_context) - if audit_exception + converge_error = converge(run_context) + save_updated_node unless converge_error + audit_error = run_audits(run_context) + if converge_error || audit_error + e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) + e.fill_backtrace + raise e end run_status.stop_clock @@ -436,9 +434,6 @@ class Chef true - # TODO get rid of resuce here, push down into sub-methods, clean up this method, return exceptions and raise new - # exception wrapping all known exceptions. sub-method should do all their own event reporting. - rescue Exception => e # CHEF-3336: Send the error first in case something goes wrong below and we don't know why Chef::Log.debug("Re-raising exception: #{e.class} - #{e.message}\n#{e.backtrace.join("\n ")}") diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index 55d6fcc893..179a1fa27e 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -386,5 +386,27 @@ class Chef end class NoAuditsProvided < RuntimeError; end + + # If a converge or audit fails, we want to wrap the output from those errors into 1 error so we can + # see both issues in the output. It is possible that nil will be provided. You must call `fill_backtrace` + # to correctly populate the backtrace with the wrapped backtraces. + class RunFailedWrappingError < RuntimeError + attr_reader :wrapped_errors + def initialize(*errors) + errors = errors.select {|e| !e.nil?} + output = "Found #{errors.size} errors, they are stored in the backtrace\n" + @wrapped_errors = errors + super output + end + + def fill_backtrace + backtrace = [] + wrapped_errors.each_with_index do |e,i| + backtrace << "#{i+1}) #{e.message}" + backtrace += e.backtrace + backtrace << "" + end + end + end end end diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 4a08b9d095..66ee6ccabe 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -151,6 +151,24 @@ class Chef unindent if @current_recipe end + def converge_failed(e) + # TODO do we want to do anything else in here? + converge_complete + end + + def audit_start(run_context) + # TODO read the number of `controls` blocks to run from the run_context.audit_runner + puts_line "Running collected audits" + end + + def audit_complete + # TODO + end + + def audit_failed(exception) + # TODO + end + # Called before action is executed on a resource. def resource_action_start(resource, action, notification_type=nil, notifier=nil) if resource.cookbook_name && resource.recipe_name -- cgit v1.2.1 From 390b858f2cfb130817573813294cb77b84f71874 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Fri, 31 Oct 2014 07:05:21 -0700 Subject: Adding error handling so saving node doesn't prevent us from running audit mode - decouples converge phase and audit phase more --- lib/chef/client.rb | 16 +++++++++++++-- lib/chef/dsl/audit.rb | 2 +- lib/chef/exceptions.rb | 11 ++++++++--- spec/unit/exceptions_spec.rb | 46 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/lib/chef/client.rb b/lib/chef/client.rb index b41db70689..6c83639ec3 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -326,6 +326,19 @@ class Chef converge_exception end + # TODO don't want to change old API + def converge_and_save(run_context) + converge_exception = converge(run_context) + unless converge_exception + begin + save_updated_node + rescue Exception => e + converge_exception = e + end + end + converge_exception + end + # TODO are failed audits going to raise exceptions, or only be handled by the reporters? def run_audits(run_context) audit_exception = nil @@ -414,8 +427,7 @@ class Chef run_context = setup_run_context - converge_error = converge(run_context) - save_updated_node unless converge_error + converge_error = converge_and_save(run_context) audit_error = run_audits(run_context) if converge_error || audit_error diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index fee89c02ca..7adcecbf14 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -24,7 +24,7 @@ class Chef # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples def controls(group_name, &group_block) - raise ::Chef::Exceptions::NoAuditsProvided("You must provide a block with audits") unless group_block + raise ::Chef::Exceptions::NoAuditsProvided unless group_block # TODO add the @example_groups list to the runner for later execution # run_context.audit_runner.register diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index 179a1fa27e..842910f2ae 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -385,7 +385,11 @@ class Chef end end - class NoAuditsProvided < RuntimeError; end + class NoAuditsProvided < RuntimeError + def initialize + super "You must provide a block with audits" + end + end # If a converge or audit fails, we want to wrap the output from those errors into 1 error so we can # see both issues in the output. It is possible that nil will be provided. You must call `fill_backtrace` @@ -402,10 +406,11 @@ class Chef def fill_backtrace backtrace = [] wrapped_errors.each_with_index do |e,i| - backtrace << "#{i+1}) #{e.message}" - backtrace += e.backtrace + backtrace << "#{i+1}) #{e.class} - #{e.message}" + backtrace += e.backtrace if e.backtrace backtrace << "" end + set_backtrace(backtrace) end end end diff --git a/spec/unit/exceptions_spec.rb b/spec/unit/exceptions_spec.rb index 6318ec9227..165c11446b 100644 --- a/spec/unit/exceptions_spec.rb +++ b/spec/unit/exceptions_spec.rb @@ -81,4 +81,50 @@ describe Chef::Exceptions do end end end + + describe Chef::Exceptions::RunFailedWrappingError do + shared_examples "RunFailedWrappingError expectations" do + it "should initialize with a default message" do + expect(e.message).to eq("Found #{num_errors} errors, they are stored in the backtrace\n") + end + + it "should provide a modified backtrace when requested" do + e.fill_backtrace + expect(e.backtrace).to eq(backtrace) + end + end + + context "initialized with nothing" do + let(:e) { Chef::Exceptions::RunFailedWrappingError.new } + let(:num_errors) { 0 } + let(:backtrace) { [] } + + include_examples "RunFailedWrappingError expectations" + end + + context "initialized with nil" do + let(:e) { Chef::Exceptions::RunFailedWrappingError.new(nil, nil) } + let(:num_errors) { 0 } + let(:backtrace) { [] } + + include_examples "RunFailedWrappingError expectations" + end + + context "initialized with 1 error and nil" do + let(:e) { Chef::Exceptions::RunFailedWrappingError.new(RuntimeError.new("foo"), nil) } + let(:num_errors) { 1 } + let(:backtrace) { ["1) RuntimeError - foo", ""] } + + include_examples "RunFailedWrappingError expectations" + end + + context "initialized with 2 errors" do + let(:e) { Chef::Exceptions::RunFailedWrappingError.new(RuntimeError.new("foo"), RuntimeError.new("bar")) } + let(:num_errors) { 2 } + let(:backtrace) { ["1) RuntimeError - foo", "", "2) RuntimeError - bar", ""] } + + include_examples "RunFailedWrappingError expectations" + end + + end end -- cgit v1.2.1 From 549b0b4d16ebb07798d9c89e3788fe26de3a4ebf Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Fri, 31 Oct 2014 15:05:10 -0700 Subject: Adding audit mode JSON formatter --- lib/chef/audit.rb | 47 ++---------------- lib/chef/audit/chef_json_formatter.rb | 88 +++++++++++++++++++++++++++++++++ lib/chef/audit/control_group_data.rb | 84 +++++++++++++++++++++++++++++++ lib/chef/audit/runner.rb | 93 +++++++++++++++++++++++++++++++++++ lib/chef/client.rb | 4 +- lib/chef/dsl/audit.rb | 5 +- lib/chef/run_context.rb | 6 +++ 7 files changed, 278 insertions(+), 49 deletions(-) create mode 100644 lib/chef/audit/chef_json_formatter.rb create mode 100644 lib/chef/audit/control_group_data.rb create mode 100644 lib/chef/audit/runner.rb diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb index 6fd067448b..43d5c11ee8 100644 --- a/lib/chef/audit.rb +++ b/lib/chef/audit.rb @@ -18,47 +18,6 @@ require 'rspec/core' -require 'chef/config' - -class Chef - class Audit - - def initialize - @configuration = RSpec::Core::Configuration.new - @world = RSpec::Core::World.new(@configuration) - @runner = RSpec::Core::Runner.new(nil, @configuration, @world) - end - - def setup - @configuration.output_stream = Chef::Config[:log_location] - @configuration.error_stream = Chef::Config[:log_location] - - configure_formatters - configure_expectation_frameworks - end - - private - # Adds formatters to RSpec. - # By default, two formatters are added: one for outputting readable text - # of audits run and one for sending JSON data back to reporting. - def configure_formatters - # TODO (future): We should allow for an audit-mode formatter config option - # and use this formatter as default/fallback if none is specified. - @configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) - # TODO: Add JSON formatter for audit reporting to analytics. - end - - def configure_expectation_frameworks - @configuration.expect_with(:rspec) do |config| - # :should is deprecated in RSpec 3+ and we have chosen to explicitly disable - # it in audits. If :should is used in an audit, this will cause the audit to - # fail with message "undefined method `should`" rather than print a deprecation - # message. - config.syntax = :expect - end - - #TODO: serverspec? - end - - end -end +require 'chef/dsl/audit' +require 'chef/audit/chef_json_formatter' +require 'chef/audit/runner' diff --git a/lib/chef/audit/chef_json_formatter.rb b/lib/chef/audit/chef_json_formatter.rb new file mode 100644 index 0000000000..5dcbdb50b7 --- /dev/null +++ b/lib/chef/audit/chef_json_formatter.rb @@ -0,0 +1,88 @@ +RSpec::Support.require_rspec_core "formatters/base_formatter" +require 'chef/audit/control_group_data' +require 'ffi_yajl' + +class Chef + class Audit + class ChefJsonFormatter < ::RSpec::Core::Formatters::BaseFormatter + ::RSpec::Core::Formatters.register self, :example_group_started, :message, :stop, :close, :example_failed + + attr_reader :control_group_data + + # TODO hopefully the runner can take care of this for us since there won't be an outer-most + # control group + @@outer_example_group_found = false + + def initialize(output) + super + end + + # Invoked for each `control`, `describe`, `context` block + def example_group_started(notification) + unless @@outer_example_group_found + @control_group_data = ControlGroupData.new(notification.group.description) + @@outer_example_group_found = true + end + end + + def example_failed(notification) + e = notification.example.metadata[:execution_result].exception + raise e unless e.kind_of? ::RSpec::Expectations::ExpectationNotMetError + end + + def message(notification) + puts "message: #{notification}" + end + + def stop(notification) + notification.examples.each do |example| + control_data = build_control_from(example) + e = example.exception + if e + control = control_group_data.example_failure(e.message, control_data) + else + control = control_group_data.example_success(control_data) + end + control.line_number = example.metadata[:line_number] + end + end + + def close(notification) + output.write FFI_Yajl::Encoder.encode(control_group_data.to_hash, pretty: true) + output.close if IO === output && output != $stdout + end + + private + + def build_control_from(example) + described_class = example.metadata[:described_class] + if described_class + resource_type = described_class.class.name.split(':')[-1] + # TODO submit github PR to expose this + resource_name = described_class.instance_variable_get(:@name) + end + + describe_groups = [] + group = example.metadata[:example_group] + # If the innermost block has a resource instead of a string, don't include it in context + describe_groups.unshift(group[:description]) if described_class.nil? + group = group[:parent_example_group] + while !group.nil? + describe_groups.unshift(group[:description]) + group = group[:parent_example_group] + end + # TODO remove this when we're no longer wrapping everything with "mysql audit" + describe_groups.shift + + { + :name => example.description, + :desc => example.full_description, + :resource_type => resource_type, + :resource_name => resource_name, + :context => describe_groups + } + end + + end + end +end diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb new file mode 100644 index 0000000000..93abfb3c21 --- /dev/null +++ b/lib/chef/audit/control_group_data.rb @@ -0,0 +1,84 @@ +class Chef + class Audit + class ControlGroupData + attr_reader :name, :status, :number_success, :number_failed, :controls + + def initialize(name) + @status = "success" + @controls = [] + @number_success = 0 + @number_failed = 0 + @name = name + end + + + def example_success(opts={}) + @number_success += 1 + control = create_control(opts) + controls << control + control + end + + def example_failure(details=nil, opts={}) + @number_failed += 1 + @status = "failure" + control = create_control(opts) + control.details = details if details + control.status = "failure" + controls << control + control + end + + def to_hash + controls.sort! {|x,y| x.line_number <=> y.line_number} + { + :control_group => { + :name => name, + :status => status, + :number_success => number_success, + :number_failed => number_failed, + :controls => controls.collect { |c| c.to_hash } + } + } + end + + private + + def create_control(opts={}) + name = opts[:name] + resource_type = opts[:resource_type] + resource_name = opts[:resource_name] + context = opts[:context] + ControlData.new(name, resource_type, resource_name, context) + end + + end + + class ControlData + attr_reader :name, :resource_type, :resource_name, :context + attr_accessor :status, :details + # TODO this only helps with debugging + attr_accessor :line_number + + def initialize(name, resource_type, resource_name, context) + @context = context + @name = name + @resource_type = resource_type + @resource_name = resource_name + end + + def to_hash + ret = { + :name => name, + :status => status, + :details => details, + :resource_type => resource_type, + :resource_name => resource_name + } + ret[:context] = context || [] + ret + end + end + + end +end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb new file mode 100644 index 0000000000..e3699266b8 --- /dev/null +++ b/lib/chef/audit/runner.rb @@ -0,0 +1,93 @@ +# +# Author:: Claire McQuin () +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/audit' +require 'chef/config' + +class Chef + class Audit + class Runner + + def initialize(run_context) + @run_context = run_context + end + + def run + setup + register_controls_groups + + # The first parameter passed to RSpec::Core::Runner.new + # is an instance of RSpec::Core::ConfigurationOptions, which is + # responsible for processing command line options passed through rspec. + # This then gets merged with the configuration. We'll just communicate + # directly with the Configuration here. + audit_runner = RSpec::Core::Runner.new(nil, configuration, world) + audit_runner.run_specs(world.ordered_example_groups) + end + + private + + # RSpec configuration and world objects are heavy, so let's wait until + # we actually need them. + def configuration + @configuration ||= RSpec::Core::Configuration.new + end + + def world + @world ||= RSpec::Core::World.new(configuration) + end + + # Configure audits before run. + # Sets up where output and error streams should stream to, adds formatters + # for people-friendly output of audit results and json for reporting. Also + # configures expectation frameworks. + def setup + configuration.output_stream = Chef::Config[:log_location] + configuration.error_stream = Chef::Config[:log_location] + + add_formatters + disable_should_syntax + end + + def add_formatters + configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) + configuration.add_formatter(ChefJsonFormatter) + end + + # Explicitly disable :should syntax. + # + # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it + # in audits. If :should is used in an audit, the audit will fail with error + # message "undefined method `:should`" rather than issue a deprecation warning. + # + # This can be removed when :should is fully removed from RSpec. + def disable_should_syntax + configuration.expect_with :rspec do |c| + c.syntax = :expect + end + end + + # Register each controls group with the world, which will handle + # the ordering of the audits that will be run. + def register_controls_groups + @run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) } + end + + end + end +end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 6c83639ec3..3818f9fad6 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -25,6 +25,7 @@ require 'chef/log' require 'chef/rest' require 'chef/api_client' require 'chef/api_client/registration' +require 'chef/audit' require 'chef/node' require 'chef/role' require 'chef/file_cache' @@ -344,7 +345,8 @@ class Chef audit_exception = nil begin @events.audit_start(run_context) - # TODO + auditor = Chef::Audit::Runner.new(run_context) + auditor.run @events.audit_complete rescue Exception => e @events.audit_failed(e) diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 7adcecbf14..948db72921 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -27,12 +27,9 @@ class Chef raise ::Chef::Exceptions::NoAuditsProvided unless group_block # TODO add the @example_groups list to the runner for later execution - # run_context.audit_runner.register - ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block) + run_context.controls_groups << ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block) end end end end - - diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 22679822a4..749e4cf388 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -50,6 +50,11 @@ class Chef # recipes, which is triggered by #load. (See also: CookbookCompiler) attr_accessor :resource_collection + attr_accessor :controls_groups + + # Chef::ProviderResolver for this run + attr_accessor :provider_resolver + # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. attr_accessor :immediate_notification_collection @@ -73,6 +78,7 @@ class Chef @node = node @cookbook_collection = cookbook_collection @resource_collection = Chef::ResourceCollection.new + @controls_groups = [] @immediate_notification_collection = Hash.new {|h,k| h[k] = []} @delayed_notification_collection = Hash.new {|h,k| h[k] = []} @definitions = Hash.new -- cgit v1.2.1 From 9f3390ffe26abbe54b51aee19a78cf2778b9b340 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Tue, 4 Nov 2014 13:34:02 -0800 Subject: Add serverspec types and matchers. --- chef.gemspec | 3 +++ lib/chef/audit.rb | 8 +++++++- lib/chef/audit/runner.rb | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/chef.gemspec b/chef.gemspec index 0461872668..2138f36114 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -45,6 +45,9 @@ Gem::Specification.new do |s| %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem, "~> 3.1" } + s.add_development_dependency "serverspec", "~> 2.3" + s.add_development_dependency "specinfra", "~> 2.4" + s.bindir = "bin" s.executables = %w( chef-client chef-solo knife chef-shell chef-apply ) diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb index 43d5c11ee8..0c79928974 100644 --- a/lib/chef/audit.rb +++ b/lib/chef/audit.rb @@ -16,7 +16,13 @@ # limitations under the License. # -require 'rspec/core' +require 'rspec' + +require 'serverspec/matcher' +require 'serverspec/helper' +require 'serverspec/subject' + +require 'specinfra' require 'chef/dsl/audit' require 'chef/audit/chef_json_formatter' diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index e3699266b8..0f439a1aa5 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -62,6 +62,7 @@ class Chef add_formatters disable_should_syntax + configure_specinfra end def add_formatters @@ -82,6 +83,12 @@ class Chef end end + def configure_specinfra + # TODO: We may need to change this based on operating system (there is a + # powershell backend) or roll our own. + Specinfra.configuration.backend = :exec + end + # Register each controls group with the world, which will handle # the ordering of the audits that will be run. def register_controls_groups -- cgit v1.2.1 From 5647ffbafcf22cd54b1edc0b02926aca90b9099c Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 5 Nov 2014 11:25:05 -0800 Subject: Adding first round of formatter integration - STDOUT doc formatter --- lib/chef/audit/audit_event_proxy.rb | 46 +++++++++++++++++++++++++++++ lib/chef/audit/chef_json_formatter.rb | 11 +------ lib/chef/audit/runner.rb | 11 +++++-- lib/chef/client.rb | 6 ++-- lib/chef/dsl/audit.rb | 1 - lib/chef/event_dispatch/base.rb | 27 ++++++++++++++---- lib/chef/formatters/doc.rb | 54 ++++++++++++++++++++++++++++++----- lib/chef/run_context.rb | 1 + 8 files changed, 128 insertions(+), 29 deletions(-) create mode 100644 lib/chef/audit/audit_event_proxy.rb diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb new file mode 100644 index 0000000000..1052821946 --- /dev/null +++ b/lib/chef/audit/audit_event_proxy.rb @@ -0,0 +1,46 @@ +RSpec::Support.require_rspec_core "formatters/base_text_formatter" + +class Chef + class Audit + class AuditEventProxy < ::RSpec::Core::Formatters::BaseFormatter + ::RSpec::Core::Formatters.register self, :example_group_started, :example_group_finished, + :example_passed, :example_failed + + # TODO I don't like this, but I don't see another way to pass this in + # see configuration.rb#L671 and formatters.rb#L129 + def self.events=(events) + @@events = events + end + + def events + @@events + end + + def initialize(output) + super + end + + def example_group_started(notification) + events.control_group_start(notification.group.description.strip) + end + + def example_group_finished(_notification) + events.control_group_end + end + + def example_passed(passed) + events.control_example_success(passed.example.description.strip) + end + + def example_failed(failed) + events.control_example_failure(failed.example.description.strip, failed.example.execution_result.exception) + end + + private + + def example_group_chain + example_group.parent_groups.reverse + end + end + end +end diff --git a/lib/chef/audit/chef_json_formatter.rb b/lib/chef/audit/chef_json_formatter.rb index 5dcbdb50b7..50dfbffbb3 100644 --- a/lib/chef/audit/chef_json_formatter.rb +++ b/lib/chef/audit/chef_json_formatter.rb @@ -5,7 +5,7 @@ require 'ffi_yajl' class Chef class Audit class ChefJsonFormatter < ::RSpec::Core::Formatters::BaseFormatter - ::RSpec::Core::Formatters.register self, :example_group_started, :message, :stop, :close, :example_failed + ::RSpec::Core::Formatters.register self, :example_group_started, :stop, :close attr_reader :control_group_data @@ -25,15 +25,6 @@ class Chef end end - def example_failed(notification) - e = notification.example.metadata[:execution_result].exception - raise e unless e.kind_of? ::RSpec::Expectations::ExpectationNotMetError - end - - def message(notification) - puts "message: #{notification}" - end - def stop(notification) notification.examples.each do |example| control_data = build_control_from(example) diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 0f439a1aa5..1408c67327 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -17,12 +17,15 @@ # require 'chef/audit' +require 'chef/audit/audit_event_proxy' require 'chef/config' class Chef class Audit class Runner + attr_reader :run_context + def initialize(run_context) @run_context = run_context end @@ -57,6 +60,8 @@ class Chef # for people-friendly output of audit results and json for reporting. Also # configures expectation frameworks. def setup + # We're setting the output stream, but that will only be used for error situations + # Our formatter forwards events to the Chef event message bus configuration.output_stream = Chef::Config[:log_location] configuration.error_stream = Chef::Config[:log_location] @@ -66,8 +71,8 @@ class Chef end def add_formatters - configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) - configuration.add_formatter(ChefJsonFormatter) + configuration.add_formatter(Chef::Audit::AuditEventProxy) + Chef::Audit::AuditEventProxy.events = run_context.events end # Explicitly disable :should syntax. @@ -92,7 +97,7 @@ class Chef # Register each controls group with the world, which will handle # the ordering of the audits that will be run. def register_controls_groups - @run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) } + run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) } end end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 3818f9fad6..bf8c83eee2 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -344,12 +344,12 @@ class Chef def run_audits(run_context) audit_exception = nil begin - @events.audit_start(run_context) + @events.audit_phase_start(run_context) auditor = Chef::Audit::Runner.new(run_context) auditor.run - @events.audit_complete + @events.audit_phase_complete rescue Exception => e - @events.audit_failed(e) + @events.audit_phase_failed(e) audit_exception = e end audit_exception diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 948db72921..a417e51b5d 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -26,7 +26,6 @@ class Chef def controls(group_name, &group_block) raise ::Chef::Exceptions::NoAuditsProvided unless group_block - # TODO add the @example_groups list to the runner for later execution run_context.controls_groups << ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block) end diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb index 555998ced4..e55f752065 100644 --- a/lib/chef/event_dispatch/base.rb +++ b/lib/chef/event_dispatch/base.rb @@ -230,15 +230,32 @@ class Chef end # Called before audit phase starts - def audit_start(run_context) + def audit_phase_start(run_context) end - # Called when the audit phase is finished. - def audit_complete + # Called when audit phase successfully finishes + def audit_phase_complete end - # Called if the audit phase fails - def audit_failed(exception) + # Called if there is an uncaught exception during the audit phase. The audit runner should + # be catching and handling errors from the examples, so this is only uncaught errors (like + # bugs in our handling code) + def audit_phase_failed(exception) + end + + def control_group_start(name) + # TODO use this only for stdout formatting, controls indentation + end + + def control_group_end + end + + def control_example_success(description) + # TODO Use this for both stdout and resource_reporter, need to pass ancestor tree for resource_reporter + # but that is ignored by stdout + end + + def control_example_failure(description, exception) end # TODO: need events for notification resolve? diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 66ee6ccabe..96ca283b9f 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -156,17 +156,57 @@ class Chef converge_complete end - def audit_start(run_context) - # TODO read the number of `controls` blocks to run from the run_context.audit_runner - puts_line "Running collected audits" + ############# + # TODO + # Make all these document printers neater + ############# + + # Called before audit phase starts + def audit_phase_start(run_context) + puts_line "" + puts_line "Audit phase starting" + end + + # Called when audit phase successfully finishes + def audit_phase_complete + puts_line "Audit phase ended" + end + + # Called if there is an uncaught exception during the audit phase. The audit runner should + # be catching and handling errors from the examples, so this is only uncaught errors (like + # bugs in our handling code) + def audit_phase_failed(error) + puts_line "Audit phase exception:" + indent + # TODO error_mapper ? + puts_line "#{error.message}" + error.backtrace.each do |l| + puts_line l + end end - def audit_complete - # TODO + def control_group_start(name) + puts_line "Control group #{name} started" + indent end - def audit_failed(exception) - # TODO + def control_group_end + unindent + end + + def control_example_success(description) + puts_line "SUCCESS - #{description}" + end + + def control_example_failure(description, error) + puts_line "FAILURE - #{description}" + indent + # TODO error_mapper ? + puts_line "#{error.message}" + # error.backtrace.each do |l| + # puts_line l + # end + unindent end # Called before action is executed on a resource. diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 749e4cf388..c6d11eaaa1 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -50,6 +50,7 @@ class Chef # recipes, which is triggered by #load. (See also: CookbookCompiler) attr_accessor :resource_collection + # The list of control groups to execute during the audit phase attr_accessor :controls_groups # Chef::ProviderResolver for this run -- cgit v1.2.1 From b9b4d3bfd6ec6196cb78094e589c094edb61b501 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Wed, 5 Nov 2014 13:35:21 -0800 Subject: add rspec/its --- chef.gemspec | 13 +++++-------- lib/chef/audit.rb | 1 + lib/chef/audit/chef_example_group.rb | 1 - lib/chef/dsl/audit.rb | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/chef.gemspec b/chef.gemspec index 2138f36114..eb7528e630 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -35,19 +35,16 @@ Gem::Specification.new do |s| s.add_dependency 'plist', '~> 3.1.0' + %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_dependency gem, "~> 3.1" } + s.add_dependency "rspec_junit_formatter", "~> 0.2.0" + s.add_dependency "serverspec", "~> 2.3" + s.add_dependency "specinfra", "~> 2.4" + s.add_development_dependency "rack" # Rake 10.2 drops Ruby 1.8 support s.add_development_dependency "rake", "~> 10.1.0" - # rspec_junit_formatter 0.2.0 drops ruby 1.8.7 support - s.add_development_dependency "rspec_junit_formatter", "~> 0.2.0" - - %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem, "~> 3.1" } - - s.add_development_dependency "serverspec", "~> 2.3" - s.add_development_dependency "specinfra", "~> 2.4" - s.bindir = "bin" s.executables = %w( chef-client chef-solo knife chef-shell chef-apply ) diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb index 0c79928974..9d0a2a70ed 100644 --- a/lib/chef/audit.rb +++ b/lib/chef/audit.rb @@ -17,6 +17,7 @@ # require 'rspec' +require 'rspec/its' require 'serverspec/matcher' require 'serverspec/helper' diff --git a/lib/chef/audit/chef_example_group.rb b/lib/chef/audit/chef_example_group.rb index cd874d57b7..482647cd03 100644 --- a/lib/chef/audit/chef_example_group.rb +++ b/lib/chef/audit/chef_example_group.rb @@ -1,4 +1,3 @@ -require 'rspec/core' class Chef class Audit diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index a417e51b5d..75a57c6bb4 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#require 'chef/audit' + require 'chef/audit/chef_example_group' class Chef -- cgit v1.2.1 From 2bb912157470f55975f2e50e3792132478639a78 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 6 Nov 2014 07:33:35 -0800 Subject: Preparing for demo - using rspec documentation formatter for output instead of the proxy --- lib/chef/audit/audit_event_proxy.rb | 4 ---- lib/chef/audit/chef_example_group.rb | 9 --------- lib/chef/audit/control_group_data.rb | 1 + lib/chef/audit/runner.rb | 18 ++++++++++++++---- lib/chef/dsl/audit.rb | 12 ++++++++---- lib/chef/formatters/doc.rb | 6 ++++-- 6 files changed, 27 insertions(+), 23 deletions(-) delete mode 100644 lib/chef/audit/chef_example_group.rb diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index 1052821946..d56c7f0ac5 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -16,10 +16,6 @@ class Chef @@events end - def initialize(output) - super - end - def example_group_started(notification) events.control_group_start(notification.group.description.strip) end diff --git a/lib/chef/audit/chef_example_group.rb b/lib/chef/audit/chef_example_group.rb deleted file mode 100644 index 482647cd03..0000000000 --- a/lib/chef/audit/chef_example_group.rb +++ /dev/null @@ -1,9 +0,0 @@ - -class Chef - class Audit - class ChefExampleGroup < ::RSpec::Core::ExampleGroup - # Can encompass tests in a `control` block or `describe` block - define_example_group_method :control - end - end -end diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index 93abfb3c21..fc758e45ba 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -15,6 +15,7 @@ class Chef def example_success(opts={}) @number_success += 1 control = create_control(opts) + control.status = "success" controls << control control end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 1408c67327..4a76b7e65b 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -18,6 +18,7 @@ require 'chef/audit' require 'chef/audit/audit_event_proxy' +require 'chef/audit/chef_json_formatter' require 'chef/config' class Chef @@ -25,6 +26,7 @@ class Chef class Runner attr_reader :run_context + private :run_context def initialize(run_context) @run_context = run_context @@ -48,11 +50,11 @@ class Chef # RSpec configuration and world objects are heavy, so let's wait until # we actually need them. def configuration - @configuration ||= RSpec::Core::Configuration.new + RSpec.configuration end def world - @world ||= RSpec::Core::World.new(configuration) + RSpec.world end # Configure audits before run. @@ -62,8 +64,14 @@ class Chef def setup # We're setting the output stream, but that will only be used for error situations # Our formatter forwards events to the Chef event message bus + # TODO so some testing to see if these output to a log file - we probably need + # to register these before any formatters are added. configuration.output_stream = Chef::Config[:log_location] configuration.error_stream = Chef::Config[:log_location] + # TODO im pretty sure I only need this because im running locally in rvmsudo + configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR))) + configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)")) + configuration.color = Chef::Config[:color] add_formatters disable_should_syntax @@ -71,8 +79,10 @@ class Chef end def add_formatters - configuration.add_formatter(Chef::Audit::AuditEventProxy) - Chef::Audit::AuditEventProxy.events = run_context.events + configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) + configuration.add_formatter(Chef::Audit::ChefJsonFormatter) + #configuration.add_formatter(Chef::Audit::AuditEventProxy) + #Chef::Audit::AuditEventProxy.events = run_context.events end # Explicitly disable :should syntax. diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 75a57c6bb4..42a1927efb 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -16,17 +16,21 @@ # limitations under the License. # -require 'chef/audit/chef_example_group' +require 'rspec/core' class Chef module DSL module Audit + # Can encompass tests in a `control` block or `describe` block + ::RSpec::Core::ExampleGroup.define_example_group_method :control + ::RSpec::Core::ExampleGroup.define_example_group_method :__controls__ + # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples - def controls(group_name, &group_block) - raise ::Chef::Exceptions::NoAuditsProvided unless group_block + def controls(*args, &block) + raise ::Chef::Exceptions::NoAuditsProvided unless block - run_context.controls_groups << ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block) + run_context.controls_groups << ::RSpec::Core::ExampleGroup.__controls__(*args, &block) end end diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 96ca283b9f..d9c9124713 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -164,18 +164,20 @@ class Chef # Called before audit phase starts def audit_phase_start(run_context) puts_line "" - puts_line "Audit phase starting" + puts_line "++ Audit phase starting ++" end # Called when audit phase successfully finishes def audit_phase_complete - puts_line "Audit phase ended" + puts_line "" + puts_line "++ Audit phase ended ++ " end # Called if there is an uncaught exception during the audit phase. The audit runner should # be catching and handling errors from the examples, so this is only uncaught errors (like # bugs in our handling code) def audit_phase_failed(error) + puts_line "" puts_line "Audit phase exception:" indent # TODO error_mapper ? -- cgit v1.2.1 From 73594ef27855e6f1dabb57fdffa04adc881f06be Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Mon, 10 Nov 2014 09:52:06 -0800 Subject: Wiring audit event proxy to send events correctly to the audit_reporter --- lib/chef/application/solo.rb | 3 + lib/chef/audit.rb | 1 - lib/chef/audit/audit_event_proxy.rb | 64 ++++++++++++---- lib/chef/audit/audit_reporter.rb | 135 ++++++++++++++++++++++++++++++++++ lib/chef/audit/chef_json_formatter.rb | 79 -------------------- lib/chef/audit/control_group_data.rb | 90 ++++++++++++++++------- lib/chef/audit/runner.rb | 7 +- lib/chef/client.rb | 26 +++++-- lib/chef/dsl/audit.rb | 2 + lib/chef/event_dispatch/base.rb | 17 ++--- lib/chef/exceptions.rb | 6 ++ lib/chef/formatters/doc.rb | 30 +------- spec/integration/solo/solo_spec.rb | 2 +- 13 files changed, 291 insertions(+), 171 deletions(-) create mode 100644 lib/chef/audit/audit_reporter.rb delete mode 100644 lib/chef/audit/chef_json_formatter.rb diff --git a/lib/chef/application/solo.rb b/lib/chef/application/solo.rb index 6e568ddbb1..50f7f5c5d4 100644 --- a/lib/chef/application/solo.rb +++ b/lib/chef/application/solo.rb @@ -211,6 +211,9 @@ class Chef::Application::Solo < Chef::Application config_fetcher = Chef::ConfigFetcher.new(Chef::Config[:json_attribs]) @chef_client_json = config_fetcher.fetch_json end + + # If we don't specify this, solo will try to perform the audits + Chef::Config[:audit_mode] = false end def setup_application diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb index 9d0a2a70ed..ed8db93d96 100644 --- a/lib/chef/audit.rb +++ b/lib/chef/audit.rb @@ -26,5 +26,4 @@ require 'serverspec/subject' require 'specinfra' require 'chef/dsl/audit' -require 'chef/audit/chef_json_formatter' require 'chef/audit/runner' diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index d56c7f0ac5..71d1e2aa50 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -3,11 +3,10 @@ RSpec::Support.require_rspec_core "formatters/base_text_formatter" class Chef class Audit class AuditEventProxy < ::RSpec::Core::Formatters::BaseFormatter - ::RSpec::Core::Formatters.register self, :example_group_started, :example_group_finished, - :example_passed, :example_failed + ::RSpec::Core::Formatters.register self, :stop, :example_group_started # TODO I don't like this, but I don't see another way to pass this in - # see configuration.rb#L671 and formatters.rb#L129 + # see rspec files configuration.rb#L671 and formatters.rb#L129 def self.events=(events) @@events = events end @@ -17,26 +16,61 @@ class Chef end def example_group_started(notification) - events.control_group_start(notification.group.description.strip) + if notification.group.parent_groups.size == 1 + # top level controls block + desc = notification.group.description + Chef::Log.debug("Entered controls block named #{desc}") + events.control_group_started(desc) + end end - def example_group_finished(_notification) - events.control_group_end + def stop(notification) + Chef::Log.info("Successfully executed all controls blocks and contained examples") + notification.examples.each do |example| + control_group_name, control_data = build_control_from(example) + e = example.exception + if e + events.control_example_failure(control_group_name, control_data, e) + else + events.control_example_success(control_group_name, control_data) + end + end end - def example_passed(passed) - events.control_example_success(passed.example.description.strip) - end + private - def example_failed(failed) - events.control_example_failure(failed.example.description.strip, failed.example.execution_result.exception) - end + def build_control_from(example) + described_class = example.metadata[:described_class] + if described_class + resource_type = described_class.class.name.split(':')[-1] + # TODO submit github PR to expose this + resource_name = described_class.instance_variable_get(:@name) + end - private + # The following code builds up the context - the list of wrapping `describe` or `control` blocks + describe_groups = [] + group = example.metadata[:example_group] + # If the innermost block has a resource instead of a string, don't include it in context + describe_groups.unshift(group[:description]) if described_class.nil? + group = group[:parent_example_group] + while !group.nil? + describe_groups.unshift(group[:description]) + group = group[:parent_example_group] + end + + # We know all of our examples each live in a top-level `controls` block - get this name now + outermost_group_desc = describe_groups.shift - def example_group_chain - example_group.parent_groups.reverse + return outermost_group_desc, { + :name => example.description, + :desc => example.full_description, + :resource_type => resource_type, + :resource_name => resource_name, + :context => describe_groups, + :line_number => example.metadata[:line_number] + } end + end end end diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb new file mode 100644 index 0000000000..b1c9d30bfc --- /dev/null +++ b/lib/chef/audit/audit_reporter.rb @@ -0,0 +1,135 @@ +# +# Auther:: Tyler Ball () +# +# Copyright:: Copyright (c) 2014 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/event_dispatch/base' +require 'chef/audit/control_group_data' + +class Chef + class Audit + class AuditReporter < EventDispatch::Base + + attr_reader :rest_client, :audit_data, :ordered_control_groups + private :rest_client, :audit_data, :ordered_control_groups + + PROTOCOL_VERSION = '0.1.0' + + def initialize(rest_client) + if Chef::Config[:audit_mode] == false + @audit_enabled = false + else + @audit_enabled = true + end + @rest_client = rest_client + # Ruby 1.9.3 and above "enumerate their values in the order that the corresponding keys were inserted." + @ordered_control_groups = Hash.new + end + + def audit_phase_start(run_status) + Chef::Log.debug("Audit Reporter starting") + @audit_data = AuditData.new(run_status.node.name, run_status.run_id) + end + + def audit_phase_complete + Chef::Log.debug("Audit Reporter completed successfully without errors") + ordered_control_groups.each do |name, control_group| + audit_data.add_control_group(control_group) + end + post_auditing_data + end + + # If the audit phase failed, its because there was some kind of error in the framework + # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. + def audit_phase_failed(error) + # The stacktrace information has already been logged elsewhere + Chef::Log.error("Audit Reporter failed - sending error to server with available example information") + ordered_control_groups.each do |name, control_group| + audit_data.add_control_group(control_group) + end + post_auditing_data(error) + end + + def control_group_started(name) + if ordered_control_groups.has_key?(name) + raise AuditControlGroupDuplicate.new(name) + end + ordered_control_groups.store(name, ControlGroupData.new(name)) + end + + def control_example_success(control_group_name, example_data) + control_group = ordered_control_groups[control_group_name] + control_group.example_success(example_data) + end + + def control_example_failure(control_group_name, example_data, error) + control_group = ordered_control_groups[control_group_name] + control_group.example_failure(example_data, error.message) + end + + def auditing_enabled? + @audit_enabled + end + + private + + def post_auditing_data(error = nil) + if auditing_enabled? + audit_history_url = "controls" + Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") + run_data = audit_data.to_hash + + if error + run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" + end + + Chef::Log.debug run_data.inspect + compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data)) + Chef::Log.debug("Sending compressed audit data...") + # Since we're posting compressed data we can not directly call post_rest which expects JSON + audit_url = rest_client.create_url(audit_history_url) + begin + puts Chef::JSONCompat.to_json_pretty(run_data) + rest_client.raw_http_request(:POST, audit_url, headers({'Content-Encoding' => 'gzip'}), compressed_data) + rescue StandardError => e + if e.respond_to? :response + error_file = "failed-audit-data.json" + Chef::FileCache.store(error_file, Chef::JSONCompat.to_json_pretty(run_data), 0640) + Chef::Log.error("Failed to post audit report to server (HTTP #{e.response.code}), saving to #{Chef::FileCache.load(error_file, false)}") + else + Chef::Log.error("Failed to post audit report to server (#{e})") + end + end + else + Chef::Log.debug("Server doesn't support audit report, skipping.") + end + end + + def headers(additional_headers = {}) + options = {'X-Ops-Audit-Report-Protocol-Version' => PROTOCOL_VERSION} + options.merge(additional_headers) + end + + def encode_gzip(data) + "".tap do |out| + Zlib::GzipWriter.wrap(StringIO.new(out)){|gz| gz << data } + end + end + + end + end +end diff --git a/lib/chef/audit/chef_json_formatter.rb b/lib/chef/audit/chef_json_formatter.rb deleted file mode 100644 index 50dfbffbb3..0000000000 --- a/lib/chef/audit/chef_json_formatter.rb +++ /dev/null @@ -1,79 +0,0 @@ -RSpec::Support.require_rspec_core "formatters/base_formatter" -require 'chef/audit/control_group_data' -require 'ffi_yajl' - -class Chef - class Audit - class ChefJsonFormatter < ::RSpec::Core::Formatters::BaseFormatter - ::RSpec::Core::Formatters.register self, :example_group_started, :stop, :close - - attr_reader :control_group_data - - # TODO hopefully the runner can take care of this for us since there won't be an outer-most - # control group - @@outer_example_group_found = false - - def initialize(output) - super - end - - # Invoked for each `control`, `describe`, `context` block - def example_group_started(notification) - unless @@outer_example_group_found - @control_group_data = ControlGroupData.new(notification.group.description) - @@outer_example_group_found = true - end - end - - def stop(notification) - notification.examples.each do |example| - control_data = build_control_from(example) - e = example.exception - if e - control = control_group_data.example_failure(e.message, control_data) - else - control = control_group_data.example_success(control_data) - end - control.line_number = example.metadata[:line_number] - end - end - - def close(notification) - output.write FFI_Yajl::Encoder.encode(control_group_data.to_hash, pretty: true) - output.close if IO === output && output != $stdout - end - - private - - def build_control_from(example) - described_class = example.metadata[:described_class] - if described_class - resource_type = described_class.class.name.split(':')[-1] - # TODO submit github PR to expose this - resource_name = described_class.instance_variable_get(:@name) - end - - describe_groups = [] - group = example.metadata[:example_group] - # If the innermost block has a resource instead of a string, don't include it in context - describe_groups.unshift(group[:description]) if described_class.nil? - group = group[:parent_example_group] - while !group.nil? - describe_groups.unshift(group[:description]) - group = group[:parent_example_group] - end - # TODO remove this when we're no longer wrapping everything with "mysql audit" - describe_groups.shift - - { - :name => example.description, - :desc => example.full_description, - :resource_type => resource_type, - :resource_name => resource_name, - :context => describe_groups - } - end - - end - end -end diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index fc758e45ba..e19a6e1a15 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -1,5 +1,29 @@ +require 'securerandom' + class Chef class Audit + class AuditData + attr_reader :node_name, :run_id, :control_groups + + def initialize(node_name, run_id) + @node_name = node_name + @run_id = run_id + @control_groups = [] + end + + def add_control_group(control_group) + control_groups << control_group + end + + def to_hash + { + :node_name => node_name, + :run_id => run_id, + :control_groups => control_groups.collect { |c| c.to_hash } + } + end + end + class ControlGroupData attr_reader :name, :status, :number_success, :number_failed, :controls @@ -12,18 +36,18 @@ class Chef end - def example_success(opts={}) + def example_success(control_data) @number_success += 1 - control = create_control(opts) + control = create_control(control_data) control.status = "success" controls << control control end - def example_failure(details=nil, opts={}) + def example_failure(control_data, details) @number_failed += 1 @status = "failure" - control = create_control(opts) + control = create_control(control_data) control.details = details if details control.status = "failure" controls << control @@ -31,53 +55,69 @@ class Chef end def to_hash + # We sort it so the examples appear in the output in the same order + # they appeared in the recipe controls.sort! {|x,y| x.line_number <=> y.line_number} - { - :control_group => { - :name => name, - :status => status, - :number_success => number_success, - :number_failed => number_failed, - :controls => controls.collect { |c| c.to_hash } - } + h = { + :name => name, + :status => status, + :number_success => number_success, + :number_failed => number_failed, + :controls => controls.collect { |c| c.to_hash } } + add_display_only_data(h) end private - def create_control(opts={}) - name = opts[:name] - resource_type = opts[:resource_type] - resource_name = opts[:resource_name] - context = opts[:context] - ControlData.new(name, resource_type, resource_name, context) + def create_control(control_data) + name = control_data[:name] + resource_type = control_data[:resource_type] + resource_name = control_data[:resource_name] + context = control_data[:context] + line_number = control_data[:line_number] + # TODO make this smarter with splat arguments so if we start passing in more control_data + # I don't have to modify code in multiple places + ControlData.new(name, resource_type, resource_name, context, line_number) + end + + # The id and control sequence number are ephemeral data - they are not needed + # to be persisted and can be regenerated at will. They are only needed + # for display purposes. + def add_display_only_data(group) + group[:id] = SecureRandom.uuid + group[:controls].collect!.with_index do |c, i| + # i is zero-indexed, and we want the display one-indexed + c[:sequence_number] = i+1 + c + end + group end end class ControlData - attr_reader :name, :resource_type, :resource_name, :context + attr_reader :name, :resource_type, :resource_name, :context, :line_number attr_accessor :status, :details - # TODO this only helps with debugging - attr_accessor :line_number - def initialize(name, resource_type, resource_name, context) + def initialize(name, resource_type, resource_name, context, line_number) @context = context @name = name @resource_type = resource_type @resource_name = resource_name + @line_number = line_number end def to_hash - ret = { + h = { :name => name, :status => status, :details => details, :resource_type => resource_type, :resource_name => resource_name } - ret[:context] = context || [] - ret + h[:context] = context || [] + h end end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 4a76b7e65b..4059741359 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -18,7 +18,6 @@ require 'chef/audit' require 'chef/audit/audit_event_proxy' -require 'chef/audit/chef_json_formatter' require 'chef/config' class Chef @@ -72,6 +71,7 @@ class Chef configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR))) configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)")) configuration.color = Chef::Config[:color] + configuration.expose_dsl_globally = false add_formatters disable_should_syntax @@ -80,9 +80,8 @@ class Chef def add_formatters configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) - configuration.add_formatter(Chef::Audit::ChefJsonFormatter) - #configuration.add_formatter(Chef::Audit::AuditEventProxy) - #Chef::Audit::AuditEventProxy.events = run_context.events + configuration.add_formatter(Chef::Audit::AuditEventProxy) + Chef::Audit::AuditEventProxy.events = run_context.events end # Explicitly disable :should syntax. diff --git a/lib/chef/client.rb b/lib/chef/client.rb index bf8c83eee2..b27a2b693d 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -44,6 +44,7 @@ require 'chef/formatters/doc' require 'chef/formatters/minimal' require 'chef/version' require 'chef/resource_reporter' +require 'chef/audit/audit_reporter' require 'chef/run_lock' require 'chef/policy_builder' require 'chef/request_id' @@ -210,6 +211,17 @@ class Chef end end + # Resource repoters send event information back to the chef server for processing. + # Can only be called after we have a @rest object + def register_reporters + [ + Chef::ResourceReporter.new(rest), + Chef::Audit::AuditReporter.new(rest) + ].each do |r| + events.register(r) + end + end + # Instantiates a Chef::Node object, possibly loading the node's prior state # when using chef-client. Delegates to policy_builder # @@ -296,9 +308,7 @@ class Chef end # We now have the client key, and should use it from now on. @rest = Chef::REST.new(config[:chef_server_url], client_name, config[:client_key]) - # TODO register this where we register all other event listeners - @resource_reporter = Chef::ResourceReporter.new(@rest) - @events.register(@resource_reporter) + register_reporters rescue Exception => e # TODO: munge exception so a semantic failure message can be given to the # user @@ -320,6 +330,7 @@ class Chef runner.converge @events.converge_complete rescue Exception => e + Chef::Log.error("Converge failed with error message #{e.message}") @events.converge_failed(e) converge_exception = e end @@ -340,15 +351,16 @@ class Chef converge_exception end - # TODO are failed audits going to raise exceptions, or only be handled by the reporters? def run_audits(run_context) audit_exception = nil begin - @events.audit_phase_start(run_context) + @events.audit_phase_start(run_status) + Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run @events.audit_phase_complete rescue Exception => e + Chef::Log.error("Audit phase failed with error message #{e.message}") @events.audit_phase_failed(e) audit_exception = e end @@ -429,8 +441,8 @@ class Chef run_context = setup_run_context - converge_error = converge_and_save(run_context) - audit_error = run_audits(run_context) + converge_error = converge_and_save(run_context) unless (Chef::Config[:audit_mode] == true) + audit_error = run_audits(run_context) unless (Chef::Config[:audit_mode] == false) if converge_error || audit_error e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 42a1927efb..1849b65633 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -29,6 +29,8 @@ class Chef # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples def controls(*args, &block) raise ::Chef::Exceptions::NoAuditsProvided unless block + name = args[0] + raise AuditNameMissing if name.nil? || name.empty? run_context.controls_groups << ::RSpec::Core::ExampleGroup.__controls__(*args, &block) end diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb index e55f752065..695e31cf2e 100644 --- a/lib/chef/event_dispatch/base.rb +++ b/lib/chef/event_dispatch/base.rb @@ -230,7 +230,7 @@ class Chef end # Called before audit phase starts - def audit_phase_start(run_context) + def audit_phase_start(run_status) end # Called when audit phase successfully finishes @@ -243,19 +243,16 @@ class Chef def audit_phase_failed(exception) end - def control_group_start(name) - # TODO use this only for stdout formatting, controls indentation + # Signifies the start of a `controls` block with a defined name + def control_group_started(name) end - def control_group_end + # An example in a `controls` block completed successfully + def control_example_success(control_group_name, example_data) end - def control_example_success(description) - # TODO Use this for both stdout and resource_reporter, need to pass ancestor tree for resource_reporter - # but that is ignored by stdout - end - - def control_example_failure(description, exception) + # An example in a `controls` block failed with the provided error + def control_example_failure(control_group_name, example_data, error) end # TODO: need events for notification resolve? diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index 842910f2ae..9fae1d566f 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -385,6 +385,12 @@ class Chef end end + class AuditControlGroupDuplicate < RuntimeError + def initialize(name) + super "Audit control group with name '#{name}' has already been defined" + end + end + class AuditNameMissing < RuntimeError; end class NoAuditsProvided < RuntimeError def initialize super "You must provide a block with audits" diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index d9c9124713..09d04f3aae 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -162,20 +162,16 @@ class Chef ############# # Called before audit phase starts - def audit_phase_start(run_context) + def audit_phase_start(run_status) puts_line "" puts_line "++ Audit phase starting ++" end - # Called when audit phase successfully finishes def audit_phase_complete puts_line "" puts_line "++ Audit phase ended ++ " end - # Called if there is an uncaught exception during the audit phase. The audit runner should - # be catching and handling errors from the examples, so this is only uncaught errors (like - # bugs in our handling code) def audit_phase_failed(error) puts_line "" puts_line "Audit phase exception:" @@ -187,30 +183,6 @@ class Chef end end - def control_group_start(name) - puts_line "Control group #{name} started" - indent - end - - def control_group_end - unindent - end - - def control_example_success(description) - puts_line "SUCCESS - #{description}" - end - - def control_example_failure(description, error) - puts_line "FAILURE - #{description}" - indent - # TODO error_mapper ? - puts_line "#{error.message}" - # error.backtrace.each do |l| - # puts_line l - # end - unindent - end - # Called before action is executed on a resource. def resource_action_start(resource, action, notification_type=nil, notifier=nil) if resource.cookbook_name && resource.recipe_name diff --git a/spec/integration/solo/solo_spec.rb b/spec/integration/solo/solo_spec.rb index cc9ba1abb2..9500e7a1ca 100644 --- a/spec/integration/solo/solo_spec.rb +++ b/spec/integration/solo/solo_spec.rb @@ -92,7 +92,7 @@ EOM # We have a timeout protection here so that if due to some bug # run_lock gets stuck we can discover it. expect { - Timeout.timeout(120) do + Timeout.timeout(1200) do chef_dir = File.join(File.dirname(__FILE__), "..", "..", "..") # Instantiate the first chef-solo run -- cgit v1.2.1 From 3a26043bcd362f600aacb19bb106c4c9cba899a3 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Fri, 31 Oct 2014 15:05:10 -0700 Subject: Adding audit mode JSON formatter First pass at DSL additions Renaming DSL methods to match the spec Creating our own example group class to simplify adding examples to the spec runner Adding logic for exceptions from converge phase not interfering with audit phase and vice-versa Adding error handling so saving node doesn't prevent us from running audit mode - decouples converge phase and audit phase more Updating for github comments Add setup phase to audit-mode. Refactor runner into own class. Fix typo tie things together Adding first round of formatter integration - STDOUT doc formatter Preparing for demo - using rspec documentation formatter for output instead of the proxy Add serverspec types and matchers. add rspec/its Add gems as core dependencies Updating with changes from demo Updating with @mcquin and @lamont comments Getting rid of unused method Wiring audit event proxy to send events correctly to the audit_reporter removing old pry debugging statement Removing unecessary todo Sending to correct server URL Fixing TODOs Adding uncaught error information --- lib/chef/run_context.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index c6d11eaaa1..0999ae57c1 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -53,9 +53,6 @@ class Chef # The list of control groups to execute during the audit phase attr_accessor :controls_groups - # Chef::ProviderResolver for this run - attr_accessor :provider_resolver - # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. attr_accessor :immediate_notification_collection -- cgit v1.2.1 From 221131fa38d77fd3e1cd914fba4dde73b749b78b Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Thu, 13 Nov 2014 14:23:44 -0800 Subject: Basic test for audit mode. --- .gitignore | 3 + .kitchen.yml | 84 +++++++++++++++++++ dev-repo/README.md | 13 +++ dev-repo/cookbooks/audit_test/.gitignore | 16 ++++ dev-repo/cookbooks/audit_test/.kitchen.yml | 16 ++++ dev-repo/cookbooks/audit_test/Berksfile | 3 + dev-repo/cookbooks/audit_test/README.md | 4 + dev-repo/cookbooks/audit_test/chefignore | 95 ++++++++++++++++++++++ dev-repo/cookbooks/audit_test/metadata.rb | 8 ++ dev-repo/cookbooks/audit_test/recipes/default.rb | 11 +++ dev-repo/dev-config.rb | 2 + dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json | 3 + 12 files changed, 258 insertions(+) create mode 100644 .kitchen.yml create mode 100644 dev-repo/README.md create mode 100644 dev-repo/cookbooks/audit_test/.gitignore create mode 100644 dev-repo/cookbooks/audit_test/.kitchen.yml create mode 100644 dev-repo/cookbooks/audit_test/Berksfile create mode 100644 dev-repo/cookbooks/audit_test/README.md create mode 100644 dev-repo/cookbooks/audit_test/chefignore create mode 100644 dev-repo/cookbooks/audit_test/metadata.rb create mode 100644 dev-repo/cookbooks/audit_test/recipes/default.rb create mode 100644 dev-repo/dev-config.rb create mode 100644 dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json diff --git a/.gitignore b/.gitignore index a9e4338e2a..30a3bd6531 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ Berksfile.lock # Vagrant Vagrantfile .vagrant/ + +# Dev Repo Local Mode Data +dev-repo/nodes/* diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000000..c9be1b56e7 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,84 @@ +driver: + name: vagrant + forward_agent: yes + customize: + cpus: 4 + memory: 4096 + synced_folders: + - ['.', '/home/vagrant/chef'] + - ['../ohai', '/home/vagrant/ohai'] + - ['../triager', '/home/vagrant/triager'] + +provisioner: + name: chef_zero + require_chef_omnibus: 12.0.0.rc.1 + +platforms: + - name: centos-5.10 + run_list: + - name: centos-6.5 + run_list: + - name: debian-7.2.0 + run_list: + - name: debian-7.4 + run_list: + - name: debian-6.0.8 + run_list: + - name: freebsd-9.2 + run_list: + - name: freebsd-10.0 + run_list: + - name: ubuntu-10.04 + run_list: + - name: ubuntu-12.04 + run_list: + - name: ubuntu-12.10 + run_list: + - name: ubuntu-13.04 + run_list: + - name: ubuntu-13.10 + run_list: + - name: ubuntu-14.04 + run_list: + # The following boxes are shared via VagrantCloud. Until kitchen-vagrant + # is updated you'll need to add the box manually: + # + # vagrant box add chef/windows-8.1-professional + # + # Please note this may require a `vagrant login` if the box is private. + # + # The following boxes are VMware only also. You can enable VMware Fusion + # as the default provider by copying `.kitchen.local.yml.vmware.example` + # over to `.kitchen.local.yml`. + # + - name: macosx-10.8 + driver: + box: chef/macosx-10.8 # private + - name: macosx-10.9 + driver: + box: chef/macosx-10.9 # private + - name: macosx-10.10 + driver: + box: chef/macosx-10.10 # private + # - name: windows-7-professional + # provisioner: + # name: windows_chef_zero + # require_chef_omnibus: 11.12.4 + # driver: + # box: chef/windows-7-professional # private + # - name: windows-8.1-professional + # provisioner: + # name: windows_chef_zero + # require_chef_omnibus: 11.12.4 + # driver: + # box: chef/windows-8.1-professional # private + # - name: windows-2008r2-standard + # provisioner: + # name: windows_chef_zero + # require_chef_omnibus: 11.12.4 + # driver: + # box: chef/windows-server-2008r2-standard # private + +suites: + - name: chef + run_list: diff --git a/dev-repo/README.md b/dev-repo/README.md new file mode 100644 index 0000000000..84fe5f77df --- /dev/null +++ b/dev-repo/README.md @@ -0,0 +1,13 @@ +# Chef Developer Repo + +This repository contains some basic cookbooks to test chef while you're hacking away. You can provision a VM using the kitchen configuration and run these tests like below: + +``` +$ kitchen converge chef-ubuntu-1210 +$ kitchen login chef-ubuntu-1210 +$ export PATH=/opt/chef/bin:/opt/chef/embedded/bin:$PATH +$ cd ~/chef +$ bundle install +$ bundle exec chef-client -z -o "recipe[audit_test::default]" -c dev-repo/dev-config.rb + +``` diff --git a/dev-repo/cookbooks/audit_test/.gitignore b/dev-repo/cookbooks/audit_test/.gitignore new file mode 100644 index 0000000000..ec2a890bd3 --- /dev/null +++ b/dev-repo/cookbooks/audit_test/.gitignore @@ -0,0 +1,16 @@ +.vagrant +Berksfile.lock +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +bin/* +.bundle/* + +.kitchen/ +.kitchen.local.yml diff --git a/dev-repo/cookbooks/audit_test/.kitchen.yml b/dev-repo/cookbooks/audit_test/.kitchen.yml new file mode 100644 index 0000000000..3775752da2 --- /dev/null +++ b/dev-repo/cookbooks/audit_test/.kitchen.yml @@ -0,0 +1,16 @@ +--- +driver: + name: vagrant + +provisioner: + name: chef_zero + +platforms: + - name: ubuntu-12.04 + - name: centos-6.4 + +suites: + - name: default + run_list: + - recipe[audit_test::default] + attributes: diff --git a/dev-repo/cookbooks/audit_test/Berksfile b/dev-repo/cookbooks/audit_test/Berksfile new file mode 100644 index 0000000000..0ac9b78cf7 --- /dev/null +++ b/dev-repo/cookbooks/audit_test/Berksfile @@ -0,0 +1,3 @@ +source "https://supermarket.getchef.com" + +metadata diff --git a/dev-repo/cookbooks/audit_test/README.md b/dev-repo/cookbooks/audit_test/README.md new file mode 100644 index 0000000000..31fb97a12d --- /dev/null +++ b/dev-repo/cookbooks/audit_test/README.md @@ -0,0 +1,4 @@ +# audit_test + +TODO: Enter the cookbook description here. + diff --git a/dev-repo/cookbooks/audit_test/chefignore b/dev-repo/cookbooks/audit_test/chefignore new file mode 100644 index 0000000000..80dc2d20ef --- /dev/null +++ b/dev-repo/cookbooks/audit_test/chefignore @@ -0,0 +1,95 @@ +# Put files/directories that should be ignored in this file when uploading +# or sharing to the community site. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +Icon? +nohup.out +ehthumbs.db +Thumbs.db + +# SASS # +######## +.sass-cache + +# EDITORS # +########### +\#* +.#* +*~ +*.sw[a-z] +*.bak +REVISION +TAGS* +tmtags +*_flymake.* +*_flymake +*.tmproj +.project +.settings +mkmf.log + +## COMPILED ## +############## +a.out +*.o +*.pyc +*.so +*.com +*.class +*.dll +*.exe +*/rdoc/ + +# Testing # +########### +.watchr +.rspec +spec/* +spec/fixtures/* +test/* +features/* +Guardfile +Procfile + +# SCM # +####### +.git +*/.git +.gitignore +.gitmodules +.gitconfig +.gitattributes +.svn +*/.bzr/* +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Cookbooks # +############# +CONTRIBUTING + +# Strainer # +############ +Colanderfile +Strainerfile +.colander +.strainer + +# Vagrant # +########### +.vagrant +Vagrantfile + +# Travis # +########## +.travis.yml diff --git a/dev-repo/cookbooks/audit_test/metadata.rb b/dev-repo/cookbooks/audit_test/metadata.rb new file mode 100644 index 0000000000..4a60104e92 --- /dev/null +++ b/dev-repo/cookbooks/audit_test/metadata.rb @@ -0,0 +1,8 @@ +name 'audit_test' +maintainer 'The Authors' +maintainer_email 'you@example.com' +license 'all_rights' +description 'Installs/Configures audit_test' +long_description 'Installs/Configures audit_test' +version '0.1.0' + diff --git a/dev-repo/cookbooks/audit_test/recipes/default.rb b/dev-repo/cookbooks/audit_test/recipes/default.rb new file mode 100644 index 0000000000..f02f24c2c9 --- /dev/null +++ b/dev-repo/cookbooks/audit_test/recipes/default.rb @@ -0,0 +1,11 @@ +# +# Cookbook Name:: audit_test +# Recipe:: default +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end diff --git a/dev-repo/dev-config.rb b/dev-repo/dev-config.rb new file mode 100644 index 0000000000..4ac411c832 --- /dev/null +++ b/dev-repo/dev-config.rb @@ -0,0 +1,2 @@ +cookbook_path "/home/vagrant/chef/dev-repo/cookbooks" +cache_path "/home/vagrant/.cache/chef" diff --git a/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json b/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json new file mode 100644 index 0000000000..17e7b8d173 --- /dev/null +++ b/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json @@ -0,0 +1,3 @@ +{ + "name": "chef-ubuntu-1210.vagrantup.com" +} \ No newline at end of file -- cgit v1.2.1 From d20d3aca4e6cc947322c347ba6b7a0f6260576b6 Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Fri, 14 Nov 2014 15:40:24 -0800 Subject: Make sure we don't close the output_stream after running rspec. --- .gitignore | 4 +- dev-repo/README.md | 13 --- dev-repo/cookbooks/audit_test/.gitignore | 16 ---- dev-repo/cookbooks/audit_test/.kitchen.yml | 16 ---- dev-repo/cookbooks/audit_test/Berksfile | 3 - dev-repo/cookbooks/audit_test/README.md | 4 - dev-repo/cookbooks/audit_test/chefignore | 95 ---------------------- dev-repo/cookbooks/audit_test/metadata.rb | 8 -- dev-repo/cookbooks/audit_test/recipes/default.rb | 11 --- dev-repo/dev-config.rb | 2 - dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json | 3 - kitchen-tests/.chef/client.rb | 9 +- kitchen-tests/cookbooks/audit_test/.gitignore | 16 ++++ kitchen-tests/cookbooks/audit_test/.kitchen.yml | 16 ++++ kitchen-tests/cookbooks/audit_test/Berksfile | 3 + kitchen-tests/cookbooks/audit_test/README.md | 4 + kitchen-tests/cookbooks/audit_test/chefignore | 95 ++++++++++++++++++++++ kitchen-tests/cookbooks/audit_test/metadata.rb | 8 ++ .../cookbooks/audit_test/recipes/default.rb | 11 +++ lib/chef/audit/audit_reporter.rb | 23 ++---- lib/chef/audit/rspec_formatter.rb | 19 +++++ lib/chef/audit/runner.rb | 1 - lib/chef/client.rb | 8 +- lib/chef/config.rb | 1 + lib/chef/monologger.rb | 2 - lib/chef/resource_reporter.rb | 3 +- lib/chef/run_context.rb | 13 ++- 27 files changed, 201 insertions(+), 206 deletions(-) delete mode 100644 dev-repo/README.md delete mode 100644 dev-repo/cookbooks/audit_test/.gitignore delete mode 100644 dev-repo/cookbooks/audit_test/.kitchen.yml delete mode 100644 dev-repo/cookbooks/audit_test/Berksfile delete mode 100644 dev-repo/cookbooks/audit_test/README.md delete mode 100644 dev-repo/cookbooks/audit_test/chefignore delete mode 100644 dev-repo/cookbooks/audit_test/metadata.rb delete mode 100644 dev-repo/cookbooks/audit_test/recipes/default.rb delete mode 100644 dev-repo/dev-config.rb delete mode 100644 dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json create mode 100644 kitchen-tests/cookbooks/audit_test/.gitignore create mode 100644 kitchen-tests/cookbooks/audit_test/.kitchen.yml create mode 100644 kitchen-tests/cookbooks/audit_test/Berksfile create mode 100644 kitchen-tests/cookbooks/audit_test/README.md create mode 100644 kitchen-tests/cookbooks/audit_test/chefignore create mode 100644 kitchen-tests/cookbooks/audit_test/metadata.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/default.rb create mode 100644 lib/chef/audit/rspec_formatter.rb diff --git a/.gitignore b/.gitignore index 30a3bd6531..ecba9f4030 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,5 @@ Berksfile.lock Vagrantfile .vagrant/ -# Dev Repo Local Mode Data -dev-repo/nodes/* +# Kitchen Tests Local Mode Data +kitchen-tests/nodes/* diff --git a/dev-repo/README.md b/dev-repo/README.md deleted file mode 100644 index 84fe5f77df..0000000000 --- a/dev-repo/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Chef Developer Repo - -This repository contains some basic cookbooks to test chef while you're hacking away. You can provision a VM using the kitchen configuration and run these tests like below: - -``` -$ kitchen converge chef-ubuntu-1210 -$ kitchen login chef-ubuntu-1210 -$ export PATH=/opt/chef/bin:/opt/chef/embedded/bin:$PATH -$ cd ~/chef -$ bundle install -$ bundle exec chef-client -z -o "recipe[audit_test::default]" -c dev-repo/dev-config.rb - -``` diff --git a/dev-repo/cookbooks/audit_test/.gitignore b/dev-repo/cookbooks/audit_test/.gitignore deleted file mode 100644 index ec2a890bd3..0000000000 --- a/dev-repo/cookbooks/audit_test/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -.vagrant -Berksfile.lock -*~ -*# -.#* -\#*# -.*.sw[a-z] -*.un~ - -# Bundler -Gemfile.lock -bin/* -.bundle/* - -.kitchen/ -.kitchen.local.yml diff --git a/dev-repo/cookbooks/audit_test/.kitchen.yml b/dev-repo/cookbooks/audit_test/.kitchen.yml deleted file mode 100644 index 3775752da2..0000000000 --- a/dev-repo/cookbooks/audit_test/.kitchen.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -driver: - name: vagrant - -provisioner: - name: chef_zero - -platforms: - - name: ubuntu-12.04 - - name: centos-6.4 - -suites: - - name: default - run_list: - - recipe[audit_test::default] - attributes: diff --git a/dev-repo/cookbooks/audit_test/Berksfile b/dev-repo/cookbooks/audit_test/Berksfile deleted file mode 100644 index 0ac9b78cf7..0000000000 --- a/dev-repo/cookbooks/audit_test/Berksfile +++ /dev/null @@ -1,3 +0,0 @@ -source "https://supermarket.getchef.com" - -metadata diff --git a/dev-repo/cookbooks/audit_test/README.md b/dev-repo/cookbooks/audit_test/README.md deleted file mode 100644 index 31fb97a12d..0000000000 --- a/dev-repo/cookbooks/audit_test/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# audit_test - -TODO: Enter the cookbook description here. - diff --git a/dev-repo/cookbooks/audit_test/chefignore b/dev-repo/cookbooks/audit_test/chefignore deleted file mode 100644 index 80dc2d20ef..0000000000 --- a/dev-repo/cookbooks/audit_test/chefignore +++ /dev/null @@ -1,95 +0,0 @@ -# Put files/directories that should be ignored in this file when uploading -# or sharing to the community site. -# Lines that start with '# ' are comments. - -# OS generated files # -###################### -.DS_Store -Icon? -nohup.out -ehthumbs.db -Thumbs.db - -# SASS # -######## -.sass-cache - -# EDITORS # -########### -\#* -.#* -*~ -*.sw[a-z] -*.bak -REVISION -TAGS* -tmtags -*_flymake.* -*_flymake -*.tmproj -.project -.settings -mkmf.log - -## COMPILED ## -############## -a.out -*.o -*.pyc -*.so -*.com -*.class -*.dll -*.exe -*/rdoc/ - -# Testing # -########### -.watchr -.rspec -spec/* -spec/fixtures/* -test/* -features/* -Guardfile -Procfile - -# SCM # -####### -.git -*/.git -.gitignore -.gitmodules -.gitconfig -.gitattributes -.svn -*/.bzr/* -*/.hg/* -*/.svn/* - -# Berkshelf # -############# -Berksfile -Berksfile.lock -cookbooks/* -tmp - -# Cookbooks # -############# -CONTRIBUTING - -# Strainer # -############ -Colanderfile -Strainerfile -.colander -.strainer - -# Vagrant # -########### -.vagrant -Vagrantfile - -# Travis # -########## -.travis.yml diff --git a/dev-repo/cookbooks/audit_test/metadata.rb b/dev-repo/cookbooks/audit_test/metadata.rb deleted file mode 100644 index 4a60104e92..0000000000 --- a/dev-repo/cookbooks/audit_test/metadata.rb +++ /dev/null @@ -1,8 +0,0 @@ -name 'audit_test' -maintainer 'The Authors' -maintainer_email 'you@example.com' -license 'all_rights' -description 'Installs/Configures audit_test' -long_description 'Installs/Configures audit_test' -version '0.1.0' - diff --git a/dev-repo/cookbooks/audit_test/recipes/default.rb b/dev-repo/cookbooks/audit_test/recipes/default.rb deleted file mode 100644 index f02f24c2c9..0000000000 --- a/dev-repo/cookbooks/audit_test/recipes/default.rb +++ /dev/null @@ -1,11 +0,0 @@ -# -# Cookbook Name:: audit_test -# Recipe:: default -# -# Copyright (c) 2014 The Authors, All Rights Reserved. - -controls "basic control" do - it "should pass" do - expect(2 - 2).to eq(0) - end -end diff --git a/dev-repo/dev-config.rb b/dev-repo/dev-config.rb deleted file mode 100644 index 4ac411c832..0000000000 --- a/dev-repo/dev-config.rb +++ /dev/null @@ -1,2 +0,0 @@ -cookbook_path "/home/vagrant/chef/dev-repo/cookbooks" -cache_path "/home/vagrant/.cache/chef" diff --git a/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json b/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json deleted file mode 100644 index 17e7b8d173..0000000000 --- a/dev-repo/nodes/chef-ubuntu-1210.vagrantup.com.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "name": "chef-ubuntu-1210.vagrantup.com" -} \ No newline at end of file diff --git a/kitchen-tests/.chef/client.rb b/kitchen-tests/.chef/client.rb index 5eb200a939..98f773d691 100644 --- a/kitchen-tests/.chef/client.rb +++ b/kitchen-tests/.chef/client.rb @@ -1,7 +1,8 @@ -chef_dir = File.expand_path(File.dirame(__FILE__)) -repo_dir = File.expand_path(Fild.join(chef_dir, '..')) +chef_dir = File.expand_path(File.dirname(__FILE__)) +repo_dir = File.expand_path(File.join(chef_dir, '..')) -log_level :info +log_level :info chef_repo_path repo_dir -local_mode true +local_mode true +cache_path "#{ENV['HOME']}/.cache/chef" diff --git a/kitchen-tests/cookbooks/audit_test/.gitignore b/kitchen-tests/cookbooks/audit_test/.gitignore new file mode 100644 index 0000000000..ec2a890bd3 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/.gitignore @@ -0,0 +1,16 @@ +.vagrant +Berksfile.lock +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +bin/* +.bundle/* + +.kitchen/ +.kitchen.local.yml diff --git a/kitchen-tests/cookbooks/audit_test/.kitchen.yml b/kitchen-tests/cookbooks/audit_test/.kitchen.yml new file mode 100644 index 0000000000..3775752da2 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/.kitchen.yml @@ -0,0 +1,16 @@ +--- +driver: + name: vagrant + +provisioner: + name: chef_zero + +platforms: + - name: ubuntu-12.04 + - name: centos-6.4 + +suites: + - name: default + run_list: + - recipe[audit_test::default] + attributes: diff --git a/kitchen-tests/cookbooks/audit_test/Berksfile b/kitchen-tests/cookbooks/audit_test/Berksfile new file mode 100644 index 0000000000..0ac9b78cf7 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/Berksfile @@ -0,0 +1,3 @@ +source "https://supermarket.getchef.com" + +metadata diff --git a/kitchen-tests/cookbooks/audit_test/README.md b/kitchen-tests/cookbooks/audit_test/README.md new file mode 100644 index 0000000000..31fb97a12d --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/README.md @@ -0,0 +1,4 @@ +# audit_test + +TODO: Enter the cookbook description here. + diff --git a/kitchen-tests/cookbooks/audit_test/chefignore b/kitchen-tests/cookbooks/audit_test/chefignore new file mode 100644 index 0000000000..80dc2d20ef --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/chefignore @@ -0,0 +1,95 @@ +# Put files/directories that should be ignored in this file when uploading +# or sharing to the community site. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +Icon? +nohup.out +ehthumbs.db +Thumbs.db + +# SASS # +######## +.sass-cache + +# EDITORS # +########### +\#* +.#* +*~ +*.sw[a-z] +*.bak +REVISION +TAGS* +tmtags +*_flymake.* +*_flymake +*.tmproj +.project +.settings +mkmf.log + +## COMPILED ## +############## +a.out +*.o +*.pyc +*.so +*.com +*.class +*.dll +*.exe +*/rdoc/ + +# Testing # +########### +.watchr +.rspec +spec/* +spec/fixtures/* +test/* +features/* +Guardfile +Procfile + +# SCM # +####### +.git +*/.git +.gitignore +.gitmodules +.gitconfig +.gitattributes +.svn +*/.bzr/* +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Cookbooks # +############# +CONTRIBUTING + +# Strainer # +############ +Colanderfile +Strainerfile +.colander +.strainer + +# Vagrant # +########### +.vagrant +Vagrantfile + +# Travis # +########## +.travis.yml diff --git a/kitchen-tests/cookbooks/audit_test/metadata.rb b/kitchen-tests/cookbooks/audit_test/metadata.rb new file mode 100644 index 0000000000..4a60104e92 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/metadata.rb @@ -0,0 +1,8 @@ +name 'audit_test' +maintainer 'The Authors' +maintainer_email 'you@example.com' +license 'all_rights' +description 'Installs/Configures audit_test' +long_description 'Installs/Configures audit_test' +version '0.1.0' + diff --git a/kitchen-tests/cookbooks/audit_test/recipes/default.rb b/kitchen-tests/cookbooks/audit_test/recipes/default.rb new file mode 100644 index 0000000000..f02f24c2c9 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/default.rb @@ -0,0 +1,11 @@ +# +# Cookbook Name:: audit_test +# Recipe:: default +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index b1c9d30bfc..a671ce2221 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -53,15 +53,9 @@ class Chef post_auditing_data end - # If the audit phase failed, its because there was some kind of error in the framework - # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. def audit_phase_failed(error) - # The stacktrace information has already been logged elsewhere - Chef::Log.error("Audit Reporter failed - sending error to server with available example information") - ordered_control_groups.each do |name, control_group| - audit_data.add_control_group(control_group) - end - post_auditing_data(error) + # TODO + raise error end def control_group_started(name) @@ -87,16 +81,13 @@ class Chef private - def post_auditing_data(error = nil) + def post_auditing_data if auditing_enabled? - audit_history_url = "controls" - Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") + node_name = audit_data.node_name + run_id = audit_data.run_id + audit_history_url = "audits/nodes/#{node_name}/runs/#{run_id}" + Chef::Log.info("Sending audit report (run-id: #{run_id})") run_data = audit_data.to_hash - - if error - run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" - end - Chef::Log.debug run_data.inspect compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data)) Chef::Log.debug("Sending compressed audit data...") diff --git a/lib/chef/audit/rspec_formatter.rb b/lib/chef/audit/rspec_formatter.rb new file mode 100644 index 0000000000..990c1cd780 --- /dev/null +++ b/lib/chef/audit/rspec_formatter.rb @@ -0,0 +1,19 @@ +require 'rspec/core' + +class Chef + class Audit + class RspecFormatter < RSpec::Core::Formatters::DocumentationFormatter + RSpec::Core::Formatters.register self, :close + + # @api public + # + # Invoked at the very end, `close` allows the formatter to clean + # up resources, e.g. open streams, etc. + # + # @param _notification [NullNotification] (Ignored) + def close(_notification) + # Normally Rspec closes the streams it's given. We don't want it for Chef. + end + end + end +end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 4059741359..e20c8b3810 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -71,7 +71,6 @@ class Chef configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR))) configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)")) configuration.color = Chef::Config[:color] - configuration.expose_dsl_globally = false add_formatters disable_should_syntax diff --git a/lib/chef/client.rb b/lib/chef/client.rb index b27a2b693d..8cadd43878 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -330,7 +330,6 @@ class Chef runner.converge @events.converge_complete rescue Exception => e - Chef::Log.error("Converge failed with error message #{e.message}") @events.converge_failed(e) converge_exception = e end @@ -351,16 +350,15 @@ class Chef converge_exception end + # TODO are failed audits going to raise exceptions, or only be handled by the reporters? def run_audits(run_context) audit_exception = nil begin @events.audit_phase_start(run_status) - Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run @events.audit_phase_complete rescue Exception => e - Chef::Log.error("Audit phase failed with error message #{e.message}") @events.audit_phase_failed(e) audit_exception = e end @@ -441,8 +439,8 @@ class Chef run_context = setup_run_context - converge_error = converge_and_save(run_context) unless (Chef::Config[:audit_mode] == true) - audit_error = run_audits(run_context) unless (Chef::Config[:audit_mode] == false) + converge_error = converge_and_save(run_context) + audit_error = run_audits(run_context) if converge_error || audit_error e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) diff --git a/lib/chef/config.rb b/lib/chef/config.rb index 4b83a0eca3..be31be937a 100644 --- a/lib/chef/config.rb +++ b/lib/chef/config.rb @@ -320,6 +320,7 @@ class Chef default :ez, false default :enable_reporting, true default :enable_reporting_url_fatals, false + default :audit_mode, nil # Policyfile is an experimental feature where a node gets its run list and # cookbook version set from a single document on the server instead of diff --git a/lib/chef/monologger.rb b/lib/chef/monologger.rb index 464b21bdd3..f7d226f82e 100644 --- a/lib/chef/monologger.rb +++ b/lib/chef/monologger.rb @@ -1,5 +1,4 @@ require 'logger' - require 'pp' #== MonoLogger @@ -89,4 +88,3 @@ class MonoLogger < Logger end - diff --git a/lib/chef/resource_reporter.rb b/lib/chef/resource_reporter.rb index 1816fc857d..a673f4aa58 100644 --- a/lib/chef/resource_reporter.rb +++ b/lib/chef/resource_reporter.rb @@ -20,7 +20,8 @@ # require 'uri' -require 'securerandom' +require 'zlib' +require 'chef/monkey_patches/securerandom' require 'chef/event_dispatch/base' class Chef diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 0999ae57c1..41fd11e6eb 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -18,6 +18,7 @@ # limitations under the License. require 'chef/resource_collection' +require 'chef/provider_resolver' require 'chef/cookbook_version' require 'chef/node' require 'chef/role' @@ -53,6 +54,9 @@ class Chef # The list of control groups to execute during the audit phase attr_accessor :controls_groups + # Chef::ProviderResolver for this run + attr_accessor :provider_resolver + # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. attr_accessor :immediate_notification_collection @@ -87,6 +91,7 @@ class Chef @node.run_context = self @cookbook_compiler = nil + @provider_resolver = Chef::ProviderResolver.new(@node) end # Triggers the compile phase of the chef run. Implemented by @@ -104,7 +109,7 @@ class Chef if nr.instance_of?(Chef::Resource) @immediate_notification_collection[nr.name] << notification else - @immediate_notification_collection[nr.declared_key] << notification + @immediate_notification_collection[nr.to_s] << notification end end @@ -115,7 +120,7 @@ class Chef if nr.instance_of?(Chef::Resource) @delayed_notification_collection[nr.name] << notification else - @delayed_notification_collection[nr.declared_key] << notification + @delayed_notification_collection[nr.to_s] << notification end end @@ -123,7 +128,7 @@ class Chef if resource.instance_of?(Chef::Resource) return @immediate_notification_collection[resource.name] else - return @immediate_notification_collection[resource.declared_key] + return @immediate_notification_collection[resource.to_s] end end @@ -131,7 +136,7 @@ class Chef if resource.instance_of?(Chef::Resource) return @delayed_notification_collection[resource.name] else - return @delayed_notification_collection[resource.declared_key] + return @delayed_notification_collection[resource.to_s] end end -- cgit v1.2.1 From c6732b92c8b501800c9009dd2f4d04042f9bca9c Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Mon, 10 Nov 2014 15:08:25 -0800 Subject: Sending to correct server URL --- lib/chef/audit/audit_reporter.rb | 12 ++++++------ lib/chef/audit/runner.rb | 1 + lib/chef/client.rb | 8 +++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index a671ce2221..6cbab82acb 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -53,9 +53,11 @@ class Chef post_auditing_data end + # If the audit phase failed, its because there was some kind of error in the framework + # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. def audit_phase_failed(error) - # TODO - raise error + # The stacktrace information has already been logged elsewhere + Chef::Log.error("Audit Reporter failed - not sending any auditing information to the server") end def control_group_started(name) @@ -83,10 +85,8 @@ class Chef def post_auditing_data if auditing_enabled? - node_name = audit_data.node_name - run_id = audit_data.run_id - audit_history_url = "audits/nodes/#{node_name}/runs/#{run_id}" - Chef::Log.info("Sending audit report (run-id: #{run_id})") + audit_history_url = "controls" + Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") run_data = audit_data.to_hash Chef::Log.debug run_data.inspect compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data)) diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index e20c8b3810..4059741359 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -71,6 +71,7 @@ class Chef configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR))) configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)")) configuration.color = Chef::Config[:color] + configuration.expose_dsl_globally = false add_formatters disable_should_syntax diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 8cadd43878..b27a2b693d 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -330,6 +330,7 @@ class Chef runner.converge @events.converge_complete rescue Exception => e + Chef::Log.error("Converge failed with error message #{e.message}") @events.converge_failed(e) converge_exception = e end @@ -350,15 +351,16 @@ class Chef converge_exception end - # TODO are failed audits going to raise exceptions, or only be handled by the reporters? def run_audits(run_context) audit_exception = nil begin @events.audit_phase_start(run_status) + Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run @events.audit_phase_complete rescue Exception => e + Chef::Log.error("Audit phase failed with error message #{e.message}") @events.audit_phase_failed(e) audit_exception = e end @@ -439,8 +441,8 @@ class Chef run_context = setup_run_context - converge_error = converge_and_save(run_context) - audit_error = run_audits(run_context) + converge_error = converge_and_save(run_context) unless (Chef::Config[:audit_mode] == true) + audit_error = run_audits(run_context) unless (Chef::Config[:audit_mode] == false) if converge_error || audit_error e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) -- cgit v1.2.1 From 8efee3e8ed41dc7cd4ae0b4a1664467dd403346d Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 12 Nov 2014 12:53:52 -0800 Subject: Adding uncaught error information --- lib/chef/audit/audit_reporter.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 6cbab82acb..b1c9d30bfc 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -57,7 +57,11 @@ class Chef # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. def audit_phase_failed(error) # The stacktrace information has already been logged elsewhere - Chef::Log.error("Audit Reporter failed - not sending any auditing information to the server") + Chef::Log.error("Audit Reporter failed - sending error to server with available example information") + ordered_control_groups.each do |name, control_group| + audit_data.add_control_group(control_group) + end + post_auditing_data(error) end def control_group_started(name) @@ -83,11 +87,16 @@ class Chef private - def post_auditing_data + def post_auditing_data(error = nil) if auditing_enabled? audit_history_url = "controls" Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") run_data = audit_data.to_hash + + if error + run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" + end + Chef::Log.debug run_data.inspect compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data)) Chef::Log.debug("Sending compressed audit data...") -- cgit v1.2.1 From 772232776ed10465708d1ddab7c7238a199f6199 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Fri, 31 Oct 2014 15:05:10 -0700 Subject: Adding audit mode JSON formatter First pass at DSL additions Renaming DSL methods to match the spec Creating our own example group class to simplify adding examples to the spec runner Adding logic for exceptions from converge phase not interfering with audit phase and vice-versa Adding error handling so saving node doesn't prevent us from running audit mode - decouples converge phase and audit phase more Updating for github comments Add setup phase to audit-mode. Refactor runner into own class. Fix typo tie things together Adding first round of formatter integration - STDOUT doc formatter Preparing for demo - using rspec documentation formatter for output instead of the proxy Add serverspec types and matchers. add rspec/its Add gems as core dependencies Updating with changes from demo Updating with @mcquin and @lamont comments Getting rid of unused method Wiring audit event proxy to send events correctly to the audit_reporter removing old pry debugging statement Removing unecessary todo Sending to correct server URL Fixing TODOs Adding uncaught error information --- lib/chef/audit/audit_reporter.rb | 98 ++++++++++++++++++++++++------------ lib/chef/audit/control_group_data.rb | 3 ++ lib/chef/audit/runner.rb | 3 +- lib/chef/client.rb | 7 +-- lib/chef/config.rb | 79 ++++++++++++++--------------- lib/chef/run_context.rb | 5 -- 6 files changed, 110 insertions(+), 85 deletions(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index b1c9d30bfc..5ed1f7bd52 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -19,22 +19,19 @@ require 'chef/event_dispatch/base' require 'chef/audit/control_group_data' +require 'time' class Chef class Audit class AuditReporter < EventDispatch::Base - attr_reader :rest_client, :audit_data, :ordered_control_groups - private :rest_client, :audit_data, :ordered_control_groups + attr_reader :rest_client, :audit_data, :ordered_control_groups, :run_status + private :rest_client, :audit_data, :ordered_control_groups, :run_status PROTOCOL_VERSION = '0.1.0' def initialize(rest_client) - if Chef::Config[:audit_mode] == false - @audit_enabled = false - else - @audit_enabled = true - end + @audit_enabled = Chef::Config[:audit_mode] @rest_client = rest_client # Ruby 1.9.3 and above "enumerate their values in the order that the corresponding keys were inserted." @ordered_control_groups = Hash.new @@ -43,25 +40,32 @@ class Chef def audit_phase_start(run_status) Chef::Log.debug("Audit Reporter starting") @audit_data = AuditData.new(run_status.node.name, run_status.run_id) + @run_status = run_status end def audit_phase_complete - Chef::Log.debug("Audit Reporter completed successfully without errors") + Chef::Log.debug("Audit Reporter completed successfully without errors.") ordered_control_groups.each do |name, control_group| audit_data.add_control_group(control_group) end - post_auditing_data end # If the audit phase failed, its because there was some kind of error in the framework # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. def audit_phase_failed(error) # The stacktrace information has already been logged elsewhere - Chef::Log.error("Audit Reporter failed - sending error to server with available example information") + Chef::Log.debug("Audit Reporter failed.") ordered_control_groups.each do |name, control_group| audit_data.add_control_group(control_group) end - post_auditing_data(error) + end + + def run_completed(node) + post_auditing_data + end + + def run_failed(error) + post_reporting_data(error) end def control_group_started(name) @@ -81,41 +85,65 @@ class Chef control_group.example_failure(example_data, error.message) end + # If @audit_enabled is nil or true, we want to run audits def auditing_enabled? - @audit_enabled + @audit_enabled != false end private def post_auditing_data(error = nil) - if auditing_enabled? - audit_history_url = "controls" - Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") - run_data = audit_data.to_hash + unless auditing_enabled? + Chef::Log.debug("Audit Reports are disabled. Skipping sending reports.") + return + end - if error - run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" - end + unless run_status + Chef::Log.debug("Run failed before audits were initialized, not sending audit report to server") + return + end - Chef::Log.debug run_data.inspect - compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data)) - Chef::Log.debug("Sending compressed audit data...") - # Since we're posting compressed data we can not directly call post_rest which expects JSON + audit_data.start_time = iso8601ify(run_status.start_time) + audit_data.end_time = iso8601ify(run_status.end_time) + + audit_history_url = "controls" + Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") + run_data = audit_data.to_hash + + if error + # TODO: Rather than a single string we might want to format the exception here similar to + # lib/chef/resource_reporter.rb#83 + run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" + end + + Chef::Log.debug "Audit Report:\n#{Chef::JSONCompat.to_json_pretty(run_data)}" + # Since we're posting compressed data we can not directly call post_rest which expects JSON + begin audit_url = rest_client.create_url(audit_history_url) - begin - puts Chef::JSONCompat.to_json_pretty(run_data) - rest_client.raw_http_request(:POST, audit_url, headers({'Content-Encoding' => 'gzip'}), compressed_data) - rescue StandardError => e - if e.respond_to? :response + rest_client.post(audit_url, run_data, headers) + rescue StandardError => e + if e.respond_to? :response + code = e.response.code.nil? ? "Exception Code Empty" : e.response.code + + # 404 error code is OK. This means the version of server we're running against doesn't support + # audit reporting. Don't alarm failure in this case. + if code == "404" + Chef::Log.debug("Server doesn't support audit reporting. Skipping report.") + return + else + # Save the audit report to local disk error_file = "failed-audit-data.json" Chef::FileCache.store(error_file, Chef::JSONCompat.to_json_pretty(run_data), 0640) - Chef::Log.error("Failed to post audit report to server (HTTP #{e.response.code}), saving to #{Chef::FileCache.load(error_file, false)}") - else - Chef::Log.error("Failed to post audit report to server (#{e})") + Chef::Log.error("Failed to post audit report to server. Saving report to #{Chef::FileCache.load(error_file, false)}") end + else + Chef::Log.error("Failed to post audit report to server (#{e})") + end + + if Chef::Config[:enable_reporting_url_fatals] + Chef::Log.error("Reporting fatals enabled. Aborting run.") + raise end - else - Chef::Log.debug("Server doesn't support audit report, skipping.") end end @@ -130,6 +158,10 @@ class Chef end end + def iso8601ify(time) + time.utc.iso8601.to_s + end + end end end diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index e19a6e1a15..e221ae94cc 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -4,6 +4,7 @@ class Chef class Audit class AuditData attr_reader :node_name, :run_id, :control_groups + attr_accessor :start_time, :end_time def initialize(node_name, run_id) @node_name = node_name @@ -19,6 +20,8 @@ class Chef { :node_name => node_name, :run_id => run_id, + :start_time => start_time, + :end_time => end_time, :control_groups => control_groups.collect { |c| c.to_hash } } end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 4059741359..0758dacd6d 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -18,6 +18,7 @@ require 'chef/audit' require 'chef/audit/audit_event_proxy' +require 'chef/audit/rspec_formatter' require 'chef/config' class Chef @@ -79,7 +80,7 @@ class Chef end def add_formatters - configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) + configuration.add_formatter(Chef::Audit::RspecFormatter) configuration.add_formatter(Chef::Audit::AuditEventProxy) Chef::Audit::AuditEventProxy.events = run_context.events end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index b27a2b693d..16315b8e08 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -330,7 +330,6 @@ class Chef runner.converge @events.converge_complete rescue Exception => e - Chef::Log.error("Converge failed with error message #{e.message}") @events.converge_failed(e) converge_exception = e end @@ -355,12 +354,10 @@ class Chef audit_exception = nil begin @events.audit_phase_start(run_status) - Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run @events.audit_phase_complete rescue Exception => e - Chef::Log.error("Audit phase failed with error message #{e.message}") @events.audit_phase_failed(e) audit_exception = e end @@ -441,8 +438,8 @@ class Chef run_context = setup_run_context - converge_error = converge_and_save(run_context) unless (Chef::Config[:audit_mode] == true) - audit_error = run_audits(run_context) unless (Chef::Config[:audit_mode] == false) + converge_error = converge_and_save(run_context) + audit_error = run_audits(run_context) if converge_error || audit_error e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) diff --git a/lib/chef/config.rb b/lib/chef/config.rb index be31be937a..7613a4fd4a 100644 --- a/lib/chef/config.rb +++ b/lib/chef/config.rb @@ -271,7 +271,7 @@ class Chef # * :fatal # These work as you'd expect. There is also a special `:auto` setting. # When set to :auto, Chef will auto adjust the log verbosity based on - # context. When a tty is available (usually because the user is running chef + # context. When a tty is available (usually becase the user is running chef # in a console), the log level is set to :warn, and output formatters are # used as the primary mode of output. When a tty is not available, the # logger is the primary mode of output, and the log level is set to :info @@ -317,7 +317,6 @@ class Chef default :why_run, false default :color, false default :client_fork, true - default :ez, false default :enable_reporting, true default :enable_reporting_url_fatals, false default :audit_mode, nil @@ -562,12 +561,10 @@ class Chef # used to update files. default :file_atomic_update, true - # There are 3 possible values for this configuration setting. - # true => file staging is done in the destination directory - # false => file staging is done via tempfiles under ENV['TMP'] - # :auto => file staging will try using destination directory if possible and - # will fall back to ENV['TMP'] if destination directory is not usable. - default :file_staging_uses_destdir, :auto + # If false file staging is will be done via tempfiles that are + # created under ENV['TMP'] otherwise tempfiles will be created in + # the directory that files are going to reside. + default :file_staging_uses_destdir, true # Exit if another run is in progress and the chef-client is unable to # get the lock before time expires. If nil, no timeout is enforced. (Exits @@ -627,44 +624,44 @@ class Chef # # If there is no 'locale -a' then we return 'en_US.UTF-8' since that is the most commonly # available English UTF-8 locale. However, all modern POSIXen should support 'locale -a'. - def self.guess_internal_locale - # https://github.com/opscode/chef/issues/2181 - # Some systems have the `locale -a` command, but the result has - # invalid characters for the default encoding. - # - # For example, on CentOS 6 with ENV['LANG'] = "en_US.UTF-8", - # `locale -a`.split fails with ArgumentError invalid UTF-8 encoding. - locales = shell_out_with_systems_locale!("locale -a").stdout.split - case - when locales.include?('C.UTF-8') - 'C.UTF-8' - when locales.include?('en_US.UTF-8'), locales.include?('en_US.utf8') - 'en_US.UTF-8' - when locales.include?('en.UTF-8') - 'en.UTF-8' - else - # Will match en_ZZ.UTF-8, en_ZZ.utf-8, en_ZZ.UTF8, en_ZZ.utf8 - guesses = locales.select { |l| l =~ /^en_.*UTF-?8$/i } - unless guesses.empty? - guessed_locale = guesses.first - # Transform into the form en_ZZ.UTF-8 - guessed_locale.gsub(/UTF-?8$/i, "UTF-8") + default :internal_locale do + begin + # https://github.com/opscode/chef/issues/2181 + # Some systems have the `locale -a` command, but the result has + # invalid characters for the default encoding. + # + # For example, on CentOS 6 with ENV['LANG'] = "en_US.UTF-8", + # `locale -a`.split fails with ArgumentError invalid UTF-8 encoding. + locales = shell_out_with_systems_locale("locale -a").stdout.split + case + when locales.include?('C.UTF-8') + 'C.UTF-8' + when locales.include?('en_US.UTF-8'), locales.include?('en_US.utf8') + 'en_US.UTF-8' + when locales.include?('en.UTF-8') + 'en.UTF-8' else - Chef::Log.warn "Please install an English UTF-8 locale for Chef to use, falling back to C locale and disabling UTF-8 support." - 'C' + # Will match en_ZZ.UTF-8, en_ZZ.utf-8, en_ZZ.UTF8, en_ZZ.utf8 + guesses = locales.select { |l| l =~ /^en_.*UTF-?8$/i } + unless guesses.empty? + guessed_locale = guesses.first + # Transform into the form en_ZZ.UTF-8 + guessed_locale.gsub(/UTF-?8$/i, "UTF-8") + else + Chef::Log.warn "Please install an English UTF-8 locale for Chef to use, falling back to C locale and disabling UTF-8 support." + 'C' + end end + rescue + if Chef::Platform.windows? + Chef::Log.debug "Defaulting to locale en_US.UTF-8 on Windows, until it matters that we do something else." + else + Chef::Log.debug "No usable locale -a command found, assuming you have en_US.UTF-8 installed." + end + 'en_US.UTF-8' end - rescue - if Chef::Platform.windows? - Chef::Log.debug "Defaulting to locale en_US.UTF-8 on Windows, until it matters that we do something else." - else - Chef::Log.debug "No usable locale -a command found, assuming you have en_US.UTF-8 installed." - end - 'en_US.UTF-8' end - default :internal_locale, guess_internal_locale - # Force UTF-8 Encoding, for when we fire up in the 'C' locale or other strange locales (e.g. # japanese windows encodings). If we do not do this, then knife upload will fail when a cookbook's # README.md has UTF-8 characters that do not encode in whatever surrounding encoding we have been diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 41fd11e6eb..8f7296822c 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -18,7 +18,6 @@ # limitations under the License. require 'chef/resource_collection' -require 'chef/provider_resolver' require 'chef/cookbook_version' require 'chef/node' require 'chef/role' @@ -54,9 +53,6 @@ class Chef # The list of control groups to execute during the audit phase attr_accessor :controls_groups - # Chef::ProviderResolver for this run - attr_accessor :provider_resolver - # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. attr_accessor :immediate_notification_collection @@ -91,7 +87,6 @@ class Chef @node.run_context = self @cookbook_compiler = nil - @provider_resolver = Chef::ProviderResolver.new(@node) end # Triggers the compile phase of the chef run. Implemented by -- cgit v1.2.1 From 7027bb8a8782f934a3239ee45e98fb7fc18d99bc Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 14 Nov 2014 12:36:54 -0800 Subject: Add a Chef::Audit::Controls object for 'controls' --- lib/chef/audit.rb | 10 +--- lib/chef/audit/controls.rb | 120 +++++++++++++++++++++++++++++++++++++++++++++ lib/chef/audit/runner.rb | 80 +----------------------------- lib/chef/client.rb | 16 +++++- lib/chef/dsl/audit.rb | 5 +- spec/unit/client_spec.rb | 9 ++++ 6 files changed, 146 insertions(+), 94 deletions(-) create mode 100644 lib/chef/audit/controls.rb diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb index ed8db93d96..1c28e7ea75 100644 --- a/lib/chef/audit.rb +++ b/lib/chef/audit.rb @@ -16,14 +16,6 @@ # limitations under the License. # -require 'rspec' -require 'rspec/its' - -require 'serverspec/matcher' -require 'serverspec/helper' -require 'serverspec/subject' - -require 'specinfra' - require 'chef/dsl/audit' +require 'chef/audit/controls' require 'chef/audit/runner' diff --git a/lib/chef/audit/controls.rb b/lib/chef/audit/controls.rb new file mode 100644 index 0000000000..61e01b70f2 --- /dev/null +++ b/lib/chef/audit/controls.rb @@ -0,0 +1,120 @@ +# +# Author:: Claire McQuin () +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/audit/audit_event_proxy' + +class Chef + class Audit + class Controls + + attr_reader :run_context + private :run_context + + ::RSpec::Core::ExampleGroup.define_example_group_method :control + ::RSpec::Core::ExampleGroup.define_example_group_method :__controls__ + + def initialize(run_context, *args, &block) + # To avoid cross-contamination between the configuration in our and other + # spec/spec_helper.rb files and the configuration for audits, we give + # each `controls` group its own RSpec. The files required here manipulate + # RSpec's global configuration, so we wait to load them. + # + # Additionally, name conflicts emerged between Chef's `package` and + # Serverspec's `package`. Requiring serverspec in here eliminates those + # namespace conflict. + # + # TODO: Once we have specs written for audit-mode, I'd like to play with + # which files can be moved around. I'm concerned about how much overhead + # we introduce by sequestering RSpec by `controls` group instead of in + # a common class or shared module. + require 'rspec' + require 'rspec/its' + require 'serverspec/matcher' + require 'serverspec/helper' + require 'serverspec/subject' + require 'specinfra' + + @run_context = run_context + configure + world.register(RSpec::Core::ExampleGroup.__controls__(*args, &block)) + end + + def run + # The first parameter passed to RSpec::Core::Runner.new + # is an instance of RSpec::Core::ConfigurationOptions, which is + # responsible for processing command line options passed through rspec. + # This then gets merged with the configuration. We'll just communicate + # directly with the Configuration here. + RSpec::Core::Runner.new(nil, configuration, world).run_specs(world.ordered_example_groups) + end + + private + def configuration + RSpec.configuration + end + + def world + RSpec.world + end + + # Sets up where output and error streams should stream to, adds formatters + # for people-friendly output of audit results and json for reporting. Also + # configures expectation frameworks. + def configure + # We're setting the output stream, but that will only be used for error situations + # Our formatter forwards events to the Chef event message bus + # TODO so some testing to see if these output to a log file - we probably need + # to register these before any formatters are added. + configuration.output_stream = Chef::Config[:log_location] + configuration.error_stream = Chef::Config[:log_location] + + add_formatters + disable_should_syntax + configure_specinfra + end + + def add_formatters + configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) + configuration.add_formatter(Chef::Audit::AuditEventProxy) + Chef::Audit::AuditEventProxy.events = run_context.events + end + + # Explicitly disable :should syntax. + # + # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it + # in audits. If :should is used in an audit, the audit will fail with error + # message "undefined method `:should`" rather than issue a deprecation warning. + # + # This can be removed when :should is fully removed from RSpec. + def disable_should_syntax + RSpec.configure do |config| + config.expect_with :rspec do |c| + c.syntax = :expect + end + end + end + + def configure_specinfra + # TODO: We may need to change this based on operating system (there is a + # powershell backend) or roll our own. + Specinfra.configuration.backend = :exec + end + + end + end +end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 0758dacd6d..bd8774b9c5 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -16,9 +16,6 @@ # limitations under the License. # -require 'chef/audit' -require 'chef/audit/audit_event_proxy' -require 'chef/audit/rspec_formatter' require 'chef/config' class Chef @@ -33,83 +30,8 @@ class Chef end def run - setup - register_controls_groups - - # The first parameter passed to RSpec::Core::Runner.new - # is an instance of RSpec::Core::ConfigurationOptions, which is - # responsible for processing command line options passed through rspec. - # This then gets merged with the configuration. We'll just communicate - # directly with the Configuration here. - audit_runner = RSpec::Core::Runner.new(nil, configuration, world) - audit_runner.run_specs(world.ordered_example_groups) - end - - private - - # RSpec configuration and world objects are heavy, so let's wait until - # we actually need them. - def configuration - RSpec.configuration - end - - def world - RSpec.world - end - - # Configure audits before run. - # Sets up where output and error streams should stream to, adds formatters - # for people-friendly output of audit results and json for reporting. Also - # configures expectation frameworks. - def setup - # We're setting the output stream, but that will only be used for error situations - # Our formatter forwards events to the Chef event message bus - # TODO so some testing to see if these output to a log file - we probably need - # to register these before any formatters are added. - configuration.output_stream = Chef::Config[:log_location] - configuration.error_stream = Chef::Config[:log_location] - # TODO im pretty sure I only need this because im running locally in rvmsudo - configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR))) - configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)")) - configuration.color = Chef::Config[:color] - configuration.expose_dsl_globally = false - - add_formatters - disable_should_syntax - configure_specinfra + run_context.controls_groups.each { |ctls_grp| ctls_grp.run } end - - def add_formatters - configuration.add_formatter(Chef::Audit::RspecFormatter) - configuration.add_formatter(Chef::Audit::AuditEventProxy) - Chef::Audit::AuditEventProxy.events = run_context.events - end - - # Explicitly disable :should syntax. - # - # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it - # in audits. If :should is used in an audit, the audit will fail with error - # message "undefined method `:should`" rather than issue a deprecation warning. - # - # This can be removed when :should is fully removed from RSpec. - def disable_should_syntax - configuration.expect_with :rspec do |c| - c.syntax = :expect - end - end - - def configure_specinfra - # TODO: We may need to change this based on operating system (there is a - # powershell backend) or roll our own. - Specinfra.configuration.backend = :exec - end - - # Register each controls group with the world, which will handle - # the ordering of the audits that will be run. - def register_controls_groups - run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) } - end - end end end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 16315b8e08..c7ddded1ff 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -330,6 +330,7 @@ class Chef runner.converge @events.converge_complete rescue Exception => e + Chef::Log.error("Converge failed with error message #{e.message}") @events.converge_failed(e) converge_exception = e end @@ -354,10 +355,12 @@ class Chef audit_exception = nil begin @events.audit_phase_start(run_status) + Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run @events.audit_phase_complete rescue Exception => e + Chef::Log.error("Audit phase failed with error message #{e.message}") @events.audit_phase_failed(e) audit_exception = e end @@ -438,8 +441,17 @@ class Chef run_context = setup_run_context - converge_error = converge_and_save(run_context) - audit_error = run_audits(run_context) + unless Chef::Config[:audit_mode] == true + converge_error = converge_and_save(run_context) + else + Chef::Log.debug("Skipping converge. Chef is configured to run audits only.") + end + + unless Chef::Config[:audit_mode] == false + audit_error = run_audits(run_context) + else + Chef::Log.debug("Skipping audits. Chef is configured to converge the node only.") + end if converge_error || audit_error e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error) diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 1849b65633..520a667cd4 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -23,16 +23,13 @@ class Chef module Audit # Can encompass tests in a `control` block or `describe` block - ::RSpec::Core::ExampleGroup.define_example_group_method :control - ::RSpec::Core::ExampleGroup.define_example_group_method :__controls__ - # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples def controls(*args, &block) raise ::Chef::Exceptions::NoAuditsProvided unless block name = args[0] raise AuditNameMissing if name.nil? || name.empty? - run_context.controls_groups << ::RSpec::Core::ExampleGroup.__controls__(*args, &block) + run_context.controls_groups << Chef::Audit::Controls.new(run_context, args, &block) end end diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index 10958d628c..eb13efbf76 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -192,6 +192,7 @@ describe Chef::Client do let(:http_cookbook_sync) { double("Chef::REST (cookbook sync)") } let(:http_node_save) { double("Chef::REST (node save)") } let(:runner) { double("Chef::Runner") } + let(:audit_runner) { double("Chef::Audit::Runner") } let(:api_client_exists?) { false } @@ -253,6 +254,13 @@ describe Chef::Client do expect_any_instance_of(Chef::ResourceReporter).to receive(:run_completed) end + def stub_for_audit + expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner) + expect(audit_runner).to receive(:run).and_return(true) + + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:audit_phase_complete) + end + def stub_for_node_save allow(node).to receive(:data_for_save).and_return(node.for_json) @@ -282,6 +290,7 @@ describe Chef::Client do stub_for_node_load stub_for_sync_cookbooks stub_for_converge + stub_for_audit stub_for_node_save stub_for_run end -- cgit v1.2.1 From 246c9dc60e1bed3b237b2454da445c3ef5c92b7e Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 21 Nov 2014 10:05:55 -0800 Subject: Use a central runner --- lib/chef/audit.rb | 21 -------- lib/chef/audit/controls.rb | 120 --------------------------------------------- lib/chef/audit/runner.rb | 62 ++++++++++++++++++++++- lib/chef/client.rb | 2 +- lib/chef/dsl/audit.rb | 4 +- lib/chef/run_context.rb | 4 +- 6 files changed, 65 insertions(+), 148 deletions(-) delete mode 100644 lib/chef/audit.rb delete mode 100644 lib/chef/audit/controls.rb diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb deleted file mode 100644 index 1c28e7ea75..0000000000 --- a/lib/chef/audit.rb +++ /dev/null @@ -1,21 +0,0 @@ -# -# Author:: Claire McQuin () -# Copyright:: Copyright (c) 2014 Chef Software, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/dsl/audit' -require 'chef/audit/controls' -require 'chef/audit/runner' diff --git a/lib/chef/audit/controls.rb b/lib/chef/audit/controls.rb deleted file mode 100644 index 61e01b70f2..0000000000 --- a/lib/chef/audit/controls.rb +++ /dev/null @@ -1,120 +0,0 @@ -# -# Author:: Claire McQuin () -# Copyright:: Copyright (c) 2014 Chef Software, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'chef/audit/audit_event_proxy' - -class Chef - class Audit - class Controls - - attr_reader :run_context - private :run_context - - ::RSpec::Core::ExampleGroup.define_example_group_method :control - ::RSpec::Core::ExampleGroup.define_example_group_method :__controls__ - - def initialize(run_context, *args, &block) - # To avoid cross-contamination between the configuration in our and other - # spec/spec_helper.rb files and the configuration for audits, we give - # each `controls` group its own RSpec. The files required here manipulate - # RSpec's global configuration, so we wait to load them. - # - # Additionally, name conflicts emerged between Chef's `package` and - # Serverspec's `package`. Requiring serverspec in here eliminates those - # namespace conflict. - # - # TODO: Once we have specs written for audit-mode, I'd like to play with - # which files can be moved around. I'm concerned about how much overhead - # we introduce by sequestering RSpec by `controls` group instead of in - # a common class or shared module. - require 'rspec' - require 'rspec/its' - require 'serverspec/matcher' - require 'serverspec/helper' - require 'serverspec/subject' - require 'specinfra' - - @run_context = run_context - configure - world.register(RSpec::Core::ExampleGroup.__controls__(*args, &block)) - end - - def run - # The first parameter passed to RSpec::Core::Runner.new - # is an instance of RSpec::Core::ConfigurationOptions, which is - # responsible for processing command line options passed through rspec. - # This then gets merged with the configuration. We'll just communicate - # directly with the Configuration here. - RSpec::Core::Runner.new(nil, configuration, world).run_specs(world.ordered_example_groups) - end - - private - def configuration - RSpec.configuration - end - - def world - RSpec.world - end - - # Sets up where output and error streams should stream to, adds formatters - # for people-friendly output of audit results and json for reporting. Also - # configures expectation frameworks. - def configure - # We're setting the output stream, but that will only be used for error situations - # Our formatter forwards events to the Chef event message bus - # TODO so some testing to see if these output to a log file - we probably need - # to register these before any formatters are added. - configuration.output_stream = Chef::Config[:log_location] - configuration.error_stream = Chef::Config[:log_location] - - add_formatters - disable_should_syntax - configure_specinfra - end - - def add_formatters - configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter) - configuration.add_formatter(Chef::Audit::AuditEventProxy) - Chef::Audit::AuditEventProxy.events = run_context.events - end - - # Explicitly disable :should syntax. - # - # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it - # in audits. If :should is used in an audit, the audit will fail with error - # message "undefined method `:should`" rather than issue a deprecation warning. - # - # This can be removed when :should is fully removed from RSpec. - def disable_should_syntax - RSpec.configure do |config| - config.expect_with :rspec do |c| - c.syntax = :expect - end - end - end - - def configure_specinfra - # TODO: We may need to change this based on operating system (there is a - # powershell backend) or roll our own. - Specinfra.configuration.backend = :exec - end - - end - end -end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index bd8774b9c5..a150336569 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -30,8 +30,68 @@ class Chef end def run - run_context.controls_groups.each { |ctls_grp| ctls_grp.run } + setup + register_controls + runner = RSpec::Core::Runner.new(nil) + runner.run_specs(RSpec.world.ordered_example_groups) end + + private + def setup + require_deps + set_streams + add_formatters + disable_should_syntax + configure_specinfra + add_example_group_methods + end + + def register_controls + run_context.controls.each do |name, group| + ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group[:args], &group[:block]) + RSpec.world.register(ctl_grp) + end + end + + def require_deps + require 'rspec' + require 'rspec/its' + require 'specinfra' + require 'serverspec/helper' + require 'serverspec/matcher' + require 'serverspec/subject' + require 'chef/audit/audit_event_proxy' + require 'chef/audit/rspec_formatter' + end + + def set_streams + RSpec.configuration.output_stream = Chef::Config[:log_location] + RSpec.configuration.error_stream = Chef::Config[:log_location] + end + + def add_formatters + RSpec.configuration.add_formatter(Chef::Audit::AuditEventProxy) + RSpec.configuration.add_formatter(Chef::Audit::RspecFormatter) + Chef::Audit::AuditEventProxy.events = run_context.events + end + + def disable_should_syntax + RSpec.configure do |config| + config.expect_with :rspec do |c| + c.syntax = :expect + end + end + end + + def configure_specinfra + Specinfra.configuration.backend = :exec + end + + def add_example_group_methods + RSpec::Core::ExampleGroup.define_example_group_method :__controls__ + RSpec::Core::ExampleGroup.define_example_group_method :control + end + end end end diff --git a/lib/chef/client.rb b/lib/chef/client.rb index c7ddded1ff..9e1d2dc207 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -25,7 +25,7 @@ require 'chef/log' require 'chef/rest' require 'chef/api_client' require 'chef/api_client/registration' -require 'chef/audit' +require 'chef/audit/runner' require 'chef/node' require 'chef/role' require 'chef/file_cache' diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 520a667cd4..44c0c56fac 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -16,8 +16,6 @@ # limitations under the License. # -require 'rspec/core' - class Chef module DSL module Audit @@ -29,7 +27,7 @@ class Chef name = args[0] raise AuditNameMissing if name.nil? || name.empty? - run_context.controls_groups << Chef::Audit::Controls.new(run_context, args, &block) + run_context.controls[:name] = { :args => args, :block => block } end end diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 8f7296822c..a724789d3c 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -51,7 +51,7 @@ class Chef attr_accessor :resource_collection # The list of control groups to execute during the audit phase - attr_accessor :controls_groups + attr_accessor :controls # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. @@ -76,7 +76,7 @@ class Chef @node = node @cookbook_collection = cookbook_collection @resource_collection = Chef::ResourceCollection.new - @controls_groups = [] + @controls = {} @immediate_notification_collection = Hash.new {|h,k| h[k] = []} @delayed_notification_collection = Hash.new {|h,k| h[k] = []} @definitions = Hash.new -- cgit v1.2.1 From 694b24cf15b287242e842d322a3eb0db901879e8 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Wed, 19 Nov 2014 16:40:04 -0800 Subject: Add Chef::Exceptions to audit error --- lib/chef/audit/audit_reporter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 5ed1f7bd52..b0eb835c0c 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -70,7 +70,7 @@ class Chef def control_group_started(name) if ordered_control_groups.has_key?(name) - raise AuditControlGroupDuplicate.new(name) + raise Chef::Exceptions::AuditControlGroupDuplicate.new(name) end ordered_control_groups.store(name, ControlGroupData.new(name)) end -- cgit v1.2.1 From 2d7e99441986ca5bed15dfa1420fddb1b1185632 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Wed, 19 Nov 2014 16:40:17 -0800 Subject: Add recipe to test include_recipe with controls. --- .../cookbooks/audit_test/recipes/include_recipe.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb diff --git a/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb b/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb new file mode 100644 index 0000000000..00bdd9c9e9 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb @@ -0,0 +1,15 @@ +# +# Cookbook Name:: audit_test +# Recipe:: include_recipe +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +include_recipe "audit_test::default" + +controls "another basic control" do + it "should also pass" do + arr = [0, 0] + arr.delete(0) + expect( arr ).to be_empty + end +end -- cgit v1.2.1 From dbe6f8f8af4d10e666908bcffe13222933adfe74 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Thu, 20 Nov 2014 16:32:38 -0800 Subject: Wrap in control block until we get top-level tests working. --- kitchen-tests/cookbooks/audit_test/recipes/default.rb | 6 ++++-- kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb | 10 ++++++---- lib/chef/dsl/audit.rb | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/kitchen-tests/cookbooks/audit_test/recipes/default.rb b/kitchen-tests/cookbooks/audit_test/recipes/default.rb index f02f24c2c9..848e085125 100644 --- a/kitchen-tests/cookbooks/audit_test/recipes/default.rb +++ b/kitchen-tests/cookbooks/audit_test/recipes/default.rb @@ -5,7 +5,9 @@ # Copyright (c) 2014 The Authors, All Rights Reserved. controls "basic control" do - it "should pass" do - expect(2 - 2).to eq(0) + control "math" do + it "should pass" do + expect(2 - 2).to eq(0) + end end end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb b/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb index 00bdd9c9e9..2ead0722e1 100644 --- a/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb +++ b/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb @@ -7,9 +7,11 @@ include_recipe "audit_test::default" controls "another basic control" do - it "should also pass" do - arr = [0, 0] - arr.delete(0) - expect( arr ).to be_empty + control "math" do + it "should also pass" do + arr = [0, 0] + arr.delete(0) + expect( arr ).to be_empty + end end end diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 44c0c56fac..72a1752df2 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -27,7 +27,7 @@ class Chef name = args[0] raise AuditNameMissing if name.nil? || name.empty? - run_context.controls[:name] = { :args => args, :block => block } + run_context.controls[name] = { :args => args, :block => block } end end -- cgit v1.2.1 From f7e9523dc1c47b935c8921225bac514ca2eb6337 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 21 Nov 2014 10:25:28 -0800 Subject: Add recipes to test include_recipe, multiple controls blocks --- kitchen-tests/cookbooks/audit_test/.gitignore | 16 ---- kitchen-tests/cookbooks/audit_test/.kitchen.yml | 16 ---- kitchen-tests/cookbooks/audit_test/Berksfile | 3 - kitchen-tests/cookbooks/audit_test/README.md | 4 - kitchen-tests/cookbooks/audit_test/chefignore | 95 ---------------------- kitchen-tests/cookbooks/audit_test/metadata.rb | 8 -- .../cookbooks/audit_test/recipes/default.rb | 13 --- .../cookbooks/audit_test/recipes/include_recipe.rb | 17 ---- 8 files changed, 172 deletions(-) delete mode 100644 kitchen-tests/cookbooks/audit_test/.gitignore delete mode 100644 kitchen-tests/cookbooks/audit_test/.kitchen.yml delete mode 100644 kitchen-tests/cookbooks/audit_test/Berksfile delete mode 100644 kitchen-tests/cookbooks/audit_test/README.md delete mode 100644 kitchen-tests/cookbooks/audit_test/chefignore delete mode 100644 kitchen-tests/cookbooks/audit_test/metadata.rb delete mode 100644 kitchen-tests/cookbooks/audit_test/recipes/default.rb delete mode 100644 kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb diff --git a/kitchen-tests/cookbooks/audit_test/.gitignore b/kitchen-tests/cookbooks/audit_test/.gitignore deleted file mode 100644 index ec2a890bd3..0000000000 --- a/kitchen-tests/cookbooks/audit_test/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -.vagrant -Berksfile.lock -*~ -*# -.#* -\#*# -.*.sw[a-z] -*.un~ - -# Bundler -Gemfile.lock -bin/* -.bundle/* - -.kitchen/ -.kitchen.local.yml diff --git a/kitchen-tests/cookbooks/audit_test/.kitchen.yml b/kitchen-tests/cookbooks/audit_test/.kitchen.yml deleted file mode 100644 index 3775752da2..0000000000 --- a/kitchen-tests/cookbooks/audit_test/.kitchen.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -driver: - name: vagrant - -provisioner: - name: chef_zero - -platforms: - - name: ubuntu-12.04 - - name: centos-6.4 - -suites: - - name: default - run_list: - - recipe[audit_test::default] - attributes: diff --git a/kitchen-tests/cookbooks/audit_test/Berksfile b/kitchen-tests/cookbooks/audit_test/Berksfile deleted file mode 100644 index 0ac9b78cf7..0000000000 --- a/kitchen-tests/cookbooks/audit_test/Berksfile +++ /dev/null @@ -1,3 +0,0 @@ -source "https://supermarket.getchef.com" - -metadata diff --git a/kitchen-tests/cookbooks/audit_test/README.md b/kitchen-tests/cookbooks/audit_test/README.md deleted file mode 100644 index 31fb97a12d..0000000000 --- a/kitchen-tests/cookbooks/audit_test/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# audit_test - -TODO: Enter the cookbook description here. - diff --git a/kitchen-tests/cookbooks/audit_test/chefignore b/kitchen-tests/cookbooks/audit_test/chefignore deleted file mode 100644 index 80dc2d20ef..0000000000 --- a/kitchen-tests/cookbooks/audit_test/chefignore +++ /dev/null @@ -1,95 +0,0 @@ -# Put files/directories that should be ignored in this file when uploading -# or sharing to the community site. -# Lines that start with '# ' are comments. - -# OS generated files # -###################### -.DS_Store -Icon? -nohup.out -ehthumbs.db -Thumbs.db - -# SASS # -######## -.sass-cache - -# EDITORS # -########### -\#* -.#* -*~ -*.sw[a-z] -*.bak -REVISION -TAGS* -tmtags -*_flymake.* -*_flymake -*.tmproj -.project -.settings -mkmf.log - -## COMPILED ## -############## -a.out -*.o -*.pyc -*.so -*.com -*.class -*.dll -*.exe -*/rdoc/ - -# Testing # -########### -.watchr -.rspec -spec/* -spec/fixtures/* -test/* -features/* -Guardfile -Procfile - -# SCM # -####### -.git -*/.git -.gitignore -.gitmodules -.gitconfig -.gitattributes -.svn -*/.bzr/* -*/.hg/* -*/.svn/* - -# Berkshelf # -############# -Berksfile -Berksfile.lock -cookbooks/* -tmp - -# Cookbooks # -############# -CONTRIBUTING - -# Strainer # -############ -Colanderfile -Strainerfile -.colander -.strainer - -# Vagrant # -########### -.vagrant -Vagrantfile - -# Travis # -########## -.travis.yml diff --git a/kitchen-tests/cookbooks/audit_test/metadata.rb b/kitchen-tests/cookbooks/audit_test/metadata.rb deleted file mode 100644 index 4a60104e92..0000000000 --- a/kitchen-tests/cookbooks/audit_test/metadata.rb +++ /dev/null @@ -1,8 +0,0 @@ -name 'audit_test' -maintainer 'The Authors' -maintainer_email 'you@example.com' -license 'all_rights' -description 'Installs/Configures audit_test' -long_description 'Installs/Configures audit_test' -version '0.1.0' - diff --git a/kitchen-tests/cookbooks/audit_test/recipes/default.rb b/kitchen-tests/cookbooks/audit_test/recipes/default.rb deleted file mode 100644 index 848e085125..0000000000 --- a/kitchen-tests/cookbooks/audit_test/recipes/default.rb +++ /dev/null @@ -1,13 +0,0 @@ -# -# Cookbook Name:: audit_test -# Recipe:: default -# -# Copyright (c) 2014 The Authors, All Rights Reserved. - -controls "basic control" do - control "math" do - it "should pass" do - expect(2 - 2).to eq(0) - end - end -end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb b/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb deleted file mode 100644 index 2ead0722e1..0000000000 --- a/kitchen-tests/cookbooks/audit_test/recipes/include_recipe.rb +++ /dev/null @@ -1,17 +0,0 @@ -# -# Cookbook Name:: audit_test -# Recipe:: include_recipe -# -# Copyright (c) 2014 The Authors, All Rights Reserved. - -include_recipe "audit_test::default" - -controls "another basic control" do - control "math" do - it "should also pass" do - arr = [0, 0] - arr.delete(0) - expect( arr ).to be_empty - end - end -end -- cgit v1.2.1 From c4333485de9dc373ad9d846332cbd897da306ff3 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 21 Nov 2014 13:02:17 -0800 Subject: Raise error if duplicate controls block given. --- lib/chef/dsl/audit.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index 72a1752df2..e22c38f587 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -16,6 +16,8 @@ # limitations under the License. # +require 'chef/exceptions' + class Chef module DSL module Audit @@ -23,9 +25,14 @@ class Chef # Can encompass tests in a `control` block or `describe` block # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples def controls(*args, &block) - raise ::Chef::Exceptions::NoAuditsProvided unless block + raise Chef::Exceptions::NoAuditsProvided unless block + name = args[0] - raise AuditNameMissing if name.nil? || name.empty? + if name.nil? || name.empty? + raise Chef::Exceptions::AuditNameMissing + elsif run_context.controls.has_key?(name) + raise Chef::Exceptions::AuditControlGroupDuplicate.new(name) + end run_context.controls[name] = { :args => args, :block => block } end -- cgit v1.2.1 From 58e1f80100273960bf5e04c99f6447736c86420c Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 21 Nov 2014 13:05:29 -0800 Subject: Add omitted config options. --- lib/chef/audit/runner.rb | 105 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 13 deletions(-) diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index a150336569..a290dd6607 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -32,28 +32,39 @@ class Chef def run setup register_controls - runner = RSpec::Core::Runner.new(nil) - runner.run_specs(RSpec.world.ordered_example_groups) + do_run end private + # Prepare to run audits: + # - Require files + # - Configure RSpec + # - Configure Specinfra/Serverspec def setup require_deps - set_streams - add_formatters - disable_should_syntax + configure_rspec configure_specinfra - add_example_group_methods - end - - def register_controls - run_context.controls.each do |name, group| - ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group[:args], &group[:block]) - RSpec.world.register(ctl_grp) - end end + # RSpec uses a global configuration object, RSpec.configuration. We found + # there was interference between the configuration for audit-mode and + # the configuration for our own spec tests in these cases: + # 1. Specinfra and Serverspec modify RSpec.configuration when loading. + # 2. Setting output/error streams. + # 3. Adding formatters. + # 4. Defining example group aliases. + # + # Moreover, Serverspec loads its DSL methods into the global namespace, + # which causes conflicts with the Chef namespace for resources and packages. + # + # We wait until we're in the audit-phase of the chef-client run to load + # these files. This helps with the namespacing problems we saw, and + # prevents Specinfra and Serverspec from modifying the RSpec configuration + # used by our spec tests. def require_deps + # TODO: We need to figure out a way to give audits its own configuration + # object. This involves finding a way to load these files w/o them adding + # to the configuration object used by our spec tests. require 'rspec' require 'rspec/its' require 'specinfra' @@ -64,17 +75,49 @@ class Chef require 'chef/audit/rspec_formatter' end + # Configure RSpec just the way we like it: + # - Set location of error and output streams + # - Add custom audit-mode formatters + # - Explicitly disable :should syntax + # - Set :color option according to chef config + # - Disable exposure of global DSL + def configure_rspec + set_streams + add_formatters + disable_should_syntax + + RSpec.configure do |c| + c.color = Chef::Config[:color] + c.expose_dsl_globally = false + end + end + + # Set the error and output streams which audit-mode will use to report + # human-readable audit information. + # + # This should always be called before #add_formatters. RSpec won't allow + # the output stream to be changed for a formatter once the formatter has + # been added. def set_streams + # TODO: Do some testing to ensure these will output/output properly to + # a file. RSpec.configuration.output_stream = Chef::Config[:log_location] RSpec.configuration.error_stream = Chef::Config[:log_location] end + # Add formatters which we use to + # 1. Output human-readable data to the output stream, + # 2. Collect JSON data to send back to the analytics server. def add_formatters RSpec.configuration.add_formatter(Chef::Audit::AuditEventProxy) RSpec.configuration.add_formatter(Chef::Audit::RspecFormatter) Chef::Audit::AuditEventProxy.events = run_context.events end + # Audit-mode uses RSpec 3. :should syntax is deprecated by default in + # RSpec 3, so we explicitly disable it here. + # + # This can be removed once :should is removed from RSpec. def disable_should_syntax RSpec.configure do |config| config.expect_with :rspec do |c| @@ -83,15 +126,51 @@ class Chef end end + # Set up the backend for Specinfra/Serverspec. def configure_specinfra + # TODO: We may need to be clever and adjust this based on operating + # system, or make it configurable. E.g., there is a PowerShell backend, + # as well as an SSH backend. Specinfra.configuration.backend = :exec end + # Iterates through the controls registered to this run_context, builds an + # example group (RSpec::Core::ExampleGroup) object per controls, and + # registers the group with the RSpec.world. + # + # We could just store an array of example groups and not use RSpec.world, + # but it may be useful later if we decide to apply our own ordering scheme + # or use example group filters. + def register_controls + add_example_group_methods + run_context.controls.each do |name, group| + ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group[:args], &group[:block]) + RSpec.world.register(ctl_grp) + end + end + + # Add example group method aliases to RSpec. + # + # __controls__: Used internally to create example groups from the controls + # saved in the run_context. + # control: Used within the context of a controls block, like RSpec's + # describe or context. def add_example_group_methods RSpec::Core::ExampleGroup.define_example_group_method :__controls__ RSpec::Core::ExampleGroup.define_example_group_method :control end + # Run the audits! + def do_run + # RSpec::Core::Runner wants to be initialized with an + # RSpec::Core::ConfigurationOptions object, which is used to process + # command line configuration arguments. We directly fiddle with the + # internal RSpec configuration object, so we give nil here and let + # RSpec pick up its own configuration and world. + runner = RSpec::Core::Runner.new(nil) + runner.run_specs(RSpec.world.ordered_example_groups) + end + end end end -- cgit v1.2.1 From 0066549c2f45ca4ed8e1e3b3e1a8990efafa1019 Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Thu, 20 Nov 2014 16:33:38 -0800 Subject: * Some primitive recipes that check basic functionality of audit mode. * Fix a minor issue where the exception class was not being found correctly. --- .../cookbooks/audit_test/recipes/default.rb | 26 ++++++++++++++++++ .../recipes/error_duplicate_control_groups.rb | 17 ++++++++++++ .../cookbooks/audit_test/recipes/error_no_block.rb | 7 +++++ .../audit_test/recipes/error_orphan_control.rb | 13 +++++++++ .../cookbooks/audit_test/recipes/failed_specs.rb | 14 ++++++++++ .../audit_test/recipes/serverspec_collision.rb | 31 ++++++++++++++++++++++ .../audit_test/recipes/with_include_recipe.rb | 16 +++++++++++ lib/chef/audit/audit_reporter.rb | 2 +- 8 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/default.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb diff --git a/kitchen-tests/cookbooks/audit_test/recipes/default.rb b/kitchen-tests/cookbooks/audit_test/recipes/default.rb new file mode 100644 index 0000000000..4f634d73c1 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/default.rb @@ -0,0 +1,26 @@ +# +# Cookbook Name:: audit_test +# Recipe:: default +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control group" do + control "basic math" do + it "should pass" do + expect(2 - 2).to eq(0) + end + end +end + +controls "control group without top level control" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end + +controls "control group with empty control" do + control "empty" +end + +controls "empty control group with block" do +end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb new file mode 100644 index 0000000000..77a4592e9d --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb @@ -0,0 +1,17 @@ +# +# Cookbook Name:: audit_test +# Recipe:: error_duplicate_control_groups +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control group" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end + +controls "basic control group" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb new file mode 100644 index 0000000000..76a8817b5d --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb @@ -0,0 +1,7 @@ +# +# Cookbook Name:: audit_test +# Recipe:: error_no_block +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "empty control group without block" diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb new file mode 100644 index 0000000000..d74acd6c6b --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb @@ -0,0 +1,13 @@ +# +# Cookbook Name:: audit_test +# Recipe:: error_orphan_control +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control group" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end + +control "orphan control" diff --git a/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb b/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb new file mode 100644 index 0000000000..3225d3983e --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb @@ -0,0 +1,14 @@ +# +# Cookbook Name:: audit_test +# Recipe:: failed_specs +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +controls "basic control group" do + control "basic math" do + # Can not write a good control :( + it "should pass" do + expect(2 - 0).to eq(0) + end + end +end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb new file mode 100644 index 0000000000..70109d84b8 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb @@ -0,0 +1,31 @@ +# +# Cookbook Name:: audit_test +# Recipe:: serverspec_collision +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +file "/tmp/audit_test_file" do + action :create + content "Welcome to audit mode." +end + +controls "file auditing" do + describe "test file" do + it "says welcome" do + expect(file("/tmp/audit_test_file")).to contain("Welcome") + end + end +end + +file "/tmp/audit_test_file_2" do + action :create + content "Bye to audit mode." +end + +controls "end file auditing" do + describe "end file" do + it "says bye" do + expect(file("/tmp/audit_test_file_2")).to contain("Bye") + end + end +end diff --git a/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb b/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb new file mode 100644 index 0000000000..ff39cde117 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb @@ -0,0 +1,16 @@ +# +# Cookbook Name:: audit_test +# Recipe:: with_include_recipe +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +include_recipe "audit_test::serverspec_collision" + +controls "basic example" do + it "should pass" do + expect(2 - 2).to eq(0) + end +end + +include_recipe "audit_test::serverspec_collision" +include_recipe "audit_test::default" diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index b0eb835c0c..af94f0968b 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -107,7 +107,7 @@ class Chef audit_data.end_time = iso8601ify(run_status.end_time) audit_history_url = "controls" - Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})") + Chef::Log.debug("Sending audit report (run-id: #{audit_data.run_id})") run_data = audit_data.to_hash if error -- cgit v1.2.1 From d223f2480cecb211a8c54edbb6ff4e7a46f9be9b Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Thu, 20 Nov 2014 17:34:49 -0800 Subject: Disable audit phase during specs in order not to break the existing rspec configuration. --- spec/spec_helper.rb | 2 ++ spec/unit/client_spec.rb | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 995be5060b..9f8cc754b7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -169,6 +169,8 @@ RSpec.configure do |config| config.before(:each) do Chef::Config.reset + + allow_any_instance_of(Chef::Audit::Runner).to receive(:run).and_return(true) end config.before(:suite) do diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index eb13efbf76..71c30ed532 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -252,6 +252,11 @@ describe Chef::Client do # updates the server with the resource history # (has its own tests, so stubbing it here.) expect_any_instance_of(Chef::ResourceReporter).to receive(:run_completed) + + # --AuditReporter#audit_phase_complete + # posts the audit data to server. + # (has its own tests, so stubbing it here.) + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:audit_phase_complete) end def stub_for_audit -- cgit v1.2.1 From b6804bbe2e8fd90a958db3685da92b3112c29536 Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Thu, 20 Nov 2014 17:56:14 -0800 Subject: Instructions on running audit mode examples. --- kitchen-tests/cookbooks/webapp/README.md | 8 +++++++- lib/chef/application/client.rb | 10 +++++++++- lib/chef/application/solo.rb | 4 ++-- lib/chef/audit/audit_reporter.rb | 3 +-- lib/chef/client.rb | 8 ++------ lib/chef/config.rb | 4 +++- spec/functional/resource/deploy_revision_spec.rb | 2 +- spec/functional/resource/git_spec.rb | 2 +- 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/kitchen-tests/cookbooks/webapp/README.md b/kitchen-tests/cookbooks/webapp/README.md index e8de6ee467..5c55542cbf 100644 --- a/kitchen-tests/cookbooks/webapp/README.md +++ b/kitchen-tests/cookbooks/webapp/README.md @@ -1,4 +1,10 @@ # webapp -TODO: Enter the cookbook description here. +This cookbook has some basic recipes to test audit mode. +In order to run these tests on your dev box: + +``` +$ bundle install +$ bundle exec chef-client -c kitchen-tests/.chef/client.rb -z -o audit_test::default +``` diff --git a/lib/chef/application/client.rb b/lib/chef/application/client.rb index 295dc2470e..b10f818cf4 100644 --- a/lib/chef/application/client.rb +++ b/lib/chef/application/client.rb @@ -241,7 +241,15 @@ class Chef::Application::Client < Chef::Application option :audit_mode, :long => "--[no-]audit-mode", :description => "If not specified, run converge and audit phase. If true, run only audit phase. If false, run only converge phase.", - :boolean => true + :boolean => true, + :proc => lambda { |set| + # Convert boolean to config options of :audit_only or :disabled + if set + :audit_only + else + :disabled + end + } IMMEDIATE_RUN_SIGNAL = "1".freeze diff --git a/lib/chef/application/solo.rb b/lib/chef/application/solo.rb index 50f7f5c5d4..c3f5444ef7 100644 --- a/lib/chef/application/solo.rb +++ b/lib/chef/application/solo.rb @@ -212,8 +212,8 @@ class Chef::Application::Solo < Chef::Application @chef_client_json = config_fetcher.fetch_json end - # If we don't specify this, solo will try to perform the audits - Chef::Config[:audit_mode] = false + # Disable auditing for solo + Chef::Config[:audit_mode] = :disabled end def setup_application diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index af94f0968b..ce4978180e 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -31,7 +31,6 @@ class Chef PROTOCOL_VERSION = '0.1.0' def initialize(rest_client) - @audit_enabled = Chef::Config[:audit_mode] @rest_client = rest_client # Ruby 1.9.3 and above "enumerate their values in the order that the corresponding keys were inserted." @ordered_control_groups = Hash.new @@ -87,7 +86,7 @@ class Chef # If @audit_enabled is nil or true, we want to run audits def auditing_enabled? - @audit_enabled != false + Chef::Config[:audit_mode] != :disabled end private diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 9e1d2dc207..aa0d6722fe 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -441,16 +441,12 @@ class Chef run_context = setup_run_context - unless Chef::Config[:audit_mode] == true + if Chef::Config[:audit_mode] != :audit_only converge_error = converge_and_save(run_context) - else - Chef::Log.debug("Skipping converge. Chef is configured to run audits only.") end - unless Chef::Config[:audit_mode] == false + if Chef::Config[:audit_mode] != :disabled audit_error = run_audits(run_context) - else - Chef::Log.debug("Skipping audits. Chef is configured to converge the node only.") end if converge_error || audit_error diff --git a/lib/chef/config.rb b/lib/chef/config.rb index 7613a4fd4a..19fa272100 100644 --- a/lib/chef/config.rb +++ b/lib/chef/config.rb @@ -319,7 +319,9 @@ class Chef default :client_fork, true default :enable_reporting, true default :enable_reporting_url_fatals, false - default :audit_mode, nil + # Possible values for :audit_mode + # :enabled, :disabled, :audit_only, + default :audit_mode, :enabled # Policyfile is an experimental feature where a node gets its run list and # cookbook version set from a single document on the server instead of diff --git a/spec/functional/resource/deploy_revision_spec.rb b/spec/functional/resource/deploy_revision_spec.rb index 7bc3da9a05..05a21c48c7 100644 --- a/spec/functional/resource/deploy_revision_spec.rb +++ b/spec/functional/resource/deploy_revision_spec.rb @@ -45,7 +45,7 @@ describe Chef::Resource::DeployRevision, :unix_only => true do before(:all) do @ohai = Ohai::System.new - @ohai.all_plugins("os") + @ohai.all_plugins(@ohai.all_plugins(["platform", "os"])) end let(:node) do diff --git a/spec/functional/resource/git_spec.rb b/spec/functional/resource/git_spec.rb index 4f462b7cb6..9d3b82f19e 100644 --- a/spec/functional/resource/git_spec.rb +++ b/spec/functional/resource/git_spec.rb @@ -92,7 +92,7 @@ E before(:all) do @ohai = Ohai::System.new - @ohai.all_plugins("os") + @ohai.all_plugins(["platform", "os"]) end context "working with pathes with special characters" do -- cgit v1.2.1 From 2b3c252f5cad689debce640f01db9705f3c57d22 Mon Sep 17 00:00:00 2001 From: Serdar Sutay Date: Fri, 21 Nov 2014 10:10:56 -0800 Subject: Test including supported serverspec helpers. Updates per PR comments. --- .travis.yml | 1 + kitchen-tests/cookbooks/audit_test/.gitignore | 16 +++ kitchen-tests/cookbooks/audit_test/.kitchen.yml | 16 +++ kitchen-tests/cookbooks/audit_test/Berksfile | 3 + kitchen-tests/cookbooks/audit_test/README.md | 12 +++ kitchen-tests/cookbooks/audit_test/chefignore | 95 ++++++++++++++++ kitchen-tests/cookbooks/audit_test/metadata.rb | 8 ++ .../audit_test/recipes/serverspec_support.rb | 37 +++++++ kitchen-tests/cookbooks/webapp/README.md | 9 +- lib/chef/audit/audit_reporter.rb | 2 +- lib/chef/formatters/doc.rb | 6 +- lib/chef/version.rb | 2 +- spec/functional/resource/deploy_revision_spec.rb | 3 +- spec/unit/client_spec.rb | 120 +++++++++++++++++---- spec/unit/dsl/audit_spec.rb | 24 +++++ 15 files changed, 317 insertions(+), 37 deletions(-) create mode 100644 kitchen-tests/cookbooks/audit_test/.gitignore create mode 100644 kitchen-tests/cookbooks/audit_test/.kitchen.yml create mode 100644 kitchen-tests/cookbooks/audit_test/Berksfile create mode 100644 kitchen-tests/cookbooks/audit_test/README.md create mode 100644 kitchen-tests/cookbooks/audit_test/chefignore create mode 100644 kitchen-tests/cookbooks/audit_test/metadata.rb create mode 100644 kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb create mode 100644 spec/unit/dsl/audit_spec.rb diff --git a/.travis.yml b/.travis.yml index 37418ab621..e9e7c2cdc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ branches: - 10-stable - 11-stable - 12-stable + - audit-mode # do not run expensive spec tests on PRs, only on branches script: " diff --git a/kitchen-tests/cookbooks/audit_test/.gitignore b/kitchen-tests/cookbooks/audit_test/.gitignore new file mode 100644 index 0000000000..ec2a890bd3 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/.gitignore @@ -0,0 +1,16 @@ +.vagrant +Berksfile.lock +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +bin/* +.bundle/* + +.kitchen/ +.kitchen.local.yml diff --git a/kitchen-tests/cookbooks/audit_test/.kitchen.yml b/kitchen-tests/cookbooks/audit_test/.kitchen.yml new file mode 100644 index 0000000000..be11e33081 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/.kitchen.yml @@ -0,0 +1,16 @@ +--- +driver: + name: vagrant + +provisioner: + name: chef_zero + +platforms: + - name: ubuntu-12.04 + - name: centos-6.5 + +suites: + - name: default + run_list: + - recipe[audit_test::default] + attributes: diff --git a/kitchen-tests/cookbooks/audit_test/Berksfile b/kitchen-tests/cookbooks/audit_test/Berksfile new file mode 100644 index 0000000000..0ac9b78cf7 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/Berksfile @@ -0,0 +1,3 @@ +source "https://supermarket.getchef.com" + +metadata diff --git a/kitchen-tests/cookbooks/audit_test/README.md b/kitchen-tests/cookbooks/audit_test/README.md new file mode 100644 index 0000000000..75e2f44808 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/README.md @@ -0,0 +1,12 @@ +# audit_test + +This cookbook has some basic recipes to test audit mode. + +In order to run these tests on your dev box: + +``` +$ bundle install +$ bundle exec chef-client -c kitchen-tests/.chef/client.rb -z -o audit_test::default -l debug +``` + +Expected JSON output for the tests will be printed to `debug` log. diff --git a/kitchen-tests/cookbooks/audit_test/chefignore b/kitchen-tests/cookbooks/audit_test/chefignore new file mode 100644 index 0000000000..80dc2d20ef --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/chefignore @@ -0,0 +1,95 @@ +# Put files/directories that should be ignored in this file when uploading +# or sharing to the community site. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +Icon? +nohup.out +ehthumbs.db +Thumbs.db + +# SASS # +######## +.sass-cache + +# EDITORS # +########### +\#* +.#* +*~ +*.sw[a-z] +*.bak +REVISION +TAGS* +tmtags +*_flymake.* +*_flymake +*.tmproj +.project +.settings +mkmf.log + +## COMPILED ## +############## +a.out +*.o +*.pyc +*.so +*.com +*.class +*.dll +*.exe +*/rdoc/ + +# Testing # +########### +.watchr +.rspec +spec/* +spec/fixtures/* +test/* +features/* +Guardfile +Procfile + +# SCM # +####### +.git +*/.git +.gitignore +.gitmodules +.gitconfig +.gitattributes +.svn +*/.bzr/* +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Cookbooks # +############# +CONTRIBUTING + +# Strainer # +############ +Colanderfile +Strainerfile +.colander +.strainer + +# Vagrant # +########### +.vagrant +Vagrantfile + +# Travis # +########## +.travis.yml diff --git a/kitchen-tests/cookbooks/audit_test/metadata.rb b/kitchen-tests/cookbooks/audit_test/metadata.rb new file mode 100644 index 0000000000..4a60104e92 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/metadata.rb @@ -0,0 +1,8 @@ +name 'audit_test' +maintainer 'The Authors' +maintainer_email 'you@example.com' +license 'all_rights' +description 'Installs/Configures audit_test' +long_description 'Installs/Configures audit_test' +version '0.1.0' + diff --git a/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb new file mode 100644 index 0000000000..0396cc0de7 --- /dev/null +++ b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb @@ -0,0 +1,37 @@ +# +# Cookbook Name:: audit_test +# Recipe:: serverspec_support +# +# Copyright (c) 2014 The Authors, All Rights Reserved. + +file "/tmp/audit_test_file" do + action :create + content "Welcome to audit mode." +end + +# package "curl" do +# action :install +# end + +controls "serverspec helpers with types" do + control "file helper" do + it "says welcome" do + expect(file("/tmp/audit_test_file")).to contain("Welcome") + end + end + + control service("com.apple.CoreRAID") do + it { is_expected.to be_enabled } + it { is_expected.not_to be_running } + end + + # describe "package helper" do + # it "works" do + # expect(package("curl")).to be_installed + # end + # end + + control package("postgresql") do + it { is_expected.to_not be_installed } + end +end diff --git a/kitchen-tests/cookbooks/webapp/README.md b/kitchen-tests/cookbooks/webapp/README.md index 5c55542cbf..f19ab46735 100644 --- a/kitchen-tests/cookbooks/webapp/README.md +++ b/kitchen-tests/cookbooks/webapp/README.md @@ -1,10 +1,3 @@ # webapp -This cookbook has some basic recipes to test audit mode. - -In order to run these tests on your dev box: - -``` -$ bundle install -$ bundle exec chef-client -c kitchen-tests/.chef/client.rb -z -o audit_test::default -``` +TODO: Enter the cookbook description here. diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index ce4978180e..21ffb62829 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -64,7 +64,7 @@ class Chef end def run_failed(error) - post_reporting_data(error) + post_auditing_data(error) end def control_group_started(name) diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 09d04f3aae..99603965a9 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -163,13 +163,11 @@ class Chef # Called before audit phase starts def audit_phase_start(run_status) - puts_line "" - puts_line "++ Audit phase starting ++" + puts_line "Starting audit phase" end def audit_phase_complete - puts_line "" - puts_line "++ Audit phase ended ++ " + puts_line "Auditing complete" end def audit_phase_failed(error) diff --git a/lib/chef/version.rb b/lib/chef/version.rb index a8fc002399..30c3394c2c 100644 --- a/lib/chef/version.rb +++ b/lib/chef/version.rb @@ -17,7 +17,7 @@ class Chef CHEF_ROOT = File.dirname(File.expand_path(File.dirname(__FILE__))) - VERSION = '12.1.0.dev.0' + VERSION = '12.2.0.alpha.0' end # diff --git a/spec/functional/resource/deploy_revision_spec.rb b/spec/functional/resource/deploy_revision_spec.rb index 05a21c48c7..e5f5341fcd 100644 --- a/spec/functional/resource/deploy_revision_spec.rb +++ b/spec/functional/resource/deploy_revision_spec.rb @@ -45,11 +45,10 @@ describe Chef::Resource::DeployRevision, :unix_only => true do before(:all) do @ohai = Ohai::System.new - @ohai.all_plugins(@ohai.all_plugins(["platform", "os"])) + @ohai.all_plugins(["platform", "os"]) end let(:node) do - Chef::Node.new.tap do |n| n.name "rspec-test" n.consume_external_attrs(@ohai.data, {}) diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index 71c30ed532..8a1246e1f6 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -187,7 +187,7 @@ describe Chef::Client do end describe "a full client run" do - shared_examples_for "a successful client run" do + shared_context "a client run" do let(:http_node_load) { double("Chef::REST (node)") } let(:http_cookbook_sync) { double("Chef::REST (cookbook sync)") } let(:http_node_save) { double("Chef::REST (node save)") } @@ -205,7 +205,10 @@ describe Chef::Client do # --Client.register # Make sure Client#register thinks the client key doesn't # exist, so it tries to register and create one. - expect(File).to receive(:exists?).with(Chef::Config[:client_key]).exactly(1).times.and_return(api_client_exists?) + expect(File).to receive(:exists?). + with(Chef::Config[:client_key]). + exactly(:once). + and_return(api_client_exists?) unless api_client_exists? # Client.register will register with the validation client name. @@ -219,7 +222,7 @@ describe Chef::Client do # previous step. expect(Chef::REST).to receive(:new). with(Chef::Config[:chef_server_url], fqdn, Chef::Config[:client_key]). - exactly(1). + exactly(:once). and_return(http_node_load) # --Client#build_node @@ -247,23 +250,12 @@ describe Chef::Client do # --Client#converge expect(Chef::Runner).to receive(:new).and_return(runner) expect(runner).to receive(:converge).and_return(true) - - # --ResourceReporter#run_completed - # updates the server with the resource history - # (has its own tests, so stubbing it here.) - expect_any_instance_of(Chef::ResourceReporter).to receive(:run_completed) - - # --AuditReporter#audit_phase_complete - # posts the audit data to server. - # (has its own tests, so stubbing it here.) - expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:audit_phase_complete) end def stub_for_audit + # -- Client#run_audits expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner) expect(audit_runner).to receive(:run).and_return(true) - - expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:audit_phase_complete) end def stub_for_node_save @@ -282,6 +274,15 @@ describe Chef::Client do # Post conditions: check that node has been filled in correctly expect(client).to receive(:run_started) expect(client).to receive(:run_completed_successfully) + + # --ResourceReporter#run_completed + # updates the server with the resource history + # (has its own tests, so stubbing it here.) + expect_any_instance_of(Chef::ResourceReporter).to receive(:run_completed) + # --AuditReporter#run_completed + # posts the audit data to server. + # (has its own tests, so stubbing it here.) + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_completed) end before do @@ -299,8 +300,12 @@ describe Chef::Client do stub_for_node_save stub_for_run end + end - it "runs ohai, sets up authentication, loads node state, synchronizes policy, and converges" do + shared_examples_for "a successful client run" do + include_context "a client run" + + it "runs ohai, sets up authentication, loads node state, synchronizes policy, converges, and runs audits" do # This is what we're testing. client.run @@ -310,16 +315,12 @@ describe Chef::Client do end end - describe "when running chef-client without fork" do - include_examples "a successful client run" end describe "when the client key already exists" do - let(:api_client_exists?) { true } - include_examples "a successful client run" end @@ -358,7 +359,6 @@ describe Chef::Client do end describe "when a permanent run list is passed as an option" do - include_examples "a successful client run" do let(:new_runlist) { "recipe[new_run_list_recipe]" } @@ -388,6 +388,84 @@ describe Chef::Client do end end + describe "when converge fails" do + include_context "a client run" do + def stub_for_converge + expect(Chef::Runner).to receive(:new).and_return(runner) + expect(runner).to receive(:converge).and_raise(Exception) + end + + def stub_for_node_save + expect(client).to_not receive(:save_updated_node) + end + + def stub_for_run + expect_any_instance_of(Chef::RunLock).to receive(:acquire) + expect_any_instance_of(Chef::RunLock).to receive(:save_pid) + expect_any_instance_of(Chef::RunLock).to receive(:release) + + # Post conditions: check that node has been filled in correctly + expect(client).to receive(:run_started) + expect(client).to receive(:run_failed) + + # --ResourceReporter#run_completed + # updates the server with the resource history + # (has its own tests, so stubbing it here.) + # TODO: What gets called here? + #expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) + # --AuditReporter#run_completed + # posts the audit data to server. + # (has its own tests, so stubbing it here.) + # TODO: What gets called here? + #expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) + end + end + + it "runs the audits and raises the error" do + expect{ client.run }.to raise_error(Exception) + end + end + + describe "when the audit phase fails" do + context "with an exception" do + include_context "a client run" do + def stub_for_audit + expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner) + expect(audit_runner).to receive(:run).and_raise(Exception) + end + + def stub_for_run + expect_any_instance_of(Chef::RunLock).to receive(:acquire) + expect_any_instance_of(Chef::RunLock).to receive(:save_pid) + expect_any_instance_of(Chef::RunLock).to receive(:release) + + # Post conditions: check that node has been filled in correctly + expect(client).to receive(:run_started) + expect(client).to receive(:run_failed) + + # --ResourceReporter#run_completed + # updates the server with the resource history + # (has its own tests, so stubbing it here.) + # TODO: What gets called here? + #expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) + # --AuditReporter#run_completed + # posts the audit data to server. + # (has its own tests, so stubbing it here.) + # TODO: What gets called here? + #expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) + end + end + + it "should save the node after converge and raise exception" do + expect{ client.run }.to raise_error(Exception) + end + end + + context "with failed audits" do + skip("because I don't think we've implemented this yet") + end + end + end diff --git a/spec/unit/dsl/audit_spec.rb b/spec/unit/dsl/audit_spec.rb new file mode 100644 index 0000000000..7ddffb4e9f --- /dev/null +++ b/spec/unit/dsl/audit_spec.rb @@ -0,0 +1,24 @@ + +require 'spec_helper' +require 'chef/dsl/audit' + +class AuditDSLTester + include Chef::DSL::Audit +end + +describe Chef::DSL::Audit do + let(:auditor) { AuditDSLTester.new } + + it "raises an error when a block of audits is not provided" do + expect{ auditor.controls "name" }.to raise_error(Chef::Exceptions::NoAuditsProvided) + end + + it "raises an error when no audit name is given" do + expect{ auditor.controls do end }.to raise_error(Chef::Exceptions::AuditNameMissing) + end + + it "raises an error if the audit name is a duplicate" do + auditor.controls "unique" do end + expect { auditor.controls "unique" do end }.to raise_error(Chef::Exceptions::AuditControlGroupDuplicate) + end +end -- cgit v1.2.1 From 19f2c6e437642db0c03b193349b13d04636cb8ee Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 2 Dec 2014 09:30:45 -0800 Subject: Updating to use audit syntax rather than control --- lib/chef/audit/audit_event_proxy.rb | 6 +++--- lib/chef/audit/control_group_data.rb | 8 ++++---- lib/chef/audit/runner.rb | 2 +- lib/chef/dsl/audit.rb | 4 ++-- lib/chef/run_context.rb | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index 71d1e2aa50..6d5591d943 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -17,15 +17,15 @@ class Chef def example_group_started(notification) if notification.group.parent_groups.size == 1 - # top level controls block + # top level `controls` block desc = notification.group.description - Chef::Log.debug("Entered controls block named #{desc}") + Chef::Log.debug("Entered `controls` block named #{desc}") events.control_group_started(desc) end end def stop(notification) - Chef::Log.info("Successfully executed all controls blocks and contained examples") + Chef::Log.info("Successfully executed all `controls` blocks and contained examples") notification.examples.each do |example| control_group_name, control_data = build_control_from(example) e = example.exception diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index e221ae94cc..969d128c1b 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -28,19 +28,19 @@ class Chef end class ControlGroupData - attr_reader :name, :status, :number_success, :number_failed, :controls + attr_reader :name, :status, :number_succeeded, :number_failed, :controls def initialize(name) @status = "success" @controls = [] - @number_success = 0 + @number_succeeded = 0 @number_failed = 0 @name = name end def example_success(control_data) - @number_success += 1 + @number_succeeded += 1 control = create_control(control_data) control.status = "success" controls << control @@ -64,7 +64,7 @@ class Chef h = { :name => name, :status => status, - :number_success => number_success, + :number_succeeded => number_succeeded, :number_failed => number_failed, :controls => controls.collect { |c| c.to_hash } } diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index a290dd6607..df6b6d682f 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -143,7 +143,7 @@ class Chef # or use example group filters. def register_controls add_example_group_methods - run_context.controls.each do |name, group| + run_context.audits.each do |name, group| ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group[:args], &group[:block]) RSpec.world.register(ctl_grp) end diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index e22c38f587..a261d38f33 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -30,11 +30,11 @@ class Chef name = args[0] if name.nil? || name.empty? raise Chef::Exceptions::AuditNameMissing - elsif run_context.controls.has_key?(name) + elsif run_context.audits.has_key?(name) raise Chef::Exceptions::AuditControlGroupDuplicate.new(name) end - run_context.controls[name] = { :args => args, :block => block } + run_context.audits[name] = { :args => args, :block => block } end end diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index a724789d3c..d14035da2f 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -50,8 +50,8 @@ class Chef # recipes, which is triggered by #load. (See also: CookbookCompiler) attr_accessor :resource_collection - # The list of control groups to execute during the audit phase - attr_accessor :controls + # The list of audits (control groups) to execute during the audit phase + attr_accessor :audits # A Hash containing the immediate notifications triggered by resources # during the converge phase of the chef run. @@ -76,7 +76,7 @@ class Chef @node = node @cookbook_collection = cookbook_collection @resource_collection = Chef::ResourceCollection.new - @controls = {} + @audits = {} @immediate_notification_collection = Hash.new {|h,k| h[k] = []} @delayed_notification_collection = Hash.new {|h,k| h[k] = []} @definitions = Hash.new -- cgit v1.2.1 From b079e015f4ebb8c5db600bd49641699cbbacdb10 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 2 Dec 2014 16:09:35 -0800 Subject: Adding cookbook and recipe information per analytics request --- lib/chef/audit/audit_event_proxy.rb | 2 +- lib/chef/audit/audit_reporter.rb | 9 +++++++-- lib/chef/audit/control_group_data.rb | 27 ++++++++++----------------- lib/chef/audit/runner.rb | 2 +- lib/chef/dsl/audit.rb | 11 ++++++++++- lib/chef/formatters/doc.rb | 27 +++++++++++++++++++-------- spec/unit/dsl/audit_spec.rb | 8 +++++++- 7 files changed, 55 insertions(+), 31 deletions(-) diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index 6d5591d943..36160db9bb 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -43,7 +43,7 @@ class Chef described_class = example.metadata[:described_class] if described_class resource_type = described_class.class.name.split(':')[-1] - # TODO submit github PR to expose this + # TODO https://github.com/serverspec/serverspec/pull/493 resource_name = described_class.instance_variable_get(:@name) end diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 21ffb62829..d022ac0c47 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -28,7 +28,7 @@ class Chef attr_reader :rest_client, :audit_data, :ordered_control_groups, :run_status private :rest_client, :audit_data, :ordered_control_groups, :run_status - PROTOCOL_VERSION = '0.1.0' + PROTOCOL_VERSION = '0.1.1' def initialize(rest_client) @rest_client = rest_client @@ -36,6 +36,10 @@ class Chef @ordered_control_groups = Hash.new end + def run_context + run_status.run_context + end + def audit_phase_start(run_status) Chef::Log.debug("Audit Reporter starting") @audit_data = AuditData.new(run_status.node.name, run_status.run_id) @@ -71,7 +75,8 @@ class Chef if ordered_control_groups.has_key?(name) raise Chef::Exceptions::AuditControlGroupDuplicate.new(name) end - ordered_control_groups.store(name, ControlGroupData.new(name)) + metadata = run_context.audits[name].metadata + ordered_control_groups.store(name, ControlGroupData.new(name, metadata)) end def control_example_success(control_group_name, example_data) diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index 969d128c1b..42a91ef5a7 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -28,14 +28,15 @@ class Chef end class ControlGroupData - attr_reader :name, :status, :number_succeeded, :number_failed, :controls + attr_reader :name, :status, :number_succeeded, :number_failed, :controls, :metadata - def initialize(name) + def initialize(name, metadata={}) @status = "success" @controls = [] @number_succeeded = 0 @number_failed = 0 @name = name + @metadata = metadata end @@ -68,20 +69,14 @@ class Chef :number_failed => number_failed, :controls => controls.collect { |c| c.to_hash } } - add_display_only_data(h) + # If there is a duplicate key, metadata will overwrite it + add_display_only_data(h).merge(metadata) end private def create_control(control_data) - name = control_data[:name] - resource_type = control_data[:resource_type] - resource_name = control_data[:resource_name] - context = control_data[:context] - line_number = control_data[:line_number] - # TODO make this smarter with splat arguments so if we start passing in more control_data - # I don't have to modify code in multiple places - ControlData.new(name, resource_type, resource_name, context, line_number) + ControlData.new(control_data) end # The id and control sequence number are ephemeral data - they are not needed @@ -103,12 +98,10 @@ class Chef attr_reader :name, :resource_type, :resource_name, :context, :line_number attr_accessor :status, :details - def initialize(name, resource_type, resource_name, context, line_number) - @context = context - @name = name - @resource_type = resource_type - @resource_name = resource_name - @line_number = line_number + def initialize(control_data={}) + control_data.each do |k, v| + self.instance_variable_set("@#{k}", v) + end end def to_hash diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index df6b6d682f..2fd33ac0de 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -144,7 +144,7 @@ class Chef def register_controls add_example_group_methods run_context.audits.each do |name, group| - ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group[:args], &group[:block]) + ctl_grp = RSpec::Core::ExampleGroup.__controls__(*group.args, &group.block) RSpec.world.register(ctl_grp) end end diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb index a261d38f33..022bbcce01 100644 --- a/lib/chef/dsl/audit.rb +++ b/lib/chef/dsl/audit.rb @@ -34,7 +34,16 @@ class Chef raise Chef::Exceptions::AuditControlGroupDuplicate.new(name) end - run_context.audits[name] = { :args => args, :block => block } + # This DSL will only work in the Recipe class because that exposes the cookbook_name + cookbook_name = self.cookbook_name + metadata = { + cookbook_name: cookbook_name, + cookbook_version: self.run_context.cookbook_collection[cookbook_name].version, + recipe_name: self.recipe_name, + line_number: block.source_location[1] + } + + run_context.audits[name] = Struct.new(:args, :block, :metadata).new(args, block, metadata) end end diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 99603965a9..4f79411ed9 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -8,7 +8,9 @@ class Chef # "specdoc" class Doc < Formatters::Base - attr_reader :start_time, :end_time + attr_reader :start_time, :end_time, :successful_audits, :failed_audits + private :successful_audits, :failed_audits + cli_name(:doc) def initialize(out, err) @@ -16,6 +18,8 @@ class Chef @updated_resources = 0 @up_to_date_resources = 0 + @successful_audits = 0 + @failed_audits = 0 @start_time = Time.now @end_time = @start_time end @@ -32,12 +36,16 @@ class Chef @up_to_date_resources + @updated_resources end + def total_audits + successful_audits + failed_audits + end + def run_completed(node) @end_time = Time.now if Chef::Config[:why_run] puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources would have been updated" else - puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources updated in #{elapsed_time} seconds" + puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources updated and #{successful_audits}/#{total_audits} audits succeeded in #{elapsed_time} seconds" end end @@ -46,7 +54,7 @@ class Chef if Chef::Config[:why_run] puts_line "Chef Client failed. #{@updated_resources} resources would have been updated" else - puts_line "Chef Client failed. #{@updated_resources} resources updated in #{elapsed_time} seconds" + puts_line "Chef Client failed. #{@updated_resources} resources updated and #{successful_audits}/#{total_audits} audits succeeded in #{elapsed_time} seconds" end end @@ -156,11 +164,6 @@ class Chef converge_complete end - ############# - # TODO - # Make all these document printers neater - ############# - # Called before audit phase starts def audit_phase_start(run_status) puts_line "Starting audit phase" @@ -181,6 +184,14 @@ class Chef end end + def control_example_success(control_group_name, example_data) + @successful_audits += 1 + end + + def control_example_failure(control_group_name, example_data, error) + @failed_audits += 1 + end + # Called before action is executed on a resource. def resource_action_start(resource, action, notification_type=nil, notifier=nil) if resource.cookbook_name && resource.recipe_name diff --git a/spec/unit/dsl/audit_spec.rb b/spec/unit/dsl/audit_spec.rb index 7ddffb4e9f..7565a42d58 100644 --- a/spec/unit/dsl/audit_spec.rb +++ b/spec/unit/dsl/audit_spec.rb @@ -8,6 +8,12 @@ end describe Chef::DSL::Audit do let(:auditor) { AuditDSLTester.new } + let(:run_context) { instance_double(Chef::RunContext, :audits => audits) } + let(:audits) { [] } + + before do + allow(auditor).to receive(:run_context).and_return(run_context) + end it "raises an error when a block of audits is not provided" do expect{ auditor.controls "name" }.to raise_error(Chef::Exceptions::NoAuditsProvided) @@ -18,7 +24,7 @@ describe Chef::DSL::Audit do end it "raises an error if the audit name is a duplicate" do - auditor.controls "unique" do end + expect(audits).to receive(:has_key?).with("unique").and_return(true) expect { auditor.controls "unique" do end }.to raise_error(Chef::Exceptions::AuditControlGroupDuplicate) end end -- cgit v1.2.1 From d454a7214b2738acd9a00e1f29737927a4a4d5d7 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 26 Nov 2014 14:12:03 -0800 Subject: Failing an audit example will now raise an error and make chef exit with a non-zero code --- lib/chef/audit/audit_reporter.rb | 2 ++ lib/chef/audit/runner.rb | 4 ++++ lib/chef/client.rb | 3 ++- lib/chef/exceptions.rb | 5 +++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index d022ac0c47..00af9984b2 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -55,6 +55,8 @@ class Chef # If the audit phase failed, its because there was some kind of error in the framework # that runs tests - normal errors are interpreted as EXAMPLE failures and captured. + # We still want to send available audit information to the server so we process the + # known control groups. def audit_phase_failed(error) # The stacktrace information has already been logged elsewhere Chef::Log.debug("Audit Reporter failed.") diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 2fd33ac0de..b3df70f705 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -35,6 +35,10 @@ class Chef do_run end + def failed_examples? + RSpec.world.reporter.failed_examples.size > 0 + end + private # Prepare to run audits: # - Require files diff --git a/lib/chef/client.rb b/lib/chef/client.rb index aa0d6722fe..5d46794745 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -358,9 +358,10 @@ class Chef Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run + raise Chef::Exceptions::AuditsFailed if auditor.failed_examples? @events.audit_phase_complete rescue Exception => e - Chef::Log.error("Audit phase failed with error message #{e.message}") + Chef::Log.error("Audit phase failed with error message: #{e.message}") @events.audit_phase_failed(e) audit_exception = e end diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index 9fae1d566f..2cf8766585 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -396,6 +396,11 @@ class Chef super "You must provide a block with audits" end end + class AuditsFailed < RuntimeError + def initialize + super "There were audit example failures. Results were still sent to the server." + end + end # If a converge or audit fails, we want to wrap the output from those errors into 1 error so we can # see both issues in the output. It is possible that nil will be provided. You must call `fill_backtrace` -- cgit v1.2.1 From 92f21502937682a3db715bf7c3a3b79e1bd784bc Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 3 Dec 2014 16:58:14 -0800 Subject: if in why_run, do not enter the audit phase - because people run why_run to see converge statistics --- DOC_CHANGES.md | 541 +-------------------------------------------- lib/chef/audit/runner.rb | 10 +- lib/chef/client.rb | 9 +- lib/chef/exceptions.rb | 4 +- lib/chef/formatters/doc.rb | 10 +- 5 files changed, 35 insertions(+), 539 deletions(-) diff --git a/DOC_CHANGES.md b/DOC_CHANGES.md index 22294a3e39..15f88abdca 100644 --- a/DOC_CHANGES.md +++ b/DOC_CHANGES.md @@ -6,540 +6,17 @@ Example Doc Change: Description of the required change. --> -### Knife now prefers to use `config.rb` rather than `knife.rb` +### Chef Why Run Mode Ignores Audit Phase -Knife will now look for `config.rb` in preference to `knife.rb` for its -configuration file. The syntax and configuration options available in -`config.rb` are identical to `knife.rb`. Also, the search path for -configuration files is unchanged. +Because most users enable `why_run` mode to determine what resources convergence will update on their system, the audit +phase is not executed. There is no way to get both `why_run` output and audit output in 1 single command. To get +audit output without performing convergence use the `--audit-mode` flag. -At this time, it is _recommended_ that users use `config.rb` instead of -`knife.rb`, but `knife.rb` is not deprecated; no warning will be emitted -when using `knife.rb`. Once third-party application developers have had -sufficient time to adapt to the change, `knife.rb` will become -deprecated and config.rb will be preferred. +#### Editors note 1 -### value_for_platform Method +The `--audit-mode` flag should be a link to the documentation for that flag -- where "platform" can be a comma-separated list, each specifying a platform, such as Red Hat, openSUSE, or Fedora, version specifies the version of that platform, and value specifies the value that will be used if the node's platform matches the value_for_platform method. If each value only has a single platform, then the syntax is like the following: -+ where platform can be a comma-separated list, each specifying a platform, such as Red Hat, openSUSE, or Fedora, version specifies either the exact version of that platform, or a constraint to match the platform's version against. The following rules apply to constraint matches: +#### Editors node 2 -+ * Exact matches take precedence no matter what, and should never throw exceptions. -+ * Matching multiple constraints raises a RuntimeError. -+ * The following constraints are allowed: <,<=,>,>=,~>. -+ -+ The following is an example of using the method with constraints: -+ -+ ```ruby -+ value_for_platform( -+ "os1" => { -+ "< 1.0" => "less than 1.0", -+ "~> 2.0" => "version 2.x", -+ ">= 3.0" => "version 3.0", -+ "3.0.1" => "3.0.1 will always use this value" } -+ ) -+ ``` - -+ If each value only has a single platform, then the syntax is like the following: - -### environment attribute to git provider - -Similar to other environment options: - -``` -environment Hash of environment variables in the form of {"ENV_VARIABLE" => "VALUE"}. -``` - -Also the `user` attribute should mention the setting of the HOME env var: - -``` -user The system user that is responsible for the checked-out code. The HOME environment variable will automatically be -set to the home directory of this user when using this option. -``` - -### Metadata `name` Attribute is Required. - -Current documentation states: - -> The name of the cookbook. This field is inferred unless specified. - -This is no longer correct as of 12.0. The `name` field is required; if -it is not specified, an error will be raised if it is not specified. - -### chef-zero port ranges - -- to avoid crashes, by default, Chef will now scan a port range and take the first available port from 8889-9999. -- to change this behavior, you can pass --chef-zero-port=PORT_RANGE (for example, 10,20,30 or 10000-20000) or modify Chef::Config.chef_zero.port to be a po -rt string, an enumerable of ports, or a single port number. - -### Encrypted Data Bags Version 3 - -Encrypted Data Bag version 3 uses [GCM](http://en.wikipedia.org/wiki/Galois/Counter_Mode) internally. Ruby 2 and OpenSSL version 1.0.1 or higher are required to use it. - -### New windows_service resource - -The windows_service resource inherits from the service resource and has all the same options but adds an action and attribute. - -action :configure_startup - sets the startup type on the resource to the value of the `startup_type` attribute -attribute startup_type - the value as a symbol that the startup type should be set to on the service, valid options :automatic, :manual, :disabled - -Note that the service resource will also continue to set the startup type to automatic or disabled, respectively, when the enabled or disabled actions are used. - -### Fetch encrypted data bag items with dsl method -DSL method `data_bag_item` now takes an optional String parameter `secret`, which is used to interact with encrypted data bag items. -If the data bag item being fetched is encrypted and no `secret` is provided, Chef looks for a secret at `Chef::Config[:encrypted_data_bag_secret]`. -If `secret` is provided, but the data bag item is not encrypted, then a regular data bag item is returned (no decryption is attempted). - -### Encrypted data bag UX -The user can now provide a secret for data bags in 4 ways. They are, in order of descending preference: -1. Provide the secret on the command line of `knife data bag` and `knife bootstrap` commands with `--secret` -1. Provide the location of a file containing the secret on the command line of `knife data bag` and `knife bootstrap` commands with `--secret-file` -1. Add the secret to your workstation config with `knife[:secret] = ...` -1. Add the location of a file containing the secret to your workstation config with `knife[:secret-file] = ...` - -When adding the secret information to your workstation config, it will not be used for writeable operations unless `--encrypt` is also passed on the command line. -Data bag read-only operations (`knife data bag show` and `knife bootstrap`) do not require `--encrypt` to be passed, and will attempt to use an available secret for decryption. -Unencrypted data bags will not attempt to be unencrypted, even if a secret is provided. -Trying to view an encrypted data bag without providing a secret will issue a warning and show the encrypted contents. -Trying to edit or create an encrypted data bag without providing a secret will fail. - -Here are some example scenarios: - -``` -# Providing `knife[:secret_file] = ...` in knife.rb will create and encrypt the data bag -knife data bag create BAG_NAME ITEM_NAME --encrypt - -# The same command ran with --secret will use the command line secret instead of the knife.rb secret -knife data bag create ANOTHER_BAG ITEM_NAME --encrypt --secret 'ANOTHER_SECRET' - -# The next two commands will fail, because they are using the wrong secret -knife data bag edit BAG_NAME --secret 'ANOTHER_SECRET' -knife data bag edit ANOTHER_BAG --encrypt - -# The next command will unencrypt the data and show it using the `knife[:secret_file]` without passing the --encrypt flag -knife data bag show BAG_NAME - -# To create an unencrypted data bag, simply do not provide `--secret`, `--secret-file` or `--encrypt` -knife data bag create UNENCRYPTED_BAG - -# If a secret is available from any of the 4 possible entries, it will be copied to a bootstrapped node, even if `--encrypt` is not present -knife bootstrap FQDN -``` - -### Enhanced search functionality: result filtering -#### Use in recipes -`Chef::Search::Query#search` can take an optional `:filter_result` argument which returns search data in the form of the Hash specified. Suppose your data looks like -```json -{"languages": { - "c": { - "gcc": { - "version": "4.6.3", - "description": "gcc version 4.6.3 (Ubuntu/Linaro 4.6.3-1ubuntu5) " - } - }, - "ruby": { - "platform": "x86_64-linux", - "version": "1.9.3", - "release_date": "2013-11-22" - }, - "perl": { - "version": "5.14.2", - "archname": "x86_64-linux-gnu-thread-multi" - }, - "python": { - "version": "2.7.3", - "builddate": "Feb 27 2014, 19:58:35" - } -}} -``` -for a node running Ubuntu named `node01`, and you want to get back only information on which versions of c and ruby you have. In a recipe you would write -```ruby -search(:node, "platform:ubuntu", :filter_result => {"c_version" => ["languages", "c", "gcc", "version"], - "ruby_version" => ["languages", "ruby", "version"]}) -``` -and receive -```ruby -[ - {"url" => "https://api.opscode.com/organization/YOUR_ORG/nodes/node01", - "data" => {"c_version" => "4.6.3", "ruby_version" => "1.9.3"}, - # snip other Ubuntu nodes -] -``` -If instead you wanted all the languages data (remember, `"languages"` is only one tiny piece of information the Chef Server stores about your node), you would have `:filter_result => {"languages" => ["languages"]}` in your search query. - -For backwards compatibility, a `partial_search` method has been added to `Chef::Search::Query` which can be used in the same way as the `partial_search` method from the [partial_search cookbook](https://supermarket.getchef.com/cookbooks/partial_search). Note that this method has been deprecated and will be removed in future versions of Chef. - -#### Use in knife -Search results can likewise be filtered by adding the `--filter-result` (or `-f`) option. Considering the node data above, you can use `knife search` with filtering to extract the c and ruby versions on your Ubuntu platforms: -```bash -$ knife search node "platform:ubuntu" --filter-result "c_version:languages.c.gcc.version, ruby_version:languages.ruby.version" -1 items found - -: - c_version: 4.6.3 - ruby_version: 1.9.3 - -$ -``` - -## Client and solo application changes - -### Unforked interval chef-client runs are disabled -Unforked interval and daemonized chef-client runs are now explicitly prohibited. Runs configured with CLI options -`--interval SEC` or `--daemonize` paired with `--no-fork`, or the equivalent config options paired with -`client_fork false` will fail immediately with error. - -### Sleep happens before converge -When configured to splay sleep or run at intervals, `chef-client` and `chef-solo` perform both splay and interval -sleeps before converging. In previous releases, chef would splay sleep then converge then interval sleep. - -### Signal handling -When sent `SIGTERM` the thread or process will: -1. if chef is not converging, exit immediately with exitstatus 3 or -1. allow chef to finish converging then exit immediately with the converge's exitstatus. - -To terminate immediately, send `SIGINT`. - -# `knife ssl check` will verify X509 properties of your trusted certificates - -When you run `knife ssl check URL (options)` knife will verify if the certificate files, with extensions `*.crt` and `*.pem` -in your `:trusted_certs_dir` have valid X509 certificate properties. Knife will generate warnings for certificates that -do not meet X509 standards. OpenSSL **will not** use these certificates in verifying SSL connections. - -## Troubleshooting -For each certificate that does not meet X509 specifications, a message will be displayed indicating why the certificate -failed to meet these specifications. You may see output similar to - -``` -There are invalid certificates in your trusted_certs_dir. -OpenSSL will not use the following certificates when verifying SSL connections: - -/path/to/your/invalid/certificate.crt: a message to help you debug -``` - -The documentation for resolving common issues with certificates is a work in progress. A few suggestions -are outlined in the following sections. If you would like to help expand this documentation, please -submit a pull request to [chef-docs](https://github.com/opscode/chef-docs) with your contribution. - -### Fetch the certificate again -If the certificate was generated by your chef server, you may want to try downloading the certificate again. -By default, the certificate is stored in the following location on the host where your chef-server runs: -`/var/opt/chef-server/nginx/ca/SERVER_HOSTNAME.crt`. Copy that file into your `:trusted_certs_dir` using SSH, -SCP, or some other secure method and run `knife ssl check URL (options)` again. - -### Generate a new certificate -If you control the trusted certificate and you suspect it is bad (e.g., you've fetched the certificate again, -but you're still getting warnings about it from `knife ssl check`), you might try generating a new certificate. - -#### Generate a certificate signing request -If you used a certificate authority (CA) to authenticate your certificate, you'll need to generate -a certificate signing request (CSR) to fetch a new certificate. - -If you don't have one already, you'll need to create an openssl configuration file. This example -configuration file is saved in our current working directory as openssl.cnf - -``` -# -# OpenSSL configuration file -# ./openssl.cnf -# - -[ req ] -default_bits = 1024 # Size of keys -default_keyfile = key.pem # name of generated keys -default_md = md5 # message digest algorithm -string_mask = nombstr # permitted characters -distinguished_name = req_distinguished_name -req_extensions = v3_req - -[ req_distinguished_name ] -# Variable name Prompt string -#--------------------- ---------------------------------- -0.organizationName = Organization Name (company) -organizationalUnitName = Organizational Unit Name (department, division) -emailAddress = Email Address -emailAddress_max = 40 -localityName = Locality Name (city, district) -stateOrProvinceName = State or Province Name (full name) -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (hostname, IP, or your name) -commonName_max = 64 - -# Default values for the above, for consistency and less typing. -# Variable name Value -#-------------------------- ------------------------------ -0.organizationName_default = My Company -localityName_default = My Town -stateOrProvinceName_default = State or Providence -countryName_default = US - -[ v3_req ] -basicConstraints = CA:FALSE # This is NOT a CA certificate -subjectKeyIdentifier = hash -``` - -You can use `openssl` to create a certificate from an existing private key -``` -$ openssl req -new -extensions v3_req -key KEYNAME.pem -out REQNAME.pem -config ./openssl.cnf -``` -or `openssl` can create a new private key simultaneously -``` -$ openssl req -new -extensions v3_req -keyout KEYNAME.pem -out REQNAME.pem -config ./openssl.cnf -``` -where `KEYNAME` is the path to your private key and `REQNAME` is the path to your CSR. - -You can verify your CSR was generated correctly -``` -$ openssl req -noout -text -in REQNAME.pem -``` - -The final step is to submit your CSR to your certificate authority (CA) for signing. - -### Generate a self-signed (root) certificate -You'll need to modify your openssl configuration file, or create a separate file, for -generating root certificates. - -``` -# -# OpenSSL configuration file -# ./openssl.cnf -# - -dir = . - -[ ca ] -default_ca = CA_default - -[ CA_default ] -serial = $dir/serial -database = $dir/certindex.txt -new_certs_dir = $dir/certs -certificate = $dir/cacert.pem -private_key = $dir/private/cakey.pem -default_days = 365 -default_md = md5 -preserve = no -email_in_dn = no -nameopt = default_ca -certopt = default_ca -policy = policy_match - -[ policy_match ] -countryName = match -stateOrProvinceName = match -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ v3_ca ] -basicConstraints = CA:TRUE # This is a CA certificate -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer:always -``` - -You can now create a root certificate. If you have a private key you would like -to use -``` -$ openssl req -new -x509 -extensions v3_ca -key KEYNAME.pem -out CERTNAME.pem -config ./openssl.cnf -``` -or `openssl` can create a new private key simultaneously -``` -$ openssl req -new -x509 -extensions v3_ca -keyout KEYNAME.pem -out CERTNAME.pem -config ./openssl.cnf -``` -where `KEYNAME` is the path to your private key and `REQNAME` is the path to your CSR. - -At this point, you should add the generated certificate to your trusted certificates as well as -replace the old server certificate. Furthermore, you should regenerate any certificates that -were signed by the previous root certificate. - -For more information and an example on how to set up your server to generate certificates -check out this post on [setting up OpenSSL to create certificates](http://www.flatmtn.com/article/setting-openssl-create-certificates). - -#### Signing certificates -Use your root certificate to sign certificate requests sent to your server -``` -$ openssl ca -out CERTNAME.pem -config ./openssl.cnf -infiles REQNAME.pem -``` -This creates the certificate `CERTNAME.pem` generated from CSR `REQNAME.pem`. You -should send `CERTNAME.pem` back to the client who generated the CSR. - -### Certificate attributes -When creating certificates and certificate signing requests, you will be prompted for -information via the command line. These are your certificate attributes. - -RDN | Name | Explanation | Examples -:---: | :---: | --- | --- -CN | Common Name | You server's FQDN, or YOUR_SERVER Certificate Authority if root certificate | mail.domain.com, *.domain.com, MyServer Certificate Authority -OU | Organizational Unit | (Optional) Additional organization information. | mail server, R&D -O | Organization | The exact name of your organization. Do not abbreviate. | DevOpsRUs Inc. -L | Locality | The city where your organization is located | Seattle -S | State or Province | The state or province where your organization is located. Do not abbreviate. | Washington -C | Country Name | 2-letter ISO abbreviation for your country. | US - | Email Address | How you or another maintainer can be reached. | maintainers@devopsr.us - -If you examine the `policy_match` section in the openssl configuration file example from the section on generating -self signed certificates, you'll see specifications that CSRs need to match the countryName, stateOrProvinceName, -and the organizationName. CSRs whose CN, S, and O values do not match those of the root certificate will not be -signed by that root certificate. You can modify these requirements as desired. - -### Key usage -A keyUsage field can be added to your `v3_req` and `v3_ca` sections of your configuration file. -Key usage extensions define the purpose of the public key contained in a certificate, limiting what -it can and cannot be used for. - -Extension | Description ---- | --- -digitalSignature | Use when the public key is used with a digital signature mechanism to support security services other than non-repudiation, certificate signing, or CRL signing. A digital signature is often used for entity authentication and data origin authentication with integrity -nonRepudiation | Use when the public key is used to verify digital signatures used to provide a non-repudiation service. Non-repudiation protects against the signing entity falsely denying some action (excluding certificate or CRL signing). -keyEncipherment | Use when a certificate will be used with a protocol that encrypts keys. -dataEncipherment | Use when the public key is used for encrypting user data, other than cryptographic keys. -keyAgreement | Use when the sender and receiver of the public key need to derive the key without using encryption. This key can then can be used to encrypt messages between the sender and receiver. Key agreement is typically used with Diffie-Hellman ciphers. -certificateSigning | Use when the subject public key is used to verify a signature on certificates. This extension can be used only in CA certificates. -cRLSigning | Use when the subject public key is to verify a signature on revocation information, such as a CRL. -encipherOnly | Use only when key agreement is also enabled. This enables the public key to be used only for enciphering data while performing key agreement. -decipherOnly | Use only when key agreement is also enabled. This enables the public key to be used only for deciphering data while performing key agreement. -[Source](http://www-01.ibm.com/support/knowledgecenter/SSKTMJ_8.0.1/com.ibm.help.domino.admin.doc/DOC/H_KEY_USAGE_EXTENSIONS_FOR_INTERNET_CERTIFICATES_1521_OVER.html) - -### Subject Alternative Names -Subject alternative names (SANs) allow you to list host names to protect with a single certificate. -To create a certificate using SANs, you'll need to add a `subjectAltName` field to your `v3_req` section -in your openssl configuration file - -``` -[ v3_req ] -basicConstraints = CA:FALSE # This is NOT a CA certificate -subjectKeyIdentifier = hash -subjectAltName = @alt_names - -[alt_names] -DNS.1 = kb.example.com -DNS.2 = helpdesk.example.org -DNS.3 = systems.example.net -IP.1 = 192.168.1.1 -IP.2 = 192.168.69.14 -``` - -### Reboot resource in core -The `reboot` resource will reboot the server, a necessary step in some installations, especially on Windows. If this resource is used with notifications, it must receive explicit `:immediate` notifications only: results of delayed notifications are undefined. Currently supported on Windows, Linux, and OS X; will work incidentally on some other Unixes. - -There are three actions: - -```ruby -reboot "app_requires_reboot" do - action :request_reboot - reason "Need to reboot when the run completes successfully." - delay_mins 5 -end - -reboot "cancel_reboot_request" do - action :cancel - reason "Cancel a previous end-of-run reboot request." -end - -reboot "now" do - action :reboot_now - reason "Cannot continue Chef run without a reboot." - delay_mins 2 -end - -# the `:immediate` is required for results to be defined. -notifies :reboot_now, "reboot[now]", :immediate -``` - -### Escape sensitive characters before globbing -Some paths contain characters reserved by glob and must be escaped so that -glob operations perform as expected. One common example is Windows file paths -separated by `"\\"`. To ensure that your globs work correctly, it is recommended -that you apply `Chef::Util::PathHelper::escape_glob` before globbing file paths. - -```ruby -path = "C:\\Users\\me\\chef-repo\\cookbooks" -Dir.exist?(path) # true -Dir.entries(path) # [".", "..", "apache2", "apt", ...] - -Dir.glob(File.join(path, "*")) # [] -Dir[File.join(path, "*")] # [] - -PathHelper = Chef::Util::PathHelper -Dir.glob(File.join(PathHelper.escape_glob(path), "*")) # ["#{path}\\apache2", "#{path}\\apt", ...] -Dir[PathHelper.escape_glob(path) + "/*"] # ["#{path}\\apache2", "#{path}\\apt", ...] -``` -## Mac OS X default package provider is now Homebrew - -Per [Chef RFC 016](https://github.com/opscode/chef-rfc/blob/master/rfc016-homebrew-osx-package-provider.md), the default provider for the `package` resource on Mac OS X is now [Homebrew](http://brew.sh). The [homebrew cookbook's](https://supermarket.getchef.com/cookbooks/homebrew) default recipe, or some other method is still required for getting homebrew installed on the system. The cookbook won't be strictly required just to install packages from homebrew on OS X, though. To use this, simply use the `package` resource, or the `homebrew_package` shortcut resource: - -```ruby -package 'emacs' -``` - -Or, - -```ruby -homebrew_package 'emacs' -``` - -The macports provider will still be available, and can be used with the shortcut resource, or by using the `provider` attribute: - -```ruby -macports_package 'emacs' -``` - -Or, - -```ruby -package 'emacs' do - provider Chef::Provider::Package::Macports -end -``` - -### Providing `homebrew_user` - -Homebrew recommends being ran as a non-root user, whereas Chef recommends being ran with root privileges. The -`homebrew_package` provider has logic to try and determine which user to install Homebrew packages as. - -By default, the `homebrew_package` provider will try to execute the homebrew command as the owner of the `/usr/local/bin/brew` -executable. If that executable does not exist, Chef will try to find it by executing `which brew`. If that cannot be -found, Chef then errors. The Homebrew recommendation is the default install, which will place the executable at -`/usr/local/bin/brew` owned by a non-root user. - -You can circumvent this by providing the `homebrew_package` a `homebrew_user` attribute, like: - -```ruby -# provided as a uid -homebrew_package 'emacs' do - homebrew_user 1001 -end - -# provided as a string -homebrew_package 'vim' do - homebrew_user 'user1' -end -``` - -Chef will then execute the Homebrew command as that user. The `homebrew_user` attribute can only be provided to the -`homebrew_package` resource, not the `package` resource. - -### Default `guard_interpreter` attribute for `powershell_script` resource - -For the `powershell_script` resource, the `guard_interpreter` attribute is set to `:powershell_script` by default. This means -that if a string is supplied to an `only_if` or `not_if` attribute of a `powersell_script` resource, the PowerShell command -interpreter (the 64-bit version) will be used to evaluate the guard. It also means that other features available to the guard -when `guard_interpreter` is set to something other than `:default`, such as inheritance of attributes and the specification of -process architectur of the guard process (i.e. 32-bit or 64-bit process) are available by default. - -In versions of Chef prior to Chef 12, the value of the attribute was `:default` by default, which uses the 32-bit version of the -`cmd.exe` (batch script language) shell to evaluate strings supplied to guards. - -### Default `guard_interpreter` attribute for `batch` resource - -For the`batch` resource, the `guard_interpreter` attribute it is set to `:batch` by default. This means -that if a string is supplied to an `only_if` or `not_if` attribute of a `batch` resource, the 64-bit version of the Windows -default command interpreter, `cmd.exe`, will be used to evaluate the guard. It also means that other features available to the guard -when `guard_interpreter` is set to something other than `:default`, such as inheritance of attributes and the specification of -process architecture of the guard process (i.e. 32-bit or 64-bit process) are available by default. - -In versions of Chef prior to Chef 12, the value of the attribute was `:default` by default, which means the 32-bit version of the -`cmd.exe` (batch script language) shell would be used to evaluate strings supplied to guards. +This probably only needs to be a bullet point added to http://docs.getchef.com/nodes.html#about-why-run-mode under the +`certain assumptions` section diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index b3df70f705..5d485a8804 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -35,10 +35,18 @@ class Chef do_run end - def failed_examples? + def failed? RSpec.world.reporter.failed_examples.size > 0 end + def num_failed + RSpec.world.reporter.failed_examples.size + end + + def num_total + RSpec.world.reporter.examples.size + end + private # Prepare to run audits: # - Require files diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 5d46794745..634773cf80 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -358,7 +358,9 @@ class Chef Chef::Log.info("Starting audit phase") auditor = Chef::Audit::Runner.new(run_context) auditor.run - raise Chef::Exceptions::AuditsFailed if auditor.failed_examples? + if auditor.failed? + raise Chef::Exceptions::AuditsFailed.new(auditor.num_failed, auditor.num_total) + end @events.audit_phase_complete rescue Exception => e Chef::Log.error("Audit phase failed with error message: #{e.message}") @@ -446,7 +448,10 @@ class Chef converge_error = converge_and_save(run_context) end - if Chef::Config[:audit_mode] != :disabled + if Chef::Config[:why_run] == true + # why_run should probably be renamed to why_converge + Chef::Log.debug("Not running audits in 'why_run' mode - this mode is used to see potential converge changes") + elsif Chef::Config[:audit_mode] != :disabled audit_error = run_audits(run_context) end diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index 2cf8766585..dabdd03802 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -397,8 +397,8 @@ class Chef end end class AuditsFailed < RuntimeError - def initialize - super "There were audit example failures. Results were still sent to the server." + def initialize(num_failed, num_total) + super "Audit phase found failures - #{num_failed}/#{num_total} audits failed" end end diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 4f79411ed9..398c61fdc8 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -45,7 +45,10 @@ class Chef if Chef::Config[:why_run] puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources would have been updated" else - puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources updated and #{successful_audits}/#{total_audits} audits succeeded in #{elapsed_time} seconds" + puts_line "Chef Client finished, #{@updated_resources}/#{total_resources} resources updated in #{elapsed_time} seconds" + if total_audits > 0 + puts_line " #{successful_audits}/#{total_audits} Audits succeeded" + end end end @@ -54,7 +57,10 @@ class Chef if Chef::Config[:why_run] puts_line "Chef Client failed. #{@updated_resources} resources would have been updated" else - puts_line "Chef Client failed. #{@updated_resources} resources updated and #{successful_audits}/#{total_audits} audits succeeded in #{elapsed_time} seconds" + puts_line "Chef Client failed. #{@updated_resources} resources updated in #{elapsed_time} seconds" + if total_audits > 0 + puts_line " #{successful_audits} Audits succeeded" + end end end -- cgit v1.2.1 From c4aef5a6fc42a8907b52ebd2ab9122017d53e185 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 5 Dec 2014 16:52:59 -0800 Subject: Add unit tests for Audit::ControlGroupData --- spec/unit/audit/control_group_data_spec.rb | 459 +++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 spec/unit/audit/control_group_data_spec.rb diff --git a/spec/unit/audit/control_group_data_spec.rb b/spec/unit/audit/control_group_data_spec.rb new file mode 100644 index 0000000000..e5e76a6011 --- /dev/null +++ b/spec/unit/audit/control_group_data_spec.rb @@ -0,0 +1,459 @@ + +require 'spec_helper' +require 'securerandom' + +describe Chef::Audit::AuditData do + + let(:node_name) { "noodles" } + let(:run_id) { SecureRandom.uuid } + let(:audit_data) { described_class.new(node_name, run_id) } + + let(:control_group_1) { double("control group 1") } + let(:control_group_2) { double("control group 2") } + + describe "#add_control_group" do + context "when no control groups have been added" do + it "stores the control group" do + audit_data.add_control_group(control_group_1) + expect(audit_data.control_groups).to include(control_group_1) + end + + end + + context "when adding additional control groups" do + + before do + audit_data.add_control_group(control_group_1) + end + + it "stores the control group" do + audit_data.add_control_group(control_group_2) + expect(audit_data.control_groups).to include(control_group_2) + end + + it "stores all control groups" do + audit_data.add_control_group(control_group_2) + expect(audit_data.control_groups).to include(control_group_1) + end + end + end + + describe "#to_hash" do + + let(:audit_data_hash) { audit_data.to_hash } + + it "returns a hash" do + expect(audit_data_hash).to be_a(Hash) + end + + it "describes a Chef::Audit::AuditData object" do + keys = [:node_name, :run_id, :start_time, :end_time, :control_groups] + expect(audit_data_hash.keys).to match_array(keys) + end + + describe ":control_groups" do + + let(:control_hash_1) { {:name => "control group 1"} } + let(:control_hash_2) { {:name => "control group 2"} } + + let(:control_groups) { audit_data_hash[:control_groups] } + + context "with no control groups added" do + it "is an empty list" do + expect(control_groups).to eq [] + end + end + + context "with one control group added" do + + before do + allow(audit_data).to receive(:control_groups).and_return([control_group_1]) + end + + it "is a one-element list containing the control group hash" do + expect(control_group_1).to receive(:to_hash).once.and_return(control_hash_1) + expect(control_groups.size).to eq 1 + expect(control_groups).to include(control_hash_1) + end + end + + context "with multiple control groups added" do + + before do + allow(audit_data).to receive(:control_groups).and_return([control_group_1, control_group_2]) + end + + it "is a list of control group hashes" do + expect(control_group_1).to receive(:to_hash).and_return(control_hash_1) + expect(control_group_2).to receive(:to_hash).and_return(control_hash_2) + expect(control_groups.size).to eq 2 + expect(control_groups).to include(control_hash_1) + expect(control_groups).to include(control_hash_2) + end + end + end + end +end + +describe Chef::Audit::ControlData do + + let(:name) { "ramen" } + let(:resource_type) { double("Service") } + let(:resource_name) { "mysql" } + let(:context) { nil } + let(:line_number) { 27 } + + let(:control_data) { described_class.new(name: name, + resource_type: resource_type, resource_name: resource_name, + context: context, line_number: line_number) } + + + describe "#to_hash" do + + let(:control_data_hash) { control_data.to_hash } + + it "returns a hash" do + expect(control_data_hash).to be_a(Hash) + end + + it "describes a Chef::Audit::ControlData object" do + keys = [:name, :resource_type, :resource_name, :context, :status, :details] + expect(control_data_hash.keys).to match_array(keys) + end + + context "when context is nil" do + + it "sets :context to an empty array" do + expect(control_data_hash[:context]).to eq [] + end + + end + + context "when context is non-nil" do + + let(:context) { ["outer"] } + + it "sets :context to its value" do + expect(control_data_hash[:context]).to eq context + end + end + end +end + +describe Chef::Audit::ControlGroupData do + + let(:name) { "balloon" } + let(:control_group_data) { described_class.new(name) } + + shared_context "control data" do + + let(:name) { "" } + let(:resource_type) { nil } + let(:resource_name) { nil } + let(:context) { nil } + let(:line_number) { 0 } + + let(:control_data) { + { + :name => name, + :resource_type => resource_type, + :resource_name => resource_name, + :context => context, + :line_number => line_number + } + } + + end + + shared_context "control" do + include_context "control data" + + let(:control) { Chef::Audit::ControlData.new(name: name, + resource_type: resource_type, resource_name: resource_name, + context: context, line_number: line_number) } + + before do + allow(Chef::Audit::ControlData).to receive(:new). + with(name: name, resource_type: resource_type, + resource_name: resource_name, context: context, + line_number: line_number). + and_return(control) + end + end + + describe "#new" do + it "has status \"success\"" do + expect(control_group_data.status).to eq "success" + end + end + + describe "#example_success" do + include_context "control" + + def notify_success + control_group_data.example_success(control_data) + end + + it "increments the number of successful audits" do + num_success = control_group_data.number_succeeded + notify_success + expect(control_group_data.number_succeeded).to eq (num_success + 1) + end + + it "does not increment the number of failed audits" do + num_failed = control_group_data.number_failed + notify_success + expect(control_group_data.number_failed).to eq (num_failed) + end + + it "marks the audit's status as success" do + notify_success + expect(control.status).to eq "success" + end + + it "does not modify its own status" do + expect(control_group_data).to_not receive(:status=) + status = control_group_data.status + notify_success + expect(control_group_data.status).to eq status + end + + it "saves the control" do + controls = control_group_data.controls + expect(controls).to_not include(control) + notify_success + expect(controls).to include(control) + end + end + + describe "#example_failure" do + include_context "control" + + let(:details) { "poop" } + + def notify_failure + control_group_data.example_failure(control_data, details) + end + + it "does not increment the number of successful audits" do + num_success = control_group_data.number_succeeded + notify_failure + expect(control_group_data.number_succeeded).to eq num_success + end + + it "increments the number of failed audits" do + num_failed = control_group_data.number_failed + notify_failure + expect(control_group_data.number_failed).to eq (num_failed + 1) + end + + it "marks the audit's status as failure" do + notify_failure + expect(control.status).to eq "failure" + end + + it "marks its own status as failure" do + notify_failure + expect(control_group_data.status).to eq "failure" + end + + it "saves the control" do + controls = control_group_data.controls + expect(controls).to_not include(control) + notify_failure + expect(controls).to include(control) + end + + context "when details are not provided" do + + let(:details) { nil } + + it "does not save details to the control" do + default_details = control.details + expect(control).to_not receive(:details=) + notify_failure + expect(control.details).to eq default_details + end + end + + context "when details are provided" do + + let(:details) { "yep that didn't work" } + + it "saves details to the control" do + notify_failure + expect(control.details).to eq details + end + end + end + + shared_examples "multiple audits" do |success_or_failure| + include_context "control" + + let(:num_success) { 0 } + let(:num_failure) { 0 } + + before do + if num_failure == 0 + num_success.times { control_group_data.example_success(control_data) } + elsif num_success == 0 + num_failure.times { control_group_data.example_failure(control_data, nil) } + end + end + + it "counts the number of successful audits" do + expect(control_group_data.number_succeeded).to eq num_success + end + + it "counts the number of failed audits" do + expect(control_group_data.number_failed).to eq num_failure + end + + it "marks its status as \"#{success_or_failure}\"" do + expect(control_group_data.status).to eq success_or_failure + end + end + + context "when all audits pass" do + include_examples "multiple audits", "success" do + let(:num_success) { 3 } + end + end + + context "when one audit fails" do + shared_examples "mixed audit results" do + include_examples "multiple audits", "failure" do + + let(:audit_results) { [] } + let(:num_success) { audit_results.count("success") } + let(:num_failure) { 1 } + + before do + audit_results.each do |result| + if result == "success" + control_group_data.example_success(control_data) + else + control_group_data.example_failure(control_data, nil) + end + end + end + end + end + + context "and it's the first audit" do + include_examples "mixed audit results" do + let(:audit_results) { ["failure", "success", "success"] } + end + end + + context "and it's an audit in the middle" do + include_examples "mixed audit results" do + let(:audit_results) { ["success", "failure", "success"] } + end + end + + context "and it's the last audit" do + include_examples "mixed audit results" do + let(:audit_results) { ["success", "success", "failure"] } + end + end + end + + context "when all audits fail" do + include_examples "multiple audits", "failure" do + let(:num_failure) { 3 } + end + end + + describe "#to_hash" do + + let(:control_group_data_hash) { control_group_data.to_hash } + + it "returns a hash" do + expect(control_group_data_hash).to be_a(Hash) + end + + it "describes a Chef::Audit::ControlGroupData object" do + keys = [:name, :status, :number_succeeded, :number_failed, + :controls, :id] + expect(control_group_data_hash.keys).to match_array(keys) + end + + describe ":controls" do + + let(:control_group_controls) { control_group_data_hash[:controls] } + + context "with no controls added" do + it "is an empty list" do + expect(control_group_controls).to eq [] + end + end + + context "with one control added" do + include_context "control" + + let(:control_list) { [control_data] } + let(:control_hash) { control.to_hash } + + before do + expect(control_group_data).to receive(:controls).twice.and_return(control_list) + expect(control_data).to receive(:to_hash).and_return(control_hash) + end + + it "is a one-element list containing the control hash" do + expect(control_group_controls.size).to eq 1 + expect(control_group_controls).to include(control_hash) + end + + it "adds a sequence number to the control" do + control_group_data.to_hash + expect(control_hash).to have_key(:sequence_number) + end + + end + + context "with multiple controls added" do + + let(:control_hash_1) { {:line_number => 27} } + let(:control_hash_2) { {:line_number => 13} } + let(:control_hash_3) { {:line_number => 35} } + + let(:control_1) { double("control 1", + :line_number => control_hash_1[:line_number], + :to_hash => control_hash_1) } + let(:control_2) { double("control 2", + :line_number => control_hash_2[:line_number], + :to_hash => control_hash_2) } + let(:control_3) { double("control 3", + :line_number => control_hash_3[:line_number], + :to_hash => control_hash_3) } + + let(:control_list) { [control_1, control_2, control_3] } + let(:ordered_control_hashes) { [control_hash_2, control_hash_1, control_hash_3] } + + before do + # Another way to do this would be to call #example_success + # or #example_failure per control hash, but we'd have to + # then stub #create_control and it's a lot of extra stubbing work. + # We can't stub the controls reader to return a list of + # controls because of the call to sort! and the following + # reading of controls. + control_group_data.instance_variable_set(:@controls, control_list) + end + + it "is a list of control group hashes ordered by line number" do + expect(control_group_controls.size).to eq 3 + expect(control_group_controls).to eq ordered_control_hashes + end + + it "assigns sequence numbers in order" do + control_group_data.to_hash + ordered_control_hashes.each_with_index do |control_hash, idx| + expect(control_hash[:sequence_number]).to eq idx + 1 + end + end + end + end + end + +end -- cgit v1.2.1 From 87031ff00b1189b67ca6308c7119344b1ef3fe44 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 5 Dec 2014 16:41:32 -0800 Subject: Add unit tests for Audit::AuditEventProxy --- spec/unit/audit/audit_event_proxy_spec.rb | 292 ++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 spec/unit/audit/audit_event_proxy_spec.rb diff --git a/spec/unit/audit/audit_event_proxy_spec.rb b/spec/unit/audit/audit_event_proxy_spec.rb new file mode 100644 index 0000000000..1fddde43f1 --- /dev/null +++ b/spec/unit/audit/audit_event_proxy_spec.rb @@ -0,0 +1,292 @@ + +require 'spec_helper' +require 'chef/audit/audit_event_proxy' + +describe Chef::Audit::AuditEventProxy do + + let(:stdout) { StringIO.new } + let(:events) { double("Chef::Events") } + let(:audit_event_proxy) { Chef::Audit::AuditEventProxy.new(stdout) } + + before do + Chef::Audit::AuditEventProxy.events = events + end + + describe "#example_group_started" do + + let(:description) { "poots" } + let(:group) { double("ExampleGroup", :parent_groups => parents, + :description => description) } + let(:notification) { double("Notification", :group => group) } + + context "when notified from a top-level example group" do + + let(:parents) { [double("ExampleGroup")] } + + it "notifies control_group_started event" do + expect(Chef::Log).to receive(:debug). + with("Entered \`controls\` block named poots") + expect(events).to receive(:control_group_started). + with(description) + audit_event_proxy.example_group_started(notification) + end + end + + context "when notified from an inner-level example group" do + + let(:parents) { [double("ExampleGroup"), double("OuterExampleGroup")] } + + it "does nothing" do + expect(events).to_not receive(:control_group_started) + audit_event_proxy.example_group_started(notification) + end + end + end + + describe "#stop" do + + let(:examples) { [] } + let(:notification) { double("Notification", :examples => examples) } + let(:exception) { nil } + let(:example) { double("Example", :exception => exception) } + let(:control_group_name) { "audit test" } + let(:control_data) { double("ControlData") } + + before do + allow(Chef::Log).to receive(:info) # silence messages to output stream + end + + it "sends a message that audits completed" do + expect(Chef::Log).to receive(:info).with("Successfully executed all \`controls\` blocks and contained examples") + audit_event_proxy.stop(notification) + end + + context "when an example succeeded" do + + let(:examples) { [example] } + let(:excpetion) { nil } + + before do + allow(audit_event_proxy).to receive(:build_control_from). + with(example). + and_return([control_group_name, control_data]) + end + + it "notifies events" do + expect(events).to receive(:control_example_success). + with(control_group_name, control_data) + audit_event_proxy.stop(notification) + end + end + + context "when an example failed" do + + let(:examples) { [example] } + let(:exception) { double("ExpectationNotMet") } + + before do + allow(audit_event_proxy).to receive(:build_control_from). + with(example). + and_return([control_group_name, control_data]) + end + + it "notifies events" do + expect(events).to receive(:control_example_failure). + with(control_group_name, control_data, exception) + audit_event_proxy.stop(notification) + end + end + + describe "#build_control_from" do + + let(:examples) { [example] } + + let(:example) { double("Example", :metadata => metadata, + :description => example_description, + :full_description => full_description, :exception => nil) } + + let(:metadata) { + { + :described_class => described_class, + :example_group => example_group, + :line_number => line + } + } + + let(:example_group) { + { + :description => group_description, + :parent_example_group => parent_group + } + } + + let(:parent_group) { + { + :description => control_group_name, + :parent_example_group => nil + } + } + + let(:line) { 27 } + + let(:control_data) { + { + :name => example_description, + :desc => full_description, + :resource_type => resource_type, + :resource_name => resource_name, + :context => context, + :line_number => line + } + } + + shared_examples "built control" do + + before do + if described_class + allow(described_class).to receive(:instance_variable_get). + with(:@name). + and_return(resource_name) + allow(described_class.class).to receive(:name). + and_return(described_class.class) + end + end + + it "returns the controls block name and example metadata for reporting" do + expect(events).to receive(:control_example_success). + with(control_group_name, control_data) + audit_event_proxy.stop(notification) + end + end + + describe "a top-level example" do + # controls "port 111" do + # it "has nobody listening" do + # expect(port("111")).to_not be_listening + # end + # end + + # Description parts + let(:group_description) { "port 111" } + let(:example_description) { "has nobody listening" } + let(:full_description) { group_description + " " + example_description } + + # Metadata fields + let(:described_class) { nil } + + # Example group (metadata[:example_group]) fields + let(:parent_group) { nil } + + # Expected returns + let(:control_group_name) { group_description } + + # Control data fields + let(:resource_type) { nil } + let(:resource_name) { nil } + let(:context) { [] } + + include_examples "built control" + end + + describe "an example with an implicit subject" do + # controls "application ports" do + # control port(111) do + # it { is_expected.to_not be_listening } + # end + # end + + # Description parts + let(:control_group_name) { "application ports" } + let(:group_description) { "#{resource_type} #{resource_name}" } + let(:example_description) { "should not be listening" } + let(:full_description) { [control_group_name, group_description, + example_description].join(" ") } + + # Metadata fields + let(:described_class) { double("Serverspec::Type::Port", + :class => "Serverspec::Type::Port") } + + # Control data fields + let(:resource_type) { "Port" } + let(:resource_name) { "111" } + let(:context) { [] } + + include_examples "built control" + end + + describe "an example in a nested context" do + # controls "application ports" do + # control "port 111" do + # it "is not listening" do + # expect(port(111)).to_not be_listening + # end + # end + # end + + # Description parts + let(:control_group_name) { "application ports" } + let(:group_description) { "port 111" } + let(:example_description) { "is not listening" } + let(:full_description) { [control_group_name, group_description, + example_description].join(" ") } + + # Metadata fields + let(:described_class) { nil } + + # Control data fields + let(:resource_type) { nil } + let(:resource_name) { nil } + let(:context) { [group_description] } + + include_examples "built control" + end + + describe "an example in a nested context including Serverspec" do + # controls "application directory" do + # control file("/tmp/audit") do + # describe file("/tmp/audit/test_file") do + # it "is a file" do + # expect(subject).to be_file + # end + # end + # end + # end + + # Description parts + let(:control_group_name) { "application directory" } + let(:outer_group_description) { "File \"tmp/audit\"" } + let(:group_description) { "#{resource_type} #{resource_name}" } + let(:example_description) { "is a file" } + let(:full_description) { [control_group_name, outer_group_description, + group_description, example_description].join(" ") } + + # Metadata parts + let(:described_class) { double("Serverspec::Type::File", + :class => "Serverspec::Type::File") } + + # Example group parts + let(:parent_group) { + { + :description => outer_group_description, + :parent_example_group => control_group + } + } + + let(:control_group) { + { + :description => control_group_name, + :parent_example_group => nil + } + } + + # Control data parts + let(:resource_type) { "File" } + let(:resource_name) { "/tmp/audit/test_file" } + let(:context) { [outer_group_description] } + + include_examples "built control" + end + end + end + +end -- cgit v1.2.1 From dd7f6cab61f8d4a2e730f2f744585a9c5f9c8a5c Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Fri, 5 Dec 2014 16:47:29 -0800 Subject: Add unit tests for Audit::AuditReporter --- spec/unit/audit/audit_reporter_spec.rb | 374 +++++++++++++++++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 spec/unit/audit/audit_reporter_spec.rb diff --git a/spec/unit/audit/audit_reporter_spec.rb b/spec/unit/audit/audit_reporter_spec.rb new file mode 100644 index 0000000000..47e0264462 --- /dev/null +++ b/spec/unit/audit/audit_reporter_spec.rb @@ -0,0 +1,374 @@ + +require 'spec_helper' + +describe Chef::Audit::AuditReporter do + + let(:rest) { double("rest") } + let(:reporter) { described_class.new(rest) } + let(:node) { double("node", :name => "sofreshsoclean") } + let(:run_id) { 0 } + let(:start_time) { Time.new(2014, 12, 3, 9, 31, 05, "-08:00") } + let(:end_time) { Time.new(2014, 12, 3, 9, 36, 14, "-08:00") } + let(:run_status) { instance_double(Chef::RunStatus, :node => node, :run_id => run_id, + :start_time => start_time, :end_time => end_time) } + + describe "#audit_phase_start" do + + it "notifies audit phase start to debug log" do + expect(Chef::Log).to receive(:debug).with(/Audit Reporter starting/) + reporter.audit_phase_start(run_status) + end + + it "initializes an AuditData object" do + expect(Chef::Audit::AuditData).to receive(:new).with(run_status.node.name, run_status.run_id) + reporter.audit_phase_start(run_status) + end + + it "saves the run status" do + reporter.audit_phase_start(run_status) + expect(reporter.instance_variable_get(:@run_status)).to eq run_status + end + end + + describe "#run_completed" do + + let(:audit_data) { Chef::Audit::AuditData.new(node.name, run_id) } + let(:run_data) { audit_data.to_hash } + + before do + allow(reporter).to receive(:auditing_enabled?).and_return(true) + allow(reporter).to receive(:run_status).and_return(run_status) + allow(rest).to receive(:create_url).and_return(true) + allow(rest).to receive(:post).and_return(true) + allow(reporter).to receive(:audit_data).and_return(audit_data) + allow(reporter).to receive(:run_status).and_return(run_status) + allow(audit_data).to receive(:to_hash).and_return(run_data) + end + + describe "a successful run with auditing enabled" do + it "sets run start and end times" do + iso_start_time = "2014-12-03T17:31:05Z" + iso_end_time = "2014-12-03T17:36:14Z" + + reporter.run_completed(node) + expect(audit_data.start_time).to eq iso_start_time + expect(audit_data.end_time).to eq iso_end_time + end + + it "posts audit data to server endpoint" do + endpoint = "api.opscode.us/orgname/controls" + headers = { + 'X-Ops-Audit-Report-Protocol-Version' => Chef::Audit::AuditReporter::PROTOCOL_VERSION + } + + expect(rest).to receive(:create_url). + with("controls"). + and_return(endpoint) + expect(rest).to receive(:post). + with(endpoint, run_data, headers) + reporter.run_completed(node) + end + + context "when unable to post to server" do + + let(:error) do + e = StandardError.new + e.set_backtrace(caller) + e + end + + before do + expect(rest).to receive(:post).and_raise(error) + allow(error).to receive(:respond_to?).and_call_original + end + + context "the error is an http error" do + + let(:response) { double("response", :code => code) } + + before do + expect(Chef::Log).to receive(:debug).with(/Sending audit report/) + expect(Chef::Log).to receive(:debug).with(/Audit Report/) + allow(error).to receive(:response).and_return(response) + expect(error).to receive(:respond_to?).with(:response).and_return(true) + end + + context "when the code is 404" do + + let(:code) { "404" } + + it "logs that the server doesn't support audit reporting" do + expect(Chef::Log).to receive(:debug).with(/Server doesn't support audit reporting/) + reporter.run_completed(node) + end + end + + shared_examples "non-404 error code" do + + it "saves the error report" do + expect(Chef::FileCache).to receive(:store). + with("failed-audit-data.json", an_instance_of(String), 0640). + and_return(true) + expect(Chef::FileCache).to receive(:load). + with("failed-audit-data.json", false). + and_return(true) + expect(Chef::Log).to receive(:error).with(/Failed to post audit report to server/) + reporter.run_completed(node) + end + + end + + context "when the code is not 404" do + include_examples "non-404 error code" do + let(:code) { "505" } + end + end + + context "when there is no code" do + include_examples "non-404 error code" do + let(:code) { nil } + end + end + + end + + context "the error is not an http error" do + + it "logs the error" do + expect(error).to receive(:respond_to?).with(:response).and_return(false) + expect(Chef::Log).to receive(:error).with(/Failed to post audit report to server/) + reporter.run_completed(node) + end + + end + + context "when reporting url fatals are enabled" do + + before do + allow(Chef::Config).to receive(:[]). + with(:enable_reporting_url_fatals). + and_return(true) + end + + it "raises the error" do + expect(error).to receive(:respond_to?).with(:response).and_return(false) + allow(Chef::Log).to receive(:error).and_return(true) + expect(Chef::Log).to receive(:error).with(/Reporting fatals enabled. Aborting run./) + expect{ reporter.run_completed(node) }.to raise_error(error) + end + + end + end + end + + context "when auditing is not enabled" do + + before do + allow(Chef::Log).to receive(:debug) + end + + it "doesn't send reports" do + expect(reporter).to receive(:auditing_enabled?).and_return(false) + expect(Chef::Log).to receive(:debug).with("Audit Reports are disabled. Skipping sending reports.") + reporter.run_completed(node) + end + + end + + context "when the run fails before audits" do + + before do + allow(Chef::Log).to receive(:debug) + end + + it "doesn't send reports" do + expect(reporter).to receive(:auditing_enabled?).and_return(true) + expect(reporter).to receive(:run_status).and_return(nil) + expect(Chef::Log).to receive(:debug).with("Run failed before audits were initialized, not sending audit report to server") + reporter.run_completed(node) + end + + end + end + + describe "#run_failed" do + + let(:audit_data) { Chef::Audit::AuditData.new(node.name, run_id) } + let(:run_data) { audit_data.to_hash } + + let(:error) { double("AuditError", :class => "Chef::Exception::AuditError", + :message => "Well that certainly didn't work", + :backtrace => ["line 0", "line 1", "line 2"]) } + + before do + allow(reporter).to receive(:auditing_enabled?).and_return(true) + allow(reporter).to receive(:run_status).and_return(run_status) + allow(reporter).to receive(:audit_data).and_return(audit_data) + allow(audit_data).to receive(:to_hash).and_return(run_data) + end + + it "adds the error information to the reported data" do + expect(rest).to receive(:create_url) + expect(rest).to receive(:post) + reporter.run_failed(error) + expect(run_data).to have_key(:error) + expect(run_data[:error]).to eq "Chef::Exception::AuditError: Well that certainly didn't work\n" + + "line 0\nline 1\nline 2" + end + + end + + shared_context "audit data" do + + let(:control_group_foo) { instance_double(Chef::Audit::ControlGroupData, + :metadata => double("foo metadata")) } + let(:control_group_bar) { instance_double(Chef::Audit::ControlGroupData, + :metadata => double("bar metadata")) } + + let(:ordered_control_groups) { + { + "foo" => control_group_foo, + "bar" => control_group_bar + } + } + + let(:audit_data) { instance_double(Chef::Audit::AuditData, + :add_control_group => true) } + + let(:run_context) { instance_double(Chef::RunContext, + :audits => ordered_control_groups) } + + before do + allow(reporter).to receive(:ordered_control_groups).and_return(ordered_control_groups) + allow(reporter).to receive(:audit_data).and_return(audit_data) + allow(reporter).to receive(:run_status).and_return(run_status) + allow(run_status).to receive(:run_context).and_return(run_context) + end + end + + describe "#audit_phase_complete" do + include_context "audit data" + + it "notifies audit phase finished to debug log" do + expect(Chef::Log).to receive(:debug).with(/Audit Reporter completed/) + reporter.audit_phase_complete + end + + it "collects audit data" do + ordered_control_groups.each do |_name, group| + expect(audit_data).to receive(:add_control_group).with(group) + end + reporter.audit_phase_complete + end + end + + describe "#audit_phase_failed" do + include_context "audit data" + + let(:error) { double("Exception") } + + it "notifies audit phase failed to debug log" do + expect(Chef::Log).to receive(:debug).with(/Audit Reporter failed/) + reporter.audit_phase_failed(error) + end + + it "collects audit data" do + ordered_control_groups.each do |_name, group| + expect(audit_data).to receive(:add_control_group).with(group) + end + reporter.audit_phase_failed(error) + end + end + + describe "#control_group_started" do + include_context "audit data" + + let(:name) { "bat" } + let(:control_group) { instance_double(Chef::Audit::ControlGroupData, + :metadata => double("metadata")) } + + before do + allow(Chef::Audit::ControlGroupData).to receive(:new). + with(name, control_group.metadata). + and_return(control_group) + end + + it "stores the control group" do + expect(ordered_control_groups).to receive(:has_key?).with(name).and_return(false) + allow(run_context.audits).to receive(:[]).with(name).and_return(control_group) + expect(ordered_control_groups).to receive(:store). + with(name, control_group). + and_call_original + reporter.control_group_started(name) + # stubbed :has_key? above, which is used by the have_key matcher, + # so instead we check the response to Hash's #key? because luckily + # #key? does not call #has_key? + expect(ordered_control_groups.key?(name)).to be true + expect(ordered_control_groups[name]).to eq control_group + end + + context "when a control group with the same name has been seen" do + it "raises an exception" do + expect(ordered_control_groups).to receive(:has_key?).with(name).and_return(true) + expect{ reporter.control_group_started(name) }.to raise_error(Chef::Exceptions::AuditControlGroupDuplicate) + end + end + end + + describe "#control_example_success" do + include_context "audit data" + + let(:name) { "foo" } + let(:example_data) { double("example data") } + + it "notifies the control group the example succeeded" do + expect(control_group_foo).to receive(:example_success).with(example_data) + reporter.control_example_success(name, example_data) + end + end + + describe "#control_example_failure" do + include_context "audit data" + + let(:name) { "bar" } + let(:example_data) { double("example data") } + let(:error) { double("Exception", :message => "oopsie") } + + it "notifies the control group the example failed" do + expect(control_group_bar).to receive(:example_failure). + with(example_data, error.message) + reporter.control_example_failure(name, example_data, error) + end + end + + describe "#auditing_enabled?" do + shared_examples "enabled?" do |true_or_false| + + it "returns #{true_or_false}" do + expect(Chef::Config).to receive(:[]). + with(:audit_mode). + and_return(audit_setting) + expect(reporter.auditing_enabled?).to be true_or_false + end + end + + context "when auditing is disabled" do + include_examples "enabled?", false do + let(:audit_setting) { :disabled } + end + end + + context "when auditing in audit-only mode" do + include_examples "enabled?", true do + let(:audit_setting) { :audit_only } + end + end + + context "when auditing is enabled" do + include_examples "enabled?", true do + let(:audit_setting) { :enabled } + end + end + end + +end -- cgit v1.2.1 From 3b3d05b91e0e6f54a2a28539c695fe7ec7c66550 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 9 Dec 2014 09:09:17 -0800 Subject: Adding audit DSL coverage --- spec/unit/dsl/audit_spec.rb | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/spec/unit/dsl/audit_spec.rb b/spec/unit/dsl/audit_spec.rb index 7565a42d58..38707127f0 100644 --- a/spec/unit/dsl/audit_spec.rb +++ b/spec/unit/dsl/audit_spec.rb @@ -2,18 +2,19 @@ require 'spec_helper' require 'chef/dsl/audit' -class AuditDSLTester +class AuditDSLTester < Chef::Recipe include Chef::DSL::Audit end -describe Chef::DSL::Audit do - let(:auditor) { AuditDSLTester.new } - let(:run_context) { instance_double(Chef::RunContext, :audits => audits) } - let(:audits) { [] } +class BadAuditDSLTester + include Chef::DSL::Audit +end - before do - allow(auditor).to receive(:run_context).and_return(run_context) - end +describe Chef::DSL::Audit do + let(:auditor) { AuditDSLTester.new("cookbook_name", "recipe_name", run_context) } + let(:run_context) { instance_double(Chef::RunContext, :audits => audits, :cookbook_collection => cookbook_collection) } + let(:audits) { {} } + let(:cookbook_collection) { {} } it "raises an error when a block of audits is not provided" do expect{ auditor.controls "name" }.to raise_error(Chef::Exceptions::NoAuditsProvided) @@ -23,8 +24,20 @@ describe Chef::DSL::Audit do expect{ auditor.controls do end }.to raise_error(Chef::Exceptions::AuditNameMissing) end - it "raises an error if the audit name is a duplicate" do - expect(audits).to receive(:has_key?).with("unique").and_return(true) - expect { auditor.controls "unique" do end }.to raise_error(Chef::Exceptions::AuditControlGroupDuplicate) + context "audits already populated" do + let(:audits) { {"unique" => {} } } + + it "raises an error if the audit name is a duplicate" do + expect { auditor.controls "unique" do end }.to raise_error(Chef::Exceptions::AuditControlGroupDuplicate) + end end + + context "included in a class without recipe DSL" do + let(:auditor) { BadAuditDSLTester.new } + + it "fails because it relies on the recipe DSL existing" do + expect { auditor.controls "unique" do end }.to raise_error(NoMethodError, /undefined method `cookbook_name'/) + end + end + end -- cgit v1.2.1 From 1d64d2371e1abe83c3fe726e46e95d924a44e15d Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 9 Dec 2014 10:22:28 -0800 Subject: Rescuing Exception blind was covering up an unexpected error --- lib/chef/exceptions.rb | 2 +- spec/unit/client_spec.rb | 125 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 100 insertions(+), 27 deletions(-) diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb index dabdd03802..b204f6ef2a 100644 --- a/lib/chef/exceptions.rb +++ b/lib/chef/exceptions.rb @@ -409,7 +409,7 @@ class Chef attr_reader :wrapped_errors def initialize(*errors) errors = errors.select {|e| !e.nil?} - output = "Found #{errors.size} errors, they are stored in the backtrace\n" + output = "Found #{errors.size} errors, they are stored in the backtrace" @wrapped_errors = errors super output end diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index 8a1246e1f6..4f6d8a0b82 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -192,7 +192,7 @@ describe Chef::Client do let(:http_cookbook_sync) { double("Chef::REST (cookbook sync)") } let(:http_node_save) { double("Chef::REST (node save)") } let(:runner) { double("Chef::Runner") } - let(:audit_runner) { double("Chef::Audit::Runner") } + let(:audit_runner) { instance_double("Chef::Audit::Runner", :failed? => false) } let(:api_client_exists?) { false } @@ -205,6 +205,7 @@ describe Chef::Client do # --Client.register # Make sure Client#register thinks the client key doesn't # exist, so it tries to register and create one. + allow(File).to receive(:exists?).and_call_original expect(File).to receive(:exists?). with(Chef::Config[:client_key]). exactly(:once). @@ -288,6 +289,7 @@ describe Chef::Client do before do Chef::Config[:client_fork] = enable_fork Chef::Config[:cache_path] = windows? ? 'C:\chef' : '/var/chef' + Chef::Config[:why_run] = false stub_const("Chef::Client::STDOUT_FD", stdout) stub_const("Chef::Client::STDERR_FD", stderr) @@ -390,9 +392,11 @@ describe Chef::Client do describe "when converge fails" do include_context "a client run" do + let(:e) { Exception.new } def stub_for_converge expect(Chef::Runner).to receive(:new).and_return(runner) - expect(runner).to receive(:converge).and_raise(Exception) + expect(runner).to receive(:converge).and_raise(e) + expect(Chef::Application).to receive(:debug_stacktrace).with an_instance_of(Chef::Exceptions::RunFailedWrappingError) end def stub_for_node_save @@ -408,30 +412,27 @@ describe Chef::Client do expect(client).to receive(:run_started) expect(client).to receive(:run_failed) - # --ResourceReporter#run_completed - # updates the server with the resource history - # (has its own tests, so stubbing it here.) - # TODO: What gets called here? - #expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) - # --AuditReporter#run_completed - # posts the audit data to server. - # (has its own tests, so stubbing it here.) - # TODO: What gets called here? - #expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) + expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) end end it "runs the audits and raises the error" do - expect{ client.run }.to raise_error(Exception) + expect{ client.run }.to raise_error(Chef::Exceptions::RunFailedWrappingError) do |error| + expect(error.wrapped_errors.size).to eq(1) + expect(error.wrapped_errors[0]).to eq(e) + end end end describe "when the audit phase fails" do context "with an exception" do include_context "a client run" do + let(:e) { Exception.new } def stub_for_audit expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner) - expect(audit_runner).to receive(:run).and_raise(Exception) + expect(audit_runner).to receive(:run).and_raise(e) + expect(Chef::Application).to receive(:debug_stacktrace).with an_instance_of(Chef::Exceptions::RunFailedWrappingError) end def stub_for_run @@ -443,26 +444,98 @@ describe Chef::Client do expect(client).to receive(:run_started) expect(client).to receive(:run_failed) - # --ResourceReporter#run_completed - # updates the server with the resource history - # (has its own tests, so stubbing it here.) - # TODO: What gets called here? - #expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) - # --AuditReporter#run_completed - # posts the audit data to server. - # (has its own tests, so stubbing it here.) - # TODO: What gets called here? - #expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) + expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) end end it "should save the node after converge and raise exception" do - expect{ client.run }.to raise_error(Exception) + expect{ client.run }.to raise_error(Chef::Exceptions::RunFailedWrappingError) do |error| + expect(error.wrapped_errors.size).to eq(1) + expect(error.wrapped_errors[0]).to eq(e) + end end end context "with failed audits" do - skip("because I don't think we've implemented this yet") + include_context "a client run" do + let(:audit_runner) do + instance_double("Chef::Audit::Runner", :run => true, :failed? => true, :num_failed => 1, :num_total => 1) + end + + def stub_for_audit + expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner) + expect(Chef::Application).to receive(:debug_stacktrace).with an_instance_of(Chef::Exceptions::RunFailedWrappingError) + end + + def stub_for_run + expect_any_instance_of(Chef::RunLock).to receive(:acquire) + expect_any_instance_of(Chef::RunLock).to receive(:save_pid) + expect_any_instance_of(Chef::RunLock).to receive(:release) + + # Post conditions: check that node has been filled in correctly + expect(client).to receive(:run_started) + expect(client).to receive(:run_failed) + + expect_any_instance_of(Chef::ResourceReporter).to receive(:run_failed) + expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_failed) + end + end + + it "should save the node after converge and raise exception" do + expect{ client.run }.to raise_error(Chef::Exceptions::RunFailedWrappingError) do |error| + expect(error.wrapped_errors.size).to eq(1) + expect(error.wrapped_errors[0]).to be_instance_of(Chef::Exceptions::AuditsFailed) + end + end + end + end + + describe "when why_run mode is enabled" do + include_context "a client run" do + + before do + Chef::Config[:why_run] = true + end + + def stub_for_audit + expect(Chef::Audit::Runner).to_not receive(:new) + end + + def stub_for_node_save + # This is how we should be mocking external calls - not letting it fall all the way through to the + # REST call + expect(node).to receive(:save) + end + + it "runs successfully without enabling the audit runner" do + client.run + + # fork is stubbed, so we can see the outcome of the run + expect(node.automatic_attrs[:platform]).to eq("example-platform") + expect(node.automatic_attrs[:platform_version]).to eq("example-platform-1.0") + end + end + end + + describe "when audits are disabled" do + include_context "a client run" do + + before do + Chef::Config[:audit_mode] = :disabled + end + + def stub_for_audit + expect(Chef::Audit::Runner).to_not receive(:new) + end + + it "runs successfully without enabling the audit runner" do + client.run + + # fork is stubbed, so we can see the outcome of the run + expect(node.automatic_attrs[:platform]).to eq("example-platform") + expect(node.automatic_attrs[:platform_version]).to eq("example-platform-1.0") + end end end -- cgit v1.2.1 From e208a23c658c323977026c9696b0582d3a0c45ad Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 9 Dec 2014 08:51:59 -0800 Subject: Adding test for recipe DSL audit additions --- chef.gemspec | 4 ++-- lib/chef/audit/audit_event_proxy.rb | 3 +-- spec/unit/audit/audit_event_proxy_spec.rb | 4 ++-- spec/unit/exceptions_spec.rb | 2 +- spec/unit/recipe_spec.rb | 7 +++++++ 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/chef.gemspec b/chef.gemspec index eb7528e630..43fb3d16fe 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -37,8 +37,8 @@ Gem::Specification.new do |s| %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_dependency gem, "~> 3.1" } s.add_dependency "rspec_junit_formatter", "~> 0.2.0" - s.add_dependency "serverspec", "~> 2.3" - s.add_dependency "specinfra", "~> 2.4" + s.add_dependency "serverspec", "~> 2.7" + s.add_dependency "specinfra", "~> 2.10" s.add_development_dependency "rack" diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index 36160db9bb..1e40258113 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -43,8 +43,7 @@ class Chef described_class = example.metadata[:described_class] if described_class resource_type = described_class.class.name.split(':')[-1] - # TODO https://github.com/serverspec/serverspec/pull/493 - resource_name = described_class.instance_variable_get(:@name) + resource_name = described_class.name end # The following code builds up the context - the list of wrapping `describe` or `control` blocks diff --git a/spec/unit/audit/audit_event_proxy_spec.rb b/spec/unit/audit/audit_event_proxy_spec.rb index 1fddde43f1..2c4a0a1b9a 100644 --- a/spec/unit/audit/audit_event_proxy_spec.rb +++ b/spec/unit/audit/audit_event_proxy_spec.rb @@ -204,7 +204,7 @@ describe Chef::Audit::AuditEventProxy do # Metadata fields let(:described_class) { double("Serverspec::Type::Port", - :class => "Serverspec::Type::Port") } + :class => "Serverspec::Type::Port", :name => resource_name) } # Control data fields let(:resource_type) { "Port" } @@ -262,7 +262,7 @@ describe Chef::Audit::AuditEventProxy do # Metadata parts let(:described_class) { double("Serverspec::Type::File", - :class => "Serverspec::Type::File") } + :class => "Serverspec::Type::File", :name => resource_name) } # Example group parts let(:parent_group) { diff --git a/spec/unit/exceptions_spec.rb b/spec/unit/exceptions_spec.rb index 165c11446b..d35ecc8ec8 100644 --- a/spec/unit/exceptions_spec.rb +++ b/spec/unit/exceptions_spec.rb @@ -85,7 +85,7 @@ describe Chef::Exceptions do describe Chef::Exceptions::RunFailedWrappingError do shared_examples "RunFailedWrappingError expectations" do it "should initialize with a default message" do - expect(e.message).to eq("Found #{num_errors} errors, they are stored in the backtrace\n") + expect(e.message).to eq("Found #{num_errors} errors, they are stored in the backtrace") end it "should provide a modified backtrace when requested" do diff --git a/spec/unit/recipe_spec.rb b/spec/unit/recipe_spec.rb index e1a42362ef..e8c1358ba2 100644 --- a/spec/unit/recipe_spec.rb +++ b/spec/unit/recipe_spec.rb @@ -484,4 +484,11 @@ describe Chef::Recipe do expect(node[:tags]).to eql([]) end end + + describe "included DSL" do + it "should include features from Chef::DSL::Audit" do + expect(recipe.singleton_class.included_modules).to include(Chef::DSL::Audit) + expect(recipe.respond_to?(:controls)).to be true + end + end end -- cgit v1.2.1 From fc161e7f145d84d558c40a3e562312100dbf44a4 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 10 Dec 2014 11:11:40 -0800 Subject: Adding simple integration test for audit mode output --- lib/chef/audit/audit_event_proxy.rb | 19 +++++++++++++++ lib/chef/audit/audit_reporter.rb | 4 +-- lib/chef/audit/control_group_data.rb | 19 +++++++++++++++ lib/chef/audit/rspec_formatter.rb | 19 +++++++++++++++ spec/integration/client/client_spec.rb | 39 ++++++++++++++++++++++++++++++ spec/unit/audit/audit_event_proxy_spec.rb | 19 +++++++++++++++ spec/unit/audit/audit_reporter_spec.rb | 19 +++++++++++++++ spec/unit/audit/control_group_data_spec.rb | 19 +++++++++++++++ 8 files changed, 155 insertions(+), 2 deletions(-) diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index 1e40258113..ff97fb2dd0 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -1,3 +1,22 @@ +# +# Auther:: Tyler Ball () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + RSpec::Support.require_rspec_core "formatters/base_text_formatter" class Chef diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 00af9984b2..407c2deeb0 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -1,7 +1,7 @@ # -# Auther:: Tyler Ball () +# Author:: Tyler Ball () # -# Copyright:: Copyright (c) 2014 Opscode, Inc. +# Copyright:: Copyright (c) 2014 Chef Software, Inc. # License:: Apache License, Version 2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb index 42a91ef5a7..204d7f8070 100644 --- a/lib/chef/audit/control_group_data.rb +++ b/lib/chef/audit/control_group_data.rb @@ -1,3 +1,22 @@ +# +# Author:: Tyler Ball () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + require 'securerandom' class Chef diff --git a/lib/chef/audit/rspec_formatter.rb b/lib/chef/audit/rspec_formatter.rb index 990c1cd780..4c4b239d34 100644 --- a/lib/chef/audit/rspec_formatter.rb +++ b/lib/chef/audit/rspec_formatter.rb @@ -1,3 +1,22 @@ +# +# Auther:: Tyler Ball () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + require 'rspec/core' class Chef diff --git a/spec/integration/client/client_spec.rb b/spec/integration/client/client_spec.rb index f4bb124781..e961b73d0d 100644 --- a/spec/integration/client/client_spec.rb +++ b/spec/integration/client/client_spec.rb @@ -239,4 +239,43 @@ EOM end end + + when_the_repository "has a cookbook with only an audit recipe" do + + before do + file 'config/client.rb', < chef_dir) + expect(result.error?).to be_falsey + expect(result.stdout).to include("Successfully executed all `controls` blocks and contained examples") + end + + it "should exit with a non-zero code when there is an audit failure" do + file 'cookbooks/audit_test/recipes/fail.rb', <<-RECIPE +controls "control group without top level control" do + it "should fail" do + expect(2 - 2).to eq(1) + end +end + RECIPE + + result = shell_out("#{chef_client} -c \"#{path_to('config/client.rb')}\" -o 'audit_test::fail'", :cwd => chef_dir) + expect(result.error?).to be_truthy + expect(result.stdout).to include("Failure/Error: expect(2 - 2).to eq(1)") + end + end + end diff --git a/spec/unit/audit/audit_event_proxy_spec.rb b/spec/unit/audit/audit_event_proxy_spec.rb index 2c4a0a1b9a..899ba468b1 100644 --- a/spec/unit/audit/audit_event_proxy_spec.rb +++ b/spec/unit/audit/audit_event_proxy_spec.rb @@ -1,3 +1,22 @@ +# +# Author:: Tyler Ball () +# Author:: Claire McQuin () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# require 'spec_helper' require 'chef/audit/audit_event_proxy' diff --git a/spec/unit/audit/audit_reporter_spec.rb b/spec/unit/audit/audit_reporter_spec.rb index 47e0264462..84d7ea82f0 100644 --- a/spec/unit/audit/audit_reporter_spec.rb +++ b/spec/unit/audit/audit_reporter_spec.rb @@ -1,3 +1,22 @@ +# +# Author:: Tyler Ball () +# Author:: Claire McQuin () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# require 'spec_helper' diff --git a/spec/unit/audit/control_group_data_spec.rb b/spec/unit/audit/control_group_data_spec.rb index e5e76a6011..e21ab066fd 100644 --- a/spec/unit/audit/control_group_data_spec.rb +++ b/spec/unit/audit/control_group_data_spec.rb @@ -1,3 +1,22 @@ +# +# Author:: Tyler Ball () +# Author:: Claire McQuin () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# require 'spec_helper' require 'securerandom' -- cgit v1.2.1 From 66727a7a3e8e986ca3499e0e8693c332ab309247 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 4 Dec 2014 18:19:03 -0800 Subject: Adding example of sandboxing --- spec/functional/audit/runner_spec.rb | 80 ++++++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 2 - spec/support/audit_helper.rb | 61 +++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 spec/functional/audit/runner_spec.rb create mode 100644 spec/support/audit_helper.rb diff --git a/spec/functional/audit/runner_spec.rb b/spec/functional/audit/runner_spec.rb new file mode 100644 index 0000000000..cb998d66aa --- /dev/null +++ b/spec/functional/audit/runner_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' +require 'spec/support/audit_helper' +require 'chef/audit/runner' +require 'chef/audit/audit_event_proxy' +require 'chef/audit/rspec_formatter' +require 'chef/run_context' +require 'pry' + +## +# This functional test ensures that our runner can be setup to not interfere with existing RSpec +# configuration and world objects. When normally running Chef, there is only 1 RSpec instance +# so this isn't needed. In unit testing the Runner should be mocked appropriately. + +describe Chef::Audit::Runner do + + let(:events) { double("events") } + let(:run_context) { instance_double(Chef::RunContext) } + let(:runner) { Chef::Audit::Runner.new(run_context) } + + # This is the first thing that gets called, and determines how the examples are ran + around(:each) do |ex| + Sandboxing.sandboxed { ex.run } + end + + describe "#configure_rspec" do + + it "adds the necessary formatters" do + # We don't expect the events to receive any calls because the AuditEventProxy that was registered from `runner.run` + # only existed in the Configuration object that got removed by the sandboxing + #expect(events).to receive(:control_example_success) + + expect(RSpec.configuration.formatters.size).to eq(0) + expect(run_context).to receive(:events).and_return(events) + expect(Chef::Audit::AuditEventProxy).to receive(:events=) + + runner.send(:add_formatters) + + expect(RSpec.configuration.formatters.size).to eq(2) + expect(RSpec.configuration.formatters[0]).to be_instance_of(Chef::Audit::AuditEventProxy) + expect(RSpec.configuration.formatters[1]).to be_instance_of(Chef::Audit::RspecFormatter) + + end + + end + + # When running these, because we are not mocking out any of the formatters we expect to get dual output on the + # command line + describe "#run" do + + before do + expect(run_context).to receive(:events).and_return(events) + end + + it "Correctly runs an empty controls block" do + expect(run_context).to receive(:audits).and_return({}) + runner.run + end + + it "Correctly runs a single successful control" do + should_pass = lambda do + it "should pass" do + expect(2 - 2).to eq(0) + end + end + + expect(run_context).to receive(:audits).and_return({ + "should pass" => {:args => [], :block => should_pass} + }) + + # TODO capture the output and verify it + runner.run + end + + it "Correctly runs a single failing control", :pending do + + end + + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9f8cc754b7..995be5060b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -169,8 +169,6 @@ RSpec.configure do |config| config.before(:each) do Chef::Config.reset - - allow_any_instance_of(Chef::Audit::Runner).to receive(:run).and_return(true) end config.before(:suite) do diff --git a/spec/support/audit_helper.rb b/spec/support/audit_helper.rb new file mode 100644 index 0000000000..d897de0453 --- /dev/null +++ b/spec/support/audit_helper.rb @@ -0,0 +1,61 @@ +# This code comes from https://github.com/rspec/rspec-core/blob/master/spec/spec_helper.rb and +# https://github.com/rspec/rspec-core/blob/master/spec/support/sandboxing.rb + +# To leverage the sandboxing use an `around` block: +# around(:each) do |ex| +# Sandboxing.sandboxed { ex.run } +# end + +# rspec-core did not include a license on Github + +# This is necessary, otherwise +class << RSpec + attr_writer :configuration, :world +end + +class NullObject + private + def method_missing(method, *args, &block) + # ignore + end +end + +module Sandboxing + def self.sandboxed(&block) + orig_load_path = $LOAD_PATH.dup + orig_config = RSpec.configuration + orig_world = RSpec.world + orig_example = RSpec.current_example + new_config = RSpec::Core::Configuration.new + new_config.expose_dsl_globally = false + new_config.expecting_with_rspec = true + new_world = RSpec::Core::World.new(new_config) + RSpec.configuration = new_config + RSpec.world = new_world + object = Object.new + object.extend(RSpec::Core::SharedExampleGroup) + + (class << RSpec::Core::ExampleGroup; self; end).class_exec do + alias_method :orig_run, :run + def run(reporter=nil) + RSpec.current_example = nil + orig_run(reporter || NullObject.new) + end + end + + RSpec::Mocks.with_temporary_scope do + object.instance_exec(&block) + end + ensure + (class << RSpec::Core::ExampleGroup; self; end).class_exec do + remove_method :run + alias_method :run, :orig_run + remove_method :orig_run + end + + RSpec.configuration = orig_config + RSpec.world = orig_world + RSpec.current_example = orig_example + $LOAD_PATH.replace(orig_load_path) + end +end -- cgit v1.2.1 From 44fc2faadb2d63c77fb1bfacb973be615a842c42 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Mon, 15 Dec 2014 14:22:35 -0800 Subject: Finishing unit and functional test coverage for audit runner --- Gemfile | 2 + lib/chef/audit/runner.rb | 5 +- spec/functional/audit/runner_spec.rb | 129 +++++++++++++++++++++-------------- spec/support/audit_helper.rb | 3 +- spec/unit/audit/runner_spec.rb | 124 +++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+), 56 deletions(-) create mode 100644 spec/unit/audit/runner_spec.rb diff --git a/Gemfile b/Gemfile index 1418235ebc..069719ffe2 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,8 @@ source "https://rubygems.org" gemspec :name => "chef" gem "activesupport", "< 4.0.0", :group => :compat_testing, :platform => "ruby" +# TODO after serverspec stops including their top level DSL we can remove this +gem "serverspec", :git => "https://github.com/tyler-ball/serverspec/" group(:docgen) do gem "yard" diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 5d485a8804..1450ef7f61 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -74,9 +74,6 @@ class Chef # prevents Specinfra and Serverspec from modifying the RSpec configuration # used by our spec tests. def require_deps - # TODO: We need to figure out a way to give audits its own configuration - # object. This involves finding a way to load these files w/o them adding - # to the configuration object used by our spec tests. require 'rspec' require 'rspec/its' require 'specinfra' @@ -143,7 +140,7 @@ class Chef # TODO: We may need to be clever and adjust this based on operating # system, or make it configurable. E.g., there is a PowerShell backend, # as well as an SSH backend. - Specinfra.configuration.backend = :exec + Specinfra.configuration.backend = :exec if Specinfra.configuration.backend != :exec end # Iterates through the controls registered to this run_context, builds an diff --git a/spec/functional/audit/runner_spec.rb b/spec/functional/audit/runner_spec.rb index cb998d66aa..8fab332167 100644 --- a/spec/functional/audit/runner_spec.rb +++ b/spec/functional/audit/runner_spec.rb @@ -1,10 +1,25 @@ +# +# Author:: Tyler Ball () +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + require 'spec_helper' require 'spec/support/audit_helper' require 'chef/audit/runner' -require 'chef/audit/audit_event_proxy' -require 'chef/audit/rspec_formatter' -require 'chef/run_context' -require 'pry' +require 'rspec/support/spec/in_sub_process' ## # This functional test ensures that our runner can be setup to not interfere with existing RSpec @@ -13,68 +28,80 @@ require 'pry' describe Chef::Audit::Runner do - let(:events) { double("events") } - let(:run_context) { instance_double(Chef::RunContext) } - let(:runner) { Chef::Audit::Runner.new(run_context) } - - # This is the first thing that gets called, and determines how the examples are ran - around(:each) do |ex| - Sandboxing.sandboxed { ex.run } - end - - describe "#configure_rspec" do - - it "adds the necessary formatters" do - # We don't expect the events to receive any calls because the AuditEventProxy that was registered from `runner.run` - # only existed in the Configuration object that got removed by the sandboxing - #expect(events).to receive(:control_example_success) - - expect(RSpec.configuration.formatters.size).to eq(0) - expect(run_context).to receive(:events).and_return(events) - expect(Chef::Audit::AuditEventProxy).to receive(:events=) + # The functional tests must be run in a sub_process. Including Serverspec includes the Serverspec DSL - this + # conflicts with our `package` DSL (among others) when we try to test `package` inside an RSpec example. + # Our DSL leverages `method_missing` while the Serverspec DSL defines a method on the RSpec::Core::ExampleGroup. + # The defined method wins our and returns before our `method_missing` DSL can be called. + # + # Running in a sub_process means the serverspec libraries will only be included in a forked process, not the main one. + include RSpec::Support::InSubProcess - runner.send(:add_formatters) + let(:events) { double("events").as_null_object } + let(:runner) { Chef::Audit::Runner.new(run_context) } + let(:stdout) { StringIO.new } - expect(RSpec.configuration.formatters.size).to eq(2) - expect(RSpec.configuration.formatters[0]).to be_instance_of(Chef::Audit::AuditEventProxy) - expect(RSpec.configuration.formatters[1]).to be_instance_of(Chef::Audit::RspecFormatter) + around(:each) do |ex| + Sandboxing.sandboxed { ex.run } + end + before do + Chef::Config[:log_location] = stdout end - end + # When running these, because we are not mocking out any of the formatters we expect to get dual output on the + # command line + describe "#run" do - # When running these, because we are not mocking out any of the formatters we expect to get dual output on the - # command line - describe "#run" do + let(:audits) { {} } + let(:run_context) { instance_double(Chef::RunContext, :events => events, :audits => audits) } + let(:controls_name) { "controls_name" } - before do - expect(run_context).to receive(:events).and_return(events) - end + it "Correctly runs an empty controls block" do + in_sub_process do + runner.run + end + end - it "Correctly runs an empty controls block" do - expect(run_context).to receive(:audits).and_return({}) - runner.run - end + context "there is a single successful control" do + let(:audits) do + should_pass = lambda do + it "should pass" do + expect(2 - 2).to eq(0) + end + end + { controls_name => Struct.new(:args, :block).new([controls_name], should_pass)} + end - it "Correctly runs a single successful control" do - should_pass = lambda do - it "should pass" do - expect(2 - 2).to eq(0) + it "correctly runs" do + in_sub_process do + runner.run + + expect(stdout.string).to match(/1 example, 0 failures/) + end end end - expect(run_context).to receive(:audits).and_return({ - "should pass" => {:args => [], :block => should_pass} - }) + context "there is a single failing control" do + let(:audits) do + should_fail = lambda do + it "should fail" do + expect(2 - 1).to eq(0) + end + end + { controls_name => Struct.new(:args, :block).new([controls_name], should_fail)} + end - # TODO capture the output and verify it - runner.run - end + it "correctly runs" do + in_sub_process do + runner.run - it "Correctly runs a single failing control", :pending do + expect(stdout.string).to match(/Failure\/Error: expect\(2 - 1\)\.to eq\(0\)/) + expect(stdout.string).to match(/1 example, 1 failure/) + expect(stdout.string).to match(/# controls_name should fail/) + end + end + end end - end - end diff --git a/spec/support/audit_helper.rb b/spec/support/audit_helper.rb index d897de0453..5744f779fc 100644 --- a/spec/support/audit_helper.rb +++ b/spec/support/audit_helper.rb @@ -8,7 +8,8 @@ # rspec-core did not include a license on Github -# This is necessary, otherwise +# Adding these as writers is necessary, otherwise we cannot set the new configuration. +# Only want to do this in the specs. class << RSpec attr_writer :configuration, :world end diff --git a/spec/unit/audit/runner_spec.rb b/spec/unit/audit/runner_spec.rb new file mode 100644 index 0000000000..ca6f51f9eb --- /dev/null +++ b/spec/unit/audit/runner_spec.rb @@ -0,0 +1,124 @@ +# +# Author:: Tyler Ball () +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'spec_helper' +require 'spec/support/audit_helper' +require 'chef/audit/runner' + +describe Chef::Audit::Runner do + + let(:events) { double("events") } + let(:run_context) { instance_double(Chef::RunContext, :events => events) } + let(:runner) { Chef::Audit::Runner.new(run_context) } + + around(:each) do |ex| + Sandboxing.sandboxed { ex.run } + end + + describe "#initialize" do + it "correctly sets the run_context during initialization" do + expect(runner.instance_variable_get(:@run_context)).to eq(run_context) + end + end + + context "during #run" do + + describe "#setup" do + let(:log_location) { File.join(Dir.tmpdir, 'audit_log') } + let(:color) { false } + + before do + Chef::Config[:log_location] = log_location + Chef::Config[:color] = color + end + + it "sets all the config values" do + runner.send(:setup) + + expect(RSpec.configuration.output_stream).to eq(log_location) + expect(RSpec.configuration.error_stream).to eq(log_location) + + expect(RSpec.configuration.formatters.size).to eq(2) + expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::AuditEventProxy)) + expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::RspecFormatter)) + expect(Chef::Audit::AuditEventProxy.class_variable_get(:@@events)).to eq(run_context.events) + + expect(RSpec.configuration.expectation_frameworks).to eq([RSpec::Matchers]) + expect(RSpec::Matchers.configuration.syntax).to eq([:expect]) + + expect(RSpec.configuration.color).to eq(color) + expect(RSpec.configuration.expose_dsl_globally?).to eq(false) + + expect(Specinfra.configuration.backend).to eq(:exec) + end + end + + describe "#register_controls" do + let(:audits) { [] } + let(:run_context) { instance_double(Chef::RunContext, :audits => audits) } + + it "adds the control group aliases" do + runner.send(:register_controls) + + expect(RSpec::Core::DSL.example_group_aliases).to include(:__controls__) + expect(RSpec::Core::DSL.example_group_aliases).to include(:control) + end + + context "audits exist" do + let(:audits) { {"audit_name" => group} } + let(:group) {Struct.new(:args, :block).new(["group_name"], nil)} + + it "sends the audits to the world" do + runner.send(:register_controls) + + expect(RSpec.world.example_groups.size).to eq(1) + # For whatever reason, `kind_of` is not working + g = RSpec.world.example_groups[0] + expect(g.ancestors).to include(RSpec::Core::ExampleGroup) + expect(g.description).to eq("group_name") + end + end + end + + describe "#do_run" do + let(:rspec_runner) { instance_double(RSpec::Core::Runner) } + + it "executes the runner" do + expect(RSpec::Core::Runner).to receive(:new).with(nil).and_return(rspec_runner) + expect(rspec_runner).to receive(:run_specs).with([]) + + runner.send(:do_run) + end + end + end + + describe "counters" do + it "correctly calculates failed?" do + expect(runner.failed?).to eq(false) + end + + it "correctly calculates num_failed" do + expect(runner.num_failed).to eq(0) + end + + it "correctly calculates num_total" do + expect(runner.num_total).to eq(0) + end + end + +end -- cgit v1.2.1 From 0587f48b65a0a5f6ed8803cf1bd0d4b78e68c8a8 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Mon, 15 Dec 2014 14:44:36 -0800 Subject: Fixing unit test adding Serverspec DSL --- Gemfile | 2 - lib/chef/audit/audit_event_proxy.rb | 3 +- lib/chef/audit/rspec_formatter.rb | 3 +- spec/functional/audit/runner_spec.rb | 100 +++++++++++++++++------------------ spec/unit/audit/runner_spec.rb | 33 +++++++----- 5 files changed, 72 insertions(+), 69 deletions(-) diff --git a/Gemfile b/Gemfile index 069719ffe2..1418235ebc 100644 --- a/Gemfile +++ b/Gemfile @@ -2,8 +2,6 @@ source "https://rubygems.org" gemspec :name => "chef" gem "activesupport", "< 4.0.0", :group => :compat_testing, :platform => "ruby" -# TODO after serverspec stops including their top level DSL we can remove this -gem "serverspec", :git => "https://github.com/tyler-ball/serverspec/" group(:docgen) do gem "yard" diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb index ff97fb2dd0..2512b8bfe2 100644 --- a/lib/chef/audit/audit_event_proxy.rb +++ b/lib/chef/audit/audit_event_proxy.rb @@ -1,6 +1,5 @@ # -# Auther:: Tyler Ball () -# +# Author:: Tyler Ball () # Copyright:: Copyright (c) 2014 Chef Software, Inc. # License:: Apache License, Version 2.0 # diff --git a/lib/chef/audit/rspec_formatter.rb b/lib/chef/audit/rspec_formatter.rb index 4c4b239d34..074a11bed3 100644 --- a/lib/chef/audit/rspec_formatter.rb +++ b/lib/chef/audit/rspec_formatter.rb @@ -1,6 +1,5 @@ # -# Auther:: Tyler Ball () -# +# Author:: Serdar Sutay () # Copyright:: Copyright (c) 2014 Chef Software, Inc. # License:: Apache License, Version 2.0 # diff --git a/spec/functional/audit/runner_spec.rb b/spec/functional/audit/runner_spec.rb index 8fab332167..89c62ae2e8 100644 --- a/spec/functional/audit/runner_spec.rb +++ b/spec/functional/audit/runner_spec.rb @@ -36,72 +36,70 @@ describe Chef::Audit::Runner do # Running in a sub_process means the serverspec libraries will only be included in a forked process, not the main one. include RSpec::Support::InSubProcess - let(:events) { double("events").as_null_object } - let(:runner) { Chef::Audit::Runner.new(run_context) } - let(:stdout) { StringIO.new } - - around(:each) do |ex| - Sandboxing.sandboxed { ex.run } - end - - before do - Chef::Config[:log_location] = stdout - end - - # When running these, because we are not mocking out any of the formatters we expect to get dual output on the - # command line - describe "#run" do - - let(:audits) { {} } - let(:run_context) { instance_double(Chef::RunContext, :events => events, :audits => audits) } - let(:controls_name) { "controls_name" } - - it "Correctly runs an empty controls block" do - in_sub_process do - runner.run - end + let(:events) { double("events").as_null_object } + let(:runner) { Chef::Audit::Runner.new(run_context) } + let(:stdout) { StringIO.new } + + around(:each) do |ex| + Sandboxing.sandboxed { ex.run } + end + + before do + Chef::Config[:log_location] = stdout + end + + describe "#run" do + + let(:audits) { {} } + let(:run_context) { instance_double(Chef::RunContext, :events => events, :audits => audits) } + let(:controls_name) { "controls_name" } + + it "Correctly runs an empty controls block" do + in_sub_process do + runner.run end + end - context "there is a single successful control" do - let(:audits) do - should_pass = lambda do - it "should pass" do - expect(2 - 2).to eq(0) - end + context "there is a single successful control" do + let(:audits) do + should_pass = lambda do + it "should pass" do + expect(2 - 2).to eq(0) end - { controls_name => Struct.new(:args, :block).new([controls_name], should_pass)} end + { controls_name => Struct.new(:args, :block).new([controls_name], should_pass)} + end - it "correctly runs" do - in_sub_process do - runner.run + it "correctly runs" do + in_sub_process do + runner.run - expect(stdout.string).to match(/1 example, 0 failures/) - end + expect(stdout.string).to match(/1 example, 0 failures/) end end + end - context "there is a single failing control" do - let(:audits) do - should_fail = lambda do - it "should fail" do - expect(2 - 1).to eq(0) - end + context "there is a single failing control" do + let(:audits) do + should_fail = lambda do + it "should fail" do + expect(2 - 1).to eq(0) end - { controls_name => Struct.new(:args, :block).new([controls_name], should_fail)} end + { controls_name => Struct.new(:args, :block).new([controls_name], should_fail)} + end - it "correctly runs" do - in_sub_process do - runner.run + it "correctly runs" do + in_sub_process do + runner.run - expect(stdout.string).to match(/Failure\/Error: expect\(2 - 1\)\.to eq\(0\)/) - expect(stdout.string).to match(/1 example, 1 failure/) - expect(stdout.string).to match(/# controls_name should fail/) - end + expect(stdout.string).to match(/Failure\/Error: expect\(2 - 1\)\.to eq\(0\)/) + expect(stdout.string).to match(/1 example, 1 failure/) + expect(stdout.string).to match(/# controls_name should fail/) end end - end + end + end diff --git a/spec/unit/audit/runner_spec.rb b/spec/unit/audit/runner_spec.rb index ca6f51f9eb..67590fecf9 100644 --- a/spec/unit/audit/runner_spec.rb +++ b/spec/unit/audit/runner_spec.rb @@ -19,8 +19,12 @@ require 'spec_helper' require 'spec/support/audit_helper' require 'chef/audit/runner' +require 'chef/audit/audit_event_proxy' +require 'chef/audit/rspec_formatter' +require 'rspec/support/spec/in_sub_process' describe Chef::Audit::Runner do + include RSpec::Support::InSubProcess let(:events) { double("events") } let(:run_context) { instance_double(Chef::RunContext, :events => events) } @@ -48,23 +52,27 @@ describe Chef::Audit::Runner do end it "sets all the config values" do - runner.send(:setup) + # This runs the Serverspec includes - we don't want these hanging around in all subsequent tests so + # we run this in a forked process. Keeps Serverspec files from getting loaded into main process. + in_sub_process do + runner.send(:setup) - expect(RSpec.configuration.output_stream).to eq(log_location) - expect(RSpec.configuration.error_stream).to eq(log_location) + expect(RSpec.configuration.output_stream).to eq(log_location) + expect(RSpec.configuration.error_stream).to eq(log_location) - expect(RSpec.configuration.formatters.size).to eq(2) - expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::AuditEventProxy)) - expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::RspecFormatter)) - expect(Chef::Audit::AuditEventProxy.class_variable_get(:@@events)).to eq(run_context.events) + expect(RSpec.configuration.formatters.size).to eq(2) + expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::AuditEventProxy)) + expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::RspecFormatter)) + expect(Chef::Audit::AuditEventProxy.class_variable_get(:@@events)).to eq(run_context.events) - expect(RSpec.configuration.expectation_frameworks).to eq([RSpec::Matchers]) - expect(RSpec::Matchers.configuration.syntax).to eq([:expect]) + expect(RSpec.configuration.expectation_frameworks).to eq([RSpec::Matchers]) + expect(RSpec::Matchers.configuration.syntax).to eq([:expect]) - expect(RSpec.configuration.color).to eq(color) - expect(RSpec.configuration.expose_dsl_globally?).to eq(false) + expect(RSpec.configuration.color).to eq(color) + expect(RSpec.configuration.expose_dsl_globally?).to eq(false) - expect(Specinfra.configuration.backend).to eq(:exec) + expect(Specinfra.configuration.backend).to eq(:exec) + end end end @@ -88,6 +96,7 @@ describe Chef::Audit::Runner do expect(RSpec.world.example_groups.size).to eq(1) # For whatever reason, `kind_of` is not working + # expect(RSpec.world.example_groups).to include(kind_of(RSpec::Core::ExampleGroup)) => FAIL g = RSpec.world.example_groups[0] expect(g.ancestors).to include(RSpec::Core::ExampleGroup) expect(g.description).to eq("group_name") -- cgit v1.2.1 From b1842523a032e96fd049d871562dc31c62e4d810 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Tue, 16 Dec 2014 10:15:24 -0800 Subject: Unit and functional tests for spec_formatter --- spec/functional/audit/rspec_formatter_spec.rb | 53 +++++++++++++++++++++++++++ spec/unit/audit/rspec_formatter_spec.rb | 29 +++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 spec/functional/audit/rspec_formatter_spec.rb create mode 100644 spec/unit/audit/rspec_formatter_spec.rb diff --git a/spec/functional/audit/rspec_formatter_spec.rb b/spec/functional/audit/rspec_formatter_spec.rb new file mode 100644 index 0000000000..43d3c2f6dd --- /dev/null +++ b/spec/functional/audit/rspec_formatter_spec.rb @@ -0,0 +1,53 @@ +# +# Author:: Tyler Ball () +# Author:: Claire McQuin () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'spec_helper' +require 'spec/support/audit_helper' +require 'chef/audit/runner' +require 'rspec/support/spec/in_sub_process' +require 'chef/audit/rspec_formatter' + +describe Chef::Audit::RspecFormatter do + include RSpec::Support::InSubProcess + + let(:events) { double("events").as_null_object } + let(:audits) { {} } + let(:run_context) { instance_double(Chef::RunContext, :events => events, :audits => audits) } + let(:runner) { Chef::Audit::Runner.new(run_context) } + + let(:output) { double("output") } + # aggressively define this so we can mock out the new call later + let!(:formatter) { Chef::Audit::RspecFormatter.new(output) } + + around(:each) do |ex| + Sandboxing.sandboxed { ex.run } + end + + it "should not close the output using our formatter" do + in_sub_process do + expect_any_instance_of(Chef::Audit::RspecFormatter).to receive(:new).and_return(formatter) + expect(formatter).to receive(:close).and_call_original + expect(output).to_not receive(:close) + + runner.run + end + end + +end diff --git a/spec/unit/audit/rspec_formatter_spec.rb b/spec/unit/audit/rspec_formatter_spec.rb new file mode 100644 index 0000000000..471473e387 --- /dev/null +++ b/spec/unit/audit/rspec_formatter_spec.rb @@ -0,0 +1,29 @@ +# +# Author:: Tyler Ball () +# Author:: Claire McQuin () +# +# Copyright:: Copyright (c) 2014 Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'spec_helper' +require 'chef/audit/rspec_formatter' + +describe Chef::Audit::RspecFormatter do + let(:formatter) { Chef::Audit::RspecFormatter.new(nil) } + it "should respond to close" do + expect(formatter).to respond_to(:close) + end +end -- cgit v1.2.1 From 2dac0859f0fa1f6260fb06d937bcd36086ea166a Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Thu, 11 Dec 2014 13:58:39 -0800 Subject: Disable audit-mode by default. * Modify command line option --audit-mode to accept parameters enabled, disabled, or audit-only. * Emit a warning if audit-mode is enabled or audit-only. --- lib/chef/application/client.rb | 50 ++++++++++++++++++++++++++++++++---------- lib/chef/config.rb | 7 +++++- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/lib/chef/application/client.rb b/lib/chef/application/client.rb index b10f818cf4..72b4848669 100644 --- a/lib/chef/application/client.rb +++ b/lib/chef/application/client.rb @@ -239,17 +239,10 @@ class Chef::Application::Client < Chef::Application end option :audit_mode, - :long => "--[no-]audit-mode", - :description => "If not specified, run converge and audit phase. If true, run only audit phase. If false, run only converge phase.", - :boolean => true, - :proc => lambda { |set| - # Convert boolean to config options of :audit_only or :disabled - if set - :audit_only - else - :disabled - end - } + :long => "--audit-mode SETTING", + :description => "Enable audit-mode with `enabled`. Disabled audit-mode with `disabled`. Skip converge and only audit with `audit-only`", + :proc => lambda { |mode| mode.gsub("-", "_").to_sym }, + :default => :disabled IMMEDIATE_RUN_SIGNAL = "1".freeze @@ -288,6 +281,19 @@ class Chef::Application::Client < Chef::Application config_fetcher = Chef::ConfigFetcher.new(Chef::Config[:json_attribs]) @chef_client_json = config_fetcher.fetch_json end + + if mode = Chef::Config[:audit_mode] + expected_modes = [:enabled, :disabled, :audit_only] + unless expected_modes.include?(mode) + Chef::Application.fatal!(unrecognized_audit_mode(mode)) + end + + unless mode == :disabled + # This should be removed when audit-mode is enabled by default/no longer + # an experimental feature. + Chef::Log.warn(audit_mode_experimental_message) + end + end end def load_config_file @@ -408,4 +414,26 @@ class Chef::Application::Client < Chef::Application "#{"\n interval = #{Chef::Config[:interval]} seconds" if Chef::Config[:interval]}" + "\nEnable chef-client interval runs by setting `:client_fork = true` in your config file or adding `--fork` to your command line options." end + + def audit_mode_settings_explaination + "\n* To enable audit mode after converge, use command line option `--audit-mode enabled` or set `:audit_mode = :enabled` in your config file." + + "\n* To disable audit mode, use command line option `--audit-mode disabled` or set `:audit_mode = :disabled` in your config file." + + "\n* To only run audit mode, use command line option `--audit-mode audit-only` or set `:audit_mode = :audit_only` in your config file." + + "\nAudit mode is disabled by default." + end + + def unrecognized_audit_mode(mode) + "Unrecognized setting #{mode} for audit mode." + audit_mode_settings_explaination + end + + def audit_mode_experimental_message + msg = if Chef::Config[:audit_mode] == :audit_only + "Chef-client has been configured to skip converge and run only audits." + else + "Chef-client has been configure to run audits after it converges." + end + msg += " Audit mode is an experimental feature currently under development. API changes may occur. Use at your own risk." + msg += audit_mode_settings_explaination + return msg + end end diff --git a/lib/chef/config.rb b/lib/chef/config.rb index 19fa272100..9bf9e9d48e 100644 --- a/lib/chef/config.rb +++ b/lib/chef/config.rb @@ -321,7 +321,12 @@ class Chef default :enable_reporting_url_fatals, false # Possible values for :audit_mode # :enabled, :disabled, :audit_only, - default :audit_mode, :enabled + # + # TODO: 11 Dec 2014: Currently audit-mode is an experimental feature + # and is disabled by default. When users choose to enable audit-mode, + # a warning is issued in application/client#reconfigure. + # This can be removed when audit-mode is enabled by default. + default :audit_mode, :disabled # Policyfile is an experimental feature where a node gets its run list and # cookbook version set from a single document on the server instead of -- cgit v1.2.1 From 1a49821ec880ddf6ba63e17ed5dfd8ed11411d65 Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Mon, 15 Dec 2014 15:24:02 -0800 Subject: Add specs for audit-mode command-line and configuration settings. --- lib/chef/application/client.rb | 11 ++-- spec/unit/application/client_spec.rb | 99 +++++++++++++++++++++++++++++++----- spec/unit/client_spec.rb | 1 + 3 files changed, 92 insertions(+), 19 deletions(-) diff --git a/lib/chef/application/client.rb b/lib/chef/application/client.rb index 72b4848669..40772c0f8f 100644 --- a/lib/chef/application/client.rb +++ b/lib/chef/application/client.rb @@ -239,10 +239,9 @@ class Chef::Application::Client < Chef::Application end option :audit_mode, - :long => "--audit-mode SETTING", - :description => "Enable audit-mode with `enabled`. Disabled audit-mode with `disabled`. Skip converge and only audit with `audit-only`", - :proc => lambda { |mode| mode.gsub("-", "_").to_sym }, - :default => :disabled + :long => "--audit-mode MODE", + :description => "Enable audit-mode with `enabled`. Disable audit-mode with `disabled`. Skip converge and only perform audits with `audit-only`", + :proc => lambda { |mo| mo.gsub("-", "_").to_sym } IMMEDIATE_RUN_SIGNAL = "1".freeze @@ -282,7 +281,7 @@ class Chef::Application::Client < Chef::Application @chef_client_json = config_fetcher.fetch_json end - if mode = Chef::Config[:audit_mode] + if mode = config[:audit_mode] || Chef::Config[:audit_mode] expected_modes = [:enabled, :disabled, :audit_only] unless expected_modes.include?(mode) Chef::Application.fatal!(unrecognized_audit_mode(mode)) @@ -430,7 +429,7 @@ class Chef::Application::Client < Chef::Application msg = if Chef::Config[:audit_mode] == :audit_only "Chef-client has been configured to skip converge and run only audits." else - "Chef-client has been configure to run audits after it converges." + "Chef-client has been configured to run audits after it converges." end msg += " Audit mode is an experimental feature currently under development. API changes may occur. Use at your own risk." msg += audit_mode_settings_explaination diff --git a/spec/unit/application/client_spec.rb b/spec/unit/application/client_spec.rb index c2d3ec0507..e5cabcbc0e 100644 --- a/spec/unit/application/client_spec.rb +++ b/spec/unit/application/client_spec.rb @@ -18,18 +18,20 @@ require 'spec_helper' describe Chef::Application::Client, "reconfigure" do + let(:app) do + a = described_class.new + a.cli_arguments = [] + a + end + before do allow(Kernel).to receive(:trap).and_return(:ok) @original_argv = ARGV.dup ARGV.clear - @app = Chef::Application::Client.new - allow(@app).to receive(:trap) - allow(@app).to receive(:configure_opt_parser).and_return(true) - allow(@app).to receive(:configure_chef).and_return(true) - allow(@app).to receive(:configure_logging).and_return(true) - @app.cli_arguments = [] + allow(app).to receive(:trap) + allow(app).to receive(:configure_logging).and_return(true) Chef::Config[:interval] = 10 Chef::Config[:once] = false @@ -60,7 +62,7 @@ Configuration settings: interval = 600 seconds Enable chef-client interval runs by setting `:client_fork = true` in your config file or adding `--fork` to your command line options." ) - @app.reconfigure + app.reconfigure end end @@ -83,7 +85,7 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config end it "should reconfigure chef-client" do - @app.reconfigure + app.reconfigure expect(Chef::Config[:interval]).to be_nil end end @@ -96,7 +98,7 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config end it "should set the interval to 1800" do - @app.reconfigure + app.reconfigure expect(Chef::Config.interval).to eq(1800) end end @@ -110,12 +112,12 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config end it "ignores the splay" do - @app.reconfigure + app.reconfigure expect(Chef::Config.splay).to be_nil end it "forces the interval to nil" do - @app.reconfigure + app.reconfigure expect(Chef::Config.interval).to be_nil end @@ -128,14 +130,85 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config let(:json_source) { "https://foo.com/foo.json" } before do + allow(app).to receive(:configure_chef).and_return(true) Chef::Config[:json_attribs] = json_source expect(Chef::ConfigFetcher).to receive(:new).with(json_source). and_return(config_fetcher) end it "reads the JSON attributes from the specified source" do - @app.reconfigure - expect(@app.chef_client_json).to eq(json_attribs) + app.reconfigure + expect(app.chef_client_json).to eq(json_attribs) + end + end + + describe "audit mode" do + shared_examples "experimental feature" do + it "emits a warning that audit mode is an experimental feature" do + expect(Chef::Log).to receive(:warn).with(/Audit mode is an experimental feature/) + app.reconfigure + end + end + + shared_examples "unrecognized setting" do + it "fatals with a message including the incorrect setting" do + expect(Chef::Application).to receive(:fatal!).with(/Unrecognized setting #{mode} for audit mode/) + app.reconfigure + end + end + + shared_context "set via config file" do + before do + Chef::Config[:audit_mode] = mode + end + end + + shared_context "set via command line" do + before do + ARGV.replace(["--audit-mode", mode]) + end + end + + describe "enabled via config file" do + include_context "set via config file" do + let(:mode) { :enabled } + include_examples "experimental feature" + end + end + + describe "enabled via command line" do + include_context "set via command line" do + let(:mode) { "enabled" } + include_examples "experimental feature" + end + end + + describe "audit_only via config file" do + include_context "set via config file" do + let(:mode) { :audit_only } + include_examples "experimental feature" + end + end + + describe "audit-only via command line" do + include_context "set via command line" do + let(:mode) { "audit-only" } + include_examples "experimental feature" + end + end + + describe "unrecognized setting via config file" do + include_context "set via config file" do + let(:mode) { :derp } + include_examples "unrecognized setting" + end + end + + describe "unrecognized setting via command line" do + include_context "set via command line" do + let(:mode) { "derp" } + include_examples "unrecognized setting" + end end end end diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index 4f6d8a0b82..2ec32b32ac 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -290,6 +290,7 @@ describe Chef::Client do Chef::Config[:client_fork] = enable_fork Chef::Config[:cache_path] = windows? ? 'C:\chef' : '/var/chef' Chef::Config[:why_run] = false + Chef::Config[:audit_mode] = :enabled stub_const("Chef::Client::STDOUT_FD", stdout) stub_const("Chef::Client::STDERR_FD", stderr) -- cgit v1.2.1 From 5531cabf5640607874fd24c5e9c6006d848ec69b Mon Sep 17 00:00:00 2001 From: Claire McQuin Date: Mon, 15 Dec 2014 15:29:49 -0800 Subject: Explicitly enable audits. --- spec/integration/client/client_spec.rb | 1 + spec/unit/application/client_spec.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/spec/integration/client/client_spec.rb b/spec/integration/client/client_spec.rb index e961b73d0d..62660bb852 100644 --- a/spec/integration/client/client_spec.rb +++ b/spec/integration/client/client_spec.rb @@ -246,6 +246,7 @@ EOM file 'config/client.rb', < Date: Tue, 16 Dec 2014 13:03:53 -0800 Subject: Unit tests for audit-mode in chef-solo. * Audits are disabled by default. * Also, updated spec file to use RSpec :let. --- lib/chef/audit/audit_reporter.rb | 2 -- lib/chef/audit/runner.rb | 9 ++--- lib/chef/client.rb | 3 +- lib/chef/formatters/doc.rb | 3 +- spec/functional/audit/runner_spec.rb | 51 +++++++++++++++++++++------ spec/unit/application/client_spec.rb | 2 +- spec/unit/application/solo_spec.rb | 67 ++++++++++++++++++++---------------- 7 files changed, 85 insertions(+), 52 deletions(-) diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 407c2deeb0..596b06b285 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -117,8 +117,6 @@ class Chef run_data = audit_data.to_hash if error - # TODO: Rather than a single string we might want to format the exception here similar to - # lib/chef/resource_reporter.rb#83 run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}" end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 1450ef7f61..4017593c55 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -108,8 +108,6 @@ class Chef # the output stream to be changed for a formatter once the formatter has # been added. def set_streams - # TODO: Do some testing to ensure these will output/output properly to - # a file. RSpec.configuration.output_stream = Chef::Config[:log_location] RSpec.configuration.error_stream = Chef::Config[:log_location] end @@ -135,12 +133,9 @@ class Chef end end - # Set up the backend for Specinfra/Serverspec. + # Set up the backend for Specinfra/Serverspec. :exec is the local system. def configure_specinfra - # TODO: We may need to be clever and adjust this based on operating - # system, or make it configurable. E.g., there is a PowerShell backend, - # as well as an SSH backend. - Specinfra.configuration.backend = :exec if Specinfra.configuration.backend != :exec + Specinfra.configuration.backend = :exec end # Iterates through the controls registered to this run_context, builds an diff --git a/lib/chef/client.rb b/lib/chef/client.rb index 634773cf80..77f63671d7 100644 --- a/lib/chef/client.rb +++ b/lib/chef/client.rb @@ -338,7 +338,8 @@ class Chef converge_exception end - # TODO don't want to change old API + # We don't want to change the old API on the `converge` method to have it perform + # saving. So we wrap it in this method. def converge_and_save(run_context) converge_exception = converge(run_context) unless converge_exception diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb index 398c61fdc8..489888db8f 100644 --- a/lib/chef/formatters/doc.rb +++ b/lib/chef/formatters/doc.rb @@ -166,7 +166,7 @@ class Chef end def converge_failed(e) - # TODO do we want to do anything else in here? + # Currently a failed converge is handled the same way as a successful converge converge_complete end @@ -183,7 +183,6 @@ class Chef puts_line "" puts_line "Audit phase exception:" indent - # TODO error_mapper ? puts_line "#{error.message}" error.backtrace.each do |l| puts_line l diff --git a/spec/functional/audit/runner_spec.rb b/spec/functional/audit/runner_spec.rb index 89c62ae2e8..aa35548f2f 100644 --- a/spec/functional/audit/runner_spec.rb +++ b/spec/functional/audit/runner_spec.rb @@ -20,6 +20,7 @@ require 'spec_helper' require 'spec/support/audit_helper' require 'chef/audit/runner' require 'rspec/support/spec/in_sub_process' +require 'tempfile' ## # This functional test ensures that our runner can be setup to not interfere with existing RSpec @@ -60,7 +61,7 @@ describe Chef::Audit::Runner do end end - context "there is a single successful control" do + shared_context "passing audit" do let(:audits) do should_pass = lambda do it "should pass" do @@ -69,17 +70,9 @@ describe Chef::Audit::Runner do end { controls_name => Struct.new(:args, :block).new([controls_name], should_pass)} end - - it "correctly runs" do - in_sub_process do - runner.run - - expect(stdout.string).to match(/1 example, 0 failures/) - end - end end - context "there is a single failing control" do + shared_context "failing audit" do let(:audits) do should_fail = lambda do it "should fail" do @@ -88,7 +81,21 @@ describe Chef::Audit::Runner do end { controls_name => Struct.new(:args, :block).new([controls_name], should_fail)} end + end + + context "there is a single successful control" do + include_context "passing audit" + it "correctly runs" do + in_sub_process do + runner.run + + expect(stdout.string).to match(/1 example, 0 failures/) + end + end + end + context "there is a single failing control" do + include_context "failing audit" it "correctly runs" do in_sub_process do runner.run @@ -100,6 +107,30 @@ describe Chef::Audit::Runner do end end + describe "log location is a file" do + let(:tmpfile) { Tempfile.new("audit") } + before do + Chef::Config[:log_location] = tmpfile.path + end + + after do + tmpfile.close + tmpfile.unlink + end + + include_context "failing audit" + it "correctly runs" do + in_sub_process do + runner.run + + contents = tmpfile.read + expect(contents).to match(/Failure\/Error: expect\(2 - 1\)\.to eq\(0\)/) + expect(contents).to match(/1 example, 1 failure/) + expect(contents).to match(/# controls_name should fail/) + end + end + end + end end diff --git a/spec/unit/application/client_spec.rb b/spec/unit/application/client_spec.rb index 3554b78c13..33af9bc5c1 100644 --- a/spec/unit/application/client_spec.rb +++ b/spec/unit/application/client_spec.rb @@ -74,7 +74,7 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config it "should not terminate" do expect(Chef::Application).not_to receive(:fatal!) - @app.reconfigure + app.reconfigure end end diff --git a/spec/unit/application/solo_spec.rb b/spec/unit/application/solo_spec.rb index 80f0bead8b..2a07ff38ad 100644 --- a/spec/unit/application/solo_spec.rb +++ b/spec/unit/application/solo_spec.rb @@ -18,13 +18,16 @@ require 'spec_helper' describe Chef::Application::Solo do + + let(:app) { Chef::Application::Solo.new } + before do allow(Kernel).to receive(:trap).and_return(:ok) - @app = Chef::Application::Solo.new - allow(@app).to receive(:configure_opt_parser).and_return(true) - allow(@app).to receive(:configure_chef).and_return(true) - allow(@app).to receive(:configure_logging).and_return(true) - allow(@app).to receive(:trap) + allow(app).to receive(:configure_opt_parser).and_return(true) + allow(app).to receive(:configure_chef).and_return(true) + allow(app).to receive(:configure_logging).and_return(true) + allow(app).to receive(:trap) + Chef::Config[:recipe_url] = false Chef::Config[:json_attribs] = false Chef::Config[:solo] = true @@ -32,10 +35,15 @@ describe Chef::Application::Solo do describe "configuring the application" do it "should set solo mode to true" do - @app.reconfigure + app.reconfigure expect(Chef::Config[:solo]).to be_truthy end + it "should set audit-mode to :disabled" do + app.reconfigure + expect(Chef::Config[:audit_mode]).to be :disabled + end + describe "when configured to not fork the client process" do before do Chef::Config[:client_fork] = false @@ -56,7 +64,7 @@ Configuration settings: interval = 600 seconds Enable chef-client interval runs by setting `:client_fork = true` in your config file or adding `--fork` to your command line options." ) - @app.reconfigure + app.reconfigure end end end @@ -68,7 +76,7 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config it "should set the interval to 1800" do Chef::Config[:interval] = nil - @app.reconfigure + app.reconfigure expect(Chef::Config[:interval]).to eq(1800) end end @@ -85,44 +93,46 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config end it "reads the JSON attributes from the specified source" do - @app.reconfigure - expect(@app.chef_client_json).to eq(json_attribs) + app.reconfigure + expect(app.chef_client_json).to eq(json_attribs) end end describe "when the recipe_url configuration option is specified" do + let(:tarfile) { StringIO.new("remote_tarball_content") } + let(:target_file) { StringIO.new } + before do Chef::Config[:cookbook_path] = "#{Dir.tmpdir}/chef-solo/cookbooks" Chef::Config[:recipe_url] = "http://junglist.gen.nz/recipes.tgz" + allow(FileUtils).to receive(:rm_rf).and_return(true) allow(FileUtils).to receive(:mkdir_p).and_return(true) - @tarfile = StringIO.new("remote_tarball_content") - allow(@app).to receive(:open).with("http://junglist.gen.nz/recipes.tgz").and_yield(@tarfile) - @target_file = StringIO.new - allow(File).to receive(:open).with("#{Dir.tmpdir}/chef-solo/recipes.tgz", "wb").and_yield(@target_file) + allow(app).to receive(:open).with("http://junglist.gen.nz/recipes.tgz").and_yield(tarfile) + allow(File).to receive(:open).with("#{Dir.tmpdir}/chef-solo/recipes.tgz", "wb").and_yield(target_file) allow(Chef::Mixin::Command).to receive(:run_command).and_return(true) end it "should create the recipes path based on the parent of the cookbook path" do expect(FileUtils).to receive(:mkdir_p).with("#{Dir.tmpdir}/chef-solo").and_return(true) - @app.reconfigure + app.reconfigure end it "should download the recipes" do - expect(@app).to receive(:open).with("http://junglist.gen.nz/recipes.tgz").and_yield(@tarfile) - @app.reconfigure + expect(app).to receive(:open).with("http://junglist.gen.nz/recipes.tgz").and_yield(tarfile) + app.reconfigure end it "should write the recipes to the target path" do - @app.reconfigure - expect(@target_file.string).to eq("remote_tarball_content") + app.reconfigure + expect(target_file.string).to eq("remote_tarball_content") end it "should untar the target file to the parent of the cookbook path" do expect(Chef::Mixin::Command).to receive(:run_command).with({:command => "tar zxvf #{Dir.tmpdir}/chef-solo/recipes.tgz -C #{Dir.tmpdir}/chef-solo"}).and_return(true) - @app.reconfigure + app.reconfigure end end end @@ -142,9 +152,9 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config end it "should fetch the recipe_url first" do - expect(@app).to receive(:fetch_recipe_tarball).ordered + expect(app).to receive(:fetch_recipe_tarball).ordered expect(Chef::ConfigFetcher).to receive(:new).ordered.and_return(config_fetcher) - @app.reconfigure + app.reconfigure end end @@ -153,18 +163,17 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config Chef::Config[:solo] = true allow(Chef::Daemon).to receive(:change_privilege) - @chef_client = double("Chef::Client") - allow(Chef::Client).to receive(:new).and_return(@chef_client) - @app = Chef::Application::Solo.new + chef_client = double("Chef::Client") + allow(Chef::Client).to receive(:new).and_return(chef_client) # this is all stuff the reconfigure method needs - allow(@app).to receive(:configure_opt_parser).and_return(true) - allow(@app).to receive(:configure_chef).and_return(true) - allow(@app).to receive(:configure_logging).and_return(true) + allow(app).to receive(:configure_opt_parser).and_return(true) + allow(app).to receive(:configure_chef).and_return(true) + allow(app).to receive(:configure_logging).and_return(true) end it "should change privileges" do expect(Chef::Daemon).to receive(:change_privilege).and_return(true) - @app.setup_application + app.setup_application end end -- cgit v1.2.1 From f68218380f137f6c6ff51406da087fd9e81304a9 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Wed, 17 Dec 2014 18:57:22 -0800 Subject: Resolving conflicsts from giant rebase --- lib/chef/audit/runner.rb | 2 -- lib/chef/config.rb | 79 ++++++++++++++++++++++--------------------- lib/chef/resource_reporter.rb | 3 +- lib/chef/run_context.rb | 8 ++--- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index 4017593c55..7ef17a4301 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -16,8 +16,6 @@ # limitations under the License. # -require 'chef/config' - class Chef class Audit class Runner diff --git a/lib/chef/config.rb b/lib/chef/config.rb index 9bf9e9d48e..453a8f83da 100644 --- a/lib/chef/config.rb +++ b/lib/chef/config.rb @@ -271,7 +271,7 @@ class Chef # * :fatal # These work as you'd expect. There is also a special `:auto` setting. # When set to :auto, Chef will auto adjust the log verbosity based on - # context. When a tty is available (usually becase the user is running chef + # context. When a tty is available (usually because the user is running chef # in a console), the log level is set to :warn, and output formatters are # used as the primary mode of output. When a tty is not available, the # logger is the primary mode of output, and the log level is set to :info @@ -317,6 +317,7 @@ class Chef default :why_run, false default :color, false default :client_fork, true + default :ez, false default :enable_reporting, true default :enable_reporting_url_fatals, false # Possible values for :audit_mode @@ -568,10 +569,12 @@ class Chef # used to update files. default :file_atomic_update, true - # If false file staging is will be done via tempfiles that are - # created under ENV['TMP'] otherwise tempfiles will be created in - # the directory that files are going to reside. - default :file_staging_uses_destdir, true + # There are 3 possible values for this configuration setting. + # true => file staging is done in the destination directory + # false => file staging is done via tempfiles under ENV['TMP'] + # :auto => file staging will try using destination directory if possible and + # will fall back to ENV['TMP'] if destination directory is not usable. + default :file_staging_uses_destdir, :auto # Exit if another run is in progress and the chef-client is unable to # get the lock before time expires. If nil, no timeout is enforced. (Exits @@ -631,44 +634,44 @@ class Chef # # If there is no 'locale -a' then we return 'en_US.UTF-8' since that is the most commonly # available English UTF-8 locale. However, all modern POSIXen should support 'locale -a'. - default :internal_locale do - begin - # https://github.com/opscode/chef/issues/2181 - # Some systems have the `locale -a` command, but the result has - # invalid characters for the default encoding. - # - # For example, on CentOS 6 with ENV['LANG'] = "en_US.UTF-8", - # `locale -a`.split fails with ArgumentError invalid UTF-8 encoding. - locales = shell_out_with_systems_locale("locale -a").stdout.split - case - when locales.include?('C.UTF-8') - 'C.UTF-8' - when locales.include?('en_US.UTF-8'), locales.include?('en_US.utf8') - 'en_US.UTF-8' - when locales.include?('en.UTF-8') - 'en.UTF-8' - else - # Will match en_ZZ.UTF-8, en_ZZ.utf-8, en_ZZ.UTF8, en_ZZ.utf8 - guesses = locales.select { |l| l =~ /^en_.*UTF-?8$/i } - unless guesses.empty? - guessed_locale = guesses.first - # Transform into the form en_ZZ.UTF-8 - guessed_locale.gsub(/UTF-?8$/i, "UTF-8") - else - Chef::Log.warn "Please install an English UTF-8 locale for Chef to use, falling back to C locale and disabling UTF-8 support." - 'C' - end - end - rescue - if Chef::Platform.windows? - Chef::Log.debug "Defaulting to locale en_US.UTF-8 on Windows, until it matters that we do something else." + def self.guess_internal_locale + # https://github.com/opscode/chef/issues/2181 + # Some systems have the `locale -a` command, but the result has + # invalid characters for the default encoding. + # + # For example, on CentOS 6 with ENV['LANG'] = "en_US.UTF-8", + # `locale -a`.split fails with ArgumentError invalid UTF-8 encoding. + locales = shell_out_with_systems_locale!("locale -a").stdout.split + case + when locales.include?('C.UTF-8') + 'C.UTF-8' + when locales.include?('en_US.UTF-8'), locales.include?('en_US.utf8') + 'en_US.UTF-8' + when locales.include?('en.UTF-8') + 'en.UTF-8' + else + # Will match en_ZZ.UTF-8, en_ZZ.utf-8, en_ZZ.UTF8, en_ZZ.utf8 + guesses = locales.select { |l| l =~ /^en_.*UTF-?8$/i } + unless guesses.empty? + guessed_locale = guesses.first + # Transform into the form en_ZZ.UTF-8 + guessed_locale.gsub(/UTF-?8$/i, "UTF-8") else - Chef::Log.debug "No usable locale -a command found, assuming you have en_US.UTF-8 installed." + Chef::Log.warn "Please install an English UTF-8 locale for Chef to use, falling back to C locale and disabling UTF-8 support." + 'C' end - 'en_US.UTF-8' end + rescue + if Chef::Platform.windows? + Chef::Log.debug "Defaulting to locale en_US.UTF-8 on Windows, until it matters that we do something else." + else + Chef::Log.debug "No usable locale -a command found, assuming you have en_US.UTF-8 installed." + end + 'en_US.UTF-8' end + default :internal_locale, guess_internal_locale + # Force UTF-8 Encoding, for when we fire up in the 'C' locale or other strange locales (e.g. # japanese windows encodings). If we do not do this, then knife upload will fail when a cookbook's # README.md has UTF-8 characters that do not encode in whatever surrounding encoding we have been diff --git a/lib/chef/resource_reporter.rb b/lib/chef/resource_reporter.rb index a673f4aa58..1816fc857d 100644 --- a/lib/chef/resource_reporter.rb +++ b/lib/chef/resource_reporter.rb @@ -20,8 +20,7 @@ # require 'uri' -require 'zlib' -require 'chef/monkey_patches/securerandom' +require 'securerandom' require 'chef/event_dispatch/base' class Chef diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index d14035da2f..6803dc5796 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -104,7 +104,7 @@ class Chef if nr.instance_of?(Chef::Resource) @immediate_notification_collection[nr.name] << notification else - @immediate_notification_collection[nr.to_s] << notification + @immediate_notification_collection[nr.declared_key] << notification end end @@ -115,7 +115,7 @@ class Chef if nr.instance_of?(Chef::Resource) @delayed_notification_collection[nr.name] << notification else - @delayed_notification_collection[nr.to_s] << notification + @delayed_notification_collection[nr.declared_key] << notification end end @@ -123,7 +123,7 @@ class Chef if resource.instance_of?(Chef::Resource) return @immediate_notification_collection[resource.name] else - return @immediate_notification_collection[resource.to_s] + return @immediate_notification_collection[resource.declared_key] end end @@ -131,7 +131,7 @@ class Chef if resource.instance_of?(Chef::Resource) return @delayed_notification_collection[resource.name] else - return @delayed_notification_collection[resource.to_s] + return @delayed_notification_collection[resource.declared_key] end end -- cgit v1.2.1 From 02cfe1d405001f6545f8559213da967aff1751c4 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 18 Dec 2014 10:23:16 -0800 Subject: No longer need to run travis on audit-mode branch --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e9e7c2cdc2..37418ab621 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ branches: - 10-stable - 11-stable - 12-stable - - audit-mode # do not run expensive spec tests on PRs, only on branches script: " -- cgit v1.2.1 From 095d2e4128121d8303887663d882ee54a5c12c7a Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 18 Dec 2014 14:52:02 -0800 Subject: Addressing review comments, adding documentation --- .kitchen.yml | 2 -- CHANGELOG.md | 1 + DOC_CHANGES.md | 9 +++++++ RELEASE_NOTES.md | 54 +++++++++++++++++++++++++++++++++++++- chef.gemspec | 1 + lib/chef/event_dispatch/base.rb | 5 ++++ spec/integration/solo/solo_spec.rb | 2 +- spec/support/audit_helper.rb | 1 + 8 files changed, 71 insertions(+), 4 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index c9be1b56e7..ed49eb3e57 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -6,8 +6,6 @@ driver: memory: 4096 synced_folders: - ['.', '/home/vagrant/chef'] - - ['../ohai', '/home/vagrant/ohai'] - - ['../triager', '/home/vagrant/triager'] provisioner: name: chef_zero diff --git a/CHANGELOG.md b/CHANGELOG.md index f93aa4c453..2ff00ab99e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ * Update Chef to use RSpec 3. * Cleaned up script and execute provider + specs * Added deprecation warnings around the use of command attribute in script resources +* Audit mode feature added - see the RELEASE_NOTES for details ## 12.0.3 * [**Phil Dibowitz**](https://github.com/jaymzh): diff --git a/DOC_CHANGES.md b/DOC_CHANGES.md index 15f88abdca..9a6c78a524 100644 --- a/DOC_CHANGES.md +++ b/DOC_CHANGES.md @@ -6,6 +6,15 @@ Example Doc Change: Description of the required change. --> +### Experimental Audit Mode Feature + +There is a new command_line flag provided for `chef-client`: `--audit-mode`. This accepts 1 of 3 arguments: + +* disabled (default) - Audits are disabled and the phase is skipped. This is the default while Audit mode is an +experimental feature. +* enabled - Audits are enabled and will be performed after the converge phase. +* audit_only - Audits are enabled and convergence is disabled. Only audits will be performed. + ### Chef Why Run Mode Ignores Audit Phase Because most users enable `why_run` mode to determine what resources convergence will update on their system, the audit diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 630aa737df..0c73b7f7c8 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,59 @@ -# Chef Client Release Notes 12.2.0: +# Chef Client Release Notes 12.1.0: # Internal API Changes in this Release +## Experimental Audit Mode Feature + +This is a new feature intended to provide _infrastructure audits_. Chef already allows you to configure your infrastructure +with code, but there are some use cases that are not covered by resource convergence. What if you want to check that +the application Chef just installed is functioning correctly? If it provides a status page an audit can check this +and validate that the application has database connectivity. + +Audits are performed by leveraging [Serverspec](http://serverspec.org/) and [RSpec](https://relishapp.com/rspec) on the +node. As such the syntax is very similar to a normal RSpec spec. + +### Syntax + +```ruby +controls "Database Audit" do + + control "postgres package" do + it "should not be installed" do + expect(package("postgresql")).to_not be_installed + end + end + + let(:p) { port(111) } + control p do + it "has nothing listening" do + expect(p).to_not be_listening + end + end + +end +``` + +Using the example above I will break down the components of an Audit: + +* `controls` - This named block contains all the audits to be performed during the audit phase. During Chef convergence + the audits will be collected and ran in a separate phase at the end of the Chef run. Any `controls` block defined in + a recipe that is ran on the node will be performed. +* `control` - This keyword describes a section of audits to perform. The name here should either be a string describing +the system under test, or a [Serverspec resource](http://serverspec.org/resource_types.html). +* `it` - Inside this block you can use [RSpec expectations](https://relishapp.com/rspec/rspec-expectations/docs) to +write the audits. You can use the Serverspec resources here or regular ruby code. Any raised errors will fail the +audit. + +### Output and error handling + +Output from the audit run will appear in your `Chef::Config[:log_location]`. If an audit fails then Chef will raise +an error and exit with a non-zero status. + +### Further reading + +More information about the audit mode can be found in its +[RFC](https://github.com/opscode/chef-rfc/blob/master/rfc035-audit-mode.md) + # End-User Changes ## OpenBSD Package provider was added diff --git a/chef.gemspec b/chef.gemspec index 43fb3d16fe..52babdc5a1 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -35,6 +35,7 @@ Gem::Specification.new do |s| s.add_dependency 'plist', '~> 3.1.0' + # Audit mode requires these, so they are non-developmental dependencies now %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_dependency gem, "~> 3.1" } s.add_dependency "rspec_junit_formatter", "~> 0.2.0" s.add_dependency "serverspec", "~> 2.7" diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb index 695e31cf2e..25dd9fd1b2 100644 --- a/lib/chef/event_dispatch/base.rb +++ b/lib/chef/event_dispatch/base.rb @@ -229,6 +229,11 @@ class Chef def converge_failed(exception) end + ################################## + # Audit Mode Events + # This phase is currently experimental and these event APIs are subject to change + ################################## + # Called before audit phase starts def audit_phase_start(run_status) end diff --git a/spec/integration/solo/solo_spec.rb b/spec/integration/solo/solo_spec.rb index 9500e7a1ca..cc9ba1abb2 100644 --- a/spec/integration/solo/solo_spec.rb +++ b/spec/integration/solo/solo_spec.rb @@ -92,7 +92,7 @@ EOM # We have a timeout protection here so that if due to some bug # run_lock gets stuck we can discover it. expect { - Timeout.timeout(1200) do + Timeout.timeout(120) do chef_dir = File.join(File.dirname(__FILE__), "..", "..", "..") # Instantiate the first chef-solo run diff --git a/spec/support/audit_helper.rb b/spec/support/audit_helper.rb index 5744f779fc..70e36f1ad2 100644 --- a/spec/support/audit_helper.rb +++ b/spec/support/audit_helper.rb @@ -7,6 +7,7 @@ # end # rspec-core did not include a license on Github +# TODO when this API is exposed publicly from rspec-core, get rid of this copy pasta # Adding these as writers is necessary, otherwise we cannot set the new configuration. # Only want to do this in the specs. -- cgit v1.2.1 From c1676b32aa08b618f4c2317676f5590388b3bc53 Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Mon, 29 Dec 2014 15:13:51 -0800 Subject: Updating for review comments --- DOC_CHANGES.md | 10 ++++++---- lib/chef/audit/audit_reporter.rb | 4 +--- lib/chef/version.rb | 2 +- spec/support/audit_helper.rb | 2 ++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/DOC_CHANGES.md b/DOC_CHANGES.md index 0c82661f34..7429baca2a 100644 --- a/DOC_CHANGES.md +++ b/DOC_CHANGES.md @@ -14,10 +14,12 @@ Previously, when a URI scheme contained all uppercase letters, Chef would reject There is a new command_line flag provided for `chef-client`: `--audit-mode`. This accepts 1 of 3 arguments: -* disabled (default) - Audits are disabled and the phase is skipped. This is the default while Audit mode is an +* `disabled` (default) - Audits are disabled and the phase is skipped. This is the default while Audit mode is an experimental feature. -* enabled - Audits are enabled and will be performed after the converge phase. -* audit_only - Audits are enabled and convergence is disabled. Only audits will be performed. +* `enabled` - Audits are enabled and will be performed after the converge phase. +* `audit-only` - Audits are enabled and convergence is disabled. Only audits will be performed. + +This can also be configured in your node's client.rb with the key `audit_mode` and a value of `:disabled`, `:enabled` or `:audit_only`. ### Chef Why Run Mode Ignores Audit Phase @@ -32,4 +34,4 @@ The `--audit-mode` flag should be a link to the documentation for that flag #### Editors node 2 This probably only needs to be a bullet point added to http://docs.getchef.com/nodes.html#about-why-run-mode under the -`certain assumptions` section \ No newline at end of file +`certain assumptions` section diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb index 596b06b285..a5dd9a6c48 100644 --- a/lib/chef/audit/audit_reporter.rb +++ b/lib/chef/audit/audit_reporter.rb @@ -127,11 +127,9 @@ class Chef rest_client.post(audit_url, run_data, headers) rescue StandardError => e if e.respond_to? :response - code = e.response.code.nil? ? "Exception Code Empty" : e.response.code - # 404 error code is OK. This means the version of server we're running against doesn't support # audit reporting. Don't alarm failure in this case. - if code == "404" + if e.response.code == "404" Chef::Log.debug("Server doesn't support audit reporting. Skipping report.") return else diff --git a/lib/chef/version.rb b/lib/chef/version.rb index 30c3394c2c..a8fc002399 100644 --- a/lib/chef/version.rb +++ b/lib/chef/version.rb @@ -17,7 +17,7 @@ class Chef CHEF_ROOT = File.dirname(File.expand_path(File.dirname(__FILE__))) - VERSION = '12.2.0.alpha.0' + VERSION = '12.1.0.dev.0' end # diff --git a/spec/support/audit_helper.rb b/spec/support/audit_helper.rb index 70e36f1ad2..8fd3f4d719 100644 --- a/spec/support/audit_helper.rb +++ b/spec/support/audit_helper.rb @@ -22,6 +22,8 @@ class NullObject end end +# TODO remove this when RSPec exposes this functionality publically +# https://github.com/rspec/rspec-core/pull/1808 module Sandboxing def self.sandboxed(&block) orig_load_path = $LOAD_PATH.dup -- cgit v1.2.1