Adding audit mode JSON formatter

First pass at DSL additions

Renaming DSL methods to match the spec

Creating our own example group class to simplify adding examples to the spec runner

Adding logic for exceptions from converge phase not interfering with audit phase and vice-versa

Adding error handling so saving node doesn't prevent us from running audit mode - decouples converge phase and audit phase more

Updating for github comments

Add setup phase to audit-mode.

Refactor runner into own class.

Fix typo

tie things together

Adding first round of formatter integration - STDOUT doc formatter

Preparing for demo - using rspec documentation formatter for output instead of the proxy

Add serverspec types and matchers.

add rspec/its

Add gems as core dependencies

Updating with changes from demo

Updating with @mcquin and @lamont comments

Getting rid of unused method

Wiring audit event proxy to send events correctly to the audit_reporter

removing old pry debugging statement

Removing unecessary todo

Sending to correct server URL

Fixing TODOs

Adding uncaught error information

15 files changed, 740 insertions, 25 deletions

diff --git a/chef.gemspec b/chef.gemspec
index 8e906a0ef1..6f3039f589 100644
--- a/chef.gemspec
+++ b/chef.gemspec
@@ -35,16 +35,16 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'plist', '~> 3.1.0'
 
+  %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_dependency gem, "~> 3.1" }
+  s.add_dependency "rspec_junit_formatter", "~> 0.2.0"
+  s.add_dependency "serverspec", "~> 2.3"
+  s.add_dependency "specinfra", "~> 2.4"
+
   s.add_development_dependency "rack"
 
   # Rake 10.2 drops Ruby 1.8 support
   s.add_development_dependency "rake", "~> 10.1.0"
 
-  # rspec_junit_formatter 0.2.0 drops ruby 1.8.7 support
-  s.add_development_dependency "rspec_junit_formatter", "~> 0.2.0"
-
-  %w(rspec-core rspec-expectations rspec-mocks).each { |gem| s.add_development_dependency gem, "~> 3.0" }
-
   s.bindir       = "bin"
   s.executables  = %w( chef-client chef-solo knife chef-shell shef chef-apply )
 
diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb
new file mode 100644
index 0000000000..ed8db93d96
--- /dev/null
+++ b/lib/chef/audit.rb
@@ -0,0 +1,29 @@
+#
+# Author:: Claire McQuin (<claire@getchef.com>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rspec'
+require 'rspec/its'
+
+require 'serverspec/matcher'
+require 'serverspec/helper'
+require 'serverspec/subject'
+
+require 'specinfra'
+
+require 'chef/dsl/audit'
+require 'chef/audit/runner'
diff --git a/lib/chef/audit/audit_event_proxy.rb b/lib/chef/audit/audit_event_proxy.rb
new file mode 100644
index 0000000000..71d1e2aa50
--- /dev/null
+++ b/lib/chef/audit/audit_event_proxy.rb
@@ -0,0 +1,76 @@
+RSpec::Support.require_rspec_core "formatters/base_text_formatter"
+
+class Chef
+  class Audit
+    class AuditEventProxy < ::RSpec::Core::Formatters::BaseFormatter
+      ::RSpec::Core::Formatters.register self, :stop, :example_group_started
+
+      # TODO I don't like this, but I don't see another way to pass this in
+      # see rspec files configuration.rb#L671 and formatters.rb#L129
+      def self.events=(events)
+        @@events = events
+      end
+
+      def events
+        @@events
+      end
+
+      def example_group_started(notification)
+        if notification.group.parent_groups.size == 1
+          # top level controls block
+          desc = notification.group.description
+          Chef::Log.debug("Entered controls block named #{desc}")
+          events.control_group_started(desc)
+        end
+      end
+
+      def stop(notification)
+        Chef::Log.info("Successfully executed all controls blocks and contained examples")
+        notification.examples.each do |example|
+          control_group_name, control_data = build_control_from(example)
+          e = example.exception
+          if e
+            events.control_example_failure(control_group_name, control_data, e)
+          else
+            events.control_example_success(control_group_name, control_data)
+          end
+        end
+      end
+
+      private
+
+      def build_control_from(example)
+        described_class = example.metadata[:described_class]
+        if described_class
+          resource_type = described_class.class.name.split(':')[-1]
+          # TODO submit github PR to expose this
+          resource_name = described_class.instance_variable_get(:@name)
+        end
+
+        # The following code builds up the context - the list of wrapping `describe` or `control` blocks
+        describe_groups = []
+        group = example.metadata[:example_group]
+        # If the innermost block has a resource instead of a string, don't include it in context
+        describe_groups.unshift(group[:description]) if described_class.nil?
+        group = group[:parent_example_group]
+        while !group.nil?
+          describe_groups.unshift(group[:description])
+          group = group[:parent_example_group]
+        end
+
+        # We know all of our examples each live in a top-level `controls` block - get this name now
+        outermost_group_desc = describe_groups.shift
+
+        return outermost_group_desc, {
+            :name => example.description,
+            :desc => example.full_description,
+            :resource_type => resource_type,
+            :resource_name => resource_name,
+            :context => describe_groups,
+            :line_number => example.metadata[:line_number]
+        }
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb
new file mode 100644
index 0000000000..b1c9d30bfc
--- /dev/null
+++ b/lib/chef/audit/audit_reporter.rb
@@ -0,0 +1,135 @@
+#
+# Auther:: Tyler Ball (<tball@getchef.com>)
+#
+# Copyright:: Copyright (c) 2014 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/event_dispatch/base'
+require 'chef/audit/control_group_data'
+
+class Chef
+  class Audit
+    class AuditReporter < EventDispatch::Base
+
+      attr_reader :rest_client, :audit_data, :ordered_control_groups
+      private :rest_client, :audit_data, :ordered_control_groups
+
+      PROTOCOL_VERSION = '0.1.0'
+
+      def initialize(rest_client)
+        if Chef::Config[:audit_mode] == false
+          @audit_enabled = false
+        else
+          @audit_enabled = true
+        end
+        @rest_client = rest_client
+        # Ruby 1.9.3 and above "enumerate their values in the order that the corresponding keys were inserted."
+        @ordered_control_groups = Hash.new
+      end
+
+      def audit_phase_start(run_status)
+        Chef::Log.debug("Audit Reporter starting")
+        @audit_data = AuditData.new(run_status.node.name, run_status.run_id)
+      end
+
+      def audit_phase_complete
+        Chef::Log.debug("Audit Reporter completed successfully without errors")
+        ordered_control_groups.each do |name, control_group|
+          audit_data.add_control_group(control_group)
+        end
+        post_auditing_data
+      end
+
+      # If the audit phase failed, its because there was some kind of error in the framework
+      # that runs tests - normal errors are interpreted as EXAMPLE failures and captured.
+      def audit_phase_failed(error)
+        # The stacktrace information has already been logged elsewhere
+        Chef::Log.error("Audit Reporter failed - sending error to server with available example information")
+        ordered_control_groups.each do |name, control_group|
+          audit_data.add_control_group(control_group)
+        end
+        post_auditing_data(error)
+      end
+
+      def control_group_started(name)
+        if ordered_control_groups.has_key?(name)
+          raise AuditControlGroupDuplicate.new(name)
+        end
+        ordered_control_groups.store(name, ControlGroupData.new(name))
+      end
+
+      def control_example_success(control_group_name, example_data)
+        control_group = ordered_control_groups[control_group_name]
+        control_group.example_success(example_data)
+      end
+
+      def control_example_failure(control_group_name, example_data, error)
+        control_group = ordered_control_groups[control_group_name]
+        control_group.example_failure(example_data, error.message)
+      end
+
+      def auditing_enabled?
+        @audit_enabled
+      end
+
+      private
+
+      def post_auditing_data(error = nil)
+        if auditing_enabled?
+          audit_history_url = "controls"
+          Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})")
+          run_data = audit_data.to_hash
+
+          if error
+            run_data[:error] = "#{error.class.to_s}: #{error.message}\n#{error.backtrace.join("\n")}"
+          end
+
+          Chef::Log.debug run_data.inspect
+          compressed_data = encode_gzip(Chef::JSONCompat.to_json(run_data))
+          Chef::Log.debug("Sending compressed audit data...")
+          # Since we're posting compressed data we can not directly call post_rest which expects JSON
+          audit_url = rest_client.create_url(audit_history_url)
+          begin
+            puts Chef::JSONCompat.to_json_pretty(run_data)
+            rest_client.raw_http_request(:POST, audit_url, headers({'Content-Encoding' => 'gzip'}), compressed_data)
+          rescue StandardError => e
+            if e.respond_to? :response
+              error_file = "failed-audit-data.json"
+              Chef::FileCache.store(error_file, Chef::JSONCompat.to_json_pretty(run_data), 0640)
+              Chef::Log.error("Failed to post audit report to server (HTTP #{e.response.code}), saving to #{Chef::FileCache.load(error_file, false)}")
+            else
+              Chef::Log.error("Failed to post audit report to server (#{e})")
+            end
+          end
+        else
+          Chef::Log.debug("Server doesn't support audit report, skipping.")
+        end
+      end
+
+      def headers(additional_headers = {})
+        options = {'X-Ops-Audit-Report-Protocol-Version' => PROTOCOL_VERSION}
+        options.merge(additional_headers)
+      end
+
+      def encode_gzip(data)
+        "".tap do |out|
+          Zlib::GzipWriter.wrap(StringIO.new(out)){|gz| gz << data }
+        end
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb
new file mode 100644
index 0000000000..e19a6e1a15
--- /dev/null
+++ b/lib/chef/audit/control_group_data.rb
@@ -0,0 +1,125 @@
+require 'securerandom'
+
+class Chef
+  class Audit
+    class AuditData
+      attr_reader :node_name, :run_id, :control_groups
+
+      def initialize(node_name, run_id)
+        @node_name = node_name
+        @run_id = run_id
+        @control_groups = []
+      end
+
+      def add_control_group(control_group)
+        control_groups << control_group
+      end
+
+      def to_hash
+        {
+            :node_name => node_name,
+            :run_id => run_id,
+            :control_groups => control_groups.collect { |c| c.to_hash }
+        }
+      end
+    end
+
+    class ControlGroupData
+      attr_reader :name, :status, :number_success, :number_failed, :controls
+
+      def initialize(name)
+        @status = "success"
+        @controls = []
+        @number_success = 0
+        @number_failed = 0
+        @name = name
+      end
+
+
+      def example_success(control_data)
+        @number_success += 1
+        control = create_control(control_data)
+        control.status = "success"
+        controls << control
+        control
+      end
+
+      def example_failure(control_data, details)
+        @number_failed += 1
+        @status = "failure"
+        control = create_control(control_data)
+        control.details = details if details
+        control.status = "failure"
+        controls << control
+        control
+      end
+
+      def to_hash
+        # We sort it so the examples appear in the output in the same order
+        # they appeared in the recipe
+        controls.sort! {|x,y| x.line_number <=> y.line_number}
+        h = {
+              :name => name,
+              :status => status,
+              :number_success => number_success,
+              :number_failed => number_failed,
+              :controls => controls.collect { |c| c.to_hash }
+        }
+        add_display_only_data(h)
+      end
+
+      private
+
+      def create_control(control_data)
+        name = control_data[:name]
+        resource_type = control_data[:resource_type]
+        resource_name = control_data[:resource_name]
+        context = control_data[:context]
+        line_number = control_data[:line_number]
+        # TODO make this smarter with splat arguments so if we start passing in more control_data
+        # I don't have to modify code in multiple places
+        ControlData.new(name, resource_type, resource_name, context, line_number)
+      end
+
+      # The id and control sequence number are ephemeral data - they are not needed
+      # to be persisted and can be regenerated at will.  They are only needed
+      # for display purposes.
+      def add_display_only_data(group)
+        group[:id] = SecureRandom.uuid
+        group[:controls].collect!.with_index do |c, i|
+          # i is zero-indexed, and we want the display one-indexed
+          c[:sequence_number] = i+1
+          c
+        end
+        group
+      end
+
+    end
+
+    class ControlData
+      attr_reader :name, :resource_type, :resource_name, :context, :line_number
+      attr_accessor :status, :details
+
+      def initialize(name, resource_type, resource_name, context, line_number)
+        @context = context
+        @name = name
+        @resource_type = resource_type
+        @resource_name = resource_name
+        @line_number = line_number
+      end
+
+      def to_hash
+        h = {
+            :name => name,
+            :status => status,
+            :details => details,
+            :resource_type => resource_type,
+            :resource_name => resource_name
+        }
+        h[:context] = context || []
+        h
+      end
+    end
+
+  end
+end
diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb
new file mode 100644
index 0000000000..4059741359
--- /dev/null
+++ b/lib/chef/audit/runner.rb
@@ -0,0 +1,114 @@
+#
+# Author:: Claire McQuin (<claire@getchef.com>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/audit'
+require 'chef/audit/audit_event_proxy'
+require 'chef/config'
+
+class Chef
+  class Audit
+    class Runner
+
+      attr_reader :run_context
+      private :run_context
+
+      def initialize(run_context)
+        @run_context = run_context
+      end
+
+      def run
+        setup
+        register_controls_groups
+
+        # The first parameter passed to RSpec::Core::Runner.new
+        # is an instance of RSpec::Core::ConfigurationOptions, which is
+        # responsible for processing command line options passed through rspec.
+        # This then gets merged with the configuration. We'll just communicate
+        # directly with the Configuration here.
+        audit_runner = RSpec::Core::Runner.new(nil, configuration, world)
+        audit_runner.run_specs(world.ordered_example_groups)
+      end
+
+      private
+
+      # RSpec configuration and world objects are heavy, so let's wait until
+      # we actually need them.
+      def configuration
+        RSpec.configuration
+      end
+
+      def world
+        RSpec.world
+      end
+
+      # Configure audits before run.
+      # Sets up where output and error streams should stream to, adds formatters
+      # for people-friendly output of audit results and json for reporting. Also
+      # configures expectation frameworks.
+      def setup
+        # We're setting the output stream, but that will only be used for error situations
+        # Our formatter forwards events to the Chef event message bus
+        # TODO so some testing to see if these output to a log file - we probably need
+        # to register these before any formatters are added.
+        configuration.output_stream = Chef::Config[:log_location]
+        configuration.error_stream  = Chef::Config[:log_location]
+        # TODO im pretty sure I only need this because im running locally in rvmsudo
+        configuration.backtrace_exclusion_patterns.push(Regexp.new("/Users".gsub("/", File::SEPARATOR)))
+        configuration.backtrace_exclusion_patterns.push(Regexp.new("(eval)"))
+        configuration.color = Chef::Config[:color]
+        configuration.expose_dsl_globally = false
+
+        add_formatters
+        disable_should_syntax
+        configure_specinfra
+      end
+
+      def add_formatters
+        configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter)
+        configuration.add_formatter(Chef::Audit::AuditEventProxy)
+        Chef::Audit::AuditEventProxy.events = run_context.events
+      end
+
+      # Explicitly disable :should syntax.
+      #
+      # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it
+      # in audits. If :should is used in an audit, the audit will fail with error
+      # message "undefined method `:should`" rather than issue a deprecation warning.
+      #
+      # This can be removed when :should is fully removed from RSpec.
+      def disable_should_syntax
+        configuration.expect_with :rspec do |c|
+          c.syntax = :expect
+        end
+      end
+
+      def configure_specinfra
+        # TODO: We may need to change this based on operating system (there is a
+        # powershell backend) or roll our own.
+        Specinfra.configuration.backend = :exec
+      end
+
+      # Register each controls group with the world, which will handle
+      # the ordering of the audits that will be run.
+      def register_controls_groups
+        run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) }
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/client.rb b/lib/chef/client.rb
index 4f37bd0ee3..16315b8e08 100644
--- a/lib/chef/client.rb
+++ b/lib/chef/client.rb
@@ -25,6 +25,7 @@ require 'chef/log'
 require 'chef/rest'
 require 'chef/api_client'
 require 'chef/api_client/registration'
+require 'chef/audit'
 require 'chef/node'
 require 'chef/role'
 require 'chef/file_cache'
@@ -43,6 +44,7 @@ require 'chef/formatters/doc'
 require 'chef/formatters/minimal'
 require 'chef/version'
 require 'chef/resource_reporter'
+require 'chef/audit/audit_reporter'
 require 'chef/run_lock'
 require 'chef/policy_builder'
 require 'chef/request_id'
@@ -209,6 +211,17 @@ class Chef
       end
     end
 
+    # Resource repoters send event information back to the chef server for processing.
+    # Can only be called after we have a @rest object
+    def register_reporters
+      [
+        Chef::ResourceReporter.new(rest),
+        Chef::Audit::AuditReporter.new(rest)
+      ].each do |r|
+        events.register(r)
+      end
+    end
+
     # Instantiates a Chef::Node object, possibly loading the node's prior state
     # when using chef-client. Delegates to policy_builder
     #
@@ -246,7 +259,6 @@ class Chef
       @policy_builder ||= Chef::PolicyBuilder.strategy.new(node_name, ohai.data, json_attribs, @override_runlist, events)
     end
 
-
     def save_updated_node
       if Chef::Config[:solo]
         # nothing to do
@@ -260,6 +272,7 @@ class Chef
 
     def run_ohai
       ohai.all_plugins
+      @events.ohai_completed(node)
     end
 
     def node_name
@@ -295,8 +308,7 @@ class Chef
       end
       # We now have the client key, and should use it from now on.
       @rest = Chef::REST.new(config[:chef_server_url], client_name, config[:client_key])
-      @resource_reporter = Chef::ResourceReporter.new(@rest)
-      @events.register(@resource_reporter)
+      register_reporters
     rescue Exception => e
       # TODO: munge exception so a semantic failure message can be given to the
       # user
@@ -307,18 +319,49 @@ class Chef
     # Converges the node.
     #
     # === Returns
-    # true:: Always returns true
+    # The thrown exception, if there was one.  If this returns nil the converge was successful.
     def converge(run_context)
-      @events.converge_start(run_context)
-      Chef::Log.debug("Converging node #{node_name}")
-      @runner = Chef::Runner.new(run_context)
-      runner.converge
-      @events.converge_complete
-      true
-    rescue Exception
-      # TODO: should this be a separate #converge_failed(exception) method?
-      @events.converge_complete
-      raise
+      converge_exception = nil
+      catch(:end_client_run_early) do
+        begin
+          @events.converge_start(run_context)
+          Chef::Log.debug("Converging node #{node_name}")
+          @runner = Chef::Runner.new(run_context)
+          runner.converge
+          @events.converge_complete
+        rescue Exception => e
+          @events.converge_failed(e)
+          converge_exception = e
+        end
+      end
+      converge_exception
+    end
+
+    # TODO don't want to change old API
+    def converge_and_save(run_context)
+      converge_exception = converge(run_context)
+      unless converge_exception
+        begin
+          save_updated_node
+        rescue Exception => e
+          converge_exception = e
+        end
+      end
+      converge_exception
+    end
+
+    def run_audits(run_context)
+      audit_exception = nil
+      begin
+        @events.audit_phase_start(run_status)
+        auditor = Chef::Audit::Runner.new(run_context)
+        auditor.run
+        @events.audit_phase_complete
+      rescue Exception => e
+        @events.audit_phase_failed(e)
+        audit_exception = e
+      end
+      audit_exception
     end
 
     # Expands the run list. Delegates to the policy_builder.
@@ -333,7 +376,6 @@ class Chef
       policy_builder.expand_run_list
     end
 
-
     def do_windows_admin_check
       if Chef::Platform.windows?
         Chef::Log.debug("Checking for administrator privileges....")
@@ -380,7 +422,7 @@ class Chef
         Chef::Log.debug("Chef-client request_id: #{request_id}")
         enforce_path_sanity
         run_ohai
-        @events.ohai_completed(node)
+
         register unless Chef::Config[:solo]
 
         load_node
@@ -396,11 +438,14 @@ class Chef
 
         run_context = setup_run_context
 
-        catch(:end_client_run_early) do
-          converge(run_context)
-        end
+        converge_error = converge_and_save(run_context)
+        audit_error = run_audits(run_context)
 
-        save_updated_node
+        if converge_error || audit_error
+          e = Chef::Exceptions::RunFailedWrappingError.new(converge_error, audit_error)
+          e.fill_backtrace
+          raise e
+        end
 
         run_status.stop_clock
         Chef::Log.info("Chef Run complete in #{run_status.elapsed_time} seconds")
@@ -411,6 +456,7 @@ class Chef
         Chef::Platform::Rebooter.reboot_if_needed!(node)
 
         true
+
       rescue Exception => e
         # CHEF-3336: Send the error first in case something goes wrong below and we don't know why
         Chef::Log.debug("Re-raising exception: #{e.class} - #{e.message}\n#{e.backtrace.join("\n  ")}")
diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb
new file mode 100644
index 0000000000..1849b65633
--- /dev/null
+++ b/lib/chef/dsl/audit.rb
@@ -0,0 +1,40 @@
+#
+# Author:: Tyler Ball (<tball@getchef.com>)
+# Copyright:: Copyright (c) 2014 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rspec/core'
+
+class Chef
+  module DSL
+    module Audit
+
+      # Can encompass tests in a `control` block or `describe` block
+      ::RSpec::Core::ExampleGroup.define_example_group_method :control
+      ::RSpec::Core::ExampleGroup.define_example_group_method :__controls__
+
+      # Adds the controls group and block (containing controls to execute) to the runner's list of pending examples
+      def controls(*args, &block)
+        raise ::Chef::Exceptions::NoAuditsProvided unless block
+        name = args[0]
+        raise AuditNameMissing if name.nil? || name.empty?
+
+        run_context.controls_groups << ::RSpec::Core::ExampleGroup.__controls__(*args, &block)
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index bfd4503097..a1306364b7 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -225,6 +225,36 @@ class Chef
       def converge_complete
       end
 
+      # Called if the converge phase fails
+      def converge_failed(exception)
+      end
+
+      # Called before audit phase starts
+      def audit_phase_start(run_status)
+      end
+
+      # Called when audit phase successfully finishes
+      def audit_phase_complete
+      end
+
+      # Called if there is an uncaught exception during the audit phase.  The audit runner should
+      # be catching and handling errors from the examples, so this is only uncaught errors (like
+      # bugs in our handling code)
+      def audit_phase_failed(exception)
+      end
+
+      # Signifies the start of a `controls` block with a defined name
+      def control_group_started(name)
+      end
+
+      # An example in a `controls` block completed successfully
+      def control_example_success(control_group_name, example_data)
+      end
+
+      # An example in a `controls` block failed with the provided error
+      def control_example_failure(control_group_name, example_data, error)
+      end
+
       # TODO: need events for notification resolve?
       # def notifications_resolved
       # end
diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb
index 25f08455fc..078f438be2 100644
--- a/lib/chef/exceptions.rb
+++ b/lib/chef/exceptions.rb
@@ -366,5 +366,40 @@ class Chef
         super "Found more than one provider for #{resource.resource_name} resource: #{classes}"
       end
     end
+
+    class AuditControlGroupDuplicate < RuntimeError
+      def initialize(name)
+        super "Audit control group with name '#{name}' has already been defined"
+      end
+    end
+    class AuditNameMissing < RuntimeError; end
+    class NoAuditsProvided < RuntimeError
+      def initialize
+        super "You must provide a block with audits"
+      end
+    end
+
+    # If a converge or audit fails, we want to wrap the output from those errors into 1 error so we can
+    # see both issues in the output.  It is possible that nil will be provided.  You must call `fill_backtrace`
+    # to correctly populate the backtrace with the wrapped backtraces.
+    class RunFailedWrappingError < RuntimeError
+      attr_reader :wrapped_errors
+      def initialize(*errors)
+        errors = errors.select {|e| !e.nil?}
+        output = "Found #{errors.size} errors, they are stored in the backtrace\n"
+        @wrapped_errors = errors
+        super output
+      end
+
+      def fill_backtrace
+        backtrace = []
+        wrapped_errors.each_with_index do |e,i|
+          backtrace << "#{i+1}) #{e.class} -  #{e.message}"
+          backtrace += e.backtrace if e.backtrace
+          backtrace << ""
+        end
+        set_backtrace(backtrace)
+      end
+    end
   end
 end
diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb
index 4a08b9d095..09d04f3aae 100644
--- a/lib/chef/formatters/doc.rb
+++ b/lib/chef/formatters/doc.rb
@@ -151,6 +151,38 @@ class Chef
         unindent if @current_recipe
       end
 
+      def converge_failed(e)
+        # TODO do we want to do anything else in here?
+        converge_complete
+      end
+
+      #############
+      # TODO
+      # Make all these document printers neater
+      #############
+
+      # Called before audit phase starts
+      def audit_phase_start(run_status)
+        puts_line ""
+        puts_line "++ Audit phase starting ++"
+      end
+
+      def audit_phase_complete
+        puts_line ""
+        puts_line "++ Audit phase ended ++ "
+      end
+
+      def audit_phase_failed(error)
+        puts_line ""
+        puts_line "Audit phase exception:"
+        indent
+        # TODO error_mapper ?
+        puts_line "#{error.message}"
+        error.backtrace.each do |l|
+          puts_line l
+        end
+      end
+
       # Called before action is executed on a resource.
       def resource_action_start(resource, action, notification_type=nil, notifier=nil)
         if resource.cookbook_name && resource.recipe_name
diff --git a/lib/chef/recipe.rb b/lib/chef/recipe.rb
index de72a8d0c4..b4046e4f16 100644
--- a/lib/chef/recipe.rb
+++ b/lib/chef/recipe.rb
@@ -24,6 +24,7 @@ require 'chef/dsl/platform_introspection'
 require 'chef/dsl/include_recipe'
 require 'chef/dsl/registry_helper'
 require 'chef/dsl/reboot_pending'
+require 'chef/dsl/audit'
 
 require 'chef/mixin/from_file'
 
@@ -40,6 +41,7 @@ class Chef
     include Chef::DSL::Recipe
     include Chef::DSL::RegistryHelper
     include Chef::DSL::RebootPending
+    include Chef::DSL::Audit
 
     include Chef::Mixin::FromFile
     include Chef::Mixin::Deprecation
diff --git a/lib/chef/resource_reporter.rb b/lib/chef/resource_reporter.rb
index a19f26125e..a673f4aa58 100644
--- a/lib/chef/resource_reporter.rb
+++ b/lib/chef/resource_reporter.rb
@@ -20,6 +20,7 @@
 #
 
 require 'uri'
+require 'zlib'
 require 'chef/monkey_patches/securerandom'
 require 'chef/event_dispatch/base'
 
diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb
index 1a2d7ba3a3..8f7296822c 100644
--- a/lib/chef/run_context.rb
+++ b/lib/chef/run_context.rb
@@ -50,6 +50,9 @@ class Chef
     # recipes, which is triggered by #load. (See also: CookbookCompiler)
     attr_accessor :resource_collection
 
+    # The list of control groups to execute during the audit phase
+    attr_accessor :controls_groups
+
     # A Hash containing the immediate notifications triggered by resources
     # during the converge phase of the chef run.
     attr_accessor :immediate_notification_collection
@@ -73,6 +76,7 @@ class Chef
       @node = node
       @cookbook_collection = cookbook_collection
       @resource_collection = Chef::ResourceCollection.new
+      @controls_groups = []
       @immediate_notification_collection = Hash.new {|h,k| h[k] = []}
       @delayed_notification_collection = Hash.new {|h,k| h[k] = []}
       @definitions = Hash.new
diff --git a/spec/unit/exceptions_spec.rb b/spec/unit/exceptions_spec.rb
index 6318ec9227..165c11446b 100644
--- a/spec/unit/exceptions_spec.rb
+++ b/spec/unit/exceptions_spec.rb
@@ -81,4 +81,50 @@ describe Chef::Exceptions do
       end
     end
   end
+
+  describe Chef::Exceptions::RunFailedWrappingError do
+    shared_examples "RunFailedWrappingError expectations" do
+      it "should initialize with a default message" do
+        expect(e.message).to eq("Found #{num_errors} errors, they are stored in the backtrace\n")
+      end
+
+      it "should provide a modified backtrace when requested" do
+        e.fill_backtrace
+        expect(e.backtrace).to eq(backtrace)
+      end
+    end
+
+    context "initialized with nothing" do
+      let(:e) { Chef::Exceptions::RunFailedWrappingError.new  }
+      let(:num_errors) { 0 }
+      let(:backtrace) { [] }
+
+      include_examples "RunFailedWrappingError expectations"
+    end
+
+    context "initialized with nil" do
+      let(:e) { Chef::Exceptions::RunFailedWrappingError.new(nil, nil)  }
+      let(:num_errors) { 0 }
+      let(:backtrace) { [] }
+
+      include_examples "RunFailedWrappingError expectations"
+    end
+
+    context "initialized with 1 error and nil" do
+      let(:e) { Chef::Exceptions::RunFailedWrappingError.new(RuntimeError.new("foo"), nil)  }
+      let(:num_errors) { 1 }
+      let(:backtrace) { ["1) RuntimeError -  foo", ""] }
+
+      include_examples "RunFailedWrappingError expectations"
+    end
+
+    context "initialized with 2 errors" do
+      let(:e) { Chef::Exceptions::RunFailedWrappingError.new(RuntimeError.new("foo"), RuntimeError.new("bar"))  }
+      let(:num_errors) { 2 }
+      let(:backtrace) { ["1) RuntimeError -  foo", "", "2) RuntimeError -  bar", ""] }
+
+      include_examples "RunFailedWrappingError expectations"
+    end
+
+  end
 end