Adding audit mode JSON formatter

7 files changed, 278 insertions, 49 deletions

diff --git a/lib/chef/audit.rb b/lib/chef/audit.rb
index 6fd067448b..43d5c11ee8 100644
--- a/lib/chef/audit.rb
+++ b/lib/chef/audit.rb
@@ -18,47 +18,6 @@
 
 require 'rspec/core'
 
-require 'chef/config'
-
-class Chef
-  class Audit
-
-    def initialize
-      @configuration = RSpec::Core::Configuration.new
-      @world = RSpec::Core::World.new(@configuration)
-      @runner = RSpec::Core::Runner.new(nil, @configuration, @world)
-    end
-
-    def setup
-      @configuration.output_stream = Chef::Config[:log_location]
-      @configuration.error_stream = Chef::Config[:log_location]
-
-      configure_formatters
-      configure_expectation_frameworks
-    end
-
-    private
-    # Adds formatters to RSpec.
-    # By default, two formatters are added: one for outputting readable text
-    # of audits run and one for sending JSON data back to reporting.
-    def configure_formatters
-      # TODO (future): We should allow for an audit-mode formatter config option
-      # and use this formatter as default/fallback if none is specified.
-      @configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter)
-      # TODO: Add JSON formatter for audit reporting to analytics.
-    end
-
-    def configure_expectation_frameworks
-      @configuration.expect_with(:rspec) do |config|
-        # :should is deprecated in RSpec 3+ and we have chosen to explicitly disable
-        # it in audits. If :should is used in an audit, this will cause the audit to
-        # fail with message "undefined method `should`" rather than print a deprecation
-        # message.
-        config.syntax = :expect
-      end
-
-      #TODO: serverspec?
-    end
-
-  end
-end
+require 'chef/dsl/audit'
+require 'chef/audit/chef_json_formatter'
+require 'chef/audit/runner'
diff --git a/lib/chef/audit/chef_json_formatter.rb b/lib/chef/audit/chef_json_formatter.rb
new file mode 100644
index 0000000000..5dcbdb50b7
--- /dev/null
+++ b/lib/chef/audit/chef_json_formatter.rb
@@ -0,0 +1,88 @@
+RSpec::Support.require_rspec_core "formatters/base_formatter"
+require 'chef/audit/control_group_data'
+require 'ffi_yajl'
+
+class Chef
+  class Audit
+    class ChefJsonFormatter < ::RSpec::Core::Formatters::BaseFormatter
+      ::RSpec::Core::Formatters.register self, :example_group_started, :message, :stop, :close, :example_failed
+
+      attr_reader :control_group_data
+
+      # TODO hopefully the runner can take care of this for us since there won't be an outer-most
+      # control group
+      @@outer_example_group_found = false
+
+      def initialize(output)
+        super
+      end
+
+      # Invoked for each `control`, `describe`, `context` block
+      def example_group_started(notification)
+        unless @@outer_example_group_found
+          @control_group_data = ControlGroupData.new(notification.group.description)
+          @@outer_example_group_found = true
+        end
+      end
+
+      def example_failed(notification)
+        e = notification.example.metadata[:execution_result].exception
+        raise e unless e.kind_of? ::RSpec::Expectations::ExpectationNotMetError
+      end
+
+      def message(notification)
+        puts "message: #{notification}"
+      end
+
+      def stop(notification)
+        notification.examples.each do |example|
+          control_data = build_control_from(example)
+          e = example.exception
+          if e
+            control = control_group_data.example_failure(e.message, control_data)
+          else
+            control = control_group_data.example_success(control_data)
+          end
+          control.line_number = example.metadata[:line_number]
+        end
+      end
+
+      def close(notification)
+        output.write FFI_Yajl::Encoder.encode(control_group_data.to_hash, pretty: true)
+        output.close if IO === output && output != $stdout
+      end
+
+      private
+
+      def build_control_from(example)
+        described_class = example.metadata[:described_class]
+        if described_class
+          resource_type = described_class.class.name.split(':')[-1]
+          # TODO submit github PR to expose this
+          resource_name = described_class.instance_variable_get(:@name)
+        end
+
+        describe_groups = []
+        group = example.metadata[:example_group]
+        # If the innermost block has a resource instead of a string, don't include it in context
+        describe_groups.unshift(group[:description]) if described_class.nil?
+        group = group[:parent_example_group]
+        while !group.nil?
+          describe_groups.unshift(group[:description])
+          group = group[:parent_example_group]
+        end
+        # TODO remove this when we're no longer wrapping everything with "mysql audit"
+        describe_groups.shift
+
+        {
+            :name => example.description,
+            :desc => example.full_description,
+            :resource_type => resource_type,
+            :resource_name => resource_name,
+            :context => describe_groups
+        }
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/audit/control_group_data.rb b/lib/chef/audit/control_group_data.rb
new file mode 100644
index 0000000000..93abfb3c21
--- /dev/null
+++ b/lib/chef/audit/control_group_data.rb
@@ -0,0 +1,84 @@
+class Chef
+  class Audit
+    class ControlGroupData
+      attr_reader :name, :status, :number_success, :number_failed, :controls
+
+      def initialize(name)
+        @status = "success"
+        @controls = []
+        @number_success = 0
+        @number_failed = 0
+        @name = name
+      end
+
+
+      def example_success(opts={})
+        @number_success += 1
+        control = create_control(opts)
+        controls << control
+        control
+      end
+
+      def example_failure(details=nil, opts={})
+        @number_failed += 1
+        @status = "failure"
+        control = create_control(opts)
+        control.details = details if details
+        control.status = "failure"
+        controls << control
+        control
+      end
+
+      def to_hash
+        controls.sort! {|x,y| x.line_number <=> y.line_number}
+        {
+            :control_group => {
+                :name => name,
+                :status => status,
+                :number_success => number_success,
+                :number_failed => number_failed,
+                :controls => controls.collect { |c| c.to_hash }
+            }
+        }
+      end
+
+      private
+
+      def create_control(opts={})
+        name = opts[:name]
+        resource_type = opts[:resource_type]
+        resource_name = opts[:resource_name]
+        context = opts[:context]
+        ControlData.new(name, resource_type, resource_name, context)
+      end
+
+    end
+
+    class ControlData
+      attr_reader :name, :resource_type, :resource_name, :context
+      attr_accessor :status, :details
+      # TODO this only helps with debugging
+      attr_accessor :line_number
+
+      def initialize(name, resource_type, resource_name, context)
+        @context = context
+        @name = name
+        @resource_type = resource_type
+        @resource_name = resource_name
+      end
+
+      def to_hash
+        ret = {
+            :name => name,
+            :status => status,
+            :details => details,
+            :resource_type => resource_type,
+            :resource_name => resource_name
+        }
+        ret[:context] = context || []
+        ret
+      end
+    end
+
+  end
+end
diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb
new file mode 100644
index 0000000000..e3699266b8
--- /dev/null
+++ b/lib/chef/audit/runner.rb
@@ -0,0 +1,93 @@
+#
+# Author:: Claire McQuin (<claire@getchef.com>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/audit'
+require 'chef/config'
+
+class Chef
+  class Audit
+    class Runner
+
+      def initialize(run_context)
+        @run_context = run_context
+      end
+
+      def run
+        setup
+        register_controls_groups
+
+        # The first parameter passed to RSpec::Core::Runner.new
+        # is an instance of RSpec::Core::ConfigurationOptions, which is
+        # responsible for processing command line options passed through rspec.
+        # This then gets merged with the configuration. We'll just communicate
+        # directly with the Configuration here.
+        audit_runner = RSpec::Core::Runner.new(nil, configuration, world)
+        audit_runner.run_specs(world.ordered_example_groups)
+      end
+
+      private
+
+      # RSpec configuration and world objects are heavy, so let's wait until
+      # we actually need them.
+      def configuration
+        @configuration ||= RSpec::Core::Configuration.new
+      end
+
+      def world
+        @world ||= RSpec::Core::World.new(configuration)
+      end
+
+      # Configure audits before run.
+      # Sets up where output and error streams should stream to, adds formatters
+      # for people-friendly output of audit results and json for reporting. Also
+      # configures expectation frameworks.
+      def setup
+        configuration.output_stream = Chef::Config[:log_location]
+        configuration.error_stream  = Chef::Config[:log_location]
+
+        add_formatters
+        disable_should_syntax
+      end
+
+      def add_formatters
+        configuration.add_formatter(RSpec::Core::Formatters::DocumentationFormatter)
+        configuration.add_formatter(ChefJsonFormatter)
+      end
+
+      # Explicitly disable :should syntax.
+      #
+      # :should is deprecated in RSpec 3 and we have chosen to explicitly disable it
+      # in audits. If :should is used in an audit, the audit will fail with error
+      # message "undefined method `:should`" rather than issue a deprecation warning.
+      #
+      # This can be removed when :should is fully removed from RSpec.
+      def disable_should_syntax
+        configuration.expect_with :rspec do |c|
+          c.syntax = :expect
+        end
+      end
+
+      # Register each controls group with the world, which will handle
+      # the ordering of the audits that will be run.
+      def register_controls_groups
+        @run_context.controls_groups.each { |ctls_grp| world.register(ctls_grp) }
+      end
+
+    end
+  end
+end
diff --git a/lib/chef/client.rb b/lib/chef/client.rb
index 6c83639ec3..3818f9fad6 100644
--- a/lib/chef/client.rb
+++ b/lib/chef/client.rb
@@ -25,6 +25,7 @@ require 'chef/log'
 require 'chef/rest'
 require 'chef/api_client'
 require 'chef/api_client/registration'
+require 'chef/audit'
 require 'chef/node'
 require 'chef/role'
 require 'chef/file_cache'
@@ -344,7 +345,8 @@ class Chef
       audit_exception = nil
       begin
         @events.audit_start(run_context)
-        # TODO
+        auditor = Chef::Audit::Runner.new(run_context)
+        auditor.run
         @events.audit_complete
       rescue Exception => e
         @events.audit_failed(e)
diff --git a/lib/chef/dsl/audit.rb b/lib/chef/dsl/audit.rb
index 7adcecbf14..948db72921 100644
--- a/lib/chef/dsl/audit.rb
+++ b/lib/chef/dsl/audit.rb
@@ -27,12 +27,9 @@ class Chef
         raise ::Chef::Exceptions::NoAuditsProvided unless group_block
 
         # TODO add the @example_groups list to the runner for later execution
-        # run_context.audit_runner.register
-        ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block)
+        run_context.controls_groups << ::Chef::Audit::ChefExampleGroup.describe(group_name, &group_block)
       end
 
     end
   end
 end
-
-
diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb
index 22679822a4..749e4cf388 100644
--- a/lib/chef/run_context.rb
+++ b/lib/chef/run_context.rb
@@ -50,6 +50,11 @@ class Chef
     # recipes, which is triggered by #load. (See also: CookbookCompiler)
     attr_accessor :resource_collection
 
+    attr_accessor :controls_groups
+
+    # Chef::ProviderResolver for this run
+    attr_accessor :provider_resolver
+
     # A Hash containing the immediate notifications triggered by resources
     # during the converge phase of the chef run.
     attr_accessor :immediate_notification_collection
@@ -73,6 +78,7 @@ class Chef
       @node = node
       @cookbook_collection = cookbook_collection
       @resource_collection = Chef::ResourceCollection.new
+      @controls_groups = []
       @immediate_notification_collection = Hash.new {|h,k| h[k] = []}
       @delayed_notification_collection = Hash.new {|h,k| h[k] = []}
       @definitions = Hash.new