Merge pull request #2465 from opscode/sersut/basic-audit-tests

Audit Tests

26 files changed, 340 insertions, 25 deletions

diff --git a/.travis.yml b/.travis.yml
index b2b002d8b7..3a81eb8881 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,7 @@ branches:
   - 10-stable
   - 11-stable
   - 12-stable
+  - audit-mode
 
 # do not run expensive spec tests on PRs, only on branches
 script: "
diff --git a/kitchen-tests/cookbooks/audit_test/.gitignore b/kitchen-tests/cookbooks/audit_test/.gitignore
new file mode 100644
index 0000000000..ec2a890bd3
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/.gitignore
@@ -0,0 +1,16 @@
+.vagrant
+Berksfile.lock
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+
+# Bundler
+Gemfile.lock
+bin/*
+.bundle/*
+
+.kitchen/
+.kitchen.local.yml
diff --git a/kitchen-tests/cookbooks/audit_test/.kitchen.yml b/kitchen-tests/cookbooks/audit_test/.kitchen.yml
new file mode 100644
index 0000000000..be11e33081
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/.kitchen.yml
@@ -0,0 +1,16 @@
+---
+driver:
+  name: vagrant
+
+provisioner:
+  name: chef_zero
+
+platforms:
+  - name: ubuntu-12.04
+  - name: centos-6.5
+
+suites:
+  - name: default
+    run_list:
+      - recipe[audit_test::default]
+    attributes:
diff --git a/kitchen-tests/cookbooks/audit_test/Berksfile b/kitchen-tests/cookbooks/audit_test/Berksfile
new file mode 100644
index 0000000000..0ac9b78cf7
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/Berksfile
@@ -0,0 +1,3 @@
+source "https://supermarket.getchef.com"
+
+metadata
diff --git a/kitchen-tests/cookbooks/audit_test/README.md b/kitchen-tests/cookbooks/audit_test/README.md
new file mode 100644
index 0000000000..75e2f44808
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/README.md
@@ -0,0 +1,12 @@
+# audit_test
+
+This cookbook has some basic recipes to test audit mode.
+
+In order to run these tests on your dev box:
+
+```
+$ bundle install
+$ bundle exec chef-client -c kitchen-tests/.chef/client.rb -z -o audit_test::default -l debug
+```
+
+Expected JSON output for the tests will be printed to `debug` log.
diff --git a/kitchen-tests/cookbooks/audit_test/chefignore b/kitchen-tests/cookbooks/audit_test/chefignore
new file mode 100644
index 0000000000..80dc2d20ef
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/chefignore
@@ -0,0 +1,95 @@
+# Put files/directories that should be ignored in this file when uploading
+# or sharing to the community site.
+# Lines that start with '# ' are comments.
+
+# OS generated files #
+######################
+.DS_Store
+Icon?
+nohup.out
+ehthumbs.db
+Thumbs.db
+
+# SASS #
+########
+.sass-cache
+
+# EDITORS #
+###########
+\#*
+.#*
+*~
+*.sw[a-z]
+*.bak
+REVISION
+TAGS*
+tmtags
+*_flymake.*
+*_flymake
+*.tmproj
+.project
+.settings
+mkmf.log
+
+## COMPILED ##
+##############
+a.out
+*.o
+*.pyc
+*.so
+*.com
+*.class
+*.dll
+*.exe
+*/rdoc/
+
+# Testing #
+###########
+.watchr
+.rspec
+spec/*
+spec/fixtures/*
+test/*
+features/*
+Guardfile
+Procfile
+
+# SCM #
+#######
+.git
+*/.git
+.gitignore
+.gitmodules
+.gitconfig
+.gitattributes
+.svn
+*/.bzr/*
+*/.hg/*
+*/.svn/*
+
+# Berkshelf #
+#############
+Berksfile
+Berksfile.lock
+cookbooks/*
+tmp
+
+# Cookbooks #
+#############
+CONTRIBUTING
+
+# Strainer #
+############
+Colanderfile
+Strainerfile
+.colander
+.strainer
+
+# Vagrant #
+###########
+.vagrant
+Vagrantfile
+
+# Travis #
+##########
+.travis.yml
diff --git a/kitchen-tests/cookbooks/audit_test/metadata.rb b/kitchen-tests/cookbooks/audit_test/metadata.rb
new file mode 100644
index 0000000000..4a60104e92
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/metadata.rb
@@ -0,0 +1,8 @@
+name             'audit_test'
+maintainer       'The Authors'
+maintainer_email 'you@example.com'
+license          'all_rights'
+description      'Installs/Configures audit_test'
+long_description 'Installs/Configures audit_test'
+version          '0.1.0'
+
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/default.rb b/kitchen-tests/cookbooks/audit_test/recipes/default.rb
new file mode 100644
index 0000000000..4f634d73c1
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/default.rb
@@ -0,0 +1,26 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: default
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+controls "basic control group" do
+  control "basic math" do
+    it "should pass" do
+      expect(2 - 2).to eq(0)
+    end
+  end
+end
+
+controls "control group without top level control" do
+  it "should pass" do
+    expect(2 - 2).to eq(0)
+  end
+end
+
+controls "control group with empty control" do
+  control "empty"
+end
+
+controls "empty control group with block" do
+end
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb
new file mode 100644
index 0000000000..77a4592e9d
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/error_duplicate_control_groups.rb
@@ -0,0 +1,17 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: error_duplicate_control_groups
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+controls "basic control group" do
+  it "should pass" do
+    expect(2 - 2).to eq(0)
+  end
+end
+
+controls "basic control group" do
+  it "should pass" do
+    expect(2 - 2).to eq(0)
+  end
+end
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb
new file mode 100644
index 0000000000..76a8817b5d
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/error_no_block.rb
@@ -0,0 +1,7 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: error_no_block
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+controls "empty control group without block"
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb b/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb
new file mode 100644
index 0000000000..d74acd6c6b
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/error_orphan_control.rb
@@ -0,0 +1,13 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: error_orphan_control
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+controls "basic control group" do
+  it "should pass" do
+    expect(2 - 2).to eq(0)
+  end
+end
+
+control "orphan control"
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb b/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb
new file mode 100644
index 0000000000..3225d3983e
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/failed_specs.rb
@@ -0,0 +1,14 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: failed_specs
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+controls "basic control group" do
+  control "basic math" do
+    # Can not write a good control :(
+    it "should pass" do
+      expect(2 - 0).to eq(0)
+    end
+  end
+end
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb
new file mode 100644
index 0000000000..70109d84b8
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_collision.rb
@@ -0,0 +1,31 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: serverspec_collision
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+file "/tmp/audit_test_file" do
+  action :create
+  content "Welcome to audit mode."
+end
+
+controls "file auditing" do
+  describe "test file" do
+    it "says welcome" do
+      expect(file("/tmp/audit_test_file")).to contain("Welcome")
+    end
+  end
+end
+
+file "/tmp/audit_test_file_2" do
+  action :create
+  content "Bye to audit mode."
+end
+
+controls "end file auditing" do
+  describe "end file" do
+    it "says bye" do
+      expect(file("/tmp/audit_test_file_2")).to contain("Bye")
+    end
+  end
+end
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb
new file mode 100644
index 0000000000..0396cc0de7
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/serverspec_support.rb
@@ -0,0 +1,37 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: serverspec_support
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+file "/tmp/audit_test_file" do
+  action :create
+  content "Welcome to audit mode."
+end
+
+# package "curl" do
+#   action :install
+# end
+
+controls "serverspec helpers with types" do
+  control "file helper" do
+    it "says welcome" do
+      expect(file("/tmp/audit_test_file")).to contain("Welcome")
+    end
+  end
+
+  control service("com.apple.CoreRAID") do
+    it { is_expected.to be_enabled }
+    it { is_expected.not_to be_running }
+  end
+
+  # describe "package helper" do
+  #   it "works" do
+  #     expect(package("curl")).to be_installed
+  #   end
+  # end
+
+  control package("postgresql") do
+    it { is_expected.to_not be_installed }
+  end
+end
diff --git a/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb b/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb
new file mode 100644
index 0000000000..ff39cde117
--- /dev/null
+++ b/kitchen-tests/cookbooks/audit_test/recipes/with_include_recipe.rb
@@ -0,0 +1,16 @@
+#
+# Cookbook Name:: audit_test
+# Recipe:: with_include_recipe
+#
+# Copyright (c) 2014 The Authors, All Rights Reserved.
+
+include_recipe "audit_test::serverspec_collision"
+
+controls "basic example" do
+  it "should pass" do
+    expect(2 - 2).to eq(0)
+  end
+end
+
+include_recipe "audit_test::serverspec_collision"
+include_recipe "audit_test::default"
diff --git a/kitchen-tests/cookbooks/webapp/README.md b/kitchen-tests/cookbooks/webapp/README.md
index e8de6ee467..f19ab46735 100644
--- a/kitchen-tests/cookbooks/webapp/README.md
+++ b/kitchen-tests/cookbooks/webapp/README.md
@@ -1,4 +1,3 @@
 # webapp
 
 TODO: Enter the cookbook description here.
-
diff --git a/lib/chef/application/client.rb b/lib/chef/application/client.rb
index 5463f504bc..6ca674d287 100644
--- a/lib/chef/application/client.rb
+++ b/lib/chef/application/client.rb
@@ -241,7 +241,15 @@ class Chef::Application::Client < Chef::Application
   option :audit_mode,
     :long           => "--[no-]audit-mode",
     :description    => "If not specified, run converge and audit phase.  If true, run only audit phase.  If false, run only converge phase.",
-    :boolean        => true
+    :boolean        => true,
+    :proc           => lambda { |set|
+      # Convert boolean to config options of :audit_only or :disabled
+      if set
+        :audit_only
+      else
+        :disabled
+      end
+    }
 
   IMMEDIATE_RUN_SIGNAL = "1".freeze
 
diff --git a/lib/chef/application/solo.rb b/lib/chef/application/solo.rb
index f433317826..798834304c 100644
--- a/lib/chef/application/solo.rb
+++ b/lib/chef/application/solo.rb
@@ -207,8 +207,8 @@ class Chef::Application::Solo < Chef::Application
       @chef_client_json = config_fetcher.fetch_json
     end
 
-    # If we don't specify this, solo will try to perform the audits
-    Chef::Config[:audit_mode] = false
+    # Disable auditing for solo
+    Chef::Config[:audit_mode] = :disabled
   end
 
   def setup_application
diff --git a/lib/chef/audit/audit_reporter.rb b/lib/chef/audit/audit_reporter.rb
index b0eb835c0c..ce4978180e 100644
--- a/lib/chef/audit/audit_reporter.rb
+++ b/lib/chef/audit/audit_reporter.rb
@@ -31,7 +31,6 @@ class Chef
       PROTOCOL_VERSION = '0.1.0'
 
       def initialize(rest_client)
-        @audit_enabled = Chef::Config[:audit_mode]
         @rest_client = rest_client
         # Ruby 1.9.3 and above "enumerate their values in the order that the corresponding keys were inserted."
         @ordered_control_groups = Hash.new
@@ -87,7 +86,7 @@ class Chef
 
       # If @audit_enabled is nil or true, we want to run audits
       def auditing_enabled?
-        @audit_enabled != false
+        Chef::Config[:audit_mode] != :disabled
       end
 
       private
@@ -107,7 +106,7 @@ class Chef
         audit_data.end_time = iso8601ify(run_status.end_time)
 
         audit_history_url = "controls"
-        Chef::Log.info("Sending audit report (run-id: #{audit_data.run_id})")
+        Chef::Log.debug("Sending audit report (run-id: #{audit_data.run_id})")
         run_data = audit_data.to_hash
 
         if error
diff --git a/lib/chef/client.rb b/lib/chef/client.rb
index 9e1d2dc207..aa0d6722fe 100644
--- a/lib/chef/client.rb
+++ b/lib/chef/client.rb
@@ -441,16 +441,12 @@ class Chef
 
         run_context = setup_run_context
 
-        unless Chef::Config[:audit_mode] == true
+        if Chef::Config[:audit_mode] != :audit_only
           converge_error = converge_and_save(run_context)
-        else
-          Chef::Log.debug("Skipping converge. Chef is configured to run audits only.")
         end
 
-        unless Chef::Config[:audit_mode] == false
+        if Chef::Config[:audit_mode] != :disabled
           audit_error = run_audits(run_context)
-        else
-          Chef::Log.debug("Skipping audits. Chef is configured to converge the node only.")
         end
 
         if converge_error || audit_error
diff --git a/lib/chef/config.rb b/lib/chef/config.rb
index 510dcd342f..2a9d44a3c5 100644
--- a/lib/chef/config.rb
+++ b/lib/chef/config.rb
@@ -319,7 +319,9 @@ class Chef
     default :client_fork, true
     default :enable_reporting, true
     default :enable_reporting_url_fatals, false
-    default :audit_mode, nil
+    # Possible values for :audit_mode
+    # :enabled, :disabled, :audit_only,
+    default :audit_mode, :enabled
 
     # Policyfile is an experimental feature where a node gets its run list and
     # cookbook version set from a single document on the server instead of
diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb
index 09d04f3aae..99603965a9 100644
--- a/lib/chef/formatters/doc.rb
+++ b/lib/chef/formatters/doc.rb
@@ -163,13 +163,11 @@ class Chef
 
       # Called before audit phase starts
       def audit_phase_start(run_status)
-        puts_line ""
-        puts_line "++ Audit phase starting ++"
+        puts_line "Starting audit phase"
       end
 
       def audit_phase_complete
-        puts_line ""
-        puts_line "++ Audit phase ended ++ "
+        puts_line "Auditing complete"
       end
 
       def audit_phase_failed(error)
diff --git a/spec/functional/resource/deploy_revision_spec.rb b/spec/functional/resource/deploy_revision_spec.rb
index 7bc3da9a05..e5f5341fcd 100644
--- a/spec/functional/resource/deploy_revision_spec.rb
+++ b/spec/functional/resource/deploy_revision_spec.rb
@@ -45,11 +45,10 @@ describe Chef::Resource::DeployRevision, :unix_only => true do
 
   before(:all) do
     @ohai = Ohai::System.new
-    @ohai.all_plugins("os")
+    @ohai.all_plugins(["platform", "os"])
   end
 
   let(:node) do
-
     Chef::Node.new.tap do |n|
       n.name "rspec-test"
       n.consume_external_attrs(@ohai.data, {})
diff --git a/spec/functional/resource/git_spec.rb b/spec/functional/resource/git_spec.rb
index 4f462b7cb6..9d3b82f19e 100644
--- a/spec/functional/resource/git_spec.rb
+++ b/spec/functional/resource/git_spec.rb
@@ -92,7 +92,7 @@ E
 
   before(:all) do
     @ohai = Ohai::System.new
-    @ohai.all_plugins("os")
+    @ohai.all_plugins(["platform", "os"])
   end
 
   context "working with pathes with special characters" do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index e3de80f3f1..2b880dc200 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -164,6 +164,8 @@ RSpec.configure do |config|
 
   config.before(:each) do
     Chef::Config.reset
+
+    allow_any_instance_of(Chef::Audit::Runner).to receive(:run).and_return(true)
   end
 
   config.before(:suite) do
diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb
index eb13efbf76..f38dee634d 100644
--- a/spec/unit/client_spec.rb
+++ b/spec/unit/client_spec.rb
@@ -255,10 +255,10 @@ describe Chef::Client do
       end
 
       def stub_for_audit
-        expect(Chef::Audit::Runner).to receive(:new).and_return(audit_runner)
-        expect(audit_runner).to receive(:run).and_return(true)
-
-        expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:audit_phase_complete)
+        # --AuditReporter#run_completed
+        #   posts the audit data to server.
+        #   (has its own tests, so stubbing it here.)
+        expect_any_instance_of(Chef::Audit::AuditReporter).to receive(:run_completed)
       end
 
       def stub_for_node_save