Native compliance phase

Signed-off-by: Lamont Granquist <lamont@scriptkiddie.org>

10 files changed, 505 insertions, 13 deletions

diff --git a/lib/chef/compliance/profile.rb b/lib/chef/compliance/profile.rb
new file mode 100644
index 0000000000..09e752b905
--- /dev/null
+++ b/lib/chef/compliance/profile.rb
@@ -0,0 +1,97 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  module Compliance
+    class Profile
+
+      # @return [Boolean] if the profile has been enabled
+      attr_accessor :enabled
+
+      # @return [String] The full path on the host to the profile inspec.yml
+      attr_reader :path
+
+      # @return [String] The name of the cookbook that the profile is in
+      attr_reader :cookbook_name
+
+      def initialize(data, path, cookbook_name)
+        @data = data
+        @path = path
+        @cookbook_name = cookbook_name
+        disable!
+        validate!
+      end
+
+      # @return [String] name of the inspec profile from parsing the inspec.yml
+      #
+      def name
+        @data["name"]
+      end
+
+      # Raises if the inspec profile is not valid.
+      #
+      def validate!
+        raise "Inspec profile at #{path} has no name" unless name
+      end
+
+      # @return [Boolean] if the profile has been enabled
+      #
+      def enabled?
+        !!@enabled
+      end
+
+      # Set the profile to being enabled
+      #
+      def enable!
+        @enabled = true
+      end
+
+      # Set the profile as being disabled
+      #
+      def disable!
+        @enabled = false
+      end
+
+      # Render the profile in a way that it can be consumed by inspec
+      #
+      def for_inspec
+        { name: name, path: File.dirname(path) }
+      end
+
+      # Helper to construct a profile object from a hash.  Since the path and
+      # cookbook_name are required this is probably not externally useful.
+      #
+      def self.from_hash(hash, path, cookbook_name)
+        new(hash, path, cookbook_name)
+      end
+
+      # Helper to consruct a profile object from a yaml string.  Since the path
+      # and cookbook_name are required this is probably not externally useful.
+      #
+      def self.from_yaml(string, path, cookbook_name)
+        from_hash(YAML.load(string), path, cookbook_name)
+      end
+
+      # @param filename [String] full path to the inspec.yml file in the cookbook
+      # @param cookbook_name [String] cookbook that the profile is in
+      #
+      def self.from_file(filename, cookbook_name)
+        from_yaml(IO.read(filename), filename, cookbook_name)
+      end
+    end
+  end
+end
diff --git a/lib/chef/compliance/profile_collection.rb b/lib/chef/compliance/profile_collection.rb
new file mode 100644
index 0000000000..2fe83b072a
--- /dev/null
+++ b/lib/chef/compliance/profile_collection.rb
@@ -0,0 +1,86 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "profile"
+
+class Chef
+  module Compliance
+    class ProfileCollection < Array
+
+      # Add a profile to the profile collection.  The cookbook_name needs to be determined by the
+      # caller and is used in the `include_profile` API to match on.  The path should be the complete
+      # path on the host of the inspec.yml file, including the filename.
+      #
+      # @param cookbook_name [String]
+      # @param path [String]
+      #
+      def from_file(cookbook_name, path)
+        self << Profile.from_file(cookbook_name, path)
+      end
+
+      # @return [Boolean] if any of the profiles are enabled
+      #
+      def using_profiles?
+        any?(&:enabled?)
+      end
+
+      # @return [Array<Profile>] inspec profiles which are enabled in a form suitable to pass to inspec
+      #
+      def for_inspec
+        select(&:enabled?).each_with_object([]) { |profile, arry| arry << profile.for_inspec }
+      end
+
+      # DSL method to enable profiles.  This matches on the name of the profile, it does not match on
+      # the filename of the profile.
+      #
+      # @example Specific profile in a cookbook
+      #
+      # include_profile "acme_cookbook::ssh-001"
+      #
+      # @example Every profile in a cookbook
+      #
+      # include_profile "acme_cookbook"
+      #
+      # @example Matching profiles by regexp in a cookbook
+      #
+      # include_profile "acme_cookbook::ssh.*"
+      #
+      # @example Matching profiles by regexp in any cookbook in the cookbook collection
+      #
+      # include_profile ".*::ssh.*"
+      #
+      def include_profile(arg)
+        (cookbook_name, profile_name) = arg.split("::")
+        profiles = nil
+
+        if profile_name.nil?
+          profiles = select { |profile| /^#{cookbook_name}$/.match?(profile.cookbook_name) }
+          if profiles.empty?
+            raise "No inspec profiles found in cookbooks matching #{cookbook_name}"
+          end
+        else
+          profiles = select { |profile| /^#{cookbook_name}$/.match?(profile.cookbook_name) && /^#{profile_name}$/.match?(profile.name) }
+          if profiles.empty?
+            raise "No inspec profiles matching #{profile_name} found in cookbooks matching #{cookbook_name}"
+          end
+        end
+
+        profiles.each(&:enable!)
+      end
+    end
+  end
+end
diff --git a/lib/chef/compliance/runner.rb b/lib/chef/compliance/runner.rb
index 14e776a6b7..993939534c 100644
--- a/lib/chef/compliance/runner.rb
+++ b/lib/chef/compliance/runner.rb
@@ -12,6 +12,8 @@ class Chef
 
       attr_accessor :run_id
       attr_reader :node
+      attr_reader :run_context
+
       def_delegators :node, :logger
 
       def enabled?
@@ -25,7 +27,9 @@ class Chef
         logger.debug("#{self.class}##{__method__}: audit cookbook? #{audit_cookbook_present}")
         logger.debug("#{self.class}##{__method__}: compliance phase attr? #{node["audit"]["compliance_phase"]}")
 
-        if node["audit"]["compliance_phase"].nil?
+        if safe_profile_collection&.using_profiles?
+          true
+        elsif node["audit"]["compliance_phase"].nil?
           inspec_profiles.any? && !audit_cookbook_present
         else
           node["audit"]["compliance_phase"]
@@ -41,6 +45,14 @@ class Chef
         self.node = node
       end
 
+      # This hook gives us the run_context immediately after it is created so that we can wire up this object to it.
+      #
+      # (see EventDispatch::Base#)
+      #
+      def cookbook_compilation_start(run_context)
+        @run_context = run_context
+      end
+
       def run_started(run_status)
         self.run_id = run_status.run_id
       end
@@ -138,19 +150,30 @@ class Chef
           reporter: ["json-automate"],
           reporter_backtrace_inclusion: node["audit"]["result_include_backtrace"],
           reporter_message_truncation: node["audit"]["result_message_limit"],
-          waiver_file: Array(node["audit"]["waiver_file"]),
+          waiver_file: waiver_files,
         }
       end
 
+      def waiver_files
+        from_attributes = Array(node["audit"]["waiver_file"])
+        from_cookbooks = safe_waiver_collection&.for_inspec || []
+
+        from_attributes + from_cookbooks
+      end
+
       def inspec_profiles
         profiles = node["audit"]["profiles"]
         unless profiles.respond_to?(:map) && profiles.all? { |_, p| p.respond_to?(:transform_keys) && p.respond_to?(:update) }
           raise "CMPL010: #{Inspec::Dist::PRODUCT_NAME} profiles specified in an unrecognized format, expected a hash of hashes."
         end
 
-        profiles.map do |name, profile|
+        from_attributes = profiles.map do |name, profile|
           profile.transform_keys(&:to_sym).update(name: name)
-        end
+        end || []
+
+        from_cookbooks = safe_profile_collection&.for_inspec || []
+
+        from_attributes + from_cookbooks
       end
 
       def load_fetchers!
@@ -316,6 +339,14 @@ class Chef
 
         @validation_passed = true
       end
+
+      def safe_profile_collection
+        run_context&.profile_collection
+      end
+
+      def safe_waiver_collection
+        run_context&.waiver_collection
+      end
     end
   end
 end
diff --git a/lib/chef/compliance/waiver.rb b/lib/chef/compliance/waiver.rb
new file mode 100644
index 0000000000..83ad53c2c3
--- /dev/null
+++ b/lib/chef/compliance/waiver.rb
@@ -0,0 +1,86 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "yaml"
+
+class Chef
+  module Compliance
+    class Waiver
+
+      # @return [Boolean] if the waiver has been enabled
+      attr_accessor :enabled
+
+      # @return [String] The name of the cookbook that the waiver is in
+      attr_accessor :cookbook_name
+
+      # @return [String] The full path on the host to the waiver yml file
+      attr_accessor :path
+
+      def initialize(data, path, cookbook_name)
+        @data = data
+        @cookbook_name = cookbook_name
+        @path = path
+        disable!
+      end
+
+      # @return [Boolean] if the waiver has been enabled
+      #
+      def enabled?
+        !!@enabled
+      end
+
+      # Set the waiver to being enabled
+      #
+      def enable!
+        @enabled = true
+      end
+
+      # Set the waiver as being disabled
+      #
+      def disable!
+        @enabled = false
+      end
+
+      # Render the waiver in a way that it can be consumed by inspec
+      #
+      def for_inspec
+        path
+      end
+
+      # Helper to construct a waiver object from a hash.  Since the path and
+      # cookbook_name are required this is probably not externally useful.
+      #
+      def self.from_hash(hash, path, cookbook_name)
+        new(hash, path, cookbook_name)
+      end
+
+      # Helper to consruct a waiver object from a yaml string.  Since the path
+      # and cookbook_name are required this is probably not externally useful.
+      #
+      def self.from_yaml(string, path, cookbook_name)
+        from_hash(YAML.load(string), path, cookbook_name)
+      end
+
+      # @param filename [String] full path to the yml file in the cookbook
+      # @param cookbook_name [String] cookbook that the waiver is in
+      #
+      def self.from_file(filename, cookbook_name)
+        from_yaml(IO.read(filename), filename, cookbook_name)
+      end
+    end
+  end
+end
diff --git a/lib/chef/compliance/waiver_collection.rb b/lib/chef/compliance/waiver_collection.rb
new file mode 100644
index 0000000000..7bf173fa75
--- /dev/null
+++ b/lib/chef/compliance/waiver_collection.rb
@@ -0,0 +1,79 @@
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "waiver"
+
+class Chef
+  module Compliance
+    class WaiverCollection < Array
+
+      # Add a waiver to the waiver collection.  The cookbook_name needs to be determined by the
+      # caller and is used in the `include_waiver` API to match on.  The path should be the complete
+      # path on the host of the yml file, including the filename.
+      #
+      # @param cookbook_name [String]
+      # @param path [String]
+      #
+      def from_file(cookbook, filename)
+        self << Waiver.from_file(cookbook, filename)
+      end
+
+      # @return [Array<Waiver>] inspec waivers which are enabled in a form suitable to pass to inspec
+      #
+      def for_inspec
+        select(&:enabled?).each_with_object([]) { |waiver, arry| arry << waiver.for_inspec }
+      end
+
+      # DSL method to enable waiver files.  This matches on the name of the profile being wavied, it
+      # does not match on the filename of the waiver file.
+      #
+      # @example Specific waiver file in a cookbook
+      #
+      # include_waiver "acme_cookbook::ssh-001"
+      #
+      # @example Every waiver file in a cookbook
+      #
+      # include_waiver "acme_cookbook"
+      #
+      # @example Matching waivers by regexp in a cookbook
+      #
+      # include_waiver "acme_cookbook::ssh.*"
+      #
+      # @example Matching waivers by regexp in any cookbook in the cookbook collection
+      #
+      # include_waiver ".*::ssh.*"
+      #
+      def include_waiver(arg)
+        (cookbook_name, waiver_name) = arg.split("::")
+        waivers = nil
+
+        if waiver_name.nil?
+          waivers = select { |waiver| /^#{cookbook_name}$/.match?(waiver.cookbook_name) }
+          if waivers.empty?
+            raise "No inspec waivers found in cookbooks matching #{cookbook_name}"
+          end
+        else
+          waivers = select { |waiver| /^#{cookbook_name}$/.match?(waiver.cookbook_name) && /^#{waiver_name}$/.match?(waiver.name) }
+          if waivers.empty?
+            raise "No inspec waivers matching #{waiver_name} found in cookbooks matching #{cookbook_name}"
+          end
+        end
+
+        waivers.each(&:enable!)
+      end
+    end
+  end
+end
diff --git a/lib/chef/dsl/compliance.rb b/lib/chef/dsl/compliance.rb
new file mode 100644
index 0000000000..8dafbb1a8e
--- /dev/null
+++ b/lib/chef/dsl/compliance.rb
@@ -0,0 +1,33 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  module DSL
+    module Compliance
+
+      # @see Chef::Compliance::ProfileCollection#include_profile
+      def include_profile(*args)
+        run_context.profile_collection.include_profile(*args)
+      end
+
+      # @see Chef::Compliance::WaiverCollection#include_waiver
+      def include_waiver(*args)
+        run_context.waiver_collection.include_waiver(*args)
+      end
+    end
+  end
+end
diff --git a/lib/chef/dsl/recipe.rb b/lib/chef/dsl/recipe.rb
index ad85a9dd91..7b08f841e6 100644
--- a/lib/chef/dsl/recipe.rb
+++ b/lib/chef/dsl/recipe.rb
@@ -18,12 +18,13 @@
 #
 
 require_relative "../exceptions"
-require_relative "resources"
+require_relative "compliance"
+require_relative "declare_resource"
 require_relative "definitions"
 require_relative "include_recipe"
 require_relative "reboot_pending"
+require_relative "resources"
 require_relative "universal"
-require_relative "declare_resource"
 require_relative "../mixin/notifying_block"
 require_relative "../mixin/lazy_module_include"
 
@@ -42,6 +43,7 @@ class Chef
     #   - it also pollutes the namespace of nearly every context, watch out.
     #
     module Recipe
+      include Chef::DSL::Compliance
       include Chef::DSL::Universal
       include Chef::DSL::DeclareResource
       include Chef::Mixin::NotifyingBlock
diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index f9504967a9..3756a29f16 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -164,7 +164,7 @@ class Chef
       # Called when LWRPs are finished loading
       def lwrp_load_complete; end
 
-      # Called when an ohai plugin file loading starts
+      # Called when ohai plugin file loading starts
       def ohai_plugin_load_start(file_count); end
 
       # Called when an ohai plugin file has been loaded
@@ -173,9 +173,15 @@ class Chef
       # Called when an ohai plugin file has an error on load.
       def ohai_plugin_file_load_failed(path, exception); end
 
-      # Called when an ohai plugin file loading has finished
+      # Called when ohai plugin file loading has finished
       def ohai_plugin_load_complete; end
 
+      # Called when compliance file loading starts
+      def compliance_load_start; end
+
+      # Called when compliance file loading ends
+      def compliance_load_complete; end
+
       # Called before attribute files are loaded
       def attribute_load_start(attribute_file_count); end
 
diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb
index 75c18f2fcf..05aff531a3 100644
--- a/lib/chef/run_context.rb
+++ b/lib/chef/run_context.rb
@@ -25,6 +25,8 @@ require_relative "log"
 require_relative "recipe"
 require_relative "run_context/cookbook_compiler"
 require_relative "event_dispatch/events_output_stream"
+require_relative "compliance/waiver_collection"
+require_relative "compliance/profile_collection"
 require_relative "train_transport"
 require_relative "exceptions"
 require "forwardable" unless defined?(Forwardable)
@@ -120,10 +122,22 @@ class Chef
 
     # Handle to the global action_collection of executed actions for reporting / data_collector /etc
     #
-    # @return [Chef::ActionCollection
+    # @return [Chef::ActionCollection]
     #
     attr_accessor :action_collection
 
+    # Handle to the global profile_collection of inspec profiles for the compliance phase
+    #
+    # @return [Chef::Compliance::ProfileCollection]
+    #
+    attr_accessor :profile_collection
+
+    # Handle to the global waiver_collection of inspec waiver files for the compliance phase
+    #
+    # @return [Chef::Compliance::WaiverCollection]
+    #
+    attr_accessor :waiver_collection
+
     # Pointer back to the Chef::Runner that created this
     #
     attr_accessor :runner
@@ -198,6 +212,8 @@ class Chef
       @loaded_attributes_hash = {}
       @reboot_info = {}
       @cookbook_compiler = nil
+      @waiver_collection = Chef::Compliance::WaiverCollection.new
+      @profile_collection = Chef::Compliance::ProfileCollection.new
 
       initialize_child_state
     end
@@ -688,6 +704,8 @@ class Chef
         node
         node=
         open_stream
+        profile_collection
+        profile_collection=
         reboot_info
         reboot_info=
         reboot_requested?
@@ -700,6 +718,8 @@ class Chef
         transport
         transport_connection
         unreachable_cookbook?
+        waiver_collection
+        waiver_collection=
       }
 
       def initialize(parent_run_context)
diff --git a/lib/chef/run_context/cookbook_compiler.rb b/lib/chef/run_context/cookbook_compiler.rb
index 27461fea9a..cbfbc76934 100644
--- a/lib/chef/run_context/cookbook_compiler.rb
+++ b/lib/chef/run_context/cookbook_compiler.rb
@@ -32,6 +32,7 @@ class Chef
       attr_reader :events
       attr_reader :run_list_expansion
       attr_reader :logger
+      attr_reader :run_context
 
       def initialize(run_context, run_list_expansion, events)
         @run_context = run_context
@@ -43,23 +44,42 @@ class Chef
 
       # Chef::Node object for the current run.
       def node
-        @run_context.node
+        run_context.node
       end
 
       # Chef::CookbookCollection object for the current run
       def cookbook_collection
-        @run_context.cookbook_collection
+        run_context.cookbook_collection
       end
 
       # Resource Definitions from the compiled cookbooks. This is populated by
       # calling #compile_resource_definitions (which is called by #compile)
       def definitions
-        @run_context.definitions
+        run_context.definitions
+      end
+
+      # The global waiver_collection hanging off of the run_context, used by
+      # compile_compliance and the compliance phase that runs inspec
+      #
+      # @returns [Chef::Compliance::WaiverCollection]
+      #
+      def waiver_collection
+        run_context.waiver_collection
+      end
+
+      # The global profile_collection hanging off of the run_context, used by
+      # compile_compliance and the compliance phase that runs inspec
+      #
+      # @returns [Chef::Compliance::ProfileCollection]
+      #
+      def profile_collection
+        run_context.profile_collection
       end
 
       # Run the compile phase of the chef run. Loads files in the following order:
       # * Libraries
       # * Ohai
+      # * Compliance Profiles/Waivers
       # * Attributes
       # * LWRPs
       # * Resource Definitions
@@ -73,6 +93,7 @@ class Chef
       def compile
         compile_libraries
         compile_ohai_plugins
+        compile_compliance
         compile_attributes
         compile_lwrps
         compile_resource_definitions
@@ -134,6 +155,18 @@ class Chef
         @events.ohai_plugin_load_complete
       end
 
+      # Loads the compliance segment files from the cookbook into the collections
+      # hanging off of the run_context, for later use in the compliance phase
+      # inspec run.
+      #
+      def compile_compliance
+        @events.compliance_load_start
+        cookbook_order.each do |cookbook|
+          load_compliance_from_cookbook(cookbook)
+        end
+        @events.compliance_load_complete
+      end
+
       # Loads attributes files from cookbooks. Attributes files are loaded
       # according to #cookbook_order; within a cookbook, +default.rb+ is loaded
       # first, then the remaining attributes files in lexical sort order.
@@ -171,7 +204,7 @@ class Chef
         run_list_expansion.recipes.each do |recipe|
 
           path = resolve_recipe(recipe)
-          @run_context.load_recipe(recipe)
+          run_context.load_recipe(recipe)
           @events.recipe_file_loaded(path, recipe)
         rescue Chef::Exceptions::RecipeNotFound => e
           @events.recipe_not_found(e)
@@ -288,6 +321,25 @@ class Chef
         end
       end
 
+      # Load the compliance segment files from a single cookbook
+      #
+      def load_compliance_from_cookbook(cookbook_name)
+        # This identifies profiles by their inspec.yml file, we recurse into subdirs so the profiles may be deeply
+        # nested in a subdir structurre for organization.  You could have profiles inside of profiles but
+        # since that is not coherently defined, you should not.
+        #
+        each_file_in_cookbook_by_segment(cookbook_name, :compliance, [ "profiles/**/inspec.yml", "profiles/**/inspec.yaml" ]) do |filename|
+          profile_collection.from_file(filename, cookbook_name)
+        end
+
+        # This identifies waiver files as any yaml files under the waivers subdir.  We recurse into subdirs as well
+        # so that waivers may be nested in subdirs for organization.  Any other files are ignored.
+        #
+        each_file_in_cookbook_by_segment(cookbook_name, :compliance, [ "waivers/**/*.yml", "waivers/**/*.yaml" ]) do |filename|
+          waiver_collection.from_file(filename, cookbook_name)
+        end
+      end
+
       def load_resource_definitions_from_cookbook(cookbook_name)
         files_in_cookbook_by_segment(cookbook_name, :definitions).each do |filename|
           next unless File.extname(filename) == ".rb"