Add inspec fetchers from audit-cookbook.

Signed-off-by: Pete Higgins <pete@peterhiggins.org>

5 files changed, 435 insertions, 1 deletions

diff --git a/lib/chef/audit/fetcher/automate.rb b/lib/chef/audit/fetcher/automate.rb
new file mode 100644
index 0000000000..e28d8ea026
--- /dev/null
+++ b/lib/chef/audit/fetcher/automate.rb
@@ -0,0 +1,69 @@
+require "uri"
+require "plugins/inspec-compliance/lib/inspec-compliance"
+
+class Chef
+  module Audit
+    module Fetcher
+      class Automate < ::InspecPlugins::Compliance::Fetcher
+        name 'chef-automate'
+
+        # it positions itself before `compliance` fetcher
+        # only load it, if you want to use audit cookbook in Chef Solo with Chef Automate
+        priority 502
+
+        CONFIG = {
+          'insecure' => true,
+          'token' => nil,
+          'server_type' => 'automate',
+          'automate' => {
+            'ent' => 'default',
+            'token_type' => 'dctoken',
+          },
+        }
+
+        def self.resolve(target)
+          uri = get_target_uri(target)
+          return nil if uri.nil?
+
+          config = CONFIG.dup
+
+          # we have detailed information available in our lockfile, no need to ask the server
+          if target.respond_to?(:key?) && target.key?(:url)
+            profile_fetch_url = target[:url]
+          else
+            # verifies that the target e.g base/ssh exists
+            base_path = "/compliance/profiles/#{uri.host}#{uri.path}"
+
+            profile_path = if target.respond_to?(:key?) && target.key?(:version)
+                             "#{base_path}/version/#{target[:version]}/tar"
+                           else
+                             "#{base_path}/tar"
+                           end
+
+            dc = Chef::Config[:data_collector]
+            url = URI(dc[:server_url])
+            url.path = profile_path
+            profile_fetch_url = url.to_s
+
+            config['token'] = dc['token']
+
+            if config['token'].nil?
+              raise 'No data-collector token set, which is required by the chef-automate fetcher. ' \
+                'Set the `data_collector.token` configuration parameter in your client.rb ' \
+                'or use the "chef-server-automate" reporter which does not require any ' \
+                'data-collector settings and uses Chef Server to fetch profiles.'
+            end
+          end
+
+          new(profile_fetch_url, config)
+        rescue URI::Error => _e
+          nil
+        end
+
+        def to_s
+          'Chef Automate for Chef Solo Fetcher'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/audit/fetcher/chef_server.rb b/lib/chef/audit/fetcher/chef_server.rb
new file mode 100644
index 0000000000..65768a561a
--- /dev/null
+++ b/lib/chef/audit/fetcher/chef_server.rb
@@ -0,0 +1,136 @@
+require 'uri'
+require "plugins/inspec-compliance/lib/inspec-compliance"
+
+# This class implements an InSpec fetcher for for Chef Server. The implementation
+# is based on the Chef Compliance fetcher and only adapts the calls to redirect
+# the requests via Chef Server.
+#
+# This implementation depends on chef-client runtime, therefore it is only executable
+# inside of a chef-client run
+
+class Chef
+  module Audit
+    module Fetcher
+      class ChefServer < ::InspecPlugins::Compliance::Fetcher
+        name 'chef-server'
+
+        # it positions itself before `compliance` fetcher
+        # only load it, if the Chef Server is integrated with Chef Compliance
+        priority 501
+
+        CONFIG = { 'insecure' => true }
+
+        # Accepts URLs to compliance profiles in one of two forms:
+        # * a String URL with a compliance scheme, like "compliance://namespace/profile_name"
+        # * a Hash with a key of `compliance` and a value like "compliance/profile_name" and optionally a `version` key with a String value
+        def self.resolve(target)
+          uri = get_target_uri(target)
+          return nil if uri.nil?
+
+          profile = uri.host + uri.path
+          profile = uri.user + '@' + profile if uri.user
+
+          version = target[:version] if target.respond_to?(:key?) && target.key?(:version)
+          new(target_url(profile, version), CONFIG)
+        rescue URI::Error => _e
+          nil
+        end
+
+        def self.target_url(profile, version = nil)
+          organization = Chef::Config[:chef_server_url].split('/').last
+          namespace, profile_name = profile.split('/')
+
+          path_parts = [""]
+          path_parts << "compliance" if chef_server_reporter? || chef_server_fetcher?
+          path_parts << "organizations/#{organization}/owners/#{namespace}/compliance/#{profile_name}"
+          path_parts << "version/#{version}" if version
+          path_parts << "tar"
+
+          target_url = URI(Chef::Config[:chef_server_url])
+          target_url.path = path_parts.compact.join("/")
+
+          Chef::Log.info("Fetching profile from: #{target_url}")
+          target_url
+        end
+
+        #
+        # We want to save compliance: in the lockfile rather than url: to
+        # make sure we go back through the ComplianceAPI handling.
+        #
+        def resolved_source
+          { compliance: chef_server_url }
+        end
+
+        # Downloads archive to temporary file using a Chef::ServerAPI
+        # client so that Chef Server's header-based authentication can be
+        # used.
+        def download_archive_to_temp
+          return @temp_archive_path unless @temp_archive_path.nil?
+
+          rest = Chef::ServerAPI.new(@target, Chef::Config.merge(ssl_verify_mode: :verify_none))
+          archive = with_http_rescue do
+            rest.streaming_request(@target)
+          end
+          @archive_type = '.tar.gz'
+
+          if archive.nil?
+            path = @target.respond_to?(:path) ? @target.path : path
+            raise "Unable to find requested profile on path: '#{path}' on the Automate system."
+          end
+
+          Inspec::Log.debug("Archive stored at temporary location: #{archive.path}")
+          @temp_archive_path = archive.path
+        end
+
+        def with_http_rescue
+          response = yield
+          if response.respond_to?(:code)
+            # handle non 200 error codes, they are not raised as Net::HTTPClientException
+            handle_http_error_code(response.code) if response.code.to_i >= 300
+          end
+          response
+        rescue Net::HTTPClientException => e
+          Chef::Log.error e
+          handle_http_error_code(e.response.code)
+        end
+
+        def handle_http_error_code(code)
+          case code
+          when /401|403/
+            Chef::Log.error 'Auth issue: see audit cookbook TROUBLESHOOTING.md'
+          when /404/
+            Chef::Log.error 'Object does not exist on remote server.'
+          when /413/
+            Chef::Log.error 'You most likely hit the erchef request size in Chef Server that defaults to ~2MB. To increase this limit see audit cookbook TROUBLESHOOTING.md OR https://docs.chef.io/config_rb_server.html'
+          when /429/
+            Chef::Log.error "This error typically means the data sent was larger than Automate's limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
+          end
+          msg = "Received HTTP error #{code}"
+          Chef::Log.error msg
+          raise msg if @raise_if_unreachable
+        end
+
+        def to_s
+          'Chef Server/Compliance Profile Loader'
+        end
+
+        CHEF_SERVER_REPORTERS = %w(chef-server chef-server-compliance chef-server-visibility chef-server-automate)
+        def self.chef_server_reporter?
+          (Array(Chef.node.attributes['audit']['reporter']) & CHEF_SERVER_REPORTERS).any?
+        end
+
+        CHEF_SERVER_FETCHERS = %w(chef-server chef-server-compliance chef-server-visibility chef-server-automate)
+        def self.chef_server_fetcher?
+          CHEF_SERVER_FETCHERS.include?(Chef.node.attributes['audit']['fetcher'])
+        end
+
+        private
+
+        def chef_server_url
+          m = %r{^#{@config['server']}/owners/(?<owner>[^/]+)/compliance/(?<id>[^/]+)/tar$}.match(@target)
+          "#{m[:owner]}/#{m[:id]}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb
index ded5e0ece3..bbe2b8ae88 100644
--- a/lib/chef/audit/runner.rb
+++ b/lib/chef/audit/runner.rb
@@ -1,10 +1,12 @@
 autoload :Inspec, 'inspec'
 
 require_relative 'default_attributes'
+require_relative 'fetcher/automate'
+require_relative 'fetcher/chef_server'
 require_relative 'reporter/audit_enforcer'
-require_relative 'reporter/json_file'
 require_relative 'reporter/automate'
 require_relative 'reporter/chef_server_automate'
+require_relative 'reporter/json_file'
 
 class Chef
   module Audit
diff --git a/spec/unit/audit/fetcher/automate_spec.rb b/spec/unit/audit/fetcher/automate_spec.rb
new file mode 100644
index 0000000000..df0225ed9e
--- /dev/null
+++ b/spec/unit/audit/fetcher/automate_spec.rb
@@ -0,0 +1,134 @@
+require 'spec_helper'
+require "chef/audit/fetcher/automate"
+
+describe Chef::Audit::Fetcher::Automate do
+  describe ".resolve" do
+    before do
+      Chef::Config[:data_collector] = {
+        server_url: 'https://automate.test/data_collector',
+        token: token,
+      }
+    end
+
+    let(:token) { "fake_token" }
+
+    context 'when target is a string' do
+      it 'should resolve a compliance URL' do
+        res = Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'raises an exception with no data collector token' do
+        Chef::Config[:data_collector].delete(:token)
+
+        expect {
+          Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name')
+        }.to raise_error(/No data-collector token set/)
+      end
+
+      it 'includes the data collector token' do
+        expect(Chef::Audit::Fetcher::Automate).to receive(:new).with(
+          "https://automate.test/compliance/profiles/namespace/profile_name/tar",
+          hash_including('token' => token),
+        ).and_call_original
+
+        res = Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'returns nil with a non-compliance URL' do
+        res = Chef::Audit::Fetcher::Automate.resolve('http://github.com/chef-cookbooks/audit')
+
+        expect(res).to eq(nil)
+      end
+    end
+
+    context 'when target is a hash' do
+      it 'should resolve a target with a version' do
+        res = Chef::Audit::Fetcher::Automate.resolve(
+          compliance: 'namespace/profile_name',
+          version: '1.2.3',
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = "https://automate.test/compliance/profiles/namespace/profile_name/version/1.2.3/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'should resolve a target without a version' do
+        res = Chef::Audit::Fetcher::Automate.resolve(
+          compliance: 'namespace/profile_name',
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'uses url key when present' do
+        res = Chef::Audit::Fetcher::Automate.resolve(
+          compliance: 'namespace/profile_name',
+          version: '1.2.3',
+          url: 'https://profile.server.test/profiles/profile_name/1.2.3'
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = 'https://profile.server.test/profiles/profile_name/1.2.3'
+        expect(res.target).to eq(expected)
+      end
+
+      it 'does not include token in the config when url key is present' do
+        expect(Chef::Audit::Fetcher::Automate).to receive(:new).with(
+          'https://profile.server.test/profiles/profile_name/1.2.3',
+          hash_including('token' => nil),
+        ).and_call_original
+
+        res = Chef::Audit::Fetcher::Automate.resolve(
+          compliance: 'namespace/profile_name',
+          version: '1.2.3',
+          url: 'https://profile.server.test/profiles/profile_name/1.2.3'
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = 'https://profile.server.test/profiles/profile_name/1.2.3'
+        expect(res.target).to eq(expected)
+      end
+
+      it 'raises an exception with no data collector token' do
+        Chef::Config[:data_collector].delete(:token)
+
+        expect {
+          Chef::Audit::Fetcher::Automate.resolve(compliance: 'namespace/profile_name')
+        }.to raise_error(/No data-collector token set/)
+      end
+
+      it 'includes the data collector token' do
+        expect(Chef::Audit::Fetcher::Automate).to receive(:new).with(
+          "https://automate.test/compliance/profiles/namespace/profile_name/tar",
+          hash_including('token' => token),
+        ).and_call_original
+
+        res = Chef::Audit::Fetcher::Automate.resolve(compliance: 'namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate)
+        expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'returns nil with a non-profile Hash' do
+        res = Chef::Audit::Fetcher::Automate.resolve(
+          profile: 'namespace/profile_name',
+          version: '1.2.3'
+        )
+
+        expect(res).to eq(nil)
+      end
+    end
+  end
+end
diff --git a/spec/unit/audit/fetcher/chef_server_spec.rb b/spec/unit/audit/fetcher/chef_server_spec.rb
new file mode 100644
index 0000000000..1781d048d4
--- /dev/null
+++ b/spec/unit/audit/fetcher/chef_server_spec.rb
@@ -0,0 +1,93 @@
+require 'spec_helper'
+require "chef/audit/fetcher/chef_server"
+
+describe Chef::Audit::Fetcher::ChefServer do
+  let(:node) do
+    Chef::Node.new.tap do |n|
+      n.default['audit'] = {}
+    end
+  end
+
+  before :each do
+    allow(Chef).to receive(:node).and_return(node)
+
+    Chef::Config[:chef_server_url] = 'http://127.0.0.1:8889/organizations/my_org'
+  end
+
+  describe ".resolve" do
+    context 'when target is a string' do
+      it 'should resolve a compliance URL' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'should add /compliance URL prefix if needed' do
+        node.default['audit']['fetcher'] = 'chef-server'
+        res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/compliance/organizations/my_org/owners/namespace/compliance/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'includes user in the URL if present' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://username@namespace/profile_name')
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/organizations/my_org/owners/username@namespace/compliance/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'returns nil with a non-compliance URL' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve('http://github.com/chef-cookbooks/audit')
+
+        expect(res).to eq(nil)
+      end
+    end
+
+    context 'when target is a hash' do
+      it 'should resolve a target with a version' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve(
+          compliance: 'namespace/profile_name',
+          version: '1.2.3',
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/version/1.2.3/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'should resolve a target without a version' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve(
+          compliance: 'namespace/profile_name',
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'includes user in the URL if present' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve(
+          compliance: 'username@namespace/profile_name',
+        )
+
+        expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer)
+        expected = "http://127.0.0.1:8889/organizations/my_org/owners/username@namespace/compliance/profile_name/tar"
+        expect(res.target).to eq(expected)
+      end
+
+      it 'returns nil with a non-profile Hash' do
+        res = Chef::Audit::Fetcher::ChefServer.resolve(
+          profile: 'namespace/profile_name',
+          version: '1.2.3'
+        )
+
+        expect(res).to eq(nil)
+      end
+    end
+  end
+end