From 9034f3a4d430ad39aa6e10a9af0a1743daba271e Mon Sep 17 00:00:00 2001 From: Pete Higgins Date: Mon, 26 Oct 2020 11:20:16 -0700 Subject: Add inspec fetchers from audit-cookbook. Signed-off-by: Pete Higgins --- lib/chef/audit/fetcher/automate.rb | 69 ++++++++++++++ lib/chef/audit/fetcher/chef_server.rb | 136 ++++++++++++++++++++++++++++ lib/chef/audit/runner.rb | 4 +- spec/unit/audit/fetcher/automate_spec.rb | 134 +++++++++++++++++++++++++++ spec/unit/audit/fetcher/chef_server_spec.rb | 93 +++++++++++++++++++ 5 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 lib/chef/audit/fetcher/automate.rb create mode 100644 lib/chef/audit/fetcher/chef_server.rb create mode 100644 spec/unit/audit/fetcher/automate_spec.rb create mode 100644 spec/unit/audit/fetcher/chef_server_spec.rb diff --git a/lib/chef/audit/fetcher/automate.rb b/lib/chef/audit/fetcher/automate.rb new file mode 100644 index 0000000000..e28d8ea026 --- /dev/null +++ b/lib/chef/audit/fetcher/automate.rb @@ -0,0 +1,69 @@ +require "uri" +require "plugins/inspec-compliance/lib/inspec-compliance" + +class Chef + module Audit + module Fetcher + class Automate < ::InspecPlugins::Compliance::Fetcher + name 'chef-automate' + + # it positions itself before `compliance` fetcher + # only load it, if you want to use audit cookbook in Chef Solo with Chef Automate + priority 502 + + CONFIG = { + 'insecure' => true, + 'token' => nil, + 'server_type' => 'automate', + 'automate' => { + 'ent' => 'default', + 'token_type' => 'dctoken', + }, + } + + def self.resolve(target) + uri = get_target_uri(target) + return nil if uri.nil? + + config = CONFIG.dup + + # we have detailed information available in our lockfile, no need to ask the server + if target.respond_to?(:key?) && target.key?(:url) + profile_fetch_url = target[:url] + else + # verifies that the target e.g base/ssh exists + base_path = "/compliance/profiles/#{uri.host}#{uri.path}" + + profile_path = if target.respond_to?(:key?) && target.key?(:version) + "#{base_path}/version/#{target[:version]}/tar" + else + "#{base_path}/tar" + end + + dc = Chef::Config[:data_collector] + url = URI(dc[:server_url]) + url.path = profile_path + profile_fetch_url = url.to_s + + config['token'] = dc['token'] + + if config['token'].nil? + raise 'No data-collector token set, which is required by the chef-automate fetcher. ' \ + 'Set the `data_collector.token` configuration parameter in your client.rb ' \ + 'or use the "chef-server-automate" reporter which does not require any ' \ + 'data-collector settings and uses Chef Server to fetch profiles.' + end + end + + new(profile_fetch_url, config) + rescue URI::Error => _e + nil + end + + def to_s + 'Chef Automate for Chef Solo Fetcher' + end + end + end + end +end diff --git a/lib/chef/audit/fetcher/chef_server.rb b/lib/chef/audit/fetcher/chef_server.rb new file mode 100644 index 0000000000..65768a561a --- /dev/null +++ b/lib/chef/audit/fetcher/chef_server.rb @@ -0,0 +1,136 @@ +require 'uri' +require "plugins/inspec-compliance/lib/inspec-compliance" + +# This class implements an InSpec fetcher for for Chef Server. The implementation +# is based on the Chef Compliance fetcher and only adapts the calls to redirect +# the requests via Chef Server. +# +# This implementation depends on chef-client runtime, therefore it is only executable +# inside of a chef-client run + +class Chef + module Audit + module Fetcher + class ChefServer < ::InspecPlugins::Compliance::Fetcher + name 'chef-server' + + # it positions itself before `compliance` fetcher + # only load it, if the Chef Server is integrated with Chef Compliance + priority 501 + + CONFIG = { 'insecure' => true } + + # Accepts URLs to compliance profiles in one of two forms: + # * a String URL with a compliance scheme, like "compliance://namespace/profile_name" + # * a Hash with a key of `compliance` and a value like "compliance/profile_name" and optionally a `version` key with a String value + def self.resolve(target) + uri = get_target_uri(target) + return nil if uri.nil? + + profile = uri.host + uri.path + profile = uri.user + '@' + profile if uri.user + + version = target[:version] if target.respond_to?(:key?) && target.key?(:version) + new(target_url(profile, version), CONFIG) + rescue URI::Error => _e + nil + end + + def self.target_url(profile, version = nil) + organization = Chef::Config[:chef_server_url].split('/').last + namespace, profile_name = profile.split('/') + + path_parts = [""] + path_parts << "compliance" if chef_server_reporter? || chef_server_fetcher? + path_parts << "organizations/#{organization}/owners/#{namespace}/compliance/#{profile_name}" + path_parts << "version/#{version}" if version + path_parts << "tar" + + target_url = URI(Chef::Config[:chef_server_url]) + target_url.path = path_parts.compact.join("/") + + Chef::Log.info("Fetching profile from: #{target_url}") + target_url + end + + # + # We want to save compliance: in the lockfile rather than url: to + # make sure we go back through the ComplianceAPI handling. + # + def resolved_source + { compliance: chef_server_url } + end + + # Downloads archive to temporary file using a Chef::ServerAPI + # client so that Chef Server's header-based authentication can be + # used. + def download_archive_to_temp + return @temp_archive_path unless @temp_archive_path.nil? + + rest = Chef::ServerAPI.new(@target, Chef::Config.merge(ssl_verify_mode: :verify_none)) + archive = with_http_rescue do + rest.streaming_request(@target) + end + @archive_type = '.tar.gz' + + if archive.nil? + path = @target.respond_to?(:path) ? @target.path : path + raise "Unable to find requested profile on path: '#{path}' on the Automate system." + end + + Inspec::Log.debug("Archive stored at temporary location: #{archive.path}") + @temp_archive_path = archive.path + end + + def with_http_rescue + response = yield + if response.respond_to?(:code) + # handle non 200 error codes, they are not raised as Net::HTTPClientException + handle_http_error_code(response.code) if response.code.to_i >= 300 + end + response + rescue Net::HTTPClientException => e + Chef::Log.error e + handle_http_error_code(e.response.code) + end + + def handle_http_error_code(code) + case code + when /401|403/ + Chef::Log.error 'Auth issue: see audit cookbook TROUBLESHOOTING.md' + when /404/ + Chef::Log.error 'Object does not exist on remote server.' + when /413/ + Chef::Log.error 'You most likely hit the erchef request size in Chef Server that defaults to ~2MB. To increase this limit see audit cookbook TROUBLESHOOTING.md OR https://docs.chef.io/config_rb_server.html' + when /429/ + Chef::Log.error "This error typically means the data sent was larger than Automate's limit (4 MB). Run InSpec locally to identify any controls producing large diffs." + end + msg = "Received HTTP error #{code}" + Chef::Log.error msg + raise msg if @raise_if_unreachable + end + + def to_s + 'Chef Server/Compliance Profile Loader' + end + + CHEF_SERVER_REPORTERS = %w(chef-server chef-server-compliance chef-server-visibility chef-server-automate) + def self.chef_server_reporter? + (Array(Chef.node.attributes['audit']['reporter']) & CHEF_SERVER_REPORTERS).any? + end + + CHEF_SERVER_FETCHERS = %w(chef-server chef-server-compliance chef-server-visibility chef-server-automate) + def self.chef_server_fetcher? + CHEF_SERVER_FETCHERS.include?(Chef.node.attributes['audit']['fetcher']) + end + + private + + def chef_server_url + m = %r{^#{@config['server']}/owners/(?[^/]+)/compliance/(?[^/]+)/tar$}.match(@target) + "#{m[:owner]}/#{m[:id]}" + end + end + end + end +end diff --git a/lib/chef/audit/runner.rb b/lib/chef/audit/runner.rb index ded5e0ece3..bbe2b8ae88 100644 --- a/lib/chef/audit/runner.rb +++ b/lib/chef/audit/runner.rb @@ -1,10 +1,12 @@ autoload :Inspec, 'inspec' require_relative 'default_attributes' +require_relative 'fetcher/automate' +require_relative 'fetcher/chef_server' require_relative 'reporter/audit_enforcer' -require_relative 'reporter/json_file' require_relative 'reporter/automate' require_relative 'reporter/chef_server_automate' +require_relative 'reporter/json_file' class Chef module Audit diff --git a/spec/unit/audit/fetcher/automate_spec.rb b/spec/unit/audit/fetcher/automate_spec.rb new file mode 100644 index 0000000000..df0225ed9e --- /dev/null +++ b/spec/unit/audit/fetcher/automate_spec.rb @@ -0,0 +1,134 @@ +require 'spec_helper' +require "chef/audit/fetcher/automate" + +describe Chef::Audit::Fetcher::Automate do + describe ".resolve" do + before do + Chef::Config[:data_collector] = { + server_url: 'https://automate.test/data_collector', + token: token, + } + end + + let(:token) { "fake_token" } + + context 'when target is a string' do + it 'should resolve a compliance URL' do + res = Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'raises an exception with no data collector token' do + Chef::Config[:data_collector].delete(:token) + + expect { + Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name') + }.to raise_error(/No data-collector token set/) + end + + it 'includes the data collector token' do + expect(Chef::Audit::Fetcher::Automate).to receive(:new).with( + "https://automate.test/compliance/profiles/namespace/profile_name/tar", + hash_including('token' => token), + ).and_call_original + + res = Chef::Audit::Fetcher::Automate.resolve('compliance://namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'returns nil with a non-compliance URL' do + res = Chef::Audit::Fetcher::Automate.resolve('http://github.com/chef-cookbooks/audit') + + expect(res).to eq(nil) + end + end + + context 'when target is a hash' do + it 'should resolve a target with a version' do + res = Chef::Audit::Fetcher::Automate.resolve( + compliance: 'namespace/profile_name', + version: '1.2.3', + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = "https://automate.test/compliance/profiles/namespace/profile_name/version/1.2.3/tar" + expect(res.target).to eq(expected) + end + + it 'should resolve a target without a version' do + res = Chef::Audit::Fetcher::Automate.resolve( + compliance: 'namespace/profile_name', + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'uses url key when present' do + res = Chef::Audit::Fetcher::Automate.resolve( + compliance: 'namespace/profile_name', + version: '1.2.3', + url: 'https://profile.server.test/profiles/profile_name/1.2.3' + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = 'https://profile.server.test/profiles/profile_name/1.2.3' + expect(res.target).to eq(expected) + end + + it 'does not include token in the config when url key is present' do + expect(Chef::Audit::Fetcher::Automate).to receive(:new).with( + 'https://profile.server.test/profiles/profile_name/1.2.3', + hash_including('token' => nil), + ).and_call_original + + res = Chef::Audit::Fetcher::Automate.resolve( + compliance: 'namespace/profile_name', + version: '1.2.3', + url: 'https://profile.server.test/profiles/profile_name/1.2.3' + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = 'https://profile.server.test/profiles/profile_name/1.2.3' + expect(res.target).to eq(expected) + end + + it 'raises an exception with no data collector token' do + Chef::Config[:data_collector].delete(:token) + + expect { + Chef::Audit::Fetcher::Automate.resolve(compliance: 'namespace/profile_name') + }.to raise_error(/No data-collector token set/) + end + + it 'includes the data collector token' do + expect(Chef::Audit::Fetcher::Automate).to receive(:new).with( + "https://automate.test/compliance/profiles/namespace/profile_name/tar", + hash_including('token' => token), + ).and_call_original + + res = Chef::Audit::Fetcher::Automate.resolve(compliance: 'namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::Automate) + expected = "https://automate.test/compliance/profiles/namespace/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'returns nil with a non-profile Hash' do + res = Chef::Audit::Fetcher::Automate.resolve( + profile: 'namespace/profile_name', + version: '1.2.3' + ) + + expect(res).to eq(nil) + end + end + end +end diff --git a/spec/unit/audit/fetcher/chef_server_spec.rb b/spec/unit/audit/fetcher/chef_server_spec.rb new file mode 100644 index 0000000000..1781d048d4 --- /dev/null +++ b/spec/unit/audit/fetcher/chef_server_spec.rb @@ -0,0 +1,93 @@ +require 'spec_helper' +require "chef/audit/fetcher/chef_server" + +describe Chef::Audit::Fetcher::ChefServer do + let(:node) do + Chef::Node.new.tap do |n| + n.default['audit'] = {} + end + end + + before :each do + allow(Chef).to receive(:node).and_return(node) + + Chef::Config[:chef_server_url] = 'http://127.0.0.1:8889/organizations/my_org' + end + + describe ".resolve" do + context 'when target is a string' do + it 'should resolve a compliance URL' do + res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'should add /compliance URL prefix if needed' do + node.default['audit']['fetcher'] = 'chef-server' + res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/compliance/organizations/my_org/owners/namespace/compliance/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'includes user in the URL if present' do + res = Chef::Audit::Fetcher::ChefServer.resolve('compliance://username@namespace/profile_name') + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/organizations/my_org/owners/username@namespace/compliance/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'returns nil with a non-compliance URL' do + res = Chef::Audit::Fetcher::ChefServer.resolve('http://github.com/chef-cookbooks/audit') + + expect(res).to eq(nil) + end + end + + context 'when target is a hash' do + it 'should resolve a target with a version' do + res = Chef::Audit::Fetcher::ChefServer.resolve( + compliance: 'namespace/profile_name', + version: '1.2.3', + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/version/1.2.3/tar" + expect(res.target).to eq(expected) + end + + it 'should resolve a target without a version' do + res = Chef::Audit::Fetcher::ChefServer.resolve( + compliance: 'namespace/profile_name', + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/organizations/my_org/owners/namespace/compliance/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'includes user in the URL if present' do + res = Chef::Audit::Fetcher::ChefServer.resolve( + compliance: 'username@namespace/profile_name', + ) + + expect(res).to be_kind_of(Chef::Audit::Fetcher::ChefServer) + expected = "http://127.0.0.1:8889/organizations/my_org/owners/username@namespace/compliance/profile_name/tar" + expect(res.target).to eq(expected) + end + + it 'returns nil with a non-profile Hash' do + res = Chef::Audit::Fetcher::ChefServer.resolve( + profile: 'namespace/profile_name', + version: '1.2.3' + ) + + expect(res).to eq(nil) + end + end + end +end -- cgit v1.2.1