Add output_locations functionality to data collector

This commit implements the recent changes to RFC-077, to support multiple output locations in the data collector.

Signed-off-by: Jon Cowie <jonlives@gmail.com>

2 files changed, 253 insertions, 33 deletions

diff --git a/lib/chef/data_collector.rb b/lib/chef/data_collector.rb
index 0a92b800a6..e84390a901 100644
--- a/lib/chef/data_collector.rb
+++ b/lib/chef/data_collector.rb
@@ -50,8 +50,8 @@ class Chef
                         "#{Chef::Config[:data_collector][:mode].inspect} modes, disabling it")
         return false
       end
-      unless data_collector_url_configured?
-        Chef::Log.debug("data collector URL is not configured, disabling data collector")
+      unless data_collector_url_configured? || data_collector_output_locations_configured?
+        Chef::Log.debug("Neither data collector URL or output locations have been configured, disabling data collector")
         return false
       end
       if solo? && !token_auth_configured?
@@ -69,6 +69,10 @@ class Chef
       !!Chef::Config[:data_collector][:server_url]
     end
 
+    def self.data_collector_output_locations_configured?
+      !!Chef::Config[:data_collector][:output_locations]
+    end
+
     def self.why_run?
       !!Chef::Config[:why_run]
     end
@@ -104,7 +108,7 @@ class Chef
 
       def initialize
         validate_data_collector_server_url!
-
+        validate_data_collector_output_locations! if data_collector_output_locations
         @all_resource_reports    = []
         @current_resource_loaded = nil
         @error_descriptions      = {}
@@ -112,7 +116,10 @@ class Chef
         @deprecations            = Set.new
         @enabled                 = true
 
-        @http = setup_http_client
+        @http = setup_http_client(data_collector_server_url)
+        if data_collector_output_locations
+          @http_output_locations = setup_http_output_locations if data_collector_output_locations[:urls]
+        end
       end
 
       # see EventDispatch::Base#run_started
@@ -125,11 +132,11 @@ class Chef
       def run_started(current_run_status)
         update_run_status(current_run_status)
 
+        message = Chef::DataCollector::Messages.run_start_message(current_run_status)
         disable_reporter_on_error do
-          send_to_data_collector(
-            Chef::DataCollector::Messages.run_start_message(current_run_status)
-          )
+          send_to_data_collector(message)
         end
+        send_to_output_locations(message) if data_collector_output_locations
       end
 
       # see EventDispatch::Base#run_completed
@@ -286,14 +293,22 @@ class Chef
       # intended to be used primarily for Chef Solo in which case no signing
       # key will be available (in which case `Chef::ServerAPI.new()` would
       # raise an exception.
-      def setup_http_client
+      def setup_http_client(url)
         if data_collector_token.nil?
-          Chef::ServerAPI.new(data_collector_server_url, validate_utf8: false)
+          Chef::ServerAPI.new(url, validate_utf8: false)
         else
-          Chef::HTTP::SimpleJSON.new(data_collector_server_url, validate_utf8: false)
+          Chef::HTTP::SimpleJSON.new(url, validate_utf8: false)
         end
       end
 
+      def setup_http_output_locations
+        http_output_locations = {}
+        Chef::Config[:data_collector][:output_locations][:urls].each do |location_url|
+          http_output_locations[location_url] = setup_http_client(location_url)
+        end
+        http_output_locations
+      end
+
       #
       # Yields to the passed-in block (which is expected to be some interaction
       # with the DataCollector endpoint). If some communication failure occurs,
@@ -309,7 +324,8 @@ class Chef
              Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse,
              Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError,
              Errno::EHOSTDOWN => e
-        disable_data_collector_reporter
+        # Do not disable data collector reporter if additional output_locations have been specified
+        disable_data_collector_reporter unless data_collector_output_locations
         code = if e.respond_to?(:response) && e.response.code
                  e.response.code.to_s
                else
@@ -332,8 +348,31 @@ class Chef
 
       def send_to_data_collector(message)
         return unless data_collector_accessible?
+        http.post(nil, message, headers) if data_collector_server_url
+      end
+
+      def send_to_output_locations(message)
+        data_collector_output_locations.each do |type, location_list|
+          method = if type == :urls
+                     method(:send_to_http_location)
+                   elsif type == :files
+                     method(:send_to_file_location)
+                   end
 
-        http.post(nil, message, headers)
+          location_list.each do |l|
+            method.call(l, message)
+          end
+        end
+      end
+
+      def send_to_file_location(file_name, message)
+        open(file_name, "a") { |f| f.puts message }
+      end
+
+      def send_to_http_location(http_url, message)
+        @http_output_locations[http_url].post(nil, message, headers) if @http_output_locations[http_url]
+      rescue
+        Chef::Log.debug("Data collector failed to send to URL location #{http_url}. Please check your configured data_collector.output_locations")
       end
 
       #
@@ -352,16 +391,18 @@ class Chef
         # we have nothing to report.
         return unless run_status
 
-        send_to_data_collector(
-          Chef::DataCollector::Messages.run_end_message(
-            run_status: run_status,
-            expanded_run_list: expanded_run_list,
-            resources: all_resource_reports,
-            status: opts[:status],
-            error_descriptions: error_descriptions,
-            deprecations: deprecations.to_a
-          )
-        )
+        message = Chef::DataCollector::Messages.run_end_message(
+                   run_status: run_status,
+                   expanded_run_list: expanded_run_list,
+                   resources: all_resource_reports,
+                   status: opts[:status],
+                   error_descriptions: error_descriptions,
+                   deprecations: deprecations.to_a
+                  )
+        disable_reporter_on_error do
+          send_to_data_collector(message)
+        end
+        send_to_output_locations(message) if data_collector_output_locations
       end
 
       def headers
@@ -379,6 +420,10 @@ class Chef
         Chef::Config[:data_collector][:server_url]
       end
 
+      def data_collector_output_locations
+        Chef::Config[:data_collector][:output_locations]
+      end
+
       def data_collector_token
         Chef::Config[:data_collector][:token]
       end
@@ -467,21 +512,55 @@ class Chef
         @current_resource_report && @current_resource_report.new_resource != new_resource
       end
 
+      def validate_and_return_uri(uri)
+        URI(uri)
+      rescue URI::InvalidURIError
+        return nil
+      end
+
+      def validate_and_create_file(file)
+        send_to_file_location(file, "")
+        return true
+      rescue Errno::ENOENT
+        return false
+      rescue Errno::EACCES
+        return false
+      end
+
       def validate_data_collector_server_url!
-        if data_collector_server_url.empty?
-          raise Chef::Exceptions::ConfigurationError,
-            "Chef::Config[:data_collector][:server_url] is empty. Please supply a valid URL."
-        end
+        unless !data_collector_server_url && data_collector_output_locations
+          uri = validate_and_return_uri(data_collector_server_url)
+          unless uri
+            raise Chef::Exceptions::ConfigurationError, "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is not a valid URI."
+          end
 
-        begin
-          uri = URI(data_collector_server_url)
-        rescue URI::InvalidURIError
-          raise Chef::Exceptions::ConfigurationError, "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is not a valid URI."
+          if uri.host.nil?
+            raise Chef::Exceptions::ConfigurationError,
+              "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is a URI with no host. Please supply a valid URL."
+          end
         end
+      end
 
-        if uri.host.nil?
+      def validate_data_collector_output_locations!
+        if data_collector_output_locations.empty?
           raise Chef::Exceptions::ConfigurationError,
-            "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is a URI with no host. Please supply a valid URL."
+                "Chef::Config[:data_collector][:output_loations] is empty. Please supply an hash of valid URLs and / or local file paths."
+        end
+
+        data_collector_output_locations.each do |type, locations|
+
+          method = if type == :urls
+                     method(:validate_and_return_uri)
+                   elsif type == :files
+                     method(:validate_and_create_file)
+                   end
+
+          locations.each do |l|
+            unless method.call(l)
+              raise Chef::Exceptions::ConfigurationError,
+                      "Chef::Config[:data_collector][:output_locations] contains the location #{l} which is not valid."
+            end
+          end
         end
       end
     end
diff --git a/spec/unit/data_collector_spec.rb b/spec/unit/data_collector_spec.rb
index f3f7ffb30f..1189e89411 100644
--- a/spec/unit/data_collector_spec.rb
+++ b/spec/unit/data_collector_spec.rb
@@ -25,9 +25,10 @@ require "chef/resource_builder"
 describe Chef::DataCollector do
 
   describe ".register_reporter?" do
-    context "when no data collector URL is configured" do
+    context "when no data collector URL or output locations are configured" do
       it "returns false" do
         Chef::Config[:data_collector][:server_url] = nil
+        Chef::Config[:data_collector][:output_locations] = nil
         expect(Chef::DataCollector.register_reporter?).to be_falsey
       end
     end
@@ -134,6 +135,109 @@ describe Chef::DataCollector do
 
       end
     end
+
+    context "when output_locations are configured" do
+      before do
+        Chef::Config[:data_collector][:output_locations] = ["http://data_collector", "/tmp/data_collector.json"]
+      end
+
+      context "when operating in why_run mode" do
+        it "returns false" do
+          Chef::Config[:why_run] = true
+          expect(Chef::DataCollector.register_reporter?).to be_falsey
+        end
+      end
+
+      context "when not operating in why_run mode" do
+
+        before do
+          Chef::Config[:why_run] = false
+          Chef::Config[:data_collector][:token] = token
+        end
+
+        context "when a token is configured" do
+
+          let(:token) { "supersecrettoken" }
+
+          context "when report is enabled for current mode" do
+            it "returns true" do
+              allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(true)
+              expect(Chef::DataCollector.register_reporter?).to be_truthy
+            end
+          end
+
+          context "when report is disabled for current mode" do
+            it "returns false" do
+              allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(false)
+              expect(Chef::DataCollector.register_reporter?).to be_falsey
+            end
+          end
+
+        end
+
+        # `Chef::Config[:data_collector][:server_url]` defaults to a URL
+        # relative to the `chef_server_url`, so we use configuration of the
+        # token to infer whether a solo/local mode user intends for data
+        # collection to be enabled.
+        context "when a token is not configured" do
+
+          let(:token) { nil }
+
+          context "when report is enabled for current mode" do
+
+            before do
+              allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(true)
+            end
+
+            context "when the current mode is solo" do
+
+              before do
+                Chef::Config[:solo] = true
+              end
+
+              it "returns true" do
+                expect(Chef::DataCollector.register_reporter?).to be(true)
+              end
+
+            end
+
+            context "when the current mode is local mode" do
+
+              before do
+                Chef::Config[:local_mode] = true
+              end
+
+              it "returns false" do
+                expect(Chef::DataCollector.register_reporter?).to be(true)
+              end
+            end
+
+            context "when the current mode is client mode" do
+
+              before do
+                Chef::Config[:local_mode] = false
+                Chef::Config[:solo] = false
+              end
+
+              it "returns true" do
+                expect(Chef::DataCollector.register_reporter?).to be_truthy
+              end
+
+            end
+
+          end
+
+          context "when report is disabled for current mode" do
+            it "returns false" do
+              allow(Chef::DataCollector).to receive(:reporter_enabled_for_current_mode?).and_return(false)
+              expect(Chef::DataCollector.register_reporter?).to be_falsey
+            end
+          end
+
+        end
+
+      end
+    end
   end
 
   describe ".reporter_enabled_for_current_mode?" do
@@ -656,6 +760,13 @@ describe Chef::DataCollector::Reporter do
       end
     end
 
+    context "when server_url is omitted but output_locations is specified" do
+      it "raises an exception" do
+        Chef::Config[:data_collector][:output_locations] = ["http://data_collector", "/tmp/data_collector.json"]
+        expect { reporter.send(:validate_data_collector_server_url!) }.not_to raise_error(Chef::Exceptions::ConfigurationError)
+      end
+    end
+
     context "when server_url is not empty" do
       context "when server_url is an invalid URI" do
         it "raises an exception" do
@@ -683,6 +794,36 @@ describe Chef::DataCollector::Reporter do
     end
   end
 
+  describe "#validate_data_collector_output_locations!" do
+    context "when output_locations is empty" do
+      it "raises an exception" do
+        Chef::Config[:data_collector][:output_locations] = {}
+        expect { reporter.send(:validate_data_collector_output_locations!) }.to raise_error(Chef::Exceptions::ConfigurationError)
+      end
+    end
+
+    context "when valid output_locations are provided" do
+      it "does not raise an exception" do
+        Chef::Config[:data_collector][:output_locations] = { :urls => ["http://data_collector"], :files => ["/tmp/data_collection.json"] }
+        expect { reporter.send(:validate_data_collector_output_locations!) }.not_to raise_error(Chef::Exceptions::ConfigurationError)
+      end
+    end
+
+    context "when output_locations contains an invalid file path" do
+      it "raises an exception" do
+        Chef::Config[:data_collector][:output_locations] = { :urls => ["http://data_collector"], :files => ["/data_collector.json"] }
+        expect { reporter.send(:validate_data_collector_output_locations!) }.to raise_error(Chef::Exceptions::ConfigurationError)
+      end
+    end
+
+    context "when output_locations contains an invalid URI" do
+      it "raises an exception" do
+        Chef::Config[:data_collector][:output_locations] = { :urls => ["this is not a url"], :files => ["/tmp/data_collection.json"] }
+        expect { reporter.send(:validate_data_collector_output_locations!) }.to raise_error(Chef::Exceptions::ConfigurationError)
+      end
+    end
+  end
+
   describe "#detect_unprocessed_resources" do
     context "when resources do not override core methods" do
       it "adds resource reports for any resources that have not yet been processed" do