:construction: WIP: taking stab at refactoring serializers into a moduleadamleff/messages_as_module

10 files changed, 372 insertions, 360 deletions

diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index da70b11b88..568467456f 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -800,10 +800,38 @@ module ChefConfig
     # the user to configure where to send their Data Collector data, what token
     # to send, and whether Data Collector should report its findings in client
     # mode vs. solo mode.
-    default :data_collector_server_url,       nil
-    default :data_collector_token,            nil
-    default :data_collector_mode,             :both
-    default :data_collector_raise_on_failure, false
+    config_context :data_collector do
+      # Full URL to the endpoint that will receive our data. If nil, the
+      # data collector will not run.
+      # Ex: http://my-data-collector.mycompany.com/ingest
+      default :server_url,       nil
+
+      # An optional pre-shared token to pass as an HTTP header (x-data-collector-token)
+      # that can be used to determine whether or not the poster of this
+      # run data should be trusted.
+      # Ex: some-uuid-here
+      default :token,            nil
+
+      # The Chef mode during which Data Collector is allowed to function. This
+      # can be used to run Data Collector only when running as Chef Solo but
+      # not when using Chef Client.
+      # Options: :solo (for both Solo Legacy Mode and Client Local Mode), :client, :both
+      default :mode,             :both
+
+      # When the Data Collector cannot send the "starting a run" message to
+      # the Data Collector server, the Data Collector will be disabled for that
+      # run. In some situations, such as highly-regulated environments, it
+      # may be more reasonable to prevent Chef from performing the actual run.
+      # In these situations, setting this value to true will cause the Chef
+      # run to raise an exception before starting any converge activities.
+      default :raise_on_failure, false
+
+      # A user-supplied Organization string that can be sent in payloads
+      # generated by the DataCollector when Chef is run in Solo mode. This
+      # allows users to associate their Solo nodes with faux organizations
+      # without the nodes being connected to an actual Chef Server.
+      default :organization, nil
+    end
 
     configurable(:http_proxy)
     configurable(:http_proxy_user)
diff --git a/lib/chef/client.rb b/lib/chef/client.rb
index 2df5b7271f..c857da1b93 100644
--- a/lib/chef/client.rb
+++ b/lib/chef/client.rb
@@ -417,13 +417,6 @@ class Chef
       end
     end
 
-    # Register the data collector reporter to send event information to the
-    # data collector server
-    # @api private
-    def register_data_collector_reporter
-      events.register(Chef::DataCollector::Reporter.new) if Chef::DataCollector.register_reporter?
-    end
-
     #
     # Callback to fire notifications that the Chef run is starting
     #
@@ -966,6 +959,12 @@ class Chef
 
       Chef::ReservedNames::Win32::Security.has_admin_privileges?
     end
+
+    # Register the data collector reporter to send event information to the
+    # data collector server
+    def register_data_collector_reporter
+      events.register(Chef::DataCollector::Reporter.new) if Chef::DataCollector.register_reporter?
+    end
   end
 end
 
diff --git a/lib/chef/data_collector.rb b/lib/chef/data_collector.rb
index c4afbfaa34..419400f684 100644
--- a/lib/chef/data_collector.rb
+++ b/lib/chef/data_collector.rb
@@ -20,15 +20,16 @@
 
 require "uri"
 require "chef/event_dispatch/base"
+require "chef/data_collector/messages"
 require "chef/data_collector/resource_report"
-require "chef/data_collector/serializers/node_update"
-require "chef/data_collector/serializers/run_end"
-require "chef/data_collector/serializers/run_start"
 
 class Chef
+
+  # == Chef::DataCollector
+  # Provides methods for determinine whether a reporter should be registered.
   class DataCollector
     def self.register_reporter?
-      Chef::Config[:data_collector_server_url] &&
+      Chef::Config[:data_collector][:server_url] &&
         !Chef::Config[:why_run] &&
         self.reporter_enabled_for_current_mode?
     end
@@ -40,9 +41,15 @@ class Chef
         acceptable_modes = [:client, :both]
       end
 
-      acceptable_modes.include?(Chef::Config[:data_collector_mode])
+      acceptable_modes.include?(Chef::Config[:data_collector][:mode])
     end
 
+    # == Chef::DataCollector::Reporter
+    # Provides an event handler that can be registered to report on Chef
+    # run data. Unlike the existing Chef::ResourceReporter event handler,
+    # the DataCollector handler is not tied to a Chef Server / Chef Reporting
+    # and exports its data through a webhook-like mechanism to a configured
+    # endpoint.
     class Reporter < EventDispatch::Base
       attr_reader :updated_resources, :status, :exception, :error_descriptions,
                   :expanded_run_list, :run_status, :http, :resource_count,
@@ -52,31 +59,44 @@ class Chef
         @updated_resources       = []
         @resource_count          = 0
         @current_resource_loaded = nil
-        @status                  = "success"
-        @exception               = nil
         @error_descriptions      = {}
         @expanded_run_list       = {}
         @http                    = Chef::HTTP.new(data_collector_server_url)
         @enabled                 = true
       end
 
+      # see EventDispatch::Base#run_started
+      # Upon receipt, we will send our run start message to the
+      # configured DataCollector endpoint. Depending on whether
+      # the user has configured raise_on_failure, if we cannot
+      # send the message, we will either disable the DataCollector
+      # Reporter for the duration of this run, or we'll raise an
+      # exception.
       def run_started(current_run_status)
         update_run_status(current_run_status)
 
         disable_reporter_on_error do
-          send_to_data_collector(Serializers::RunStart.new(run_status))
+          send_to_data_collector(Chef::DataCollector::Messages.run_start_message(run_status).to_json)
         end
       end
 
+      # see EventDispatch::Base#run_completed
+      # Upon receipt, we will send our run completion message to the
+      # configured DataCollector endpoint.
       def run_completed(node)
-        send_run_completion
+        send_run_completion(status: "success")
       end
 
+      # see EventDispatch::Base#run_failed
       def run_failed(exception)
-        update_exception(exception)
-        send_run_completion
+        send_run_completion(status: "failure")
       end
 
+      # see EventDispatch::Base#resource_current_state_loaded
+      # Create a new ResourceReport instance that we'll use to track
+      # the state of this resource during the run. Nested resources are
+      # ignored as they are assumed to be an inline resource of a custom
+      # resource, and we only care about tracking top-level resources.
       def resource_current_state_loaded(new_resource, action, current_resource)
         return if nested_resource?(new_resource)
         update_current_resource_report(
@@ -88,11 +108,19 @@ class Chef
         )
       end
 
+      # see EventDispatch::Base#resource_up_to_date
+      # Mark our ResourceReport status accordingly, and increment the total
+      # resource count.
       def resource_up_to_date(new_resource, action)
         current_resource_report.up_to_date unless nested_resource?(new_resource)
         increment_resource_count
       end
 
+      # see EventDispatch::Base#resource_skipped
+      # Increment the total resource count. If this is a top-level resource,
+      # we also create a ResourceReport instance (because a skipped resource
+      # does not trigger the resource_current_state_loaded event), and flag
+      # it as skipped.
       def resource_skipped(new_resource, action, conditional)
         increment_resource_count
         return if nested_resource?(new_resource)
@@ -106,11 +134,18 @@ class Chef
         current_resource_report.skipped(conditional)
       end
 
+      # see EventDispatch::Base#resource_updated
+      # Flag the current ResourceReport instance as updated (as long as it's
+      # a top-level resource) and increment the total resource count.
       def resource_updated(new_resource, action)
         current_resource_report.updated unless nested_resource?(new_resource)
         increment_resource_count
       end
 
+      # see EventDispatch::Base#resource_failed
+      # Flag the current ResourceReport as failed and supply the exception as
+      # long as it's a top-level resource, increment the total resource count,
+      # and update the run error text with the proper Formatter.
       def resource_failed(new_resource, action, exception)
         current_resource_report.failed(exception) unless nested_resource?(new_resource)
         increment_resource_count
@@ -123,6 +158,10 @@ class Chef
         )
       end
 
+      # see EventDispatch::Base#resource_completed
+      # Mark the ResourceReport instance as finished (for timing details)
+      # and add it to the list of resources encountered during this run.
+      # This marks the end of this resource during this run.
       def resource_completed(new_resource)
         if current_resource_report && !nested_resource?(new_resource)
           current_resource_report.finish
@@ -131,10 +170,16 @@ class Chef
         end
       end
 
+      # see EventDispatch::Base#run_list_expanded
+      # The expanded run list is stored for later use by the run_completed
+      # event and message.
       def run_list_expanded(run_list_expansion)
         @expanded_run_list = run_list_expansion
       end
 
+      # see EventDispatch::Base#run_list_expand_failed
+      # The run error text is updated with the output of the appropriate
+      # formatter.
       def run_list_expand_failed(node, exception)
         update_error_description(
           Formatters::ErrorMapper.run_list_expand_failed(
@@ -144,6 +189,9 @@ class Chef
         )
       end
 
+      # see EventDispatch::Base#cookbook_resolution_failed
+      # The run error text is updated with the output of the appropriate
+      # formatter.
       def cookbook_resolution_failed(expanded_run_list, exception)
         update_error_description(
           Formatters::ErrorMapper.cookbook_resolution_failed(
@@ -153,6 +201,9 @@ class Chef
         )
       end
 
+      # see EventDispatch::Base#cookbook_sync_failed
+      # The run error text is updated with the output of the appropriate
+      # formatter.
       def cookbook_sync_failed(cookbooks, exception)
         update_error_description(
           Formatters::ErrorMapper.cookbook_sync_failed(
@@ -164,6 +215,15 @@ class Chef
 
       private
 
+      #
+      # Yields to the passed-in block (which is expected to be some interaction
+      # with the DataCollector endpoint). If some communication failure occurs,
+      # either disable any future communications to the DataCollector endpoint, or
+      # raise an exception (if the user has set
+      # Chef::Config.data_collector.raise_on_failure to true.)
+      #
+      # @param block [Proc] A ruby block to run. Ignored if a command is given.
+      #
       def disable_reporter_on_error
         yield
       rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
@@ -180,7 +240,7 @@ class Chef
               "URL: #{data_collector_server_url} " \
               "Exception: #{code} -- #{e.message} "
 
-        if Chef::Config[:data_collector_raise_on_failure]
+        if Chef::Config[:data_collector][:raise_on_failure]
           Chef::Log.error(msg)
           raise
         else
@@ -195,22 +255,32 @@ class Chef
         http.post(nil, message.to_json, headers)
       end
 
-      def send_run_completion
+      #
+      # Send any messages to the DataCollector endpoint that are necessary to
+      # indicate the run has completed. Currently, two messages are sent:
+      #
+      # - An "action" message with the node object indicating it's been updated
+      # - An "run_converge" (i.e. RunEnd) message with details about the run,
+      #   what resources were modified/up-to-date/skipped, etc.
+      #
+      # @param opts [Hash] Additional details about the run, such as its success/failure.
+      #
+      def send_run_completion(opts)
         # If run_status is nil we probably failed before the client triggered
         # the run_started callback. In this case we'll skip updating because
         # we have nothing to report.
         return unless run_status
 
-        send_to_data_collector(Serializers::NodeUpdate.new(run_status))
+        send_to_data_collector(Chef::DataCollector::Messages.node_update_message(run_status).to_json)
         send_to_data_collector(
-          Serializers::RunEnd.new(
+          Chef::DataCollector::Messages.run_end_message(
             run_status: run_status,
             expanded_run_list: expanded_run_list,
             total_resource_count: resource_count,
             updated_resources: updated_resources,
-            status: status,
+            status: opts[:status],
             error_descriptions: error_descriptions
-          )
+          ).to_json
         )
       end
 
@@ -222,11 +292,11 @@ class Chef
       end
 
       def data_collector_server_url
-        Chef::Config[:data_collector_server_url]
+        Chef::Config[:data_collector][:server_url]
       end
 
       def data_collector_token
-        Chef::Config[:data_collector_token]
+        Chef::Config[:data_collector][:token]
       end
 
       def increment_resource_count
@@ -246,19 +316,10 @@ class Chef
         @enabled
       end
 
-      def update_status(status)
-        @status = status
-      end
-
       def update_run_status(run_status)
         @run_status = run_status
       end
 
-      def update_exception(ex)
-        @exception = ex
-        update_status("failure")
-      end
-
       def update_current_resource_report(resource_report)
         @current_resource_report = resource_report
       end
diff --git a/lib/chef/data_collector/messages.rb b/lib/chef/data_collector/messages.rb
new file mode 100644
index 0000000000..c77ed1cb47
--- /dev/null
+++ b/lib/chef/data_collector/messages.rb
@@ -0,0 +1,231 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io)
+# Author:: Ryan Cragun (<ryan@chef.io>)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+require "securerandom"
+
+class Chef
+  class DataCollector
+    module Messages
+      def self.run_start_message(run_status)
+        {
+          "chef_server_fqdn"  => chef_server_fqdn(run_status),
+          "entity_uuid"       => node_uuid,
+          "id"                => run_status.run_id,
+          "message_version"   => "1.0.0",
+          "message_type"      => "run_start",
+          "node_name"         => run_status.node.name,
+          "organization_name" => organization,
+          "run_id"            => run_status.run_id,
+          "source"            => collector_source,
+          "start_time"        => run_status.start_time.utc.iso8601,
+        }
+      end
+
+      def self.run_end_message(reporter_data)
+        run_status = reporter_data[:run_status]
+
+        message = {
+          "chef_server_fqdn"       => chef_server_fqdn(run_status),
+          "entity_uuid"            => node_uuid,
+          "expanded_run_list"      => reporter_data[:expanded_run_list],
+          "id"                     => run_status.run_id,
+          "message_version"        => "1.0.0",
+          "message_type"           => "run_converge",
+          "node_name"              => run_status.node.name,
+          "organization_name"      => organization,
+          "resources"              => reporter_data[:updated_resources].map(&:for_json),
+          "run_id"                 => run_status.run_id,
+          "run_list"               => run_status.node.run_list.for_json,
+          "start_time"             => run_status.start_time.utc.iso8601,
+          "end_time"               => run_status.end_time.utc.iso8601,
+          "source"                 => collector_source,
+          "status"                 => reporter_data[:status],
+          "total_resource_count"   => reporter_data[:total_resource_count],
+          "updated_resource_count" => reporter_data[:updated_resources].count,
+        }
+
+        message["error"] = {
+          "class"       => run_status.exception.class,
+          "message"     => run_status.exception.message,
+          "backtrace"   => run_status.exception.backtrace,
+          "description" => reporter_data[:error_descriptions],
+        } if run_status.exception
+
+        message
+      end
+
+      def self.node_update_message(run_status)
+        {
+          "entity_name"       => run_status.node.name,
+          "entity_type"       => "node",
+          "entity_uuid"       => node_uuid,
+          "id"                => SecureRandom.uuid,
+          "message_version"   => "1.0.0",
+          "message_type"      => "action",
+          "organization_name" => organization,
+          "recorded_at"       => Time.now.utc.iso8601,
+          "remote_hostname"   => run_status.node.fqdn,
+          "requestor_name"    => run_status.node.name,
+          "requestor_type"    => "client",
+          "service_hostname"  => chef_server_fqdn(run_status),
+          "source"            => collector_source,
+          "task"              => "update",
+          "user_agent"        => Chef::HTTP::HTTPRequest::DEFAULT_UA,
+          "data"              => run_status.node,
+        }
+      end
+
+      #
+      # Fully-qualified domain name of the Chef Server configured in Chef::Config
+      # If the chef_server_url cannot be parsed as a URI, the node["fqdn"] attribute
+      # will be returned, or "localhost" if the run_status is unavailable to us.
+      #
+      # @param run_status [Chef::RunStatus] The RunStatus object for this Chef Run.
+      #
+      # @return [String] FQDN of the configured Chef Server, or node/localhost if not found.
+      #
+      def chef_server_fqdn(run_status)
+        if !Chef::Config[:chef_server_url].nil?
+          URI(Chef::Config[:chef_server_url]).host
+        elsif run_status
+          run_status.node["fqdn"]
+        else
+          "localhost"
+        end
+      end
+
+      #
+      # The organization name the node is associated with. For Chef Solo runs, a
+      # user-configured organization string is returned, or the string "chef_solo"
+      # if such a string is not configured.
+      #
+      # @return [String] Organization to which the node is associated
+      #
+      def organization
+        solo_run? ? data_collector_organization : chef_server_organization
+      end
+
+      #
+      # Returns the user-configured organization, or "chef_solo" if none is configured.
+      #
+      # This is only used when Chef is run in Solo mode.
+      #
+      # @return [String] Data-collector-specific organization used when running in Chef Solo
+      #
+      def data_collector_organization
+        Chef::Config[:data_collector][:organization] || "chef_solo"
+      end
+
+      #
+      # Return the organization assumed by the configured chef_server_url.
+      #
+      # We must parse this from the Chef::Config[:chef_server_url] because a node
+      # has no knowledge of an organization or to which organization is belongs.
+      #
+      # If we cannot determine the organization, we return "unknown_organization"
+      #
+      # @return [String] shortname of the Chef Server organization
+      #
+      def chef_server_organization
+        return "unknown_organization" unless Chef::Config[:chef_server_url]
+
+        Chef::Config[:chef_server_url].match(%r{/+organizations/+(\w+)}).nil? ? "unknown_organization" : $1
+      end
+
+      #
+      # The source of the data collecting during this run, used by the
+      # DataCollector endpoint to determine if Chef was in Solo mode or not.
+      #
+      # @return [String] "chef_solo" if in Solo mode, "chef_client" if in Client mode
+      #
+      def collector_source
+        solo_run? ? "chef_solo" : "chef_client"
+      end
+
+      #
+      # If we're running in Solo (legacy) mode, or in Solo (formerly
+      # "Chef Client Local Mode"), we're considered to be in a "solo run".
+      #
+      # @return [Boolean] Whether we're in a solo run or not
+      #
+      def solo_run?
+        Chef::Config[:solo] || Chef::Config[:local_mode]
+      end
+
+      #
+      # Returns a UUID that uniquely identifies this node for reporting reasons.
+      #
+      # The node is read in from disk if it exists, or it's generated if it does
+      # does not exist.
+      #
+      # @return [String] UUID for the node
+      #
+      def node_uuid
+        read_node_uuid || generate_node_uuid
+      end
+
+      #
+      # Generates a UUID for the node via SecureRandom.uuid and writes out
+      # metadata file so the UUID persists between runs.
+      #
+      # @return [String] UUID for the node
+      #
+      def generate_node_uuid
+        uuid = SecureRandom.uuid
+        update_metadata("node_uuid", uuid)
+
+        uuid
+      end
+
+      #
+      # Reads in the node UUID from the node metadata file
+      #
+      # @return [String] UUID for the node
+      #
+      def read_node_uuid
+        metadata["node_uuid"]
+      end
+
+      #
+      # Returns the DataCollector metadata for this node
+      #
+      # If the metadata file does not exist in the file cache path,
+      # an empty hash will be returned.
+      #
+      # @return [Hash] DataCollector metadata for this node
+      #
+      def metadata
+        JSON.load(Chef::FileCache.load(metadata_filename))
+      rescue Chef::Exceptions::FileNotFound
+        {}
+      end
+
+      def update_metadata(key, value)
+        metadata[key] = value
+        Chef::FileCache.store(metadata_filename, metadata.to_json, 0644)
+      end
+
+      def metadata_filename
+        "data_collector_metadata.json"
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/chef/data_collector/serializers/base.rb b/lib/chef/data_collector/serializers/base.rb
deleted file mode 100644
index 139d2f1dd7..0000000000
--- a/lib/chef/data_collector/serializers/base.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-#
-# Author:: Adam Leff (<adamleff@chef.io)
-# Author:: Ryan Cragun (<ryan@chef.io>)
-#
-# Copyright:: Copyright 2012-2016, Chef Software Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "json"
-require "securerandom"
-
-class Chef
-  class DataCollector
-    class Serializers
-      class Base
-        def document
-          raise "#{self.class} does not implement the #document method, which should return a hash of data to send"
-        end
-
-        def message_type
-          raise "#{self.class} does not implement the #message_type method, which return a string containing the message type"
-        end
-
-        def to_json
-          document.to_json
-        end
-
-        def chef_server_fqdn
-          return URI(Chef::Config[:chef_server_url]).host unless Chef::Config[:chef_server_url].nil?
-          return "localhost" unless defined?(run_status)
-
-          run_status.node["fqdn"]
-        end
-
-        def organization
-          solo_run? ? data_collector_organization : chef_server_organization
-        end
-
-        def data_collector_organization
-          Chef::Config[:data_collector_organization] || "chef_solo"
-        end
-
-        def chef_server_organization
-          return nil unless Chef::Config[:chef_server_url]
-
-          Chef::Config[:chef_server_url].match(%r{/organizations/(\w+)}).nil? ? "unknown_organization" : $1
-        end
-
-        def collector_source
-          solo_run? ? "chef_solo" : "chef_client"
-        end
-
-        def solo_run?
-          Chef::Config[:solo] || Chef::Config[:local_mode]
-        end
-
-        def node_uuid
-          read_node_uuid || generate_node_uuid
-        end
-
-        def generate_node_uuid
-          uuid = SecureRandom.uuid
-          update_metadata("node_uuid", uuid)
-
-          uuid
-        end
-
-        def read_node_uuid
-          metadata["node_uuid"]
-        end
-
-        def metadata
-          @metadata ||= JSON.load(Chef::FileCache.load(metadata_filename))
-        rescue Chef::Exceptions::FileNotFound
-          @metadata = {}
-        end
-
-        def update_metadata(key, value)
-          metadata[key] = value
-          Chef::FileCache.store(metadata_filename, metadata.to_json, 0644)
-        end
-
-        def metadata_filename
-          "data_collector_metadata.json"
-        end
-      end
-    end
-  end
-end
diff --git a/lib/chef/data_collector/serializers/node_update.rb b/lib/chef/data_collector/serializers/node_update.rb
deleted file mode 100644
index 0befb89b15..0000000000
--- a/lib/chef/data_collector/serializers/node_update.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-#
-# Author:: Adam Leff (<adamleff@chef.io)
-# Author:: Ryan Cragun (<ryan@chef.io>)
-#
-# Copyright:: Copyright 2012-2016, Chef Software Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "securerandom"
-require "chef/data_collector/serializers/base"
-
-class Chef
-  class DataCollector
-    class Serializers
-      class NodeUpdate < Base
-
-        attr_reader :run_status
-
-        def initialize(run_status)
-          @run_status = run_status
-        end
-
-        def message_type
-          "action"
-        end
-
-        def node
-          run_status.node
-        end
-
-        def document
-          {
-            "entity_name"       => node.name,
-            "entity_type"       => "node",
-            "entity_uuid"       => node_uuid,
-            "id"                => SecureRandom.uuid,
-            "message_version"   => "1.0.0",
-            "message_type"      => message_type,
-            "organization_name" => organization,
-            "recorded_at"       => Time.now.utc.iso8601,
-            "remote_hostname"   => node["fqdn"],
-            "requestor_name"    => node.name,
-            "requestor_type"    => "client",
-            "service_hostname"  => chef_server_fqdn,
-            "source"            => collector_source,
-            "task"              => "update",
-            "user_agent"        => Chef::HTTP::HTTPRequest::DEFAULT_UA,
-            "data"              => node,
-          }
-        end
-      end
-    end
-  end
-end
diff --git a/lib/chef/data_collector/serializers/run_end.rb b/lib/chef/data_collector/serializers/run_end.rb
deleted file mode 100644
index 1d1d586aa7..0000000000
--- a/lib/chef/data_collector/serializers/run_end.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Author:: Adam Leff (<adamleff@chef.io)
-# Author:: Ryan Cragun (<ryan@chef.io>)
-#
-# Copyright:: Copyright 2012-2016, Chef Software Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "securerandom"
-require "chef/data_collector/serializers/base"
-
-class Chef
-  class DataCollector
-    class Serializers
-      class RunEnd < Base
-        attr_reader :error_descriptions
-        attr_reader :expanded_run_list
-        attr_reader :run_status
-        attr_reader :status
-        attr_reader :total_resource_count
-        attr_reader :updated_resources
-
-        def initialize(opts)
-          @error_descriptions   = opts[:error_descriptions]
-          @expanded_run_list    = opts[:expanded_run_list]
-          @run_status           = opts[:run_status]
-          @total_resource_count = opts[:total_resource_count]
-          @updated_resources    = opts[:updated_resources]
-          @status               = opts[:status]
-        end
-
-        def message_type
-          "run_converge"
-        end
-
-        def document
-          document = {
-            "chef_server_fqdn"       => chef_server_fqdn,
-            "entity_uuid"            => node_uuid,
-            "expanded_run_list"      => expanded_run_list,
-            "id"                     => run_status.run_id,
-            "message_version"        => "1.0.0",
-            "message_type"           => message_type,
-            "node_name"              => run_status.node.name,
-            "organization_name"      => organization,
-            "resources"              => updated_resources.map(&:for_json),
-            "run_id"                 => run_status.run_id,
-            "run_list"               => run_status.node.run_list.for_json,
-            "start_time"             => run_status.start_time.utc.iso8601,
-            "end_time"               => run_status.end_time.utc.iso8601,
-            "source"                 => collector_source,
-            "status"                 => status,
-            "total_resource_count"   => total_resource_count,
-            "updated_resource_count" => updated_resources.count,
-          }
-
-          document["error"] = formatted_exception if run_status.exception
-
-          document
-        end
-
-        def formatted_exception
-          {
-            "class"       => run_status.exception.class,
-            "message"     => run_status.exception.message,
-            "backtrace"   => run_status.exception.backtrace,
-            "description" => error_descriptions,
-          }
-        end
-      end
-    end
-  end
-end
diff --git a/lib/chef/data_collector/serializers/run_start.rb b/lib/chef/data_collector/serializers/run_start.rb
deleted file mode 100644
index 5c202b37e2..0000000000
--- a/lib/chef/data_collector/serializers/run_start.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Author:: Adam Leff (<adamleff@chef.io)
-# Author:: Ryan Cragun (<ryan@chef.io>)
-#
-# Copyright:: Copyright 2012-2016, Chef Software Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "chef/data_collector/serializers/base"
-
-class Chef
-  class DataCollector
-    class Serializers
-      class RunStart < Base
-
-        attr_reader :run_status
-
-        def initialize(run_status)
-          @run_status = run_status
-        end
-
-        def message_type
-          "run_start"
-        end
-
-        def document
-          {
-            "chef_server_fqdn"  => chef_server_fqdn,
-            "entity_uuid"       => node_uuid,
-            "id"                => run_status.run_id,
-            "message_version"   => "1.0.0",
-            "message_type"      => message_type,
-            "node_name"         => run_status.node.name,
-            "organization_name" => organization,
-            "run_id"            => run_status.run_id,
-            "source"            => collector_source,
-            "start_time"        => run_status.start_time.utc.iso8601,
-          }
-        end
-      end
-    end
-  end
-end
diff --git a/spec/unit/data_collector/serializers/base_spec.rb b/spec/unit/data_collector/serializers/base_spec.rb
index dd0e4146db..350b65c753 100644
--- a/spec/unit/data_collector/serializers/base_spec.rb
+++ b/spec/unit/data_collector/serializers/base_spec.rb
@@ -66,7 +66,7 @@ describe Chef::DataCollector::Serializers::Base do
   describe '#data_collector_organization' do
     context "when the org is specified in the config" do
       it "returns the org from the config" do
-        Chef::Config[:data_collector_organization] = "org1"
+        Chef::Config[:data_collector][:organization] = "org1"
         expect(serializer.data_collector_organization).to eq("org1")
       end
     end
diff --git a/spec/unit/data_collector_spec.rb b/spec/unit/data_collector_spec.rb
index 75207c6728..e7cf491ce8 100644
--- a/spec/unit/data_collector_spec.rb
+++ b/spec/unit/data_collector_spec.rb
@@ -25,14 +25,14 @@ describe Chef::DataCollector do
   describe ".register_reporter?" do
     context "when no data collector URL is configured" do
       it "returns false" do
-        Chef::Config[:data_collector_server_url] = nil
+        Chef::Config[:data_collector][:server_url] = nil
         expect(Chef::DataCollector.register_reporter?).to be_falsey
       end
     end
 
     context "when a data collector URL is configured" do
       before do
-        Chef::Config[:data_collector_server_url] = "http://data_collector"
+        Chef::Config[:data_collector][:server_url] = "http://data_collector"
       end
 
       context "when operating in why_run mode" do
@@ -73,21 +73,21 @@ describe Chef::DataCollector do
 
       context "when data_collector_mode is :solo" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :solo
+          Chef::Config[:data_collector][:mode] = :solo
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
 
       context "when data_collector_mode is :client" do
         it "returns false" do
-          Chef::Config[:data_collector_mode] = :client
+          Chef::Config[:data_collector][:mode] = :client
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false)
         end
       end
 
       context "when data_collector_mode is :both" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :both
+          Chef::Config[:data_collector][:mode] = :both
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
@@ -101,21 +101,21 @@ describe Chef::DataCollector do
 
       context "when data_collector_mode is :solo" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :solo
+          Chef::Config[:data_collector][:mode] = :solo
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
 
       context "when data_collector_mode is :client" do
         it "returns false" do
-          Chef::Config[:data_collector_mode] = :client
+          Chef::Config[:data_collector][:mode] = :client
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false)
         end
       end
 
       context "when data_collector_mode is :both" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :both
+          Chef::Config[:data_collector][:mode] = :both
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
@@ -129,21 +129,21 @@ describe Chef::DataCollector do
 
       context "when data_collector_mode is :solo" do
         it "returns false" do
-          Chef::Config[:data_collector_mode] = :solo
+          Chef::Config[:data_collector][:mode] = :solo
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(false)
         end
       end
 
       context "when data_collector_mode is :client" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :client
+          Chef::Config[:data_collector][:mode] = :client
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
 
       context "when data_collector_mode is :both" do
         it "returns true" do
-          Chef::Config[:data_collector_mode] = :both
+          Chef::Config[:data_collector][:mode] = :both
           expect(Chef::DataCollector.reporter_enabled_for_current_mode?).to eq(true)
         end
       end
@@ -500,7 +500,7 @@ describe Chef::DataCollector::Reporter do
 
         context "when raise-on-failure is enabled" do
           it "logs an error and raises" do
-            Chef::Config[:data_collector_raise_on_failure] = true
+            Chef::Config[:data_collector][:raise_on_failure] = true
             expect(Chef::Log).to receive(:error)
             expect { reporter.send(:disable_reporter_on_error) { raise exception_class.new("bummer") } }.to raise_error(exception_class)
           end
@@ -508,7 +508,7 @@ describe Chef::DataCollector::Reporter do
 
         context "when raise-on-failure is disabled" do
           it "logs a warning and does not raise an exception" do
-            Chef::Config[:data_collector_raise_on_failure] = false
+            Chef::Config[:data_collector][:raise_on_failure] = false
             expect(Chef::Log).to receive(:warn)
             expect { reporter.send(:disable_reporter_on_error) { raise exception_class.new("bummer") } }.not_to raise_error
           end