Creation of the new DataCollector reporter

The DataCollector reporter is a new method for exporting data about your
Chef run. The details of this new feature can be found in
[RFC 077](https://github.com/chef/chef-rfc/blob/master/rfc077-mode-agnostic-data-collection.md).

Using the existing `EventDispatch` mechanics, the DataCollector reporter
collects data about a Chef run (when it starts, when it ends, what
resources were modified, etc.) and then POSTs them to a Data Collector
server URL that can be specified in your Chef configuration.

While similar functionality exists using the `ResourceReporter` and Chef
Reporting, a new implementation was done to decouple the reporting of this
data from requiring the use of a Chef Server (in the case of Chef Reporting),
opening the door to users being able to implement their own webhook-style
receiver to receive these messages and analyze them accordingly.

3 files changed, 370 insertions, 0 deletions

diff --git a/lib/chef/data_collector/messages.rb b/lib/chef/data_collector/messages.rb
new file mode 100644
index 0000000000..b6114a8bec
--- /dev/null
+++ b/lib/chef/data_collector/messages.rb
@@ -0,0 +1,125 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io)
+# Author:: Ryan Cragun (<ryan@chef.io>)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+require "securerandom"
+require_relative "messages/helpers"
+
+class Chef
+  class DataCollector
+    module Messages
+      extend Helpers
+
+      #
+      # Message payload that is sent to the DataCollector server at the
+      # start of a Chef run.
+      #
+      # @param run_status [Chef::RunStatus] The RunStatus instance for this node/run.
+      #
+      # @return [Hash] A hash containing the run start message data.
+      #
+      def self.run_start_message(run_status)
+        {
+          "chef_server_fqdn"  => chef_server_fqdn(run_status),
+          "entity_uuid"       => node_uuid,
+          "id"                => run_status.run_id,
+          "message_version"   => "1.0.0",
+          "message_type"      => "run_start",
+          "node_name"         => run_status.node.name,
+          "organization_name" => organization,
+          "run_id"            => run_status.run_id,
+          "source"            => collector_source,
+          "start_time"        => run_status.start_time.utc.iso8601,
+        }
+      end
+
+      #
+      # Message payload that is sent to the DataCollector server at the
+      # end of a Chef run.
+      #
+      # @param reporter_data [Hash] Data supplied by the Reporter, such as run_status, resource counts, etc.
+      #
+      # @return [Hash] A hash containing the run end message data.
+      #
+      def self.run_end_message(reporter_data)
+        run_status = reporter_data[:run_status]
+
+        message = {
+          "chef_server_fqdn"       => chef_server_fqdn(run_status),
+          "entity_uuid"            => node_uuid,
+          "expanded_run_list"      => reporter_data[:expanded_run_list],
+          "id"                     => run_status.run_id,
+          "message_version"        => "1.0.0",
+          "message_type"           => "run_converge",
+          "node_name"              => run_status.node.name,
+          "organization_name"      => organization,
+          "resources"              => reporter_data[:updated_resources].map(&:for_json),
+          "run_id"                 => run_status.run_id,
+          "run_list"               => run_status.node.run_list.for_json,
+          "start_time"             => run_status.start_time.utc.iso8601,
+          "end_time"               => run_status.end_time.utc.iso8601,
+          "source"                 => collector_source,
+          "status"                 => reporter_data[:status],
+          "total_resource_count"   => reporter_data[:total_resource_count],
+          "updated_resource_count" => reporter_data[:updated_resources].count,
+        }
+
+        message["error"] = {
+          "class"       => run_status.exception.class,
+          "message"     => run_status.exception.message,
+          "backtrace"   => run_status.exception.backtrace,
+          "description" => reporter_data[:error_descriptions],
+        } if run_status.exception
+
+        message
+      end
+
+      #
+      # Message payload that is sent to the DataCollector server at the
+      # end of a Chef run.
+      #
+      # @param run_status [Chef::RunStatus] The RunStatus instance for this node/run.
+      #
+      # @return [Hash] A hash containing the node object and related metadata.
+      #
+      def self.node_update_message(run_status)
+        {
+          "entity_name"       => run_status.node.name,
+          "entity_type"       => "node",
+          "entity_uuid"       => node_uuid,
+          "id"                => SecureRandom.uuid,
+          "message_version"   => "1.1.0",
+          "message_type"      => "action",
+          "organization_name" => organization,
+          "recorded_at"       => Time.now.utc.iso8601,
+          "remote_hostname"   => run_status.node["fqdn"],
+          "requestor_name"    => run_status.node.name,
+          "requestor_type"    => "client",
+          "run_id"            => run_status.run_id,
+          "service_hostname"  => chef_server_fqdn(run_status),
+          "source"            => collector_source,
+          "task"              => "update",
+          "user_agent"        => Chef::HTTP::HTTPRequest::DEFAULT_UA,
+          "data"              => run_status.node,
+        }
+      end
+    end
+  end
+end
diff --git a/lib/chef/data_collector/messages/helpers.rb b/lib/chef/data_collector/messages/helpers.rb
new file mode 100644
index 0000000000..3e52f80047
--- /dev/null
+++ b/lib/chef/data_collector/messages/helpers.rb
@@ -0,0 +1,161 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io)
+# Author:: Ryan Cragun (<ryan@chef.io>)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  class DataCollector
+    module Messages
+      module Helpers
+        #
+        # Fully-qualified domain name of the Chef Server configured in Chef::Config
+        # If the chef_server_url cannot be parsed as a URI, the node["fqdn"] attribute
+        # will be returned, or "localhost" if the run_status is unavailable to us.
+        #
+        # @param run_status [Chef::RunStatus] The RunStatus object for this Chef Run.
+        #
+        # @return [String] FQDN of the configured Chef Server, or node/localhost if not found.
+        #
+        def chef_server_fqdn(run_status)
+          if !Chef::Config[:chef_server_url].nil?
+            URI(Chef::Config[:chef_server_url]).host
+          elsif !Chef::Config[:node_name].nil?
+            Chef::Config[:node_name]
+          else
+            "localhost"
+          end
+        end
+
+        #
+        # The organization name the node is associated with. For Chef Solo runs, a
+        # user-configured organization string is returned, or the string "chef_solo"
+        # if such a string is not configured.
+        #
+        # @return [String] Organization to which the node is associated
+        #
+        def organization
+          solo_run? ? data_collector_organization : chef_server_organization
+        end
+
+        #
+        # Returns the user-configured organization, or "chef_solo" if none is configured.
+        #
+        # This is only used when Chef is run in Solo mode.
+        #
+        # @return [String] Data-collector-specific organization used when running in Chef Solo
+        #
+        def data_collector_organization
+          Chef::Config[:data_collector][:organization] || "chef_solo"
+        end
+
+        #
+        # Return the organization assumed by the configured chef_server_url.
+        #
+        # We must parse this from the Chef::Config[:chef_server_url] because a node
+        # has no knowledge of an organization or to which organization is belongs.
+        #
+        # If we cannot determine the organization, we return "unknown_organization"
+        #
+        # @return [String] shortname of the Chef Server organization
+        #
+        def chef_server_organization
+          return "unknown_organization" unless Chef::Config[:chef_server_url]
+
+          Chef::Config[:chef_server_url].match(%r{/+organizations/+(\w+)}).nil? ? "unknown_organization" : $1
+        end
+
+        #
+        # The source of the data collecting during this run, used by the
+        # DataCollector endpoint to determine if Chef was in Solo mode or not.
+        #
+        # @return [String] "chef_solo" if in Solo mode, "chef_client" if in Client mode
+        #
+        def collector_source
+          solo_run? ? "chef_solo" : "chef_client"
+        end
+
+        #
+        # If we're running in Solo (legacy) mode, or in Solo (formerly
+        # "Chef Client Local Mode"), we're considered to be in a "solo run".
+        #
+        # @return [Boolean] Whether we're in a solo run or not
+        #
+        def solo_run?
+          Chef::Config[:solo] || Chef::Config[:local_mode]
+        end
+
+        #
+        # Returns a UUID that uniquely identifies this node for reporting reasons.
+        #
+        # The node is read in from disk if it exists, or it's generated if it does
+        # does not exist.
+        #
+        # @return [String] UUID for the node
+        #
+        def node_uuid
+          read_node_uuid || generate_node_uuid
+        end
+
+        #
+        # Generates a UUID for the node via SecureRandom.uuid and writes out
+        # metadata file so the UUID persists between runs.
+        #
+        # @return [String] UUID for the node
+        #
+        def generate_node_uuid
+          uuid = SecureRandom.uuid
+          update_metadata("node_uuid", uuid)
+
+          uuid
+        end
+
+        #
+        # Reads in the node UUID from the node metadata file
+        #
+        # @return [String] UUID for the node
+        #
+        def read_node_uuid
+          metadata["node_uuid"]
+        end
+
+        #
+        # Returns the DataCollector metadata for this node
+        #
+        # If the metadata file does not exist in the file cache path,
+        # an empty hash will be returned.
+        #
+        # @return [Hash] DataCollector metadata for this node
+        #
+        def metadata
+          JSON.load(Chef::FileCache.load(metadata_filename))
+        rescue Chef::Exceptions::FileNotFound
+          {}
+        end
+
+        def update_metadata(key, value)
+          metadata[key] = value
+          Chef::FileCache.store(metadata_filename, metadata.to_json, 0644)
+        end
+
+        def metadata_filename
+          "data_collector_metadata.json"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/data_collector/resource_report.rb b/lib/chef/data_collector/resource_report.rb
new file mode 100644
index 0000000000..1793fe2c9d
--- /dev/null
+++ b/lib/chef/data_collector/resource_report.rb
@@ -0,0 +1,84 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io>)
+# Author:: Ryan Cragun (<ryan@chef.io>)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  class DataCollector
+    class ResourceReport
+
+      attr_reader :action, :current_resource, :elapsed_time, :new_resource, :status
+      attr_accessor :conditional, :exception
+
+      def initialize(new_resource, action, current_resource = nil)
+        @new_resource     = new_resource
+        @action           = action
+        @current_resource = current_resource
+      end
+
+      def skipped(conditional)
+        @status      = "skipped"
+        @conditional = conditional
+      end
+
+      def updated
+        @status = "updated"
+      end
+
+      def failed(exception)
+        @current_resource = nil
+        @status           = "failed"
+        @exception        = exception
+      end
+
+      def up_to_date
+        @status = "up-to-date"
+      end
+
+      def finish
+        @elapsed_time = new_resource.elapsed_time
+      end
+
+      def to_hash
+        hash = {
+          "type"     => new_resource.resource_name.to_sym,
+          "name"     => new_resource.name.to_s,
+          "id"       => new_resource.identity.to_s,
+          "after"    => new_resource.state_for_resource_reporter,
+          "before"   => current_resource ? current_resource.state_for_resource_reporter : {},
+          "duration" => (elapsed_time * 1000).to_i.to_s,
+          "delta"    => new_resource.respond_to?(:diff) ? new_resource.diff : "",
+          "result"   => action.to_s,
+          "status"   => status,
+        }
+
+        if new_resource.cookbook_name
+          hash["cookbook_name"]    = new_resource.cookbook_name
+          hash["cookbook_version"] = new_resource.cookbook_version.version
+        end
+
+        hash["conditional"]   = conditional.to_text if status == "skipped"
+        hash["error_message"] = exception.message unless exception.nil?
+
+        hash
+      end
+      alias :to_h :to_hash
+      alias :for_json :to_hash
+    end
+  end
+end