Creation of the new DataCollector reporter

The DataCollector reporter is a new method for exporting data about your
Chef run. The details of this new feature can be found in
[RFC 077](https://github.com/chef/chef-rfc/blob/master/rfc077-mode-agnostic-data-collection.md).

Using the existing `EventDispatch` mechanics, the DataCollector reporter
collects data about a Chef run (when it starts, when it ends, what
resources were modified, etc.) and then POSTs them to a Data Collector
server URL that can be specified in your Chef configuration.

While similar functionality exists using the `ResourceReporter` and Chef
Reporting, a new implementation was done to decouple the reporting of this
data from requiring the use of a Chef Server (in the case of Chef Reporting),
opening the door to users being able to implement their own webhook-style
receiver to receive these messages and analyze them accordingly.

20 files changed, 448 insertions, 0 deletions

diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore b/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore
new file mode 100644
index 0000000000..041413b040
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore
@@ -0,0 +1,2 @@
+nodes/
+tmp/
diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb
new file mode 100644
index 0000000000..68fc3af2dd
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb
@@ -0,0 +1,3 @@
+name 'acceptance-cookbook'
+depends "kitchen_acceptance"
+depends "data-collector-test"
diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb
new file mode 100644
index 0000000000..7f4be2c7f7
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb
@@ -0,0 +1,2 @@
+log "Running 'destroy' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "destroy"
diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb
new file mode 100644
index 0000000000..c707e874f0
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb
@@ -0,0 +1,2 @@
+log "Running 'provision' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "converge"
diff --git a/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb
new file mode 100644
index 0000000000..e4a547272b
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb
@@ -0,0 +1,2 @@
+log "Running 'verify' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "verify"
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/.gitignore b/acceptance/data-collector/.acceptance/data-collector-test/.gitignore
new file mode 100644
index 0000000000..ec2a890bd3
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/.gitignore
@@ -0,0 +1,16 @@
+.vagrant
+Berksfile.lock
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+
+# Bundler
+Gemfile.lock
+bin/*
+.bundle/*
+
+.kitchen/
+.kitchen.local.yml
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/Berksfile b/acceptance/data-collector/.acceptance/data-collector-test/Berksfile
new file mode 100644
index 0000000000..34fea2166b
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/Berksfile
@@ -0,0 +1,3 @@
+source 'https://supermarket.chef.io'
+
+metadata
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb
new file mode 100644
index 0000000000..3fb2c730b0
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb
@@ -0,0 +1,85 @@
+require "json"
+require "sinatra"
+
+class Chef
+  class Node
+    # dummy class for JSON parsing
+  end
+end
+
+module ApiHelpers
+  def self.payload_type(payload)
+    message_type = payload["message_type"]
+    status       = payload["status"]
+
+    message_type == "run_converge" ? "#{message_type}.#{status}" : message_type
+  end
+end
+
+class Counter
+  def self.reset
+    @@counters = Hash.new { |h, k| h[k] = 0 }
+  end
+
+  def self.increment(payload)
+    counter_name = ApiHelpers.payload_type(payload)
+    @@counters[counter_name] += 1
+  end
+
+  def self.to_json
+    @@counters.to_json
+  end
+end
+
+class MessageCache
+  include ApiHelpers
+
+  def self.reset
+    @@message_cache = {}
+  end
+
+  def self.store(payload)
+    cache_key = ApiHelpers.payload_type(payload)
+
+    @@message_cache[cache_key] = payload
+  end
+
+  def self.fetch(cache_key)
+    @@message_cache[cache_key].to_json
+  end
+end
+
+Counter.reset
+
+get "/" do
+  "Data Collector API server"
+end
+
+get "/reset-counters" do
+  Counter.reset
+  "counters reset"
+end
+
+get "/counters" do
+  Counter.to_json
+end
+
+get "/cache/:key" do |cache_key|
+  MessageCache.fetch(cache_key)
+end
+
+get "/reset-cache" do
+  MessageCache.reset
+  "cache reset"
+end
+
+post "/data-collector/v0" do
+  body = request.body.read
+  payload = JSON.load(body)
+
+  Counter.increment(payload)
+  MessageCache.store(payload)
+
+  status 201
+  "message received"
+end
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile b/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile
new file mode 100644
index 0000000000..94fc334d88
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile
@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "sinatra"
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb
new file mode 100644
index 0000000000..89f3555be1
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb
@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :both
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb
new file mode 100644
index 0000000000..8e3f0c4845
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb
@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :client
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb
new file mode 100644
index 0000000000..f8374107ea
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb
@@ -0,0 +1,2 @@
+chef_server_url "http://localhost:8889"
+node_name       "data-collector-test"
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb
new file mode 100644
index 0000000000..904e952e85
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb
@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :solo
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru b/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru
new file mode 100644
index 0000000000..81cf29d9fb
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru
@@ -0,0 +1,2 @@
+require_relative "./api"
+run Sinatra::Application
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb b/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb
new file mode 100644
index 0000000000..dbd376aa83
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb
@@ -0,0 +1,7 @@
+name 'data-collector-test'
+maintainer 'Adam Leff'
+maintainer_email 'adamleff@chef.io'
+license 'Apache 2.0'
+description 'Installs/Configures data-collector-test'
+long_description 'Installs/Configures data-collector-test'
+version '0.1.0'
diff --git a/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb b/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb
new file mode 100644
index 0000000000..20b945db9b
--- /dev/null
+++ b/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb
@@ -0,0 +1,38 @@
+api_root_dir = "/var/opt/data_collector_api"
+
+directory api_root_dir do
+  recursive true
+end
+
+cookbook_file ::File.join(api_root_dir, "Gemfile") do
+  source "apigemfile"
+end
+
+cookbook_file ::File.join(api_root_dir, "config.ru")
+
+cookbook_file ::File.join(api_root_dir, "api.rb")
+
+execute "bundle install --binstubs" do
+  cwd api_root_dir
+end
+
+pid_file    = "/var/run/api.pid"
+running_pid = ::File.exist?(pid_file) ? ::File.read(pid_file).strip : nil
+
+execute "kill existing API process" do
+  command "kill #{running_pid}"
+  not_if { running_pid.nil? }
+end
+
+execute "start API" do
+  command "bin/rackup -D -P #{pid_file}"
+  cwd api_root_dir
+end
+
+directory "/etc/chef"
+
+["both-mode", "client-mode", "no-endpoint", "solo-mode"].each do |config_file|
+  cookbook_file "/etc/chef/#{config_file}.rb" do
+    source "client-rb-#{config_file}.rb"
+  end
+end
diff --git a/acceptance/data-collector/.kitchen.yml b/acceptance/data-collector/.kitchen.yml
new file mode 100644
index 0000000000..f719e3ea69
--- /dev/null
+++ b/acceptance/data-collector/.kitchen.yml
@@ -0,0 +1,9 @@
+verifier:
+  name: busser
+
+suites:
+  - name: default
+    includes: 
+      - ubuntu-14.04
+    run_list:
+      - recipe[data-collector-test::default]
diff --git a/acceptance/data-collector/Berksfile b/acceptance/data-collector/Berksfile
new file mode 100644
index 0000000000..b8f003071b
--- /dev/null
+++ b/acceptance/data-collector/Berksfile
@@ -0,0 +1,3 @@
+source "https://supermarket.chef.io"
+
+cookbook "data-collector-test", path: File.join(File.expand_path(File.dirname(__FILE__)), ".acceptance", "data-collector-test")
diff --git a/acceptance/data-collector/Berksfile.lock b/acceptance/data-collector/Berksfile.lock
new file mode 100644
index 0000000000..39f4ce30dc
--- /dev/null
+++ b/acceptance/data-collector/Berksfile.lock
@@ -0,0 +1,6 @@
+DEPENDENCIES
+  data-collector-test
+    path: .acceptance/data-collector-test
+
+GRAPH
+  data-collector-test (0.1.0)
diff --git a/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb b/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb
new file mode 100644
index 0000000000..be15b96429
--- /dev/null
+++ b/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb
@@ -0,0 +1,251 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+require "serverspec"
+
+set :backend, :exec
+
+class Chef
+  class Node
+    # dummy class for parsing JSON action messages
+  end
+end
+
+shared_examples_for "reset counters" do
+  it "resets the counters" do
+    expect(command("curl http://localhost:9292/reset-counters").exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "reset cache" do
+  it "resets the message cache" do
+    expect(command("curl http://localhost:9292/reset-cache").exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "successful chef run" do |cmd|
+  include_examples "reset counters"
+  include_examples "reset cache"
+
+  it "runs chef and expects a zero exit status" do
+    expect(command(cmd).exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "unsuccessful chef run" do |cmd|
+  include_examples "reset counters"
+  include_examples "reset cache"
+
+  it "runs chef and expects a non-zero exit status" do
+    expect(command(cmd).exit_status).not_to eq(0)
+  end
+end
+
+shared_examples_for "counter checks" do |counters_to_check|
+  counters_to_check.each do |counter, value|
+    it "counter #{counter} should equal #{value}" do
+      counter_values = JSON.load(command("curl http://localhost:9292/counters").stdout)
+      expect(counter_values[counter]).to eq(value)
+    end
+  end
+end
+
+shared_examples_for "run_start payload check" do
+  describe "run_start message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        id
+        message_version
+        message_type
+        node_name
+        organization_name
+        run_id
+        source
+        start_time
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_start").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_start").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "run_converge.success payload check" do
+  describe "run_converge success message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        id
+        end_time
+        expanded_run_list
+        message_type
+        message_version
+        node_name
+        organization_name
+        resources
+        run_id
+        run_list
+        source
+        start_time
+        status
+        total_resource_count
+        updated_resource_count
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.success").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.success").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "run_converge.failure payload check" do
+  describe "run_converge failure message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        error
+        id
+        end_time
+        expanded_run_list
+        message_type
+        message_version
+        node_name
+        organization_name
+        resources
+        run_id
+        run_list
+        source
+        start_time
+        status
+        total_resource_count
+        updated_resource_count
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.failure").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.failure").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "node-update payload check" do
+  describe "node update message" do
+    let(:required_fields) do
+      %w{
+        entity_name
+        entity_type
+        entity_uuid
+        id
+        message_type
+        message_version
+        organization_name
+        recorded_at
+        remote_hostname
+        requestor_name
+        requestor_type
+        run_id
+        service_hostname
+        source
+        task
+        user_agent
+      }
+    end
+    let(:optional_fields) { %{data} }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+describe "CCR with no data collector URL configured" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/no-endpoint.rb"
+  include_examples "counter checks", { "run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil }
+end
+
+describe "CCR, local mode, config in solo mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/solo-mode.rb"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
+  include_examples "run_start payload check"
+  include_examples "run_converge.success payload check"
+  include_examples "node-update payload check"
+end
+
+describe "CCR, local mode, config in client mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/client-mode.rb"
+  include_examples "counter checks", { "run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil }
+end
+
+describe "CCR, local mode, config in both mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/both-mode.rb"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
+  include_examples "run_start payload check"
+  include_examples "run_converge.success payload check"
+  include_examples "node-update payload check"
+end
+
+describe "CCR, local mode, config in solo mode, failed run" do
+  include_examples "unsuccessful chef run", "chef-client -z -c /etc/chef/solo-mode.rb -r 'recipe[cookbook-that-does-not-exist::default]'"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => nil, "run_converge.failure" => 1 }
+  include_examples "run_start payload check"
+  include_examples "run_converge.failure payload check"
+  include_examples "node-update payload check"
+end