Merge pull request #3966 from chef/kyleen/addExpandedRunList

Report expanded run list json tree to reporting

7 files changed, 83 insertions, 3 deletions

diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index 1c9a58be23..585a3db174 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -384,6 +384,9 @@ class Chef
       def deprecation(message, location=caller(2..2)[0])
       end
 
+      def run_list_expanded(run_list_expansion)
+      end
+
       # An uncategorized message. This supports the case that a user needs to
       # pass output that doesn't fit into one of the callbacks above. Note that
       # there's no semantic information about the content or importance of the
diff --git a/lib/chef/json_compat.rb b/lib/chef/json_compat.rb
index d0b3b4c7f8..5e9f29a60b 100644
--- a/lib/chef/json_compat.rb
+++ b/lib/chef/json_compat.rb
@@ -41,6 +41,7 @@ class Chef
     CHEF_RESOURCECOLLECTION = "Chef::ResourceCollection".freeze
     CHEF_RESOURCESET        = "Chef::ResourceCollection::ResourceSet".freeze
     CHEF_RESOURCELIST       = "Chef::ResourceCollection::ResourceList".freeze
+    CHEF_RUNLISTEXPANSION   = "Chef::RunListExpansion".freeze
 
     class <<self
 
diff --git a/lib/chef/policy_builder/expand_node_object.rb b/lib/chef/policy_builder/expand_node_object.rb
index 933960a429..2c6d644e42 100644
--- a/lib/chef/policy_builder/expand_node_object.rb
+++ b/lib/chef/policy_builder/expand_node_object.rb
@@ -154,6 +154,7 @@ class Chef
         Chef::Log.info("Run List expands to [#{@expanded_run_list_with_versions.join(', ')}]")
 
         events.node_load_completed(node, @expanded_run_list_with_versions, Chef::Config)
+        events.run_list_expanded(@run_list_expansion)
 
         node
       end
diff --git a/lib/chef/resource_reporter.rb b/lib/chef/resource_reporter.rb
index 7d13a5a5ce..1175b0afb3 100644
--- a/lib/chef/resource_reporter.rb
+++ b/lib/chef/resource_reporter.rb
@@ -112,6 +112,7 @@ class Chef
       @exception = nil
       @rest_client = rest_client
       @error_descriptions = {}
+      @expanded_run_list = {}
     end
 
     def run_started(run_status)
@@ -217,6 +218,10 @@ class Chef
       end
     end
 
+    def run_list_expanded(run_list_expansion)
+      @expanded_run_list = run_list_expansion
+    end
+
     def post_reporting_data
       if reporting_enabled?
         run_data = prepare_run_data
@@ -271,6 +276,7 @@ class Chef
       run_data["data"] = {}
       run_data["start_time"] = start_time.to_s
       run_data["end_time"] = end_time.to_s
+      run_data["expanded_run_list"] = Chef::JSONCompat.to_json(@expanded_run_list)
 
       if exception
         exception_data = {}
diff --git a/lib/chef/run_list/run_list_expansion.rb b/lib/chef/run_list/run_list_expansion.rb
index 46b45f1d9e..64e4326fb8 100644
--- a/lib/chef/run_list/run_list_expansion.rb
+++ b/lib/chef/run_list/run_list_expansion.rb
@@ -22,6 +22,7 @@ require 'chef/mixin/deep_merge'
 
 require 'chef/role'
 require 'chef/rest'
+require 'chef/json_compat'
 
 class Chef
   class RunList
@@ -54,6 +55,13 @@ class Chef
       # * Duplicate roles are not shown.
       attr_reader :run_list_trace
 
+      # Like run list trace but instead of saving the entries as strings it saves their objects
+      # The to_json method uses this list to construct json.
+      attr_reader :better_run_list_trace
+
+      attr_reader :all_missing_roles
+      attr_reader :role_errors
+
       def initialize(environment, run_list_items, source=nil)
         @environment = environment
         @missing_roles_with_including_role = Array.new
@@ -68,6 +76,9 @@ class Chef
 
         @applied_roles = {}
         @run_list_trace = Hash.new {|h, key| h[key] = [] }
+        @better_run_list_trace = Hash.new {|h, key| h[key] = [] }
+        @all_missing_roles = {}
+        @role_errors = {}
       end
 
       # Did we find any errors (expanding roles)?
@@ -124,6 +135,7 @@ class Chef
       def role_not_found(name, included_by)
         Chef::Log.error("Role #{name} (included by '#{included_by}') is in the runlist but does not exist. Skipping expand.")
         @missing_roles_with_including_role << [name, included_by]
+        @all_missing_roles[name] = true
         nil
       end
 
@@ -131,6 +143,15 @@ class Chef
         @missing_roles_with_including_role.map {|item| item.first }
       end
 
+      def to_json(*a)
+        Chef::JSONCompat.to_json(to_hash, *a)
+      end
+
+      def to_hash
+        seen_items = {:recipe => {}, :role => {}}
+        {:id => @environment, :run_list => convert_run_list_trace('top level', seen_items)}
+      end
+
       private
 
       # these methods modifies internal state based on arguments, so hide it.
@@ -140,8 +161,10 @@ class Chef
       end
 
       def expand_run_list_items(items, included_by="top level")
+
         if entry = items.shift
           @run_list_trace[included_by.to_s] << entry.to_s
+          @better_run_list_trace[included_by.to_s] <<  entry
 
           case entry.type
           when :recipe
@@ -156,8 +179,26 @@ class Chef
         end
       end
 
+      # Recursive helper to decode the non-nested hash form back into a tree
+      def convert_run_list_trace(base, seen_items)
+        @better_run_list_trace[base].map do |item|
+          skipped = seen_items[item.type][item.name]
+          seen_items[item.type][item.name] = true
+          case item.type
+            when :recipe
+                {:type => 'recipe', :name => item.name, :version => item.version, :skipped => !!skipped}
+            when :role
+              error = @role_errors[item.name]
+              missing = @all_missing_roles[item.name]
+              {:type => :role, :name => item.name, :children => (missing || error || skipped) ? [] : convert_run_list_trace(item.to_s, seen_items),
+               :missing => missing, :error => error, :skipped => skipped}
+          end
+        end
+      end
+
     end
 
+
     # Expand a run list from disk. Suitable for chef-solo
     class RunListExpansionFromDisk < RunListExpansion
 
@@ -184,8 +225,14 @@ class Chef
         else
           raise
         end
+      rescue Exception => e
+        @role_errors[name] = e.to_s
+        raise
       end
+
     end
 
   end
 end
+
+
diff --git a/spec/unit/resource_reporter_spec.rb b/spec/unit/resource_reporter_spec.rb
index 4f3a085584..f2c0b8fd8b 100644
--- a/spec/unit/resource_reporter_spec.rb
+++ b/spec/unit/resource_reporter_spec.rb
@@ -50,6 +50,9 @@ describe Chef::ResourceReporter do
     @events = Chef::EventDispatch::Dispatcher.new
     @run_context = Chef::RunContext.new(@node, {}, @events)
     @run_status = Chef::RunStatus.new(@node, @events)
+    @run_list = Chef::RunList.new
+    @run_list << 'recipe[lobster]' << 'role[rage]' << 'recipe[fist]'
+    @expansion = Chef::RunList::RunListExpansion.new("_default", @run_list.run_list_items)
     @run_id = @run_status.run_id
     allow(Time).to receive(:now).and_return(@start_time, @end_time)
   end
@@ -424,6 +427,10 @@ describe Chef::ResourceReporter do
         expect(@report["run_list"]).to eq(Chef::JSONCompat.to_json(@run_status.node.run_list))
       end
 
+      it "includes the expanded_run_list" do
+        expect(@report).to have_key("expanded_run_list")
+      end
+
       it "includes the end_time" do
         expect(@report).to have_key("end_time")
         expect(@report["end_time"]).to eq(@run_status.end_time.to_s)
diff --git a/spec/unit/run_list/run_list_expansion_spec.rb b/spec/unit/run_list/run_list_expansion_spec.rb
index 859219d346..a7df9e749b 100644
--- a/spec/unit/run_list/run_list_expansion_spec.rb
+++ b/spec/unit/run_list/run_list_expansion_spec.rb
@@ -21,7 +21,7 @@ require 'spec_helper'
 describe Chef::RunList::RunListExpansion do
   before do
     @run_list = Chef::RunList.new
-    @run_list << 'recipe[lobster]' << 'role[rage]' << 'recipe[fist]'
+    @run_list << 'recipe[lobster::mastercookbook@0.1.0]' << 'role[rage]' << 'recipe[fist@0.1]'
     @expansion = Chef::RunList::RunListExpansion.new("_default", @run_list.run_list_items)
   end
 
@@ -59,7 +59,7 @@ describe Chef::RunList::RunListExpansion do
     end
 
     it "has the correct list of recipes for the given environment" do
-      expect(@expansion.recipes).to eq(["lobster", "prod-only", "fist"])
+      expect(@expansion.recipes).to eq(["lobster::mastercookbook", "prod-only", "fist"])
     end
 
   end
@@ -82,19 +82,34 @@ describe Chef::RunList::RunListExpansion do
   describe "after expanding a run list" do
     before do
       @first_role = Chef::Role.new
+      @first_role.name('rage')
       @first_role.run_list('role[mollusk]')
       @first_role.default_attributes({'foo' => 'bar'})
       @first_role.override_attributes({'baz' => 'qux'})
       @second_role = Chef::Role.new
+      @second_role.name('rage')
       @second_role.run_list('recipe[crabrevenge]')
       @second_role.default_attributes({'foo' => 'boo'})
       @second_role.override_attributes({'baz' => 'bux'})
       allow(@expansion).to receive(:fetch_role).and_return(@first_role, @second_role)
       @expansion.expand
+      @json = '{"id":"_default","run_list":[{"type":"recipe","name":"lobster::mastercookbook","version":"0.1.0",'
+      .concat(
+'"skipped":false},{"type":"role","name":"rage","children":[{"type":"role","name":"mollusk","children":[],"missing":null,'
+      .concat(
+'"error":null,"skipped":null},{"type":"recipe","name":"crabrevenge","version":null,"skipped":false}],"missing":null,'
+      .concat(
+'"error":null,"skipped":null},{"type":"recipe","name":"fist","version":"0.1","skipped":false}]}')))
+
+    end
+
+    it "produces json tree upon tracing expansion" do
+      jsonRunList = @expansion.to_json
+      expect(jsonRunList).to eq(@json)
     end
 
     it "has the ordered list of recipes" do
-      expect(@expansion.recipes).to eq(['lobster', 'crabrevenge', 'fist'])
+      expect(@expansion.recipes).to eq(['lobster::mastercookbook', 'crabrevenge', 'fist'])
     end
 
     it "has the merged attributes from the roles with outer roles overriding inner" do