lazily load cookbook_files and templatestm/even_lazier

Fixes: #6051

Signed-off-by: Thom May <thom@chef.io>

5 files changed, 34 insertions, 13 deletions

diff --git a/lib/chef/cookbook/remote_file_vendor.rb b/lib/chef/cookbook/remote_file_vendor.rb
index cfd7789311..668d74c9ce 100644
--- a/lib/chef/cookbook/remote_file_vendor.rb
+++ b/lib/chef/cookbook/remote_file_vendor.rb
@@ -62,7 +62,7 @@ class Chef
         # If the checksums are different between on-disk (current) and on-server
         # (remote, per manifest), do the update. This will also execute if there
         # is no current checksum.
-        if current_checksum != found_manifest_record["checksum"]
+        if found_manifest_record[:lazy] || current_checksum != found_manifest_record["checksum"]
           raw_file = @rest.streaming_request(found_manifest_record[:url])
 
           Chef::Log.debug("Storing updated #{cache_filename} in the cache.")
diff --git a/lib/chef/cookbook/synchronizer.rb b/lib/chef/cookbook/synchronizer.rb
index 625f3b4f20..ed6b1d9700 100644
--- a/lib/chef/cookbook/synchronizer.rb
+++ b/lib/chef/cookbook/synchronizer.rb
@@ -71,6 +71,8 @@ class Chef
 
       @cookbook_full_file_paths = {}
       @remove_obsoleted_files = true
+
+      @lazy_files = {}
     end
 
     def cache
@@ -98,15 +100,22 @@ class Chef
     end
 
     def files
-      exclude = unless Chef::Config[:no_lazy_load]
-                  [ :files, :templates ]
-                else
-                  []
-                end
+      lazy = unless Chef::Config[:no_lazy_load]
+               %w{ files templates }
+             else
+               []
+             end
 
       @files ||= cookbooks.inject([]) do |memo, cookbook|
-        cookbook.each_file(excluded_parts: exclude) do |manifest_record|
-          memo << CookbookFile.new(cookbook, manifest_record)
+        cookbook.each_file do |manifest_record|
+          part = manifest_record[:name].split("/")[0]
+          if lazy.include?(part)
+            manifest_record[:lazy] = true
+            @lazy_files[cookbook] ||= []
+            @lazy_files[cookbook] << manifest_record
+          else
+            memo << CookbookFile.new(cookbook, manifest_record)
+          end
         end
         memo
       end
@@ -230,6 +239,10 @@ class Chef
       @cookbook_full_file_paths.each do |cookbook, full_paths|
         cookbook.all_files = full_paths
       end
+
+      @lazy_files.each do |cookbook, lazy_files|
+        cookbook.cookbook_manifest.add_files_to_manifest(lazy_files)
+      end
     end
 
     def ensure_cookbook_paths
diff --git a/lib/chef/cookbook_manifest.rb b/lib/chef/cookbook_manifest.rb
index 125f353d21..526c183339 100644
--- a/lib/chef/cookbook_manifest.rb
+++ b/lib/chef/cookbook_manifest.rb
@@ -160,6 +160,14 @@ class Chef
       @manifest_records_by_path = extract_manifest_records_by_path(@manifest)
     end
 
+    # @api private
+    # takes a list of hashes
+    def add_files_to_manifest(files)
+      manifest[:all_files].concat(Array(files))
+      @checksums = extract_checksums_from_manifest(@manifest)
+      @manifest_records_by_path = extract_manifest_records_by_path(@manifest)
+    end
+
     def files_for(part)
       return root_files if part.to_s == "root_files"
       manifest[:all_files].select do |file|
diff --git a/spec/integration/client/client_spec.rb b/spec/integration/client/client_spec.rb
index 39c8b24f88..de12b8ba8e 100644
--- a/spec/integration/client/client_spec.rb
+++ b/spec/integration/client/client_spec.rb
@@ -496,7 +496,6 @@ EOM
     end
   end
 
-
   when_the_repository "has a cookbook with an ohai plugin" do
     before do
       file "cookbooks/x/recipes/default.rb", <<-RECIPE
diff --git a/spec/unit/cookbook/synchronizer_spec.rb b/spec/unit/cookbook/synchronizer_spec.rb
index 77e64482da..6578a9e670 100644
--- a/spec/unit/cookbook/synchronizer_spec.rb
+++ b/spec/unit/cookbook/synchronizer_spec.rb
@@ -118,6 +118,7 @@ describe Chef::CookbookSynchronizer do
 
   let(:synchronizer) do
     Chef::Config[:no_lazy_load] = no_lazy_load
+    Chef::Config[:file_cache_path] = "/file-cache"
     Chef::CookbookSynchronizer.new(cookbook_manifest, events)
   end
 
@@ -292,7 +293,7 @@ describe Chef::CookbookSynchronizer do
     # Current file has fff000, want abc123
     expect(Chef::CookbookVersion).to receive(:checksum_cookbook_file).
       with("/file-cache/cookbooks/cookbook_a/recipes/default.rb").
-      and_return("fff000")
+      and_return("fff000").at_least(:once)
 
     # Fetch and copy default.rb attribute file
     expect(server_api).to receive(:streaming_request).
@@ -308,7 +309,7 @@ describe Chef::CookbookSynchronizer do
     # Current file has fff000, want abc456
     expect(Chef::CookbookVersion).to receive(:checksum_cookbook_file).
       with("/file-cache/cookbooks/cookbook_a/attributes/default.rb").
-      and_return("fff000")
+      and_return("fff000").at_least(:once)
   end
 
   def setup_no_lazy_files_and_templates_chksum_mismatch_expectations
@@ -365,12 +366,12 @@ describe Chef::CookbookSynchronizer do
     # Current file has abc123, want abc123
     expect(Chef::CookbookVersion).to receive(:checksum_cookbook_file).
       with("/file-cache/cookbooks/cookbook_a/recipes/default.rb").
-      and_return("abc123")
+      and_return("abc123").at_least(:once)
 
     # Current file has abc456, want abc456
     expect(Chef::CookbookVersion).to receive(:checksum_cookbook_file).
       with("/file-cache/cookbooks/cookbook_a/attributes/default.rb").
-      and_return("abc456")
+      and_return("abc456").at_least(:once)
 
     # :load called twice
     expect(file_cache).to receive(:load).