diff options
| -rw-r--r-- | lib/chef/cookbook/cookbook_processor.rb | 131 | ||||
| -rw-r--r-- | lib/chef/cookbook_version.rb | 15 | ||||
| -rw-r--r-- | lib/chef/run_context.rb | 41 | ||||
| -rw-r--r-- | lib/chef/run_context/cookbook_compiler.rb | 183 | ||||
| -rw-r--r-- | spec/unit/recipe_spec.rb | 23 | ||||
| -rw-r--r-- | spec/unit/run_context/cookbook_compiler_spec.rb | 2 |
6 files changed, 215 insertions, 180 deletions
diff --git a/lib/chef/cookbook/cookbook_processor.rb b/lib/chef/cookbook/cookbook_processor.rb new file mode 100644 index 0000000000..714d3fdb44 --- /dev/null +++ b/lib/chef/cookbook/cookbook_processor.rb @@ -0,0 +1,131 @@ +class Chef + class Cookbook + class CookbookProcessor + def initialize(run_context, cookbook_version) + @run_context = run_context + @cookbook_version = cookbook_version + end + + attr_reader :run_context + attr_reader :cookbook_version + + def compile_libraries + files_by_segment(:libraries).each do |filename| + begin + Chef::Log.debug("Loading cookbook #{name}'s library file: #{filename}") + Kernel.load(filename) + events.library_file_loaded(filename) + rescue Exception => e + events.library_file_load_failed(filename, e) + raise + end + end + end + + def compile_attributes + list_of_attr_files = files_by_segment(:attributes).dup + if default_file = list_of_attr_files.find { |path| File.basename(path) == "default.rb" } + list_of_attr_files.delete(default_file) + list_of_attr_files.shift(default_file) + end + + list_of_attr_files.each do |filename| + compile_attribute_file(filename) + end + end + + def compile_lwrps + files_by_segment(:providers).each do |filename| + compile_lwrp_provider(filename) + end + files_by_segment(:resources).each do |filename| + compile_lwrp_resource(filename) + end + end + + def compile_definitions + files_by_segment(:definitions).each do |filename| + compile_definition(filename) + end + end + + def compile_recipe(recipe_name) + if run_context.loaded_fully_qualified_recipe?(name, recipe_name) + Chef::Log.debug("I am not loading #{recipe_name}, because I have already seen it.") + return false + end + + run_context.loaded_recipe(name, recipe_name) + + unless cookbook_version.recipe_filenames_by_name.has_key?(recipe_name) + raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}" + end + + Chef::Log.debug("Found recipe #{recipe_name} in cookbook #{name}") + recipe = Chef::Recipe.new(name, recipe_name, run_context) + recipe_filename = cookbook_version.recipe_filenames_by_name[recipe_name] + + unless recipe_filename + raise Chef::Exceptions::RecipeNotFound, "could not find #{recipe_name} files for cookbook #{name}" + end + + recipe.from_file(recipe_filename) + recipe + end + + protected + + # Lists the local paths to files in +cookbook+ of type +segment+ + # (attribute, recipe, etc.), sorted lexically. + def files_by_segment(cookbook, segment) + cookbook_version.segment_filenames(segment).sort + end + + def compile_attribute_file(filename) + Chef::Log.debug("Node #{node.name} loading cookbook #{name}'s attribute file #{filename}") + attr_file_basename = ::File.basename(filename, ".rb") + node.include_attribute("#{name}::#{attr_file_basename}") + rescue Exception => e + events.attribute_file_load_failed(filename, e) + raise + end + + def compile_lwrp_provider(filename) + Chef::Log.debug("Loading cookbook #{name}'s providers from #{filename}") + Chef::Provider::LWRPBase.build_from_file(name, filename, self) + events.lwrp_file_loaded(filename) + rescue Exception => e + events.lwrp_file_load_failed(filename, e) + raise + end + + def compile_lwrp_resource(filename) + Chef::Log.debug("Loading cookbook #{name}'s resources from #{filename}") + Chef::Resource::LWRPBase.build_from_file(name, filename, self) + events.lwrp_file_loaded(filename) + rescue Exception => e + events.lwrp_file_load_failed(filename, e) + raise + end + + def compile_definition(filename) + Chef::Log.debug("Loading cookbook #{name}'s definitions from #{filename}") + definition_list = Chef::ResourceDefinitionList.new + definition_list.from_file(filename) + definitions.merge!(definition_list.defines) do |key, oldval, newval| + Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}") + newval + end + events.definition_file_loaded(filename) + rescue Exception => e + events.definition_file_load_failed(filename, e) + raise + end + + extend Forwardable + def_delegators :run_context, :events, :node + def_delegators :cookbook_version, :name + + end + end +end diff --git a/lib/chef/cookbook_version.rb b/lib/chef/cookbook_version.rb index 8d302eeec2..d87c707e18 100644 --- a/lib/chef/cookbook_version.rb +++ b/lib/chef/cookbook_version.rb @@ -213,20 +213,7 @@ class Chef # called from DSL def load_recipe(recipe_name, run_context) - unless recipe_filenames_by_name.has_key?(recipe_name) - raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}" - end - - Chef::Log.debug("Found recipe #{recipe_name} in cookbook #{name}") - recipe = Chef::Recipe.new(name, recipe_name, run_context) - recipe_filename = recipe_filenames_by_name[recipe_name] - - unless recipe_filename - raise Chef::Exceptions::RecipeNotFound, "could not find #{recipe_name} files for cookbook #{name}" - end - - recipe.from_file(recipe_filename) - recipe + run_context.load_recipe(recipe_name) end def segment_filenames(segment) diff --git a/lib/chef/run_context.rb b/lib/chef/run_context.rb index 44b05f0cc0..be8911ca42 100644 --- a/lib/chef/run_context.rb +++ b/lib/chef/run_context.rb @@ -87,14 +87,14 @@ class Chef @node.run_context = self @node.set_cookbook_attribute - @cookbook_compiler = nil end # Triggers the compile phase of the chef run. Implemented by # Chef::RunContext::CookbookCompiler def load(run_list_expansion) - @cookbook_compiler = CookbookCompiler.new(self, run_list_expansion, events) - @cookbook_compiler.compile + raise "Cannot call load() twice" if cookbook_compiler + self.cookbook_compiler = CookbookCompiler.new(self, run_list_expansion) + cookbook_compiler.compile end # Adds an immediate notification to the @@ -149,30 +149,7 @@ class Chef # Evaluates the recipe +recipe_name+. Used by DSL::IncludeRecipe def load_recipe(recipe_name, current_cookbook: nil) - Chef::Log.debug("Loading Recipe #{recipe_name} via include_recipe") - - cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name, current_cookbook: current_cookbook) - - if unreachable_cookbook?(cookbook_name) # CHEF-4367 - Chef::Log.warn(<<-ERROR_MESSAGE) -MissingCookbookDependency: -Recipe `#{recipe_name}` is not in the run_list, and cookbook '#{cookbook_name}' -is not a dependency of any cookbook in the run_list. To load this recipe, -first add a dependency on cookbook '#{cookbook_name}' in the cookbook you're -including it from in that cookbook's metadata. -ERROR_MESSAGE - end - - - if loaded_fully_qualified_recipe?(cookbook_name, recipe_short_name) - Chef::Log.debug("I am not loading #{recipe_name}, because I have already seen it.") - false - else - loaded_recipe(cookbook_name, recipe_short_name) - node.loaded_recipe(cookbook_name, recipe_short_name) - cookbook = cookbook_collection[cookbook_name] - cookbook.load_recipe(recipe_short_name, self) - end + cookbook_compiler.compile_recipe(recipe_name, current_cookbook: current_cookbook) end def load_recipe_file(recipe_file) @@ -255,7 +232,7 @@ ERROR_MESSAGE # Used to raise an error when attempting to load a recipe belonging to a # cookbook that is not in the dependency graph. See also: CHEF-4367 def unreachable_cookbook?(cookbook_name) - @cookbook_compiler.unreachable_cookbook?(cookbook_name) + cookbook_compiler.unreachable_cookbook?(cookbook_name) end # Open a stream object that can be printed into and will dispatch to events @@ -300,11 +277,15 @@ ERROR_MESSAGE @reboot_info.size > 0 end - private - + # @api private def loaded_recipe(cookbook, recipe) @loaded_recipes["#{cookbook}::#{recipe}"] = true + node.loaded_recipe(cookbook, recipe) end + private + + attr_accessor :cookbook_compiler + end end diff --git a/lib/chef/run_context/cookbook_compiler.rb b/lib/chef/run_context/cookbook_compiler.rb index abe5afa7ae..82a380e367 100644 --- a/lib/chef/run_context/cookbook_compiler.rb +++ b/lib/chef/run_context/cookbook_compiler.rb @@ -16,12 +16,14 @@ # limitations under the License. # -require 'set' require 'chef/log' require 'chef/recipe' require 'chef/resource/lwrp_base' require 'chef/provider/lwrp_base' require 'chef/resource_definition_list' +require 'chef/cookbook/cookbook_processor' +require 'forwardable' +require 'set' class Chef class RunContext @@ -29,30 +31,21 @@ class Chef # Implements the compile phase of the chef run by loading/eval-ing files # from cookbooks in the correct order and in the correct context. class CookbookCompiler - attr_reader :events + extend Forwardable + attr_reader :run_context attr_reader :run_list_expansion + def_delegators :run_context, :events, :node, :cookbook_collection, :definitions - def initialize(run_context, run_list_expansion, events) + def initialize(run_context, run_list_expansion, events=nil) + if events + Chef::Log.deprecation("events is no longer a separate parameter to CookbookCompiler.new and will be removed in Chef 13.") + if events != run_context.events + raise "The run context event sink is different from the one passed in to CookbookCompiler.new!" + end + end @run_context = run_context - @events = events @run_list_expansion = run_list_expansion - @cookbook_order = nil - end - - # Chef::Node object for the current run. - def node - @run_context.node - end - - # Chef::CookbookCollection object for the current run - def cookbook_collection - @run_context.cookbook_collection - end - - # Resource Definitions from the compiled cookbooks. This is populated by - # calling #compile_resource_definitions (which is called by #compile) - def definitions - @run_context.definitions + @cookbook_version_compilers = {} end # Run the compile phase of the chef run. Loads files in the following order: @@ -72,7 +65,7 @@ class Chef compile_attributes compile_lwrps compile_resource_definitions - compile_recipes + compile_run_list end # Extracts the cookbook names from the expanded run list, then iterates @@ -94,60 +87,81 @@ class Chef # Loads library files from cookbooks according to #cookbook_order. def compile_libraries - @events.library_load_start(count_files_by_segment(:libraries)) + events.library_load_start(count_files_by_segment(:libraries)) cookbook_order.each do |cookbook| - load_libraries_from_cookbook(cookbook) + compiler_for(cookbook).compile_libraries end - @events.library_load_complete + events.library_load_complete end # Loads attributes files from cookbooks. Attributes files are loaded # according to #cookbook_order; within a cookbook, +default.rb+ is loaded # first, then the remaining attributes files in lexical sort order. def compile_attributes - @events.attribute_load_start(count_files_by_segment(:attributes)) + events.attribute_load_start(count_files_by_segment(:attributes)) cookbook_order.each do |cookbook| - load_attributes_from_cookbook(cookbook) + compiler_for(cookbook).compile_attributes end - @events.attribute_load_complete + events.attribute_load_complete end # Loads LWRPs according to #cookbook_order. Providers are loaded before # resources on a cookbook-wise basis. def compile_lwrps lwrp_file_count = count_files_by_segment(:providers) + count_files_by_segment(:resources) - @events.lwrp_load_start(lwrp_file_count) + events.lwrp_load_start(lwrp_file_count) cookbook_order.each do |cookbook| - load_lwrps_from_cookbook(cookbook) + compiler_for(cookbook).compile_lwrps end - @events.lwrp_load_complete + events.lwrp_load_complete end # Loads resource definitions according to #cookbook_order def compile_resource_definitions - @events.definition_load_start(count_files_by_segment(:definitions)) + events.definition_load_start(count_files_by_segment(:definitions)) cookbook_order.each do |cookbook| - load_resource_definitions_from_cookbook(cookbook) + compiler_for(cookbook).compile_resource_definitions end - @events.definition_load_complete + events.definition_load_complete end # Iterates over the expanded run_list, loading each recipe in turn. - def compile_recipes - @events.recipe_load_start(run_list_expansion.recipes.size) + def compile_run_list + events.recipe_load_start(run_list_expansion.recipes.size) run_list_expansion.recipes.each do |recipe| + cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name) begin - @run_context.load_recipe(recipe) + compiler_for(cookbook_name).compile_recipe(recipe_short_name) rescue Chef::Exceptions::RecipeNotFound => e - @events.recipe_not_found(e) + events.recipe_not_found(e) raise rescue Exception => e - path = resolve_recipe(recipe) - @events.recipe_file_load_failed(path, e) + cookbook, path = resolve_recipe(recipe) + events.recipe_file_load_failed(cookbooks[], e) raise end end - @events.recipe_load_complete + events.recipe_load_complete + end + + alias :compile_recipes :compile_run_list + + def compile_recipe(recipe_name, current_cookbook: nil) + Chef::Log.debug("Loading Recipe #{recipe_name}") + + cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name, current_cookbook: current_cookbook) + + if unreachable_cookbook?(cookbook_name) # CHEF-4367 + Chef::Log.warn(<<-ERROR_MESSAGE) + MissingCookbookDependency: + Recipe `#{recipe_name}` is not in the run_list, and cookbook '#{cookbook_name}' + is not a dependency of any cookbook in the run_list. To load this recipe, + first add a dependency on cookbook '#{cookbook_name}' in the cookbook you're + including it from in that cookbook's metadata. + ERROR_MESSAGE + end + + compiler_for(cookbook_name).compile_recipe(recipe_short_name) end # Whether or not a cookbook is reachable from the set of cookbook given @@ -163,86 +177,6 @@ class Chef private - def load_attributes_from_cookbook(cookbook_name) - list_of_attr_files = files_in_cookbook_by_segment(cookbook_name, :attributes).dup - if default_file = list_of_attr_files.find {|path| File.basename(path) == "default.rb" } - list_of_attr_files.delete(default_file) - load_attribute_file(cookbook_name.to_s, default_file) - end - - list_of_attr_files.each do |filename| - load_attribute_file(cookbook_name.to_s, filename) - end - end - - def load_attribute_file(cookbook_name, filename) - Chef::Log.debug("Node #{node.name} loading cookbook #{cookbook_name}'s attribute file #{filename}") - attr_file_basename = ::File.basename(filename, ".rb") - node.include_attribute("#{cookbook_name}::#{attr_file_basename}") - rescue Exception => e - @events.attribute_file_load_failed(filename, e) - raise - end - - def load_libraries_from_cookbook(cookbook_name) - files_in_cookbook_by_segment(cookbook_name, :libraries).each do |filename| - begin - Chef::Log.debug("Loading cookbook #{cookbook_name}'s library file: #{filename}") - Kernel.load(filename) - @events.library_file_loaded(filename) - rescue Exception => e - @events.library_file_load_failed(filename, e) - raise - end - end - end - - def load_lwrps_from_cookbook(cookbook_name) - files_in_cookbook_by_segment(cookbook_name, :providers).each do |filename| - load_lwrp_provider(cookbook_name, filename) - end - files_in_cookbook_by_segment(cookbook_name, :resources).each do |filename| - load_lwrp_resource(cookbook_name, filename) - end - end - - def load_lwrp_provider(cookbook_name, filename) - Chef::Log.debug("Loading cookbook #{cookbook_name}'s providers from #{filename}") - Chef::Provider::LWRPBase.build_from_file(cookbook_name, filename, self) - @events.lwrp_file_loaded(filename) - rescue Exception => e - @events.lwrp_file_load_failed(filename, e) - raise - end - - def load_lwrp_resource(cookbook_name, filename) - Chef::Log.debug("Loading cookbook #{cookbook_name}'s resources from #{filename}") - Chef::Resource::LWRPBase.build_from_file(cookbook_name, filename, self) - @events.lwrp_file_loaded(filename) - rescue Exception => e - @events.lwrp_file_load_failed(filename, e) - raise - end - - - def load_resource_definitions_from_cookbook(cookbook_name) - files_in_cookbook_by_segment(cookbook_name, :definitions).each do |filename| - begin - Chef::Log.debug("Loading cookbook #{cookbook_name}'s definitions from #{filename}") - resourcelist = Chef::ResourceDefinitionList.new - resourcelist.from_file(filename) - definitions.merge!(resourcelist.defines) do |key, oldval, newval| - Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}") - newval - end - @events.definition_file_loaded(filename) - rescue Exception => e - @events.definition_file_load_failed(filename, e) - raise - end - end - end - # Builds up the list of +ordered_cookbooks+ by first recursing through the # dependencies of +cookbook+, and then adding +cookbook+ to the list of # +ordered_cookbooks+. A cookbook is skipped if it appears in @@ -280,11 +214,16 @@ class Chef # Given a +recipe_name+, finds the file associated with the recipe. def resolve_recipe(recipe_name) + Chef::Recipe.parse_recipe_name(recipe_name, current_cookbook: current_cookbook) cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name) cookbook = cookbook_collection[cookbook_name] - cookbook.recipe_filenames_by_name[recipe_short_name] + [ cookbook, cookbook.recipe_filenames_by_name[recipe_short_name] ] end + def compiler_for(cookbook_name) + cookbook_version = cookbook_collection[cookbook_name] + @cookbook_version_compilers[cookbook_version] ||= Chef::Cookbook::CookbookProcessor.new(run_context, cookbook_version) + end end diff --git a/spec/unit/recipe_spec.rb b/spec/unit/recipe_spec.rb index 17ea498fe3..05d375de13 100644 --- a/spec/unit/recipe_spec.rb +++ b/spec/unit/recipe_spec.rb @@ -517,6 +517,9 @@ describe Chef::Recipe do end describe "include_recipe" do + before do + run_context.load(Chef::RunList::RunListExpansion.new("_default", [])) + end it "should evaluate another recipe with include_recipe" do expect(node).to receive(:loaded_recipe).with(:openldap, "gigantor") allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) @@ -545,9 +548,8 @@ describe Chef::Recipe do it "should not include the same recipe twice" do expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) recipe.include_recipe "openldap" - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) recipe.include_recipe "openldap" end @@ -555,7 +557,7 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.include_recipe "::default" end @@ -563,9 +565,8 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.include_recipe "::default" - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) openldap_recipe.include_recipe "openldap::default" end @@ -573,9 +574,8 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.include_recipe "openldap::default" - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) openldap_recipe.include_recipe "::default" end @@ -583,9 +583,8 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.include_recipe "::default" - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) openldap_recipe.openldap_includer("do it").run_action(:run) end @@ -593,9 +592,8 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.openldap_includer("do it").run_action(:run) - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) openldap_recipe.include_recipe "::default" end @@ -603,9 +601,8 @@ describe Chef::Recipe do openldap_recipe = Chef::Recipe.new("openldap", "test", run_context) expect(node).to receive(:loaded_recipe).with(:openldap, "default").exactly(:once) allow(run_context).to receive(:unreachable_cookbook?).with(:openldap).and_return(false) - expect(cookbook_collection[:openldap]).to receive(:load_recipe).with("default", run_context) + expect_any_instance_of(Chef::Recipe).to receive(:from_file).with(match(/default.rb$/)).exactly(:once) openldap_recipe.openldap_includer("do it").run_action(:run) - expect(cookbook_collection[:openldap]).not_to receive(:load_recipe).with("default", run_context) openldap_recipe.openldap_includer("do it").run_action(:run) end end diff --git a/spec/unit/run_context/cookbook_compiler_spec.rb b/spec/unit/run_context/cookbook_compiler_spec.rb index 20ec1d2ef7..6364976548 100644 --- a/spec/unit/run_context/cookbook_compiler_spec.rb +++ b/spec/unit/run_context/cookbook_compiler_spec.rb @@ -48,7 +48,7 @@ describe Chef::RunContext::CookbookCompiler do let(:run_list_expansion) { node.run_list.expand('_default') } let(:compiler) do - Chef::RunContext::CookbookCompiler.new(run_context, run_list_expansion, events) + Chef::RunContext::CookbookCompiler.new(run_context, run_list_expansion) end |