diff options
Diffstat (limited to 'lib/chef/cookbook')
| -rw-r--r-- | lib/chef/cookbook/chefignore.rb | 66 | ||||
| -rw-r--r-- | lib/chef/cookbook/cookbook_collection.rb | 45 | ||||
| -rw-r--r-- | lib/chef/cookbook/cookbook_version_loader.rb | 173 | ||||
| -rw-r--r-- | lib/chef/cookbook/file_system_file_vendor.rb | 56 | ||||
| -rw-r--r-- | lib/chef/cookbook/file_vendor.rb | 48 | ||||
| -rw-r--r-- | lib/chef/cookbook/metadata.rb | 629 | ||||
| -rw-r--r-- | lib/chef/cookbook/remote_file_vendor.rb | 84 | ||||
| -rw-r--r-- | lib/chef/cookbook/synchronizer.rb | 216 | ||||
| -rw-r--r-- | lib/chef/cookbook/syntax_check.rb | 136 |
9 files changed, 1453 insertions, 0 deletions
diff --git a/lib/chef/cookbook/chefignore.rb b/lib/chef/cookbook/chefignore.rb new file mode 100644 index 0000000000..e9d54639e4 --- /dev/null +++ b/lib/chef/cookbook/chefignore.rb @@ -0,0 +1,66 @@ +#-- +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Copyright:: Copyright (c) 2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Chef + class Cookbook + class Chefignore + + COMMENTS_AND_WHITESPACE = /^\w*(?:#.*)?$/ + + attr_reader :ignores + + def initialize(ignore_file_or_repo) + @ignore_file = find_ignore_file(ignore_file_or_repo) + @ignores = parse_ignore_file + end + + def remove_ignores_from(file_list) + Array(file_list).inject([]) do |unignored, file| + ignored?(file) ? unignored : unignored << file + end + end + + def ignored?(file_name) + @ignores.any? {|glob| File.fnmatch?(glob, file_name)} + end + + private + + def parse_ignore_file + ignore_globs = [] + if File.exist?(@ignore_file) && File.readable?(@ignore_file) + File.foreach(@ignore_file) do |line| + ignore_globs << line.strip unless line =~ COMMENTS_AND_WHITESPACE + end + else + Chef::Log.debug("No chefignore file found at #@ignore_file no files will be ignored") + end + ignore_globs + end + + def find_ignore_file(path) + if File.basename(path) =~ /chefignore/ + path + else + File.join(path, 'chefignore') + end + end + end + end +end + diff --git a/lib/chef/cookbook/cookbook_collection.rb b/lib/chef/cookbook/cookbook_collection.rb new file mode 100644 index 0000000000..ae63abfc93 --- /dev/null +++ b/lib/chef/cookbook/cookbook_collection.rb @@ -0,0 +1,45 @@ +#-- +# Author:: Tim Hinderliter (<tim@opscode.com>) +# Author:: Christopher Walters (<cw@opscode.com>) +# Copyright:: Copyright (c) 2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/mash' + +class Chef + # == Chef::CookbookCollection + # This class is the consistent interface for a node to obtain its + # cookbooks by name. + # + # This class is basically a glorified Hash, but since there are + # several ways this cookbook information is collected, + # (e.g. CookbookLoader for solo, hash of auto-vivified Cookbook + # objects for lazily-loaded remote cookbooks), it gets transformed + # into this. + class CookbookCollection < Mash + + # The input is a mapping of cookbook name to CookbookVersion objects. We + # simply extract them + def initialize(cookbook_versions={}) + super() do |hash, key| + raise Chef::Exceptions::CookbookNotFound, "Cookbook #{key} not found. " << + "If you're loading #{key} from another cookbook, make sure you configure the dependency in your metadata" + end + cookbook_versions.each{ |cookbook_name, cookbook_version| self[cookbook_name] = cookbook_version } + end + + end +end diff --git a/lib/chef/cookbook/cookbook_version_loader.rb b/lib/chef/cookbook/cookbook_version_loader.rb new file mode 100644 index 0000000000..48de17cc5a --- /dev/null +++ b/lib/chef/cookbook/cookbook_version_loader.rb @@ -0,0 +1,173 @@ + +require 'chef/config' +require 'chef/cookbook_version' +require 'chef/cookbook/chefignore' +require 'chef/cookbook/metadata' + +class Chef + class Cookbook + class CookbookVersionLoader + + FILETYPES_SUBJECT_TO_IGNORE = [ :attribute_filenames, + :definition_filenames, + :recipe_filenames, + :template_filenames, + :file_filenames, + :library_filenames, + :resource_filenames, + :provider_filenames] + + + attr_reader :cookbook_name + attr_reader :cookbook_settings + attr_reader :metadata_filenames + + def initialize(path, chefignore=nil) + @cookbook_path = File.expand_path( path ) + @cookbook_name = File.basename( path ) + @chefignore = chefignore + @metadata = Hash.new + @relative_path = /#{Regexp.escape(@cookbook_path)}\/(.+)$/ + @cookbook_settings = { + :attribute_filenames => {}, + :definition_filenames => {}, + :recipe_filenames => {}, + :template_filenames => {}, + :file_filenames => {}, + :library_filenames => {}, + :resource_filenames => {}, + :provider_filenames => {}, + :root_filenames => {} + } + + @metadata_filenames = [] + end + + def load_cookbooks + load_as(:attribute_filenames, 'attributes', '*.rb') + load_as(:definition_filenames, 'definitions', '*.rb') + load_as(:recipe_filenames, 'recipes', '*.rb') + load_as(:library_filenames, 'libraries', '*.rb') + load_recursively_as(:template_filenames, "templates", "*") + load_recursively_as(:file_filenames, "files", "*") + load_recursively_as(:resource_filenames, "resources", "*.rb") + load_recursively_as(:provider_filenames, "providers", "*.rb") + load_root_files + + remove_ignored_files + + if File.exists?(File.join(@cookbook_path, "metadata.rb")) + @metadata_filenames << File.join(@cookbook_path, "metadata.rb") + elsif File.exists?(File.join(@cookbook_path, "metadata.json")) + @metadata_filenames << File.join(@cookbook_path, "metadata.json") + end + + if empty? + Chef::Log.warn "found a directory #{cookbook_name} in the cookbook path, but it contains no cookbook files. skipping." + end + @cookbook_settings + end + + def cookbook_version + return nil if empty? + + Chef::CookbookVersion.new(@cookbook_name.to_sym).tap do |c| + c.root_dir = @cookbook_path + c.attribute_filenames = cookbook_settings[:attribute_filenames].values + c.definition_filenames = cookbook_settings[:definition_filenames].values + c.recipe_filenames = cookbook_settings[:recipe_filenames].values + c.template_filenames = cookbook_settings[:template_filenames].values + c.file_filenames = cookbook_settings[:file_filenames].values + c.library_filenames = cookbook_settings[:library_filenames].values + c.resource_filenames = cookbook_settings[:resource_filenames].values + c.provider_filenames = cookbook_settings[:provider_filenames].values + c.root_filenames = cookbook_settings[:root_filenames].values + c.metadata_filenames = @metadata_filenames + c.metadata = metadata(c) + end + end + + # Generates the Cookbook::Metadata object + def metadata(cookbook_version) + @metadata = Chef::Cookbook::Metadata.new(cookbook_version) + @metadata_filenames.each do |metadata_file| + case metadata_file + when /\.rb$/ + apply_ruby_metadata(metadata_file) + when /\.json$/ + apply_json_metadata(metadata_file) + else + raise RuntimeError, "Invalid metadata file: #{metadata_file} for cookbook: #{cookbook_version}" + end + end + @metadata + end + + def empty? + cookbook_settings.inject(true) do |all_empty, files| + all_empty && files.last.empty? + end + end + + def merge!(other_cookbook_loader) + other_cookbook_settings = other_cookbook_loader.cookbook_settings + @cookbook_settings.each do |file_type, file_list| + file_list.merge!(other_cookbook_settings[file_type]) + end + @metadata_filenames.concat(other_cookbook_loader.metadata_filenames) + end + + def chefignore + @chefignore ||= Chefignore.new(File.basename(@cookbook_path)) + end + + def load_root_files + Dir.glob(File.join(@cookbook_path, '*'), File::FNM_DOTMATCH).each do |file| + next if File.directory?(file) + @cookbook_settings[:root_filenames][file[@relative_path, 1]] = file + end + end + + def load_recursively_as(category, category_dir, glob) + file_spec = File.join(@cookbook_path, category_dir, '**', glob) + Dir.glob(file_spec, File::FNM_DOTMATCH).each do |file| + next if File.directory?(file) + @cookbook_settings[category][file[@relative_path, 1]] = file + end + end + + def load_as(category, *path_glob) + Dir[File.join(@cookbook_path, *path_glob)].each do |file| + @cookbook_settings[category][file[@relative_path, 1]] = file + end + end + + def remove_ignored_files + @cookbook_settings.each_value do |file_list| + file_list.reject! do |relative_path, full_path| + chefignore.ignored?(relative_path) + end + end + end + + def apply_ruby_metadata(file) + begin + @metadata.from_file(file) + rescue JSON::ParserError + Chef::Log.error("Error evaluating metadata.rb for #@cookbook_name in " + file) + raise + end + end + + def apply_json_metadata(file) + begin + @metadata.from_json(IO.read(file)) + rescue JSON::ParserError + Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + file) + raise + end + end + + end + end +end diff --git a/lib/chef/cookbook/file_system_file_vendor.rb b/lib/chef/cookbook/file_system_file_vendor.rb new file mode 100644 index 0000000000..8896e3ed30 --- /dev/null +++ b/lib/chef/cookbook/file_system_file_vendor.rb @@ -0,0 +1,56 @@ +#-- +# Author:: Christopher Walters (<cw@opscode.com>) +# Author:: Tim Hinderliter (<tim@opscode.com>) +# Copyright:: Copyright (c) 2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/cookbook/file_vendor' + +class Chef + class Cookbook + # == Chef::Cookbook::FileSystemFileVendor + # This FileVendor loads files from Chef::Config.cookbook_path. The + # thing that's sort of janky about this FileVendor implementation is + # that it basically takes only the cookbook's name from the manifest + # and throws the rest away then re-builds the list of files on the + # disk. This is due to the manifest not having the on-disk file + # locations, since in the chef-client case, that information is + # non-sensical. + class FileSystemFileVendor < FileVendor + + def initialize(manifest, *repo_paths) + @cookbook_name = manifest[:cookbook_name] + @repo_paths = repo_paths.flatten + raise ArgumentError, "You must specify at least one repo path" if @repo_paths.empty? + end + + # Implements abstract base's requirement. It looks in the + # Chef::Config.cookbook_path file hierarchy for the requested + # file. + def get_filename(filename) + location = @repo_paths.inject(nil) do |memo, basepath| + candidate_location = File.join(basepath, @cookbook_name, filename) + memo = candidate_location if File.exist?(candidate_location) + memo + end + raise "File #{filename} does not exist for cookbook #{@cookbook_name}" unless location + + location + end + + end + end +end diff --git a/lib/chef/cookbook/file_vendor.rb b/lib/chef/cookbook/file_vendor.rb new file mode 100644 index 0000000000..38eab185ca --- /dev/null +++ b/lib/chef/cookbook/file_vendor.rb @@ -0,0 +1,48 @@ +# +# Author:: Christopher Walters (<cw@opscode.com>) +# Author:: Tim Hinderliter (<tim@opscode.com>) +# Copyright:: Copyright (c) 2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class Chef + class Cookbook + # == Chef::Cookbook::FileVendor + # This class handles fetching of cookbook files based on specificity. + class FileVendor + + def self.on_create(&block) + @instance_creator = block + end + + # Factory method that creates the appropriate kind of + # Cookbook::FileVendor to serve the contents of the manifest + def self.create_from_manifest(manifest) + raise "Must call Chef::Cookbook::FileVendor.on_create before calling create_from_manifest factory" unless defined?(@instance_creator) + @instance_creator.call(manifest) + end + + # Gets the on-disk location for the given cookbook file. + # + # Subclasses are responsible for determining exactly how the + # files are obtained and where they are stored. + def get_filename(filename) + raise NotImplemented, "Subclasses must implement this method" + end + + end + end +end diff --git a/lib/chef/cookbook/metadata.rb b/lib/chef/cookbook/metadata.rb new file mode 100644 index 0000000000..8398de442c --- /dev/null +++ b/lib/chef/cookbook/metadata.rb @@ -0,0 +1,629 @@ +# +# Author:: Adam Jacob (<adam@opscode.com>) +# Author:: AJ Christensen (<aj@opscode.com>) +# Author:: Seth Falcon (<seth@opscode.com>) +# Copyright:: Copyright 2008-2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/mash' +require 'chef/mixin/from_file' +require 'chef/mixin/params_validate' +require 'chef/mixin/check_helper' +require 'chef/log' +require 'chef/version_class' +require 'chef/version_constraint' + +class Chef + class Cookbook + + # == Chef::Cookbook::Metadata + # Chef::Cookbook::Metadata provides a convenient DSL for declaring metadata + # about Chef Cookbooks. + class Metadata + + NAME = 'name'.freeze + DESCRIPTION = 'description'.freeze + LONG_DESCRIPTION = 'long_description'.freeze + MAINTAINER = 'maintainer'.freeze + MAINTAINER_EMAIL = 'maintainer_email'.freeze + LICENSE = 'license'.freeze + PLATFORMS = 'platforms'.freeze + DEPENDENCIES = 'dependencies'.freeze + RECOMMENDATIONS = 'recommendations'.freeze + SUGGESTIONS = 'suggestions'.freeze + CONFLICTING = 'conflicting'.freeze + PROVIDING = 'providing'.freeze + REPLACING = 'replacing'.freeze + ATTRIBUTES = 'attributes'.freeze + GROUPINGS = 'groupings'.freeze + RECIPES = 'recipes'.freeze + VERSION = 'version'.freeze + + COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer, + :maintainer_email, :license, :platforms, :dependencies, + :recommendations, :suggestions, :conflicting, :providing, + :replacing, :attributes, :groupings, :recipes, :version] + + VERSION_CONSTRAINTS = {:depends => DEPENDENCIES, + :recommends => RECOMMENDATIONS, + :suggests => SUGGESTIONS, + :conflicts => CONFLICTING, + :provides => PROVIDING, + :replaces => REPLACING } + + include Chef::Mixin::CheckHelper + include Chef::Mixin::ParamsValidate + include Chef::Mixin::FromFile + + attr_reader :cookbook, + :platforms, + :dependencies, + :recommendations, + :suggestions, + :conflicting, + :providing, + :replacing, + :attributes, + :groupings, + :recipes, + :version + + # Builds a new Chef::Cookbook::Metadata object. + # + # === Parameters + # cookbook<String>:: An optional cookbook object + # maintainer<String>:: An optional maintainer + # maintainer_email<String>:: An optional maintainer email + # license<String>::An optional license. Default is Apache v2.0 + # + # === Returns + # metadata<Chef::Cookbook::Metadata> + def initialize(cookbook=nil, maintainer='YOUR_COMPANY_NAME', maintainer_email='YOUR_EMAIL', license='none') + @cookbook = cookbook + @name = cookbook ? cookbook.name : "" + @long_description = "" + self.maintainer(maintainer) + self.maintainer_email(maintainer_email) + self.license(license) + self.description('A fabulous new cookbook') + @platforms = Mash.new + @dependencies = Mash.new + @recommendations = Mash.new + @suggestions = Mash.new + @conflicting = Mash.new + @providing = Mash.new + @replacing = Mash.new + @attributes = Mash.new + @groupings = Mash.new + @recipes = Mash.new + @version = Version.new "0.0.0" + if cookbook + @recipes = cookbook.fully_qualified_recipe_names.inject({}) do |r, e| + e = self.name if e =~ /::default$/ + r[e] = "" + self.provides e + r + end + end + end + + def ==(other) + COMPARISON_FIELDS.inject(true) do |equal_so_far, field| + equal_so_far && other.respond_to?(field) && (other.send(field) == send(field)) + end + end + + # Sets the cookbooks maintainer, or returns it. + # + # === Parameters + # maintainer<String>:: The maintainers name + # + # === Returns + # maintainer<String>:: Returns the current maintainer. + def maintainer(arg=nil) + set_or_return( + :maintainer, + arg, + :kind_of => [ String ] + ) + end + + # Sets the maintainers email address, or returns it. + # + # === Parameters + # maintainer_email<String>:: The maintainers email address + # + # === Returns + # maintainer_email<String>:: Returns the current maintainer email. + def maintainer_email(arg=nil) + set_or_return( + :maintainer_email, + arg, + :kind_of => [ String ] + ) + end + + # Sets the current license, or returns it. + # + # === Parameters + # license<String>:: The current license. + # + # === Returns + # license<String>:: Returns the current license + def license(arg=nil) + set_or_return( + :license, + arg, + :kind_of => [ String ] + ) + end + + # Sets the current description, or returns it. Should be short - one line only! + # + # === Parameters + # description<String>:: The new description + # + # === Returns + # description<String>:: Returns the description + def description(arg=nil) + set_or_return( + :description, + arg, + :kind_of => [ String ] + ) + end + + # Sets the current long description, or returns it. Might come from a README, say. + # + # === Parameters + # long_description<String>:: The new long description + # + # === Returns + # long_description<String>:: Returns the long description + def long_description(arg=nil) + set_or_return( + :long_description, + arg, + :kind_of => [ String ] + ) + end + + # Sets the current cookbook version, or returns it. Can be two or three digits, seperated + # by dots. ie: '2.1', '1.5.4' or '0.9'. + # + # === Parameters + # version<String>:: The curent version, as a string + # + # === Returns + # version<String>:: Returns the current version + def version(arg=nil) + if arg + @version = Chef::Version.new(arg) + end + + @version.to_s + end + + # Sets the name of the cookbook, or returns it. + # + # === Parameters + # name<String>:: The curent cookbook name. + # + # === Returns + # name<String>:: Returns the current cookbook name. + def name(arg=nil) + set_or_return( + :name, + arg, + :kind_of => [ String ] + ) + end + + # Adds a supported platform, with version checking strings. + # + # === Parameters + # platform<String>,<Symbol>:: The platform (like :ubuntu or :mac_os_x) + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has + # the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def supports(platform, *version_args) + version = new_args_format(:supports, platform, version_args) + validate_version_constraint(:supports, platform, version) + @platforms[platform] = version + @platforms[platform] + end + + # Adds a dependency on another cookbook, with version checking strings. + # + # === Parameters + # cookbook<String>:: The cookbook + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has + # the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def depends(cookbook, *version_args) + version = new_args_format(:depends, cookbook, version_args) + validate_version_constraint(:depends, cookbook, version) + @dependencies[cookbook] = version + @dependencies[cookbook] + end + + # Adds a recommendation for another cookbook, with version checking strings. + # + # === Parameters + # cookbook<String>:: The cookbook + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has + # the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def recommends(cookbook, *version_args) + version = new_args_format(:recommends, cookbook, version_args) + validate_version_constraint(:recommends, cookbook, version) + @recommendations[cookbook] = version + @recommendations[cookbook] + end + + # Adds a suggestion for another cookbook, with version checking strings. + # + # === Parameters + # cookbook<String>:: The cookbook + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has the + # formx.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def suggests(cookbook, *version_args) + version = new_args_format(:suggests, cookbook, version_args) + validate_version_constraint(:suggests, cookbook, version) + @suggestions[cookbook] = version + @suggestions[cookbook] + end + + # Adds a conflict for another cookbook, with version checking strings. + # + # === Parameters + # cookbook<String>:: The cookbook + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has + # the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def conflicts(cookbook, *version_args) + version = new_args_format(:conflicts, cookbook, version_args) + validate_version_constraint(:conflicts, cookbook, version) + @conflicting[cookbook] = version + @conflicting[cookbook] + end + + # Adds a recipe, definition, or resource provided by this cookbook. + # + # Recipes are specified as normal + # Definitions are followed by (), and can include :params for prototyping + # Resources are the stringified version (service[apache2]) + # + # === Parameters + # recipe, definition, resource<String>:: The thing we provide + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has + # the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def provides(cookbook, *version_args) + version = new_args_format(:provides, cookbook, version_args) + validate_version_constraint(:provides, cookbook, version) + @providing[cookbook] = version + @providing[cookbook] + end + + # Adds a cookbook that is replaced by this one, with version checking strings. + # + # === Parameters + # cookbook<String>:: The cookbook we replace + # version<String>:: A version constraint of the form "OP VERSION", + # where OP is one of < <= = > >= ~> and VERSION has the form x.y.z or x.y. + # + # === Returns + # versions<Array>:: Returns the list of versions for the platform + def replaces(cookbook, *version_args) + version = new_args_format(:replaces, cookbook, version_args) + validate_version_constraint(:replaces, cookbook, version) + @replacing[cookbook] = version + @replacing[cookbook] + end + + # Adds a description for a recipe. + # + # === Parameters + # recipe<String>:: The recipe + # description<String>:: The description of the recipe + # + # === Returns + # description<String>:: Returns the current description + def recipe(name, description) + @recipes[name] = description + end + + # Adds an attribute )hat a user needs to configure for this cookbook. Takes + # a name (with the / notation for a nested attribute), followed by any of + # these options + # + # display_name<String>:: What a UI should show for this attribute + # description<String>:: A hint as to what this attr is for + # choice<Array>:: An array of choices to present to the user. + # calculated<Boolean>:: If true, the default value is calculated by the recipe and cannot be displayed. + # type<String>:: "string" or "array" - default is "string" ("hash" is supported for backwards compatibility) + # required<String>:: Whether this attr is 'required', 'recommended' or 'optional' - default 'optional' (true/false values also supported for backwards compatibility) + # recipes<Array>:: An array of recipes which need this attr set. + # default<String>,<Array>,<Hash>:: The default value + # + # === Parameters + # name<String>:: The name of the attribute ('foo', or 'apache2/log_dir') + # options<Hash>:: The description of the options + # + # === Returns + # options<Hash>:: Returns the current options hash + def attribute(name, options) + validate( + options, + { + :display_name => { :kind_of => String }, + :description => { :kind_of => String }, + :choice => { :kind_of => [ Array ], :default => [] }, + :calculated => { :equal_to => [ true, false ], :default => false }, + :type => { :equal_to => [ "string", "array", "hash", "symbol" ], :default => "string" }, + :required => { :equal_to => [ "required", "recommended", "optional", true, false ], :default => "optional" }, + :recipes => { :kind_of => [ Array ], :default => [] }, + :default => { :kind_of => [ String, Array, Hash ] } + } + ) + options[:required] = remap_required_attribute(options[:required]) unless options[:required].nil? + validate_string_array(options[:choice]) + validate_calculated_default_rule(options) + validate_choice_default_rule(options) + + @attributes[name] = options + @attributes[name] + end + + def grouping(name, options) + validate( + options, + { + :title => { :kind_of => String }, + :description => { :kind_of => String } + } + ) + @groupings[name] = options + @groupings[name] + end + + def to_hash + { + NAME => self.name, + DESCRIPTION => self.description, + LONG_DESCRIPTION => self.long_description, + MAINTAINER => self.maintainer, + MAINTAINER_EMAIL => self.maintainer_email, + LICENSE => self.license, + PLATFORMS => self.platforms, + DEPENDENCIES => self.dependencies, + RECOMMENDATIONS => self.recommendations, + SUGGESTIONS => self.suggestions, + CONFLICTING => self.conflicting, + PROVIDING => self.providing, + REPLACING => self.replacing, + ATTRIBUTES => self.attributes, + GROUPINGS => self.groupings, + RECIPES => self.recipes, + VERSION => self.version + } + end + + def to_json(*a) + self.to_hash.to_json(*a) + end + + def self.from_hash(o) + cm = self.new() + cm.from_hash(o) + cm + end + + def from_hash(o) + @name = o[NAME] if o.has_key?(NAME) + @description = o[DESCRIPTION] if o.has_key?(DESCRIPTION) + @long_description = o[LONG_DESCRIPTION] if o.has_key?(LONG_DESCRIPTION) + @maintainer = o[MAINTAINER] if o.has_key?(MAINTAINER) + @maintainer_email = o[MAINTAINER_EMAIL] if o.has_key?(MAINTAINER_EMAIL) + @license = o[LICENSE] if o.has_key?(LICENSE) + @platforms = o[PLATFORMS] if o.has_key?(PLATFORMS) + @dependencies = handle_deprecated_constraints(o[DEPENDENCIES]) if o.has_key?(DEPENDENCIES) + @recommendations = handle_deprecated_constraints(o[RECOMMENDATIONS]) if o.has_key?(RECOMMENDATIONS) + @suggestions = handle_deprecated_constraints(o[SUGGESTIONS]) if o.has_key?(SUGGESTIONS) + @conflicting = handle_deprecated_constraints(o[CONFLICTING]) if o.has_key?(CONFLICTING) + @providing = o[PROVIDING] if o.has_key?(PROVIDING) + @replacing = handle_deprecated_constraints(o[REPLACING]) if o.has_key?(REPLACING) + @attributes = o[ATTRIBUTES] if o.has_key?(ATTRIBUTES) + @groupings = o[GROUPINGS] if o.has_key?(GROUPINGS) + @recipes = o[RECIPES] if o.has_key?(RECIPES) + @version = o[VERSION] if o.has_key?(VERSION) + self + end + + def self.from_json(string) + o = Chef::JSONCompat.from_json(string) + self.from_hash(o) + end + + def self.validate_json(json_str) + o = Chef::JSONCompat.from_json(json_str) + metadata = new() + VERSION_CONSTRAINTS.each do |method_name, hash_key| + if constraints = o[hash_key] + constraints.each do |cb_name, constraints| + metadata.send(method_name, cb_name, *Array(constraints)) + end + end + end + true + end + + def from_json(string) + o = Chef::JSONCompat.from_json(string) + from_hash(o) + end + + private + + def new_args_format(caller_name, dep_name, version_constraints) + if version_constraints.empty? + ">= 0.0.0" + elsif version_constraints.size == 1 + version_constraints.first + else + msg=<<-OBSOLETED +The dependency specification syntax you are using is no longer valid. You may not +specify more than one version constraint for a particular cookbook. +Consult http://wiki.opscode.com/display/chef/Metadata for the updated syntax. + +Called by: #{caller_name} '#{dep_name}', #{version_constraints.map {|vc| vc.inspect}.join(", ")} +Called from: +#{caller[0...5].map {|line| " " + line}.join("\n")} +OBSOLETED + raise Exceptions::ObsoleteDependencySyntax, msg + end + end + + def validate_version_constraint(caller_name, dep_name, constraint_str) + Chef::VersionConstraint.new(constraint_str) + rescue Chef::Exceptions::InvalidVersionConstraint => e + Log.debug(e) + + msg=<<-INVALID +The version constraint syntax you are using is not valid. If you recently +upgraded to Chef 0.10.0, be aware that you no may longer use "<<" and ">>" for +'less than' and 'greater than'; use '<' and '>' instead. +Consult http://wiki.opscode.com/display/chef/Metadata for more information. + +Called by: #{caller_name} '#{dep_name}', '#{constraint_str}' +Called from: +#{caller[0...5].map {|line| " " + line}.join("\n")} +INVALID + raise Exceptions::InvalidVersionConstraint, msg + end + # Verify that the given array is an array of strings + # + # Raise an exception if the members of the array are not Strings + # + # === Parameters + # arry<Array>:: An array to be validated + def validate_string_array(arry) + if arry.kind_of?(Array) + arry.each do |choice| + validate( {:choice => choice}, {:choice => {:kind_of => String}} ) + end + end + end + + # For backwards compatibility, remap Boolean values to String + # true is mapped to "required" + # false is mapped to "optional" + # + # === Parameters + # required_attr<String><Boolean>:: The value of options[:required] + # + # === Returns + # required_attr<String>:: "required", "recommended", or "optional" + def remap_required_attribute(value) + case value + when true + value = "required" + when false + value = "optional" + end + value + end + + def validate_calculated_default_rule(options) + calculated_conflict = ((options[:default].is_a?(Array) && !options[:default].empty?) || + (options[:default].is_a?(String) && !options[:default] != "")) && + options[:calculated] == true + raise ArgumentError, "Default cannot be specified if calculated is true!" if calculated_conflict + end + + def validate_choice_default_rule(options) + return if !options[:choice].is_a?(Array) || options[:choice].empty? + + if options[:default].is_a?(String) && options[:default] != "" + raise ArgumentError, "Default must be one of your choice values!" if options[:choice].index(options[:default]) == nil + end + + if options[:default].is_a?(Array) && !options[:default].empty? + options[:default].each do |val| + raise ArgumentError, "Default values must be a subset of your choice values!" if options[:choice].index(val) == nil + end + end + end + + # This method translates version constraint strings from + # cookbooks with the old format. + # + # Before we began respecting version constraints, we allowed + # multiple constraints to be placed on cookbooks, as well as the + # << and >> operators, which are now just < and >. For + # specifications with more than one constraint, we return an + # empty array (otherwise, we're silently abiding only part of + # the contract they have specified to us). If there is only one + # constraint, we are replacing the old << and >> with the new < + # and >. + def handle_deprecated_constraints(specification) + specification.inject(Mash.new) do |acc, (cb, constraints)| + constraints = Array(constraints) + acc[cb] = (constraints.empty? || constraints.size > 1) ? [] : constraints.first.gsub(/>>/, '>').gsub(/<</, '<') + acc + end + end + + end + + #== Chef::Cookbook::MinimalMetadata + # MinimalMetadata is a duck type of Cookbook::Metadata, used + # internally by Chef Server when determining the optimal set of + # cookbooks for a node. + # + # MinimalMetadata objects typically contain only enough information + # to solve the cookbook collection for a run list, but not enough to + # generate the proper response + class MinimalMetadata < Metadata + def initialize(name, params) + @name = name + from_hash(params) + end + end + + + end +end diff --git a/lib/chef/cookbook/remote_file_vendor.rb b/lib/chef/cookbook/remote_file_vendor.rb new file mode 100644 index 0000000000..49de62cf65 --- /dev/null +++ b/lib/chef/cookbook/remote_file_vendor.rb @@ -0,0 +1,84 @@ +# +# Author:: Tim Hinderliter (<tim@opscode.com>) +# Copyright:: Copyright (c) 2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/cookbook/file_vendor' + +class Chef + class Cookbook + # == Chef::Cookbook::RemoteFileVendor + # This FileVendor loads files by either fetching them from the local cache, or + # if not available, loading them from the remote server. + class RemoteFileVendor < FileVendor + + def initialize(manifest, rest) + @manifest = manifest + @cookbook_name = @manifest[:cookbook_name] + @rest = rest + end + + # Implements abstract base's requirement. It looks in the + # Chef::Config.cookbook_path file hierarchy for the requested + # file. + def get_filename(filename) + if filename =~ /([^\/]+)\/(.+)$/ + segment = $1 + else + raise "get_filename: Cannot determine segment/filename for incoming filename #{filename}" + end + + raise "No such segment #{segment} in cookbook #{@cookbook_name}" unless @manifest[segment] + found_manifest_record = @manifest[segment].find {|manifest_record| manifest_record[:path] == filename } + raise "No such file #{filename} in #{@cookbook_name}" unless found_manifest_record + + cache_filename = File.join("cookbooks", @cookbook_name, found_manifest_record['path']) + + # update valid_cache_entries so the upstream cache cleaner knows what + # we've used. + validate_cached_copy(cache_filename) + + current_checksum = nil + if Chef::FileCache.has_key?(cache_filename) + current_checksum = Chef::CookbookVersion.checksum_cookbook_file(Chef::FileCache.load(cache_filename, false)) + end + + # If the checksums are different between on-disk (current) and on-server + # (remote, per manifest), do the update. This will also execute if there + # is no current checksum. + if current_checksum != found_manifest_record['checksum'] + raw_file = @rest.get_rest(found_manifest_record[:url], true) + + Chef::Log.debug("Storing updated #{cache_filename} in the cache.") + Chef::FileCache.move_to(raw_file.path, cache_filename) + else + Chef::Log.debug("Not fetching #{cache_filename}, as the cache is up to date.") + Chef::Log.debug("current checksum: #{current_checksum}; manifest checksum: #{found_manifest_record['checksum']})") + end + + full_path_cache_filename = Chef::FileCache.load(cache_filename, false) + + # return the filename, not the contents (second argument= false) + full_path_cache_filename + end + + def validate_cached_copy(cache_filename) + CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename) + end + + end + end +end diff --git a/lib/chef/cookbook/synchronizer.rb b/lib/chef/cookbook/synchronizer.rb new file mode 100644 index 0000000000..54cadb941c --- /dev/null +++ b/lib/chef/cookbook/synchronizer.rb @@ -0,0 +1,216 @@ +require 'chef/client' +require 'singleton' + +class Chef + + # Keep track of the filenames that we use in both eager cookbook + # downloading (during sync_cookbooks) and lazy (during the run + # itself, through FileVendor). After the run is over, clean up the + # cache. + class CookbookCacheCleaner + + # Setup a notification to clear the valid_cache_entries when a Chef client + # run starts + Chef::Client.when_run_starts do |run_status| + instance.reset! + end + + # Register a notification to cleanup unused files from cookbooks + Chef::Client.when_run_completes_successfully do |run_status| + instance.cleanup_file_cache + end + + include Singleton + + def initialize + reset! + end + + def reset! + @valid_cache_entries = {} + end + + def mark_file_as_valid(cache_path) + @valid_cache_entries[cache_path] = true + end + + def cache + Chef::FileCache + end + + def cleanup_file_cache + unless Chef::Config[:solo] + # Delete each file in the cache that we didn't encounter in the + # manifest. + cache.find(File.join(%w{cookbooks ** *})).each do |cache_filename| + unless @valid_cache_entries[cache_filename] + Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by chef-client.") + cache.delete(cache_filename) + end + end + end + end + + end + + # Synchronizes the locally cached copies of cookbooks with the files on the + # server. + class CookbookSynchronizer + EAGER_SEGMENTS = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup + EAGER_SEGMENTS.delete(:files) + EAGER_SEGMENTS.delete(:templates) + EAGER_SEGMENTS.freeze + + def initialize(cookbooks_by_name, events) + @cookbooks_by_name, @events = cookbooks_by_name, events + end + + def cache + Chef::FileCache + end + + def cookbook_names + @cookbooks_by_name.keys + end + + def cookbooks + @cookbooks_by_name.values + end + + def cookbook_count + @cookbooks_by_name.size + end + + def have_cookbook?(cookbook_name) + @cookbooks_by_name.key?(cookbook_name) + end + + # Synchronizes all the cookbooks from the chef-server. + #) + # === Returns + # true:: Always returns true + def sync_cookbooks + Chef::Log.info("Loading cookbooks [#{cookbook_names.sort.join(', ')}]") + Chef::Log.debug("Cookbooks detail: #{cookbooks.inspect}") + + clear_obsoleted_cookbooks + + @events.cookbook_sync_start(cookbook_count) + + # Synchronize each of the node's cookbooks, and add to the + # valid_cache_entries hash. + cookbooks.each do |cookbook| + sync_cookbook(cookbook) + end + + rescue Exception => e + @events.cookbook_sync_failed(cookbooks, e) + raise + else + @events.cookbook_sync_complete + true + end + + # Iterates over cached cookbooks' files, removing files belonging to + # cookbooks that don't appear in +cookbook_hash+ + def clear_obsoleted_cookbooks + @events.cookbook_clean_start + # Remove all cookbooks no longer relevant to this node + cache.find(File.join(%w{cookbooks ** *})).each do |cache_file| + cache_file =~ /^cookbooks\/([^\/]+)\// + unless have_cookbook?($1) + Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.") + cache.delete(cache_file) + @events.removed_cookbook_file(cache_file) + end + end + @events.cookbook_clean_complete + end + + # Sync the eagerly loaded files contained by +cookbook+ + # + # === Arguments + # cookbook<Chef::Cookbook>:: The cookbook to update + # valid_cache_entries<Hash>:: Out-param; Added to this hash are the files that + # were referred to by this cookbook + def sync_cookbook(cookbook) + Chef::Log.debug("Synchronizing cookbook #{cookbook.name}") + + # files and templates are lazily loaded, and will be done later. + + EAGER_SEGMENTS.each do |segment| + segment_filenames = Array.new + cookbook.manifest[segment].each do |manifest_record| + + cache_filename = sync_file_in_cookbook(cookbook, manifest_record) + # make the segment filenames a full path. + full_path_cache_filename = cache.load(cache_filename, false) + segment_filenames << full_path_cache_filename + end + + # replace segment filenames with a full-path one. + if segment.to_sym == :recipes + cookbook.recipe_filenames = segment_filenames + elsif segment.to_sym == :attributes + cookbook.attribute_filenames = segment_filenames + else + cookbook.segment_filenames(segment).replace(segment_filenames) + end + end + @events.synchronized_cookbook(cookbook.name) + end + + # Sync an individual file if needed. If there is an up to date copy + # locally, nothing is done. + # + # === Arguments + # file_manifest::: A Hash of the form {"path" => 'relative/path', "url" => "location to fetch the file"} + # === Returns + # Path to the cached file as a String + def sync_file_in_cookbook(cookbook, file_manifest) + cache_filename = File.join("cookbooks", cookbook.name, file_manifest['path']) + mark_cached_file_valid(cache_filename) + + # If the checksums are different between on-disk (current) and on-server + # (remote, per manifest), do the update. This will also execute if there + # is no current checksum. + if !cached_copy_up_to_date?(cache_filename, file_manifest['checksum']) + download_file(file_manifest['url'], cache_filename) + @events.updated_cookbook_file(cookbook.name, cache_filename) + else + Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.") + end + + cache_filename + end + + def cached_copy_up_to_date?(local_path, expected_checksum) + if cache.has_key?(local_path) + current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false)) + expected_checksum == current_checksum + else + false + end + end + + # Unconditionally download the file from the given URL. File will be + # downloaded to the path +destination+ which is relative to the Chef file + # cache root. + def download_file(url, destination) + raw_file = server_api.get_rest(url, true) + + Chef::Log.info("Storing updated #{destination} in the cache.") + cache.move_to(raw_file.path, destination) + end + + # Marks the given file as valid (non-stale). + def mark_cached_file_valid(cache_filename) + CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename) + end + + def server_api + Chef::REST.new(Chef::Config[:chef_server_url]) + end + + end +end diff --git a/lib/chef/cookbook/syntax_check.rb b/lib/chef/cookbook/syntax_check.rb new file mode 100644 index 0000000000..bf7c45e252 --- /dev/null +++ b/lib/chef/cookbook/syntax_check.rb @@ -0,0 +1,136 @@ +# +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Copyright:: Copyright (c) 2010 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/checksum_cache' +require 'chef/mixin/shell_out' + +class Chef + class Cookbook + # == Chef::Cookbook::SyntaxCheck + # Encapsulates the process of validating the ruby syntax of files in Chef + # cookbooks. + class SyntaxCheck + include Chef::Mixin::ShellOut + + attr_reader :cookbook_path + + # Creates a new SyntaxCheck given the +cookbook_name+ and a +cookbook_path+. + # If no +cookbook_path+ is given, +Chef::Config.cookbook_path+ is used. + def self.for_cookbook(cookbook_name, cookbook_path=nil) + cookbook_path ||= Chef::Config.cookbook_path + unless cookbook_path + raise ArgumentError, "Cannot find cookbook #{cookbook_name} unless Chef::Config.cookbook_path is set or an explicit cookbook path is given" + end + new(File.join(cookbook_path, cookbook_name.to_s)) + end + + # Create a new SyntaxCheck object + # === Arguments + # cookbook_path::: the (on disk) path to the cookbook + def initialize(cookbook_path) + @cookbook_path = cookbook_path + end + + def cache + Chef::ChecksumCache.instance + end + + def ruby_files + Dir[File.join(cookbook_path, '**', '*.rb')] + end + + def untested_ruby_files + ruby_files.reject do |file| + if validated?(file) + Chef::Log.debug("Ruby file #{file} is unchanged, skipping syntax check") + true + else + false + end + end + end + + def template_files + Dir[File.join(cookbook_path, '**', '*.erb')] + end + + def untested_template_files + template_files.reject do |file| + if validated?(file) + Chef::Log.debug("Template #{file} is unchanged, skipping syntax check") + true + else + false + end + end + end + + def validated?(file) + !!cache.lookup_checksum(cache_key(file), File.stat(file)) + end + + def validated(file) + cache.generate_checksum(cache_key(file), file, File.stat(file)) + end + + def cache_key(file) + @cache_keys ||= {} + @cache_keys[file] ||= cache.generate_key(file, "chef-test") + end + + def validate_ruby_files + untested_ruby_files.each do |ruby_file| + return false unless validate_ruby_file(ruby_file) + validated(ruby_file) + end + end + + def validate_templates + untested_template_files.each do |template| + return false unless validate_template(template) + validated(template) + end + end + + def validate_template(erb_file) + Chef::Log.debug("Testing template #{erb_file} for syntax errors...") + result = shell_out("erubis -x #{erb_file} | ruby -c") + result.error! + true + rescue Mixlib::ShellOut::ShellCommandFailed + file_relative_path = erb_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1] + Chef::Log.fatal("Erb template #{file_relative_path} has a syntax error:") + result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) } + false + end + + def validate_ruby_file(ruby_file) + Chef::Log.debug("Testing #{ruby_file} for syntax errors...") + result = shell_out("ruby -c #{ruby_file}") + result.error! + true + rescue Mixlib::ShellOut::ShellCommandFailed + file_relative_path = ruby_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1] + Chef::Log.fatal("Cookbook file #{file_relative_path} has a ruby syntax error:") + result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) } + false + end + + end + end +end |