From 8f62f18a24e3213ef4b2f13a5abf0135bf6c2429 Mon Sep 17 00:00:00 2001 From: Thom May Date: Tue, 21 Mar 2017 10:31:40 +0000 Subject: RFC 67: Remove cookbook segments This implements RFC 67, which removes cookbook segments, and moves to a single list of all the files contained in a cookbook. This allows us to move forward with better audit modes and also proper shipping of ohai plugins. Signed-off-by: Thom May --- Gemfile | 2 +- Gemfile.lock | 14 +- acceptance/Gemfile.lock | 10 +- chef.gemspec | 2 +- kitchen-tests/Berksfile.lock | 6 +- kitchen-tests/Gemfile.lock | 29 +-- .../chef_server/chef_server_root_dir.rb | 4 +- .../chef_server/cookbook_artifacts_dir.rb | 8 +- .../file_system/chef_server/cookbook_dir.rb | 39 ++-- .../file_system/chef_server/cookbooks_dir.rb | 6 +- .../chef_server/versioned_cookbooks_dir.rb | 6 +- .../chef_repository_file_system_cookbook_dir.rb | 7 +- lib/chef/cookbook/cookbook_version_loader.rb | 85 +-------- lib/chef/cookbook/file_system_file_vendor.rb | 2 +- lib/chef/cookbook/manifest_v0.rb | 63 +++++++ lib/chef/cookbook/manifest_v2.rb | 41 +++++ lib/chef/cookbook/remote_file_vendor.rb | 6 +- lib/chef/cookbook/synchronizer.rb | 45 ++--- lib/chef/cookbook_manifest.rb | 156 +++++++++------- lib/chef/cookbook_site_streaming_uploader.rb | 16 +- lib/chef/cookbook_uploader.rb | 2 +- lib/chef/cookbook_version.rb | 202 ++++++--------------- lib/chef/http/socketless_chef_zero_client.rb | 1 + lib/chef/knife/cookbook_download.rb | 7 +- lib/chef/knife/cookbook_show.rb | 10 +- lib/chef/policy_builder/expand_node_object.rb | 3 +- lib/chef/policy_builder/policyfile.rb | 13 +- lib/chef/run_context/cookbook_compiler.rb | 2 +- .../knife/chef_repository_file_system_spec.rb | 97 ---------- spec/integration/knife/cookbook_download_spec.rb | 24 --- spec/integration/knife/cookbook_show_spec.rb | 35 ++-- spec/support/shared/context/client.rb | 2 +- spec/unit/client_spec.rb | 4 +- spec/unit/cookbook/cookbook_version_loader_spec.rb | 41 +++-- spec/unit/cookbook/file_vendor_spec.rb | 29 ++- spec/unit/cookbook/metadata_spec.rb | 5 +- spec/unit/cookbook/synchronizer_spec.rb | 11 +- spec/unit/cookbook_loader_spec.rb | 28 +-- spec/unit/cookbook_manifest_spec.rb | 56 ++---- spec/unit/cookbook_site_streaming_uploader_spec.rb | 4 - spec/unit/cookbook_spec.rb | 24 +-- spec/unit/cookbook_uploader_spec.rb | 2 +- .../unit/cookbook_version_file_specificity_spec.rb | 83 ++++++--- spec/unit/cookbook_version_spec.rb | 147 ++------------- spec/unit/http/socketless_chef_zero_client_spec.rb | 3 +- spec/unit/knife/cookbook_download_spec.rb | 74 +++++--- spec/unit/knife/cookbook_show_spec.rb | 51 +++++- spec/unit/policy_builder/policyfile_spec.rb | 24 +-- spec/unit/run_context/cookbook_compiler_spec.rb | 4 +- 49 files changed, 654 insertions(+), 881 deletions(-) create mode 100644 lib/chef/cookbook/manifest_v0.rb create mode 100644 lib/chef/cookbook/manifest_v2.rb diff --git a/Gemfile b/Gemfile index 49fc34986c..343c1f3f9f 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ gem "ohai", git: "https://github.com/chef/ohai.git" gem "chef-config", path: File.expand_path("../chef-config", __FILE__) if File.exist?(File.expand_path("../chef-config", __FILE__)) gem "rake" gem "bundler" -gem "cheffish" # required for rspec tests +gem "cheffish", "~> 13" # required for rspec tests group(:omnibus_package) do gem "appbundler" diff --git a/Gemfile.lock b/Gemfile.lock index 32fb91889e..8d9447bee3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/chef/chef-server - revision: 867fb0245d5c162195cdf33f9e923c732cf512c3 + revision: b670bd5f2ed1f56a39ab0c211efd4204b6978422 specs: oc-chef-pedant (2.2.0) activesupport (>= 4.2.7.1, < 6.0) @@ -117,7 +117,7 @@ PATH addressable bundler (>= 1.10) chef-config (= 13.0.92) - chef-zero (>= 4.8) + chef-zero (>= 13.0) diff-lcs (~> 1.2, >= 1.2.4) erubis (~> 2.7) ffi-yajl (~> 2.2) @@ -146,7 +146,7 @@ PATH addressable bundler (>= 1.10) chef-config (= 13.0.92) - chef-zero (>= 4.8) + chef-zero (>= 13.0) diff-lcs (~> 1.2, >= 1.2.4) erubis (~> 2.7) ffi (~> 1.9) @@ -222,14 +222,14 @@ GEM logify (~> 0.1) mime-types chef-sugar (3.4.0) - chef-zero (5.3.2) + chef-zero (13.0.0) ffi-yajl (~> 2.2) hashie (>= 2.0, < 4.0) mixlib-log (~> 1.3) rack (~> 2.0) uuidtools (~> 2.1) - cheffish (5.0.1) - chef-zero (~> 5.0) + cheffish (13.0.0) + chef-zero (~> 13.0) net-ssh chefspec (5.4.0) chef (>= 12.0) @@ -571,7 +571,7 @@ DEPENDENCIES chef! chef-config! chef-sugar - cheffish + cheffish (~> 13) chefspec chefstyle! cucumber (>= 2.4.0) diff --git a/acceptance/Gemfile.lock b/acceptance/Gemfile.lock index dfb3e68b67..eb139cc225 100644 --- a/acceptance/Gemfile.lock +++ b/acceptance/Gemfile.lock @@ -12,13 +12,13 @@ GEM addressable (2.5.1) public_suffix (~> 2.0, >= 2.0.2) artifactory (2.8.1) - aws-sdk (2.8.14) - aws-sdk-resources (= 2.8.14) - aws-sdk-core (2.8.14) + aws-sdk (2.9.1) + aws-sdk-resources (= 2.9.1) + aws-sdk-core (2.9.1) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sdk-resources (2.8.14) - aws-sdk-core (= 2.8.14) + aws-sdk-resources (2.9.1) + aws-sdk-core (= 2.9.1) aws-sigv4 (1.0.0) berkshelf (5.6.4) addressable (~> 2.3, >= 2.3.4) diff --git a/chef.gemspec b/chef.gemspec index a44c882089..eab62b8eaa 100644 --- a/chef.gemspec +++ b/chef.gemspec @@ -32,7 +32,7 @@ Gem::Specification.new do |s| s.add_dependency "erubis", "~> 2.7" s.add_dependency "diff-lcs", "~> 1.2", ">= 1.2.4" - s.add_dependency "chef-zero", ">= 4.8" + s.add_dependency "chef-zero", ">= 13.0" s.add_dependency "plist", "~> 3.2" s.add_dependency "iniparse", "~> 1.4" diff --git a/kitchen-tests/Berksfile.lock b/kitchen-tests/Berksfile.lock index 620c23e5ba..5231491687 100644 --- a/kitchen-tests/Berksfile.lock +++ b/kitchen-tests/Berksfile.lock @@ -57,7 +57,7 @@ GRAPH build-essential (8.0.0) mingw (>= 1.1) seven_zip (>= 0.0.0) - chef-client (7.2.0) + chef-client (7.2.1) cron (>= 1.7.0) logrotate (>= 1.9.0) windows (>= 1.42.0) @@ -76,7 +76,7 @@ GRAPH windows (>= 2.0) inifile_chef_gem (0.1.0) build-essential (>= 0.0.0) - iptables (4.0.0) + iptables (4.0.1) logrotate (2.1.0) compat_resource (>= 0.0.0) mariadb (1.3.0) @@ -118,7 +118,7 @@ GRAPH ubuntu (2.0.1) apt (>= 0.0.0) users (4.0.3) - windows (3.0.3) + windows (3.0.4) ohai (>= 4.0.0) xml (3.1.1) build-essential (>= 0.0.0) diff --git a/kitchen-tests/Gemfile.lock b/kitchen-tests/Gemfile.lock index a9981abe91..8881c6b839 100644 --- a/kitchen-tests/Gemfile.lock +++ b/kitchen-tests/Gemfile.lock @@ -1,16 +1,16 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.5.0) + addressable (2.5.1) public_suffix (~> 2.0, >= 2.0.2) artifactory (2.8.1) - aws-sdk (2.8.13) - aws-sdk-resources (= 2.8.13) - aws-sdk-core (2.8.13) + aws-sdk (2.9.1) + aws-sdk-resources (= 2.9.1) + aws-sdk-core (2.9.1) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sdk-resources (2.8.13) - aws-sdk-core (= 2.8.13) + aws-sdk-resources (2.9.1) + aws-sdk-core (= 2.9.1) aws-sigv4 (1.0.0) berkshelf (5.6.4) addressable (~> 2.3, >= 2.3.4) @@ -73,7 +73,8 @@ GEM hitimes (1.2.4) hitimes (1.2.4-x86-mingw32) httpclient (2.8.3) - inspec (1.17.0) + inspec (1.18.0) + addressable (~> 2.5) faraday (>= 0.9.0) hashie (~> 3.4) json (>= 1.8, < 3.0) @@ -113,7 +114,7 @@ GEM little-plugger (~> 1.1) multi_json (~> 1.10) method_source (0.8.2) - mini_portile2 (2.1.0) + mini_portile (0.6.2) minitar (0.6.1) mixlib-archive (0.4.1) mixlib-log @@ -140,10 +141,10 @@ GEM net-ssh-gateway (1.3.0) net-ssh (>= 2.6.5) nio4r (2.0.0) - nokogiri (1.7.1) - mini_portile2 (~> 2.1.0) - nokogiri (1.7.1-x86-mingw32) - mini_portile2 (~> 2.1.0) + nokogiri (1.6.6.4) + mini_portile (~> 0.6.0) + nokogiri (1.6.6.4-x86-mingw32) + mini_portile (~> 0.6.0) nori (2.6.0) octokit (4.6.2) sawyer (~> 0.8.0, >= 0.5.3) @@ -203,7 +204,7 @@ GEM solve (3.1.0) molinillo (>= 0.5) semverse (>= 1.1, < 3.0) - sslshake (1.0.13) + sslshake (1.1.0) test-kitchen (1.16.0) mixlib-install (>= 1.2, < 3.0) mixlib-shellout (>= 1.2, < 3.0) @@ -217,7 +218,7 @@ GEM hitimes toml (0.1.2) parslet (~> 1.5.0) - train (0.22.1) + train (0.23.0) docker-api (~> 1.26) json (>= 1.8, < 3.0) mixlib-shellout (~> 2.0) diff --git a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb index 5030a0733f..63ce71ef40 100644 --- a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb +++ b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb @@ -103,11 +103,11 @@ class Chef end def get_json(path) - Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key, :api_version => "0").get(path) + chef_rest.get(path) end def chef_rest - Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key) + Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key, :api_version => "0") end def api_path diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb index 0b82a64a0a..e4df7858a7 100644 --- a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb +++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb @@ -56,7 +56,7 @@ class Chef # to make this work. So instead, we make a temporary cookbook # symlinking back to real cookbook, and upload the proxy. def upload_cookbook(other, options) - cookbook_name, dash, identifier = other.name.rpartition("-") + cookbook_name, _, identifier = other.name.rpartition("-") Dir.mktmpdir do |temp_cookbooks_path| proxy_cookbook_path = "#{temp_cookbooks_path}/#{cookbook_name}" @@ -73,7 +73,7 @@ class Chef cookbook_to_upload.freeze_version if options[:freeze] # Instantiate a new uploader based on the proxy loader - uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: root.chef_rest, policy_mode: true) + uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: chef_rest, policy_mode: true) with_actual_cookbooks_dir(temp_cookbooks_path) do uploader.upload_cookbooks @@ -92,6 +92,10 @@ class Chef end end + def chef_rest + Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions)) + end + def can_have_child?(name, is_dir) is_dir && name.include?("-") end diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb index 8f5faf2183..64488ed705 100644 --- a/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb +++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb @@ -49,18 +49,6 @@ class Chef attr_reader :cookbook_name, :version - COOKBOOK_SEGMENT_INFO = { - :attributes => { :ruby_only => true }, - :definitions => { :ruby_only => true }, - :recipes => { :ruby_only => true }, - :libraries => { :recursive => true }, - :templates => { :recursive => true }, - :files => { :recursive => true }, - :resources => { :ruby_only => true, :recursive => true }, - :providers => { :ruby_only => true, :recursive => true }, - :root_files => {}, - } - def add_child(child) @children << child end @@ -80,34 +68,29 @@ class Chef end def can_have_child?(name, is_dir) - # A cookbook's root may not have directories unless they are segment directories - return name != "root_files" && COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) if is_dir + return name != "root_files" if is_dir true end def children if @children.nil? @children = [] - manifest = chef_object.manifest - COOKBOOK_SEGMENT_INFO.each do |segment, segment_info| - next unless manifest.has_key?(segment) - - # Go through each file in the manifest for the segment, and - # add cookbook subdirs and files for it. - manifest[segment].each do |segment_file| - parts = segment_file[:path].split("/") + manifest = chef_object.cookbook_manifest + manifest.by_parent_directory.each do |segment, files| + files.each do |file| + parts = file[:path].split("/") # Get or create the path to the file container = self parts[0, parts.length - 1].each do |part| old_container = container container = old_container.children.find { |child| part == child.name } if !container - container = CookbookSubdir.new(part, old_container, segment_info[:ruby_only], segment_info[:recursive]) + container = CookbookSubdir.new(part, old_container, false, true) old_container.add_child(container) end end # Create the file itself - container.add_child(CookbookFile.new(parts[parts.length - 1], container, segment_file)) + container.add_child(CookbookFile.new(parts[parts.length - 1], container, file)) end end @children = @children.sort_by { |c| c.name } @@ -165,7 +148,11 @@ class Chef end def rest - parent.rest + Chef::ServerAPI.new(parent.rest.url, parent.rest.options.merge(version_class: Chef::CookbookManifestVersions)) + end + + def chef_rest + Chef::ServerAPI.new(parent.chef_rest.url, parent.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions)) end def chef_object @@ -187,7 +174,7 @@ class Chef old_retry_count = Chef::Config[:http_retry_count] begin Chef::Config[:http_retry_count] = 0 - @chef_object ||= Chef::CookbookVersion.from_hash(root.get_json(api_path)) + @chef_object ||= Chef::CookbookVersion.from_hash(chef_rest.get(api_path)) ensure Chef::Config[:http_retry_count] = old_retry_count end diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb index 631562d7ef..4e8e68e364 100644 --- a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb +++ b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb @@ -74,13 +74,17 @@ class Chef def upload_cookbook(other, options) cookbook_to_upload = other.chef_object cookbook_to_upload.freeze_version if options[:freeze] - uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest) + uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => chef_rest) with_actual_cookbooks_dir(other.parent.file_path) do uploader.upload_cookbooks end end + def chef_rest + Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions)) + end + # Work around the fact that CookbookUploader doesn't understand chef_repo_path (yet) def with_actual_cookbooks_dir(actual_cookbook_path) old_cookbook_path = Chef::Config.cookbook_path diff --git a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb index 172405763a..8da3718136 100644 --- a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb +++ b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb @@ -78,7 +78,7 @@ class Chef cookbook_to_upload.freeze_version if options[:freeze] # Instantiate a new uploader based on the proxy loader - uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest) + uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => chef_rest) with_actual_cookbooks_dir(temp_cookbooks_path) do uploader.upload_cookbooks @@ -97,6 +97,10 @@ class Chef end end + def chef_rest + Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions)) + end + def can_have_child?(name, is_dir) is_dir && name =~ Chef::ChefFS::FileSystem::ChefServer::VersionedCookbookDir::VALID_VERSIONED_COOKBOOK_NAME end diff --git a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb index 31b538b9ce..b296901dd1 100644 --- a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb +++ b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb @@ -97,9 +97,9 @@ class Chef end def can_have_child?(name, is_dir) - if is_dir + if is_dir && !%w{ root_files .. . }.include?(name) # Only the given directories will be uploaded. - return Chef::ChefFS::FileSystem::ChefServer::CookbookDir::COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) && name != "root_files" + return true elsif name == Chef::Cookbook::CookbookVersionLoader::UPLOADED_COOKBOOK_VERSION_FILE return false end @@ -128,8 +128,7 @@ class Chef protected def make_child_entry(child_name) - segment_info = Chef::ChefFS::FileSystem::ChefServer::CookbookDir::COOKBOOK_SEGMENT_INFO[child_name.to_sym] || {} - ChefRepositoryFileSystemCookbookEntry.new(child_name, self, nil, segment_info[:ruby_only], segment_info[:recursive]) + ChefRepositoryFileSystemCookbookEntry.new(child_name, self, nil, false, true) end def cookbook_version diff --git a/lib/chef/cookbook/cookbook_version_loader.rb b/lib/chef/cookbook/cookbook_version_loader.rb index b9de9a7482..35dac27fa5 100644 --- a/lib/chef/cookbook/cookbook_version_loader.rb +++ b/lib/chef/cookbook/cookbook_version_loader.rb @@ -9,15 +9,6 @@ class Chef class Cookbook class CookbookVersionLoader - FILETYPES_SUBJECT_TO_IGNORE = [ :attribute_filenames, - :definition_filenames, - :recipe_filenames, - :template_filenames, - :file_filenames, - :library_filenames, - :resource_filenames, - :provider_filenames] - UPLOADED_COOKBOOK_VERSION_FILE = ".uploaded-cookbook-version.json".freeze attr_reader :cookbook_settings @@ -44,16 +35,7 @@ class Chef @relative_path = /#{Regexp.escape(@cookbook_path)}\/(.+)$/ @metadata_loaded = false @cookbook_settings = { - :all_files => {}, - :attribute_filenames => {}, - :definition_filenames => {}, - :recipe_filenames => {}, - :template_filenames => {}, - :file_filenames => {}, - :library_filenames => {}, - :resource_filenames => {}, - :provider_filenames => {}, - :root_filenames => {}, + :all_files => {}, } @metadata_filenames = [] @@ -84,16 +66,6 @@ class Chef remove_ignored_files - load_as(:attribute_filenames, "attributes", "*.rb") - load_as(:definition_filenames, "definitions", "*.rb") - load_as(:recipe_filenames, "recipes", "*.rb") - load_recursively_as(:library_filenames, "libraries", "*") - load_recursively_as(:template_filenames, "templates", "*") - load_recursively_as(:file_filenames, "files", "*") - load_recursively_as(:resource_filenames, "resources", "*.rb") - load_recursively_as(:provider_filenames, "providers", "*.rb") - load_root_files - if empty? Chef::Log.warn "Found a directory #{cookbook_name} in the cookbook path, but it contains no cookbook files. skipping." end @@ -126,16 +98,6 @@ class Chef Chef::CookbookVersion.new(cookbook_name, *cookbook_paths).tap do |c| c.all_files = cookbook_settings[:all_files].values - c.attribute_filenames = cookbook_settings[:attribute_filenames].values - c.definition_filenames = cookbook_settings[:definition_filenames].values - c.recipe_filenames = cookbook_settings[:recipe_filenames].values - c.template_filenames = cookbook_settings[:template_filenames].values - c.file_filenames = cookbook_settings[:file_filenames].values - c.library_filenames = cookbook_settings[:library_filenames].values - c.resource_filenames = cookbook_settings[:resource_filenames].values - c.provider_filenames = cookbook_settings[:provider_filenames].values - c.root_filenames = cookbook_settings[:root_filenames].values - c.metadata_filenames = metadata_filenames c.metadata = metadata c.freeze_version if @frozen @@ -253,51 +215,6 @@ class Chef end end - def load_root_files - select_files_by_glob(File.join(Chef::Util::PathHelper.escape_glob_dir(cookbook_path), "*"), File::FNM_DOTMATCH).each do |file| - file = Chef::Util::PathHelper.cleanpath(file) - next if File.directory?(file) - next if File.basename(file) == UPLOADED_COOKBOOK_VERSION_FILE - name = Chef::Util::PathHelper.relative_path_from(@cookbook_path, file) - cookbook_settings[:root_filenames][name] = file - end - end - - def load_recursively_as(category, category_dir, glob) - glob_pattern = File.join(Chef::Util::PathHelper.escape_glob_dir(cookbook_path, category_dir), "**", glob) - select_files_by_glob(glob_pattern, File::FNM_DOTMATCH).each do |file| - file = Chef::Util::PathHelper.cleanpath(file) - name = Chef::Util::PathHelper.relative_path_from(@cookbook_path, file) - cookbook_settings[category][name] = file - end - end - - def load_as(category, *path_glob) - glob_pattern = File.join(Chef::Util::PathHelper.escape_glob_dir(cookbook_path), *path_glob) - select_files_by_glob(glob_pattern).each do |file| - file = Chef::Util::PathHelper.cleanpath(file) - name = Chef::Util::PathHelper.relative_path_from(@cookbook_path, file) - cookbook_settings[category][name] = file - end - end - - # Mimic Dir.glob inside a cookbook by running `File.fnmatch?` against - # `cookbook_settings[:all_files]`. - # - # @param pattern [String] a glob string passed to `File.fnmatch?` - # @param option [Integer] Option flag to control globbing behavior. These - # are constants defined on `File`, such as `File::FNM_DOTMATCH`. - # `File.fnmatch?` and `Dir.glob` only take one option argument, if you - # need to combine options, you must `|` the constants together. To make - # `File.fnmatch?` behave like `Dir.glob`, `File::FNM_PATHNAME` is - # always enabled. - def select_files_by_glob(pattern, option = 0) - combined_opts = option | File::FNM_PATHNAME - cookbook_settings[:all_files].values.select do |path| - File.fnmatch?(pattern, path, combined_opts) - end - end - def remove_ignored_files cookbook_settings[:all_files].reject! do |relative_path, full_path| chefignore.ignored?(relative_path) diff --git a/lib/chef/cookbook/file_system_file_vendor.rb b/lib/chef/cookbook/file_system_file_vendor.rb index 8088ed00cd..1f43095ea3 100644 --- a/lib/chef/cookbook/file_system_file_vendor.rb +++ b/lib/chef/cookbook/file_system_file_vendor.rb @@ -35,7 +35,7 @@ class Chef attr_reader :repo_paths def initialize(manifest, *repo_paths) - @cookbook_name = manifest[:cookbook_name] + @cookbook_name = manifest.name @repo_paths = repo_paths.flatten raise ArgumentError, "You must specify at least one repo path" if repo_paths.empty? end diff --git a/lib/chef/cookbook/manifest_v0.rb b/lib/chef/cookbook/manifest_v0.rb new file mode 100644 index 0000000000..3e50d86071 --- /dev/null +++ b/lib/chef/cookbook/manifest_v0.rb @@ -0,0 +1,63 @@ +# Author:: Daniel DeLeo () +# Copyright:: Copyright 2015-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "chef/json_compat" +require "chef/mixin/versioned_api" + +class Chef + class Cookbook + class ManifestV0 + extend Chef::Mixin::VersionedAPI + + minimum_api_version 0 + + COOKBOOK_SEGMENTS = %w{ resources providers recipes definitions libraries attributes files templates root_files } + + def self.from_hash(hash) + response = Mash.new + response[:all_files] = COOKBOOK_SEGMENTS.inject([]) do |memo, segment| + next memo if hash[segment].nil? || hash[segment].empty? + hash[segment].each do |file| + file["name"] = "#{segment}/#{file["name"]}" unless segment == "root_files" + memo << file + end + memo + end + response + end + + def self.to_hash(manifest) + result = manifest.manifest.dup + result.delete("all_files") + + files = manifest.by_parent_directory + files.keys.inject(result) do |memo, parent| + if COOKBOOK_SEGMENTS.include?(parent) + memo[parent] ||= [] + files[parent].each do |file| + file["name"] = file["name"].split("/")[1] unless parent == "root_files" + file.delete("full_path") + memo[parent] << file + end + end + memo + end + + result.merge({ "frozen?" => manifest.frozen_version?, "chef_type" => "cookbook_version" }) + end + end + end +end diff --git a/lib/chef/cookbook/manifest_v2.rb b/lib/chef/cookbook/manifest_v2.rb new file mode 100644 index 0000000000..59b5c9afb0 --- /dev/null +++ b/lib/chef/cookbook/manifest_v2.rb @@ -0,0 +1,41 @@ +# Copyright:: Copyright 2015-2016, Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "chef/json_compat" +require "chef/mixin/versioned_api" + +class Chef + class Cookbook + class ManifestV2 + extend Chef::Mixin::VersionedAPI + + minimum_api_version 2 + + def self.from_hash(hash) + Chef::Log.debug "processing manifest: #{hash}" + Mash.new hash + end + + def self.to_hash(manifest) + result = manifest.manifest.dup + result["all_files"].map! { |file| file.delete("full_path"); file } + result["frozen?"] = manifest.frozen_version? + result["chef_type"] = "cookbook_version" + result.to_hash + end + + end + end +end diff --git a/lib/chef/cookbook/remote_file_vendor.rb b/lib/chef/cookbook/remote_file_vendor.rb index e63d094dc4..cfd7789311 100644 --- a/lib/chef/cookbook/remote_file_vendor.rb +++ b/lib/chef/cookbook/remote_file_vendor.rb @@ -30,7 +30,7 @@ class Chef def initialize(manifest, rest) @manifest = manifest - @cookbook_name = @manifest[:cookbook_name] || @manifest[:name] + @cookbook_name = @manifest.name @rest = rest end @@ -44,8 +44,8 @@ class Chef raise "get_filename: Cannot determine segment/filename for incoming filename #{filename}" end - raise "No such segment #{segment} in cookbook #{@cookbook_name}" unless @manifest[segment] - found_manifest_record = @manifest[segment].find { |manifest_record| manifest_record[:path] == filename } + raise "No such segment #{segment} in cookbook #{@cookbook_name}" unless @manifest.files_for(segment) + found_manifest_record = @manifest.files_for(segment).find { |manifest_record| manifest_record[:path] == filename } raise "No such file #{filename} in #{@cookbook_name}" unless found_manifest_record cache_filename = File.join("cookbooks", @cookbook_name, found_manifest_record["path"]) diff --git a/lib/chef/cookbook/synchronizer.rb b/lib/chef/cookbook/synchronizer.rb index bb44bc3d5c..625f3b4f20 100644 --- a/lib/chef/cookbook/synchronizer.rb +++ b/lib/chef/cookbook/synchronizer.rb @@ -62,18 +62,11 @@ class Chef # Synchronizes the locally cached copies of cookbooks with the files on the # server. class CookbookSynchronizer - CookbookFile = Struct.new(:cookbook, :segment, :manifest_record) + CookbookFile = Struct.new(:cookbook, :manifest_record) attr_accessor :remove_obsoleted_files def initialize(cookbooks_by_name, events) - @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup - unless Chef::Config[:no_lazy_load] - @eager_segments.delete(:files) - @eager_segments.delete(:templates) - end - @eager_segments.freeze - @cookbooks_by_name, @events = cookbooks_by_name, events @cookbook_full_file_paths = {} @@ -101,15 +94,19 @@ class Chef end def cookbook_segment(cookbook_name, segment) - @cookbooks_by_name[cookbook_name].manifest[segment] + @cookbooks_by_name[cookbook_name].files_for(segment) end def files + exclude = unless Chef::Config[:no_lazy_load] + [ :files, :templates ] + else + [] + end + @files ||= cookbooks.inject([]) do |memo, cookbook| - @eager_segments.each do |segment| - cookbook.manifest[segment].each do |manifest_record| - memo << CookbookFile.new(cookbook, segment, manifest_record) - end + cookbook.each_file(excluded_parts: exclude) do |manifest_record| + memo << CookbookFile.new(cookbook, manifest_record) end memo end @@ -162,8 +159,10 @@ class Chef @events.cookbook_sync_start(cookbook_count) queue.process(Chef::Config[:cookbook_sync_threads]) + # Ensure that cookbooks know where they're rooted at, for manifest purposes. + ensure_cookbook_paths # Update the full file paths in the manifest - update_cookbook_filenames() + update_cookbook_filenames rescue Exception => e @events.cookbook_sync_failed(cookbooks, e) @@ -176,9 +175,8 @@ class Chef # Saves the full_path to the file of the cookbook to be updated # in the manifest later def save_full_file_path(file, full_path) - @cookbook_full_file_paths[file.cookbook] ||= {} - @cookbook_full_file_paths[file.cookbook][file.segment] ||= [ ] - @cookbook_full_file_paths[file.cookbook][file.segment] << full_path + @cookbook_full_file_paths[file.cookbook] ||= [] + @cookbook_full_file_paths[file.cookbook] << full_path end # remove cookbooks that are not referenced in the expanded run_list at all @@ -229,10 +227,15 @@ class Chef end def update_cookbook_filenames - @cookbook_full_file_paths.each do |cookbook, file_segments| - file_segments.each do |segment, full_paths| - cookbook.replace_segment_filenames(segment, full_paths) - end + @cookbook_full_file_paths.each do |cookbook, full_paths| + cookbook.all_files = full_paths + end + end + + def ensure_cookbook_paths + cookbooks.each do |cookbook| + cb_dir = File.join(Chef::Config[:file_cache_path], "cookbooks", cookbook.name) + cookbook.root_paths = Array(cb_dir) end end diff --git a/lib/chef/cookbook_manifest.rb b/lib/chef/cookbook_manifest.rb index d6de9dd029..125f353d21 100644 --- a/lib/chef/cookbook_manifest.rb +++ b/lib/chef/cookbook_manifest.rb @@ -15,7 +15,10 @@ # limitations under the License. require "forwardable" +require "chef/mixin/versioned_api" require "chef/util/path_helper" +require "chef/cookbook/manifest_v0" +require "chef/cookbook/manifest_v2" require "chef/log" class Chef @@ -24,17 +27,11 @@ class Chef # to a Chef Server. class CookbookManifest - # Duplicates the same constant in CookbookVersion. We cannot remove it - # there because it is treated by other code as part of CookbookVersion's - # public API (also used in some deprecated methods). - COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ].freeze - extend Forwardable attr_reader :cookbook_version def_delegator :@cookbook_version, :root_paths - def_delegator :@cookbook_version, :segment_filenames def_delegator :@cookbook_version, :name def_delegator :@cookbook_version, :identifier def_delegator :@cookbook_version, :metadata @@ -56,10 +53,7 @@ class Chef # the format used by the `cookbook_artifacts` endpoint (for policyfiles). # Setting this option also changes the behavior of #save_url and # #force_save_url such that CookbookVersions will be uploaded to the new - # `cookbook_artifacts` API. This endpoint is currently under active - # development and the format is expected to change frequently, therefore - # the result of #manifest, #to_hash, and #to_json will not be stable when - # `policy_mode` is enabled. + # `cookbook_artifacts` API. def initialize(cookbook_version, policy_mode: false) @cookbook_version = cookbook_version @policy_mode = !!policy_mode @@ -126,10 +120,7 @@ class Chef end def to_hash - result = manifest.dup - result["frozen?"] = frozen_version? - result["chef_type"] = "cookbook_version" - result.to_hash + CookbookManifestVersions.to_hash(self) end def to_json(*a) @@ -164,15 +155,48 @@ class Chef # make the corresponding changes to the cookbook_version object. Required # to provide backward compatibility with CookbookVersion#manifest= method. def update_from(new_manifest) - @manifest = Mash.new new_manifest + @manifest = Chef::CookbookManifestVersions.from_hash(new_manifest) @checksums = extract_checksums_from_manifest(@manifest) @manifest_records_by_path = extract_manifest_records_by_path(@manifest) + end + + def files_for(part) + return root_files if part.to_s == "root_files" + manifest[:all_files].select do |file| + seg = file[:name].split("/")[0] + part.to_s == seg + end + end + + def each_file(excluded_parts: [], &block) + excluded_parts = Array(excluded_parts).map { |p| p.to_s } - COOKBOOK_SEGMENTS.each do |segment| - next unless @manifest.has_key?(segment) - filenames = @manifest[segment].map { |manifest_record| manifest_record["name"] } + manifest[:all_files].each do |file| + seg = file[:name].split("/")[0] + next if excluded_parts.include?(seg) + yield file if block_given? + end + end + + def by_parent_directory + @by_parent_directory ||= + manifest[:all_files].inject({}) do |memo, file| + parts = file[:name].split("/") + parent = if parts.length == 1 + "root_files" + else + parts[0] + end + + memo[parent] ||= [] + memo[parent] << file + memo + end + end - cookbook_version.replace_segment_filenames(segment, filenames) + def root_files + manifest[:all_files].select do |file| + file[:name].split("/").length == 1 end end @@ -186,15 +210,7 @@ class Chef # See #preferred_manifest_record for a description an individual manifest record. def generate_manifest manifest = Mash.new({ - :recipes => Array.new, - :definitions => Array.new, - :libraries => Array.new, - :attributes => Array.new, - :files => Array.new, - :templates => Array.new, - :resources => Array.new, - :providers => Array.new, - :root_files => Array.new, + all_files: Array.new, }) @checksums = {} @@ -203,24 +219,24 @@ class Chef raise "Cookbook #{name} does not have root_paths! Cannot generate manifest." end - COOKBOOK_SEGMENTS.each do |segment| - segment_filenames(segment).each do |segment_file| - next if File.directory?(segment_file) + @cookbook_version.all_files.each do |file| + next if File.directory?(file) - path, specificity = parse_segment_file_from_root_paths(segment, segment_file) - file_name = File.basename(path) + name, path, specificity = parse_file_from_root_paths(file) - csum = checksum_cookbook_file(segment_file) - @checksums[csum] = segment_file - rs = Mash.new({ - :name => file_name, - :path => path, - :checksum => csum, - :specificity => specificity, - }) + csum = checksum_cookbook_file(file) + @checksums[csum] = file + rs = Mash.new({ + :name => name, + :path => path, + :checksum => csum, + :specificity => specificity, + # full_path is not a part of the normal manifest, but is very useful to keep around. + # uploaders should strip this out. + :full_path => file, + }) - manifest[segment] << rs - end + manifest[:all_files] << rs end manifest[:metadata] = metadata @@ -238,38 +254,42 @@ class Chef @manifest = manifest end - def parse_segment_file_from_root_paths(segment, segment_file) + def parse_file_from_root_paths(file) root_paths.each do |root_path| - pathname = Chef::Util::PathHelper.relative_path_from(root_path, segment_file) + pathname = Chef::Util::PathHelper.relative_path_from(root_path, file) parts = pathname.each_filename.take(2) # Check if path is actually under root_path next if parts[0] == ".." - if segment == :templates || segment == :files + + # if we have a root_file, such as metadata.rb, the first part will be "." + return [ pathname.to_s, pathname.to_s, "default" ] if parts.length == 1 + + segment = parts[0] + + name = File.join(segment, pathname.basename.to_s) + + if segment == "templates" || segment == "files" # Check if pathname looks like files/foo or templates/foo (unscoped) if pathname.each_filename.to_a.length == 2 # Use root_default in case the same path exists at root_default and default - return [ pathname.to_s, "root_default" ] + return [ name, pathname.to_s, "root_default" ] else - return [ pathname.to_s, parts[1] ] + return [ name, pathname.to_s, parts[1] ] end else - return [ pathname.to_s, "default" ] + return [ name, pathname.to_s, "default" ] end end - Chef::Log.error("Cookbook file #{segment_file} not under cookbook root paths #{root_paths.inspect}.") - raise "Cookbook file #{segment_file} not under cookbook root paths #{root_paths.inspect}." + Chef::Log.error("Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}.") + raise "Cookbook file #{file} not under cookbook root paths #{root_paths.inspect}." end def extract_checksums_from_manifest(manifest) - checksums = {} - COOKBOOK_SEGMENTS.each do |segment| - next unless manifest.has_key?(segment) - manifest[segment].each do |manifest_record| - checksums[manifest_record[:checksum]] = nil - end + manifest[:all_files].inject({}) do |memo, manifest_record| + memo[manifest_record[:checksum]] = nil + memo end - checksums end def checksum_cookbook_file(filepath) @@ -277,14 +297,20 @@ class Chef end def extract_manifest_records_by_path(manifest) - manifest_records_by_path = {} - COOKBOOK_SEGMENTS.each do |segment| - next unless manifest.has_key?(segment) - manifest[segment].each do |manifest_record| - manifest_records_by_path[manifest_record[:path]] = manifest_record - end + manifest[:all_files].inject({}) do |memo, manifest_record| + memo[manifest_record[:path]] = manifest_record + memo end - manifest_records_by_path end + + end + class CookbookManifestVersions + + extend Chef::Mixin::VersionedAPIFactory + add_versioned_api_class Chef::Cookbook::ManifestV0 + add_versioned_api_class Chef::Cookbook::ManifestV2 + + def_versioned_delegator :from_hash + def_versioned_delegator :to_hash end end diff --git a/lib/chef/cookbook_site_streaming_uploader.rb b/lib/chef/cookbook_site_streaming_uploader.rb index c0e85ff984..1641992eac 100644 --- a/lib/chef/cookbook_site_streaming_uploader.rb +++ b/lib/chef/cookbook_site_streaming_uploader.rb @@ -43,15 +43,13 @@ class Chef FileUtils.mkdir_p(tmp_cookbook_dir) Chef::Log.debug("Staging at #{tmp_cookbook_dir}") checksums_to_on_disk_paths = cookbook.checksums - Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment| - cookbook.manifest[segment].each do |manifest_record| - path_in_cookbook = manifest_record[:path] - on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]] - dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook) - FileUtils.mkdir_p(File.dirname(dest)) - Chef::Log.debug("Staging #{on_disk_path} to #{dest}") - FileUtils.cp(on_disk_path, dest) - end + cookbook.each_file do |manifest_record| + path_in_cookbook = manifest_record[:path] + on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]] + dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook) + FileUtils.mkdir_p(File.dirname(dest)) + Chef::Log.debug("Staging #{on_disk_path} to #{dest}") + FileUtils.cp(on_disk_path, dest) end # First, generate metadata diff --git a/lib/chef/cookbook_uploader.rb b/lib/chef/cookbook_uploader.rb index bb75234563..5e11314190 100644 --- a/lib/chef/cookbook_uploader.rb +++ b/lib/chef/cookbook_uploader.rb @@ -40,7 +40,7 @@ class Chef def initialize(cookbooks, opts = {}) @opts = opts @cookbooks = Array(cookbooks) - @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url]) + @rest = opts[:rest] || Chef::ServerAPI.new(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions) @concurrency = opts[:concurrency] || 10 @policy_mode = opts[:policy_mode] || false end diff --git a/lib/chef/cookbook_version.rb b/lib/chef/cookbook_version.rb index 7353941bb1..738f98929e 100644 --- a/lib/chef/cookbook_version.rb +++ b/lib/chef/cookbook_version.rb @@ -37,45 +37,23 @@ class Chef class CookbookVersion include Comparable + extend Forwardable + + def_delegator :@cookbook_manifest, :files_for + def_delegator :@cookbook_manifest, :each_file COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ] - attr_accessor :all_files + attr_reader :all_files attr_accessor :root_paths - attr_accessor :definition_filenames - attr_accessor :template_filenames - attr_accessor :file_filenames - attr_accessor :library_filenames - attr_accessor :resource_filenames - attr_accessor :provider_filenames - attr_accessor :root_filenames attr_accessor :name - attr_accessor :metadata_filenames - - def status=(new_status) - Chef.deprecated(:internal_api, "Deprecated method `status' called. This method will be removed.") - @status = new_status - end - - def status - Chef.deprecated(:internal_api, "Deprecated method `status' called. This method will be removed.") - @status - end # A Chef::Cookbook::Metadata object. It has a setter that fixes up the # metadata to add descriptions of the recipes contained in this # CookbookVersion. attr_reader :metadata - # attribute_filenames also has a setter that has non-default - # functionality. - attr_reader :attribute_filenames - - # recipe_filenames also has a setter that has non-default - # functionality. - attr_reader :recipe_filenames - attr_reader :recipe_filenames_by_name attr_reader :attribute_filenames_by_short_filename @@ -96,6 +74,11 @@ class Chef root_paths[0] end + def all_files=(files) + @all_files = Array(files) + cookbook_manifest.reset! + end + # This is the one and only method that knows how cookbook files' # checksums are generated. def self.checksum_cookbook_file(filepath) @@ -118,23 +101,10 @@ class Chef @root_paths = root_paths @frozen = false - @attribute_filenames = Array.new - @definition_filenames = Array.new - @template_filenames = Array.new - @file_filenames = Array.new - @recipe_filenames = Array.new - @recipe_filenames_by_name = Hash.new - @library_filenames = Array.new - @resource_filenames = Array.new - @provider_filenames = Array.new - @metadata_filenames = Array.new - @root_filenames = Array.new - @all_files = Array.new - # deprecated - @status = :ready @file_vendor = nil + @cookbook_manifest = Chef::CookbookManifest.new(self) @metadata = Chef::Cookbook::Metadata.new @chef_server_rest = chef_server_rest end @@ -163,10 +133,12 @@ class Chef "#{name}-#{version}" end - def attribute_filenames=(*filenames) - @attribute_filenames = filenames.flatten - @attribute_filenames_by_short_filename = filenames_by_name(attribute_filenames) - attribute_filenames + def attribute_filenames_by_short_filename + @attribute_filenames_by_short_filename ||= filenames_by_name(files_for("attributes")) + end + + def recipe_filenames_by_name + @recipe_filenames_by_name ||= filenames_by_name(files_for("recipes")) end def metadata=(metadata) @@ -175,14 +147,14 @@ class Chef @metadata end - ## BACKCOMPAT/DEPRECATED - Remove these and fix breakage before release [DAN - 5/20/2010]## - alias :attribute_files :attribute_filenames - alias :attribute_files= :attribute_filenames= - def manifest cookbook_manifest.manifest end + def manifest=(new_manifest) + cookbook_manifest.update_from(new_manifest) + end + # Returns a hash of checksums to either nil or the on disk path (which is # done by generate_manifest). def checksums @@ -193,29 +165,16 @@ class Chef cookbook_manifest.manifest_records_by_path end - def manifest=(new_manifest) - cookbook_manifest.update_from(new_manifest) - end - # Return recipe names in the form of cookbook_name::recipe_name def fully_qualified_recipe_names - results = Array.new - recipe_filenames_by_name.each_key do |rname| - results << "#{name}::#{rname}" + files_for("recipes").inject([]) do |memo, recipe| + rname = recipe[:name].split("/")[1] + rname = File.basename(rname, ".rb") + memo << "#{name}::#{rname}" + memo end - results end - def recipe_filenames=(*filenames) - @recipe_filenames = filenames.flatten - @recipe_filenames_by_name = filenames_by_name(recipe_filenames) - recipe_filenames - end - - ## BACKCOMPAT/DEPRECATED - Remove these and fix breakage before release [DAN - 5/20/2010]## - alias :recipe_files :recipe_filenames - alias :recipe_files= :recipe_filenames= - # called from DSL def load_recipe(recipe_name, run_context) unless recipe_filenames_by_name.has_key?(recipe_name) @@ -235,41 +194,7 @@ class Chef end def segment_filenames(segment) - unless COOKBOOK_SEGMENTS.include?(segment) - raise ArgumentError, "invalid segment #{segment}: must be one of #{COOKBOOK_SEGMENTS.join(', ')}" - end - - case segment.to_sym - when :resources - @resource_filenames - when :providers - @provider_filenames - when :recipes - @recipe_filenames - when :libraries - @library_filenames - when :definitions - @definition_filenames - when :attributes - @attribute_filenames - when :files - @file_filenames - when :templates - @template_filenames - when :root_files - @root_filenames - end - end - - def replace_segment_filenames(segment, filenames) - case segment.to_sym - when :recipes - self.recipe_filenames = filenames - when :attributes - self.attribute_filenames = filenames - else - segment_filenames(segment).replace(filenames) - end + files_for(segment).map { |f| f["full_path"] || File.join(root_dir, f["path"]) } end # Query whether a template file +template_filename+ is available. File @@ -349,7 +274,7 @@ class Chef filenames_by_pref = Hash.new preferences.each { |pref| filenames_by_pref[pref] = Array.new } - manifest[segment].each do |manifest_record| + files_for(segment).each do |manifest_record| manifest_record_path = manifest_record[:path] # find the NON SPECIFIC filenames, but prefer them by filespecificity. @@ -389,7 +314,7 @@ class Chef records_by_pref = Hash.new preferences.each { |pref| records_by_pref[pref] = Array.new } - manifest[segment].each do |manifest_record| + files_for(segment).each do |manifest_record| manifest_record_path = manifest_record[:path] # extract the preference part from the path. @@ -469,12 +394,22 @@ class Chef end private :preferences_for_path + def display + output = Mash.new + output["cookbook_name"] = name + output["name"] = full_name + output["frozen?"] = frozen_version? + output["metadata"] = metadata.to_hash + output["version"] = version + output.merge(cookbook_manifest.by_parent_directory) + end + def self.from_hash(o) cookbook_version = new(o["cookbook_name"] || o["name"]) # We want the Chef::Cookbook::Metadata object to always be inflated - cookbook_version.metadata = Chef::Cookbook::Metadata.from_hash(o["metadata"]) cookbook_version.manifest = o + cookbook_version.metadata = Chef::Cookbook::Metadata.from_hash(o["metadata"]) cookbook_version.identifier = o["identifier"] if o.key?("identifier") # We don't need the following step when we decide to stop supporting deprecated operators in the metadata (e.g. <<, >>) @@ -488,33 +423,6 @@ class Chef from_hash(o) end - # @deprecated This method was used by the Ruby Chef Server and is no longer - # needed. There is no replacement. - def generate_manifest_with_urls - Chef.deprecated(:internal_api, "Deprecated method #generate_manifest_with_urls.") - - rendered_manifest = manifest.dup - COOKBOOK_SEGMENTS.each do |segment| - if rendered_manifest.has_key?(segment) - rendered_manifest[segment].each do |manifest_record| - url_options = { :cookbook_name => name.to_s, :cookbook_version => version, :checksum => manifest_record["checksum"] } - manifest_record["url"] = yield(url_options) - end - end - end - rendered_manifest - end - - def to_hash - # TODO: this should become deprecated when the API for CookbookManifest becomes stable - cookbook_manifest.to_hash - end - - def to_json(*a) - # TODO: this should become deprecated when the API for CookbookManifest becomes stable - cookbook_manifest.to_json - end - def metadata_json_file File.join(root_paths[0], "metadata.json") end @@ -533,22 +441,12 @@ class Chef # REST API ## - def save_url - # TODO: this should become deprecated when the API for CookbookManifest becomes stable - cookbook_manifest.save_url - end - - def force_save_url - # TODO: this should become deprecated when the API for CookbookManifest becomes stable - cookbook_manifest.force_save_url - end - def chef_server_rest @chef_server_rest ||= chef_server_rest end def self.chef_server_rest - Chef::ServerAPI.new(Chef::Config[:chef_server_url]) + Chef::ServerAPI.new(Chef::Config[:chef_server_url], { version_class: Chef::CookbookManifestVersions }) end def destroy @@ -602,12 +500,12 @@ class Chef Chef::Version.new(version) <=> Chef::Version.new(other.version) end - private - def cookbook_manifest @cookbook_manifest ||= CookbookManifest.new(self) end + private + def find_preferred_manifest_record(node, segment, filename) preferences = preferences_for_path(node, segment, filename) @@ -615,15 +513,21 @@ class Chef preferences.find { |preferred_filename| manifest_records_by_path[preferred_filename] } end - # For each filename, produce a mapping of base filename (i.e. recipe name + # For each manifest record, produce a mapping of base filename (i.e. recipe name + # or attribute file) to on disk location + def relative_paths_by_name(records) + records.select { |record| record[:name] =~ /\.rb$/ }.inject({}) { |memo, record| memo[File.basename(record[:name], ".rb")] = record[:path]; memo } + end + + # For each manifest record, produce a mapping of base filename (i.e. recipe name # or attribute file) to on disk location - def filenames_by_name(filenames) - filenames.select { |filename| filename =~ /\.rb$/ }.inject({}) { |memo, filename| memo[File.basename(filename, ".rb")] = filename; memo } + def filenames_by_name(records) + records.select { |record| record[:name] =~ /\.rb$/ }.inject({}) { |memo, record| memo[File.basename(record[:name], ".rb")] = record[:full_path]; memo } end def file_vendor unless @file_vendor - @file_vendor = Chef::Cookbook::FileVendor.create_from_manifest(manifest) + @file_vendor = Chef::Cookbook::FileVendor.create_from_manifest(cookbook_manifest) end @file_vendor end diff --git a/lib/chef/http/socketless_chef_zero_client.rb b/lib/chef/http/socketless_chef_zero_client.rb index d2f1f45b1d..296330815a 100644 --- a/lib/chef/http/socketless_chef_zero_client.rb +++ b/lib/chef/http/socketless_chef_zero_client.rb @@ -170,6 +170,7 @@ class Chef "QUERY_STRING" => url.query, "SERVER_PORT" => url.port, "HTTP_HOST" => "localhost:#{url.port}", + "HTTP_X_OPS_SERVER_API_VERSION" => headers["X-Ops-Server-API-Version"], "rack.url_scheme" => "chefzero", "rack.input" => StringIO.new(body_str), } diff --git a/lib/chef/knife/cookbook_download.rb b/lib/chef/knife/cookbook_download.rb index 741f444093..b745e77f27 100644 --- a/lib/chef/knife/cookbook_download.rb +++ b/lib/chef/knife/cookbook_download.rb @@ -70,7 +70,7 @@ class Chef ui.info("Downloading #{@cookbook_name} cookbook version #{@version}") cookbook = Chef::CookbookVersion.load(@cookbook_name, @version) - manifest = cookbook.manifest + manifest = cookbook.cookbook_manifest basedir = File.join(config[:download_directory], "#{@cookbook_name}-#{cookbook.version}") if File.exists?(basedir) @@ -83,10 +83,9 @@ class Chef end end - Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment| - next unless manifest.has_key?(segment) + manifest.by_parent_directory.each do |segment, files| ui.info("Downloading #{segment}") - manifest[segment].each do |segment_file| + files.each do |segment_file| dest = File.join(basedir, segment_file["path"].gsub("/", File::SEPARATOR)) Chef::Log.debug("Downloading #{segment_file['path']} to #{dest}") FileUtils.mkdir_p(File.dirname(dest)) diff --git a/lib/chef/knife/cookbook_show.rb b/lib/chef/knife/cookbook_show.rb index d0c930de0a..1d9983632d 100644 --- a/lib/chef/knife/cookbook_show.rb +++ b/lib/chef/knife/cookbook_show.rb @@ -76,9 +76,13 @@ class Chef pretty_print(temp_file.read) when 3 # We are showing a specific part of the cookbook - output(cookbook.manifest[segment]) - when 2 # We are showing the whole cookbook data - output(cookbook) + if segment == "metadata" + output(cookbook.metadata) + else + output(cookbook.files_for(segment)) + end + when 2 # We are showing the whole cookbook + output(cookbook.display) when 1 # We are showing the cookbook versions (all of them) env = config[:environment] api_endpoint = env ? "environments/#{env}/cookbooks/#{cookbook_name}" : "cookbooks/#{cookbook_name}" diff --git a/lib/chef/policy_builder/expand_node_object.rb b/lib/chef/policy_builder/expand_node_object.rb index dbed44a002..26f39e8b73 100644 --- a/lib/chef/policy_builder/expand_node_object.rb +++ b/lib/chef/policy_builder/expand_node_object.rb @@ -239,7 +239,8 @@ class Chef end def api_service - @api_service ||= Chef::ServerAPI.new(config[:chef_server_url]) + @api_service ||= Chef::ServerAPI.new(config[:chef_server_url], + { version_class: Chef::CookbookManifestVersions }) end def config diff --git a/lib/chef/policy_builder/policyfile.rb b/lib/chef/policy_builder/policyfile.rb index f0009eac6c..f84e1dc68e 100644 --- a/lib/chef/policy_builder/policyfile.rb +++ b/lib/chef/policy_builder/policyfile.rb @@ -149,7 +149,7 @@ class Chef # # @return [Chef::RunContext] def setup_run_context(specific_recipes = nil) - Chef::Cookbook::FileVendor.fetch_from_remote(http_api) + Chef::Cookbook::FileVendor.fetch_from_remote(api_service) sync_cookbooks cookbook_collection = Chef::CookbookCollection.new(cookbooks_to_sync) cookbook_collection.validate! @@ -267,7 +267,7 @@ class Chef # @api private def policy - @policy ||= http_api.get(policyfile_location) + @policy ||= api_service.get(policyfile_location) rescue Net::HTTPServerException => e raise ConfigurationError, "Error loading policyfile from `#{policyfile_location}': #{e.class} - #{e.message}" end @@ -452,8 +452,9 @@ class Chef end # @api private - def http_api - @api_service ||= Chef::ServerAPI.new(config[:chef_server_url]) + def api_service + @api_service ||= Chef::ServerAPI.new(config[:chef_server_url], + { version_class: Chef::CookbookManifestVersions }) end # @api private @@ -496,7 +497,7 @@ class Chef def compat_mode_manifest_for(cookbook_name, lock_data) xyz_version = lock_data["dotted_decimal_identifier"] rel_url = "cookbooks/#{cookbook_name}/#{xyz_version}" - inflate_cbv_object(http_api.get(rel_url)) + inflate_cbv_object(api_service.get(rel_url)) rescue Exception => e message = "Error loading cookbook #{cookbook_name} at version #{xyz_version} from #{rel_url}: #{e.class} - #{e.message}" err = Chef::Exceptions::CookbookNotFound.new(message) @@ -507,7 +508,7 @@ class Chef def artifact_manifest_for(cookbook_name, lock_data) identifier = lock_data["identifier"] rel_url = "cookbook_artifacts/#{cookbook_name}/#{identifier}" - inflate_cbv_object(http_api.get(rel_url)) + inflate_cbv_object(api_service.get(rel_url)) rescue Exception => e message = "Error loading cookbook #{cookbook_name} with identifier #{identifier} from #{rel_url}: #{e.class} - #{e.message}" err = Chef::Exceptions::CookbookNotFound.new(message) diff --git a/lib/chef/run_context/cookbook_compiler.rb b/lib/chef/run_context/cookbook_compiler.rb index b2a8d236a3..94635be03d 100644 --- a/lib/chef/run_context/cookbook_compiler.rb +++ b/lib/chef/run_context/cookbook_compiler.rb @@ -268,7 +268,7 @@ class Chef # Lists the local paths to files in +cookbook+ of type +segment+ # (attribute, recipe, etc.), sorted lexically. def files_in_cookbook_by_segment(cookbook, segment) - cookbook_collection[cookbook].segment_filenames(segment).sort + cookbook_collection[cookbook].files_for(segment).map { |record| record[:full_path] }.sort end # Yields the name, as a symbol, of each cookbook depended on by diff --git a/spec/integration/knife/chef_repository_file_system_spec.rb b/spec/integration/knife/chef_repository_file_system_spec.rb index cc538c98c0..222d3aee8a 100644 --- a/spec/integration/knife/chef_repository_file_system_spec.rb +++ b/spec/integration/knife/chef_repository_file_system_spec.rb @@ -158,103 +158,6 @@ EOM end end - when_the_repository "has extraneous subdirectories and files under a cookbook" do - before do - directory "cookbooks/cookbook1" do - file "a.rb", "" - file "blarghle/blah.rb", "" - directory "attributes" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "definitions" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "recipes" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "libraries" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "templates" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "files" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "resources" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - directory "providers" do - file "a.rb", "" - file "b.json", {} - file "c/d.rb", "" - file "c/e.json", {} - end - end - end - - it "knife list --local -Rfp / should NOT return them" do - knife("list --local -Rfp /").should_succeed <= 0.0.0 + x::x: >= 0.0.0 recipes: - recommendations: - replacing: + x: + x::x: source_url: - suggestions: version: 1.0.0 name: x-1.0.0 -providers: recipes: checksum: 4631b34cf58de10c5ef1304889941b2e - name: default.rb + name: recipes/default.rb path: recipes/default.rb specificity: default url: http://127.0.0.1:8900/file_store/checksums/4631b34cf58de10c5ef1304889941b2e checksum: d41d8cd98f00b204e9800998ecf8427e - name: x.rb + name: recipes/x.rb path: recipes/x.rb specificity: default url: http://127.0.0.1:8900/file_store/checksums/d41d8cd98f00b204e9800998ecf8427e -resources: root_files: checksum: 8226671f751ba102dea6a6b6bd32fa8d name: metadata.rb path: metadata.rb specificity: default url: http://127.0.0.1:8900/file_store/checksums/8226671f751ba102dea6a6b6bd32fa8d -templates: version: 1.0.0 EOM end @@ -99,11 +89,9 @@ EOM knife("cookbook show x 1.0.0 metadata").should_succeed <= 0.0.0 + x::x: >= 0.0.0 recipes: -recommendations: -replacing: + x: + x::x: source_url: -suggestions: version: 1.0.0 EOM end @@ -126,13 +115,13 @@ EOM it "knife cookbook show x 1.0.0 recipes shows all the recipes" do knife("cookbook show x 1.0.0 recipes").should_succeed < [] }). and_return({}) diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index ec3f70b9b0..7ffc17c4fc 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -188,7 +188,7 @@ describe Chef::Client do # ---Client#sync_cookbooks -- downloads the list of cookbooks to sync # expect_any_instance_of(Chef::CookbookSynchronizer).to receive(:sync_cookbooks) - expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url]).and_return(http_cookbook_sync) + expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions).and_return(http_cookbook_sync) expect(http_cookbook_sync).to receive(:post). with("environments/_default/cookbook_versions", { :run_list => ["override_recipe"] }). and_return({}) @@ -222,7 +222,7 @@ describe Chef::Client do # ---Client#sync_cookbooks -- downloads the list of cookbooks to sync # expect_any_instance_of(Chef::CookbookSynchronizer).to receive(:sync_cookbooks) - expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url]).and_return(http_cookbook_sync) + expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions).and_return(http_cookbook_sync) expect(http_cookbook_sync).to receive(:post). with("environments/_default/cookbook_versions", { :run_list => ["new_run_list_recipe"] }). and_return({}) diff --git a/spec/unit/cookbook/cookbook_version_loader_spec.rb b/spec/unit/cookbook/cookbook_version_loader_spec.rb index 786e17f35b..40a054abee 100644 --- a/spec/unit/cookbook/cookbook_version_loader_spec.rb +++ b/spec/unit/cookbook/cookbook_version_loader_spec.rb @@ -40,42 +40,45 @@ describe Chef::Cookbook::CookbookVersionLoader do File.join(cookbook_path, cookbook_relative_path) end + def full_paths_for_part(part) + loaded_cookbook.files_for(part).inject([]) { |memo, f| memo << f[:full_path]; memo } + end + it "loads attribute files of the cookbook" do - expect(loaded_cookbook.attribute_filenames).to include(full_path("/attributes/default.rb")) - expect(loaded_cookbook.attribute_filenames).to include(full_path("/attributes/smokey.rb")) + expect(full_paths_for_part("attributes")).to include(full_path("/attributes/default.rb")) + expect(full_paths_for_part("attributes")).to include(full_path("/attributes/smokey.rb")) end it "loads definition files" do - expect(loaded_cookbook.definition_filenames).to include(full_path("/definitions/client.rb")) - expect(loaded_cookbook.definition_filenames).to include(full_path("/definitions/server.rb")) + expect(full_paths_for_part("definitions")).to include(full_path("/definitions/client.rb")) + expect(full_paths_for_part("definitions")).to include(full_path("/definitions/server.rb")) end it "loads recipes" do - expect(loaded_cookbook.recipe_filenames).to include(full_path("/recipes/default.rb")) - expect(loaded_cookbook.recipe_filenames).to include(full_path("/recipes/gigantor.rb")) - expect(loaded_cookbook.recipe_filenames).to include(full_path("/recipes/one.rb")) - expect(loaded_cookbook.recipe_filenames).to include(full_path("/recipes/return.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("/recipes/default.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("/recipes/gigantor.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("/recipes/one.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("/recipes/return.rb")) end it "loads libraries" do - expect(loaded_cookbook.library_filenames).to include(full_path("/libraries/openldap.rb")) - expect(loaded_cookbook.library_filenames).to include(full_path("/libraries/openldap/version.rb")) + expect(full_paths_for_part("libraries")).to include(full_path("/libraries/openldap.rb")) + expect(full_paths_for_part("libraries")).to include(full_path("/libraries/openldap/version.rb")) end it "loads static files in the files/ dir" do - expect(loaded_cookbook.file_filenames).to include(full_path("/files/default/remotedir/remotesubdir/remote_subdir_file1.txt")) - expect(loaded_cookbook.file_filenames).to include(full_path("/files/default/remotedir/remotesubdir/remote_subdir_file2.txt")) + expect(full_paths_for_part("files")).to include(full_path("/files/default/remotedir/remotesubdir/remote_subdir_file1.txt")) + expect(full_paths_for_part("files")).to include(full_path("/files/default/remotedir/remotesubdir/remote_subdir_file2.txt")) end it "loads files that start with a ." do - expect(loaded_cookbook.file_filenames).to include(full_path("/files/default/.dotfile")) - expect(loaded_cookbook.file_filenames).to include(full_path("/files/default/.ssh/id_rsa")) - expect(loaded_cookbook.file_filenames).to include(full_path("/files/default/remotedir/.a_dotdir/.a_dotfile_in_a_dotdir")) + expect(full_paths_for_part("files")).to include(full_path("/files/default/.dotfile")) + expect(full_paths_for_part("files")).to include(full_path("/files/default/.ssh/id_rsa")) + expect(full_paths_for_part("files")).to include(full_path("/files/default/remotedir/.a_dotdir/.a_dotfile_in_a_dotdir")) end it "loads root files that start with a ." do expect(loaded_cookbook.all_files).to include(full_path(".root_dotfile")) - expect(loaded_cookbook.root_filenames).to include(full_path(".root_dotfile")) end it "loads all unignored files, even if they don't match a segment type" do @@ -97,9 +100,9 @@ describe Chef::Cookbook::CookbookVersionLoader do let(:cookbook_path) { File.join(CHEF_SPEC_DATA, "kitchen/openldap") } it "skips ignored files" do - expect(loaded_cookbook.recipe_filenames).to include(full_path("recipes/gigantor.rb")) - expect(loaded_cookbook.recipe_filenames).to include(full_path("recipes/woot.rb")) - expect(loaded_cookbook.recipe_filenames).to_not include(full_path("recipes/ignoreme.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("recipes/gigantor.rb")) + expect(full_paths_for_part("recipes")).to include(full_path("recipes/woot.rb")) + expect(full_paths_for_part("recipes")).to_not include(full_path("recipes/ignoreme.rb")) end end diff --git a/spec/unit/cookbook/file_vendor_spec.rb b/spec/unit/cookbook/file_vendor_spec.rb index 164fbd8177..557e1b8775 100644 --- a/spec/unit/cookbook/file_vendor_spec.rb +++ b/spec/unit/cookbook/file_vendor_spec.rb @@ -16,6 +16,7 @@ # limitations under the License. # require "spec_helper" +require "chef/cookbook_version" describe Chef::Cookbook::FileVendor do @@ -25,6 +26,12 @@ describe Chef::Cookbook::FileVendor do let(:http) { double("Chef::ServerAPI") } + # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest + let(:manifest) do + cbv = Chef::CookbookVersion.new("bob", Array(Dir.tmpdir)) + cbv.cookbook_manifest + end + before do file_vendor_class.fetch_from_remote(http) end @@ -39,8 +46,11 @@ describe Chef::Cookbook::FileVendor do context "with a manifest from a cookbook version" do - # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest - let(:manifest) { { :cookbook_name => "bob", :name => "bob-1.2.3" } } + # # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest + # let(:manifest) do + # cbv = Chef::CookbookVersion.new("bob", Array(Dir.tmpdir)) + # cbv.cookbook_manifest + # end it "creates a RemoteFileVendor for a given manifest" do file_vendor = file_vendor_class.create_from_manifest(manifest) @@ -53,9 +63,6 @@ describe Chef::Cookbook::FileVendor do context "with a manifest from a cookbook artifact" do - # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest - let(:manifest) { { :name => "bob" } } - it "creates a RemoteFileVendor for a given manifest" do file_vendor = file_vendor_class.create_from_manifest(manifest) expect(file_vendor).to be_a_kind_of(Chef::Cookbook::RemoteFileVendor) @@ -70,8 +77,10 @@ describe Chef::Cookbook::FileVendor do let(:cookbook_path) { %w{/var/chef/cookbooks /var/chef/other_cookbooks} } - # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest - let(:manifest) { { :cookbook_name => "bob" } } + let(:manifest) do + cbv = Chef::CookbookVersion.new("bob", Array(Dir.tmpdir)) + cbv.cookbook_manifest + end before do file_vendor_class.fetch_from_disk(cookbook_path) @@ -97,8 +106,10 @@ describe Chef::Cookbook::FileVendor do context "when vendoring a cookbook with a name mismatch" do let(:cookbook_path) { File.join(CHEF_SPEC_DATA, "cookbooks") } - # A manifest is a Hash of the format defined by Chef::CookbookVersion#manifest - let(:manifest) { { :cookbook_name => "name-mismatch" } } + let(:manifest) do + cbv = Chef::CookbookVersion.new("name-mismatch", Array(Dir.tmpdir)) + cbv.cookbook_manifest + end before do file_vendor_class.fetch_from_disk(cookbook_path) diff --git a/spec/unit/cookbook/metadata_spec.rb b/spec/unit/cookbook/metadata_spec.rb index d1117127f1..4a18244ea0 100644 --- a/spec/unit/cookbook/metadata_spec.rb +++ b/spec/unit/cookbook/metadata_spec.rb @@ -765,7 +765,10 @@ describe Chef::Cookbook::Metadata do describe "recipes" do let(:cookbook) do c = Chef::CookbookVersion.new("test_cookbook") - c.recipe_files = [ "default.rb", "enlighten.rb" ] + c.manifest = { all_files: [ + { name: "recipes/default.rb", path: "recipes/default.rb", checksum: "my_only_friend" }, + { name: "recipes/enlighten.rb", path: "recipes/enlighten.rb", checksum: "my_only_friend" }, + ] } c end diff --git a/spec/unit/cookbook/synchronizer_spec.rb b/spec/unit/cookbook/synchronizer_spec.rb index 82876273e7..77e64482da 100644 --- a/spec/unit/cookbook/synchronizer_spec.rb +++ b/spec/unit/cookbook/synchronizer_spec.rb @@ -62,6 +62,7 @@ describe Chef::CookbookSynchronizer do let(:cookbook_a_default_recipe) do { "path" => "recipes/default.rb", + "name" => "recipes/default.rb", "url" => "http://chef.example.com/abc123", "checksum" => "abc123", } @@ -70,6 +71,7 @@ describe Chef::CookbookSynchronizer do let(:cookbook_a_default_attrs) do { "path" => "attributes/default.rb", + "name" => "attributes/default.rb", "url" => "http://chef.example.com/abc456", "checksum" => "abc456", } @@ -78,6 +80,7 @@ describe Chef::CookbookSynchronizer do let(:cookbook_a_template) do { "path" => "templates/default/apache2.conf.erb", + "name" => "templates/apache2.conf.erb", "url" => "http://chef.example.com/ffffff", "checksum" => "abc125", } @@ -86,18 +89,14 @@ describe Chef::CookbookSynchronizer do let(:cookbook_a_file) do { "path" => "files/default/megaman.conf", + "name" => "files/megaman.conf", "url" => "http://chef.example.com/megaman.conf", "checksum" => "abc124", } end let(:cookbook_a_manifest) do - segments = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ] - cookbook_a_manifest = segments.inject({}) { |h, segment| h[segment.to_s] = []; h } - cookbook_a_manifest["recipes"] = [ cookbook_a_default_recipe ] - cookbook_a_manifest["attributes"] = [ cookbook_a_default_attrs ] - cookbook_a_manifest["templates"] = [ cookbook_a_template ] - cookbook_a_manifest["files"] = [ cookbook_a_file ] + cookbook_a_manifest = { all_files: [ cookbook_a_default_recipe, cookbook_a_default_attrs, cookbook_a_template, cookbook_a_file ] } cookbook_a_manifest end diff --git a/spec/unit/cookbook_loader_spec.rb b/spec/unit/cookbook_loader_spec.rb index eef5d2afd5..dd731b53d3 100644 --- a/spec/unit/cookbook_loader_spec.rb +++ b/spec/unit/cookbook_loader_spec.rb @@ -31,6 +31,10 @@ describe Chef::CookbookLoader do let(:cookbook_loader) { Chef::CookbookLoader.new(repo_paths) } + def full_paths_for_part(cb, part) + cookbook_loader[cb].files_for(part).inject([]) { |memo, f| memo << f[:full_path]; memo } + end + it "checks each directory only once when loading (CHEF-3487)" do cookbook_paths = [] repo_paths.each do |repo_path| @@ -112,61 +116,61 @@ describe Chef::CookbookLoader do end it "should allow you to override an attribute file via cookbook_path" do - expect(cookbook_loader[:openldap].attribute_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "attributes").detect do |f| f =~ /cookbooks\/openldap\/attributes\/default.rb/ end).not_to eql(nil) - expect(cookbook_loader[:openldap].attribute_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "attributes").detect do |f| f =~ /kitchen\/openldap\/attributes\/default.rb/ end).to eql(nil) end it "should load different attribute files from deeper paths" do - expect(cookbook_loader[:openldap].attribute_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "attributes").detect do |f| f =~ /kitchen\/openldap\/attributes\/robinson.rb/ end).not_to eql(nil) end it "should allow you to override a definition file via cookbook_path" do - expect(cookbook_loader[:openldap].definition_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "definitions").detect do |f| f =~ /cookbooks\/openldap\/definitions\/client.rb/ end).not_to eql(nil) - expect(cookbook_loader[:openldap].definition_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "definitions").detect do |f| f =~ /kitchen\/openldap\/definitions\/client.rb/ end).to eql(nil) end it "should load definition files from deeper paths" do - expect(cookbook_loader[:openldap].definition_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "definitions").detect do |f| f =~ /kitchen\/openldap\/definitions\/drewbarrymore.rb/ end).not_to eql(nil) end it "should allow you to override a recipe file via cookbook_path" do - expect(cookbook_loader[:openldap].recipe_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "recipes").detect do |f| f =~ /cookbooks\/openldap\/recipes\/gigantor.rb/ end).not_to eql(nil) - expect(cookbook_loader[:openldap].recipe_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "recipes").detect do |f| f =~ /kitchen\/openldap\/recipes\/gigantor.rb/ end).to eql(nil) end it "should load recipe files from deeper paths" do - expect(cookbook_loader[:openldap].recipe_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "recipes").detect do |f| f =~ /kitchen\/openldap\/recipes\/woot.rb/ end).not_to eql(nil) end it "should allow you to have an 'ignore' file, which skips loading files in later cookbooks" do - expect(cookbook_loader[:openldap].recipe_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "recipes").detect do |f| f =~ /kitchen\/openldap\/recipes\/ignoreme.rb/ end).to eql(nil) end it "should find files that start with a ." do - expect(cookbook_loader[:openldap].file_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "files").detect do |f| f =~ /\.dotfile$/ end).to match(/\.dotfile$/) - expect(cookbook_loader[:openldap].file_filenames.detect do |f| + expect(full_paths_for_part(:openldap, "files").detect do |f| f =~ /\.ssh\/id_rsa$/ end).to match(/\.ssh\/id_rsa$/) end diff --git a/spec/unit/cookbook_manifest_spec.rb b/spec/unit/cookbook_manifest_spec.rb index acf0ade9f9..d77c07e0f7 100644 --- a/spec/unit/cookbook_manifest_spec.rb +++ b/spec/unit/cookbook_manifest_spec.rb @@ -81,11 +81,6 @@ describe Chef::CookbookManifest do expect(cookbook_manifest.frozen_version?).to be(false) end - it "delegates `segment_filenames' to cookbook_version" do - expect(cookbook_version).to receive(:segment_filenames).with(:recipes).and_return([]) - expect(cookbook_manifest.segment_filenames(:recipes)).to eq([]) - end - end context "when given an empty cookbook" do @@ -101,15 +96,7 @@ describe Chef::CookbookManifest do "frozen?" => false, - "recipes" => [], - "definitions" => [], - "libraries" => [], - "attributes" => [], - "files" => [], - "templates" => [], - "resources" => [], - "providers" => [], - "root_files" => [], + "all_files" => [], } end @@ -123,16 +110,7 @@ describe Chef::CookbookManifest do let(:cookbook_root) { File.join(CHEF_SPEC_DATA, "cb_version_cookbooks", "tatft") } - let(:attribute_filenames) { Dir[File.join(cookbook_root, "attributes", "**", "*.rb")] } - let(:definition_filenames) { Dir[File.join(cookbook_root, "definitions", "**", "*.rb")] } - let(:file_filenames) { Dir[File.join(cookbook_root, "files", "**", "*.tgz")] } - let(:recipe_filenames) { Dir[File.join(cookbook_root, "recipes", "**", "*.rb")] } - let(:template_filenames) { Dir[File.join(cookbook_root, "templates", "**", "*.erb")] } - let(:library_filenames) { Dir[File.join(cookbook_root, "libraries", "**", "*.rb")] } - let(:resource_filenames) { Dir[File.join(cookbook_root, "resources", "**", "*.rb")] } - let(:provider_filenames) { Dir[File.join(cookbook_root, "providers", "**", "*.rb")] } - let(:root_filenames) { Array(File.join(cookbook_root, "README.rdoc")) } - let(:metadata_filenames) { Array(File.join(cookbook_root, "metadata.json")) } + let(:all_files) { Dir[File.join(cookbook_root, "**", "**")].reject { |f| File.directory? f } } let(:match_md5) { /[0-9a-f]{32}/ } @@ -141,8 +119,15 @@ describe Chef::CookbookManifest do relative_path = Pathname.new(path).relative_path_from(Pathname.new(cookbook_root)).to_s + parts = relative_path.split("/") + name = if %w{templates files}.include?(parts[0]) && parts.length == 3 + File.join(parts[0], parts[2]) + else + relative_path + end + { - "name" => File.basename(path), + "name" => name, "path" => relative_path, "checksum" => Chef::Digester.generate_md5_checksum_for_file(path), "specificity" => "default", @@ -161,29 +146,12 @@ describe Chef::CookbookManifest do "frozen?" => false, - "recipes" => map_to_file_specs(recipe_filenames), - "definitions" => map_to_file_specs(definition_filenames), - "libraries" => map_to_file_specs(library_filenames), - "attributes" => map_to_file_specs(attribute_filenames), - "files" => map_to_file_specs(file_filenames), - "templates" => map_to_file_specs(template_filenames), - "resources" => map_to_file_specs(resource_filenames), - "providers" => map_to_file_specs(provider_filenames), - "root_files" => map_to_file_specs(root_filenames), + "all_files" => map_to_file_specs(all_files), } end before do - cookbook_version.attribute_filenames = attribute_filenames - cookbook_version.definition_filenames = definition_filenames - cookbook_version.file_filenames = file_filenames - cookbook_version.recipe_filenames = recipe_filenames - cookbook_version.template_filenames = template_filenames - cookbook_version.library_filenames = library_filenames - cookbook_version.resource_filenames = resource_filenames - cookbook_version.provider_filenames = provider_filenames - cookbook_version.root_filenames = root_filenames - cookbook_version.metadata_filenames = metadata_filenames + cookbook_version.all_files = all_files end it "converts the CookbookVersion to a ruby Hash representation" do diff --git a/spec/unit/cookbook_site_streaming_uploader_spec.rb b/spec/unit/cookbook_site_streaming_uploader_spec.rb index 10963386dd..0e9c277b11 100644 --- a/spec/unit/cookbook_site_streaming_uploader_spec.rb +++ b/spec/unit/cookbook_site_streaming_uploader_spec.rb @@ -49,10 +49,6 @@ describe Chef::CookbookSiteStreamingUploader do cookbook = @loader[:openldap] files_count = Dir.glob(File.join(@cookbook_repo, cookbook.name.to_s, "**", "*"), File::FNM_DOTMATCH).count { |file| File.file?(file) } - # The fixture cookbook contains a spec/spec_helper.rb file, which is not - # a part of any cookbook segment, so it is not uploaded. - files_count -= 1 - expect(Tempfile).to receive(:new).with("chef-#{cookbook.name}-build").and_return(FakeTempfile.new("chef-#{cookbook.name}-build")) expect(FileUtils).to receive(:mkdir_p).exactly(files_count + 1).times expect(FileUtils).to receive(:cp).exactly(files_count).times diff --git a/spec/unit/cookbook_spec.rb b/spec/unit/cookbook_spec.rb index 33b4a3ccd8..ac3a1373ea 100644 --- a/spec/unit/cookbook_spec.rb +++ b/spec/unit/cookbook_spec.rb @@ -19,7 +19,8 @@ require "spec_helper" describe Chef::CookbookVersion do -# COOKBOOK_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "data", "cookbooks", "openldap")) + COOKBOOK_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "data", "cookbooks", "openldap")) + before(:each) do @cookbook_repo = File.expand_path(File.join(File.dirname(__FILE__), "..", "data", "cookbooks")) cl = Chef::CookbookLoader.new(@cookbook_repo) @@ -37,26 +38,21 @@ describe Chef::CookbookVersion do end it "should allow you to set the list of attribute files and create the mapping from short names to paths" do - @cookbook.attribute_filenames = [ "attributes/one.rb", "attributes/two.rb" ] - expect(@cookbook.attribute_filenames).to eq([ "attributes/one.rb", "attributes/two.rb" ]) - expect(@cookbook.attribute_filenames_by_short_filename.keys.sort).to eql(%w{one two}) - expect(@cookbook.attribute_filenames_by_short_filename["one"]).to eq("attributes/one.rb") - expect(@cookbook.attribute_filenames_by_short_filename["two"]).to eq("attributes/two.rb") + expect(@cookbook.attribute_filenames_by_short_filename.keys.sort).to eql(%w{default smokey}) + expect(@cookbook.attribute_filenames_by_short_filename["default"]).to eq(File.join(COOKBOOK_PATH, "attributes/default.rb")) + expect(@cookbook.attribute_filenames_by_short_filename["smokey"]).to eq(File.join(COOKBOOK_PATH, "attributes/smokey.rb")) end it "should allow you to set the list of recipe files and create the mapping of recipe short name to filename" do - @cookbook.recipe_filenames = [ "recipes/one.rb", "recipes/two.rb" ] - expect(@cookbook.recipe_filenames).to eq([ "recipes/one.rb", "recipes/two.rb" ]) - expect(@cookbook.recipe_filenames_by_name.keys.sort).to eql(%w{one two}) - expect(@cookbook.recipe_filenames_by_name["one"]).to eq("recipes/one.rb") - expect(@cookbook.recipe_filenames_by_name["two"]).to eq("recipes/two.rb") + expect(@cookbook.recipe_filenames_by_name.keys.sort).to eql(%w{default gigantor one return}) + expect(@cookbook.recipe_filenames_by_name["one"]).to eq(File.join(COOKBOOK_PATH, "recipes/one.rb")) + expect(@cookbook.recipe_filenames_by_name["gigantor"]).to eq(File.join(COOKBOOK_PATH, "recipes/gigantor.rb")) end it "should generate a list of recipes by fully-qualified name" do - @cookbook.recipe_filenames = [ "recipes/one.rb", "/recipes/two.rb", "three.rb" ] expect(@cookbook.fully_qualified_recipe_names.include?("openldap::one")).to eq(true) - expect(@cookbook.fully_qualified_recipe_names.include?("openldap::two")).to eq(true) - expect(@cookbook.fully_qualified_recipe_names.include?("openldap::three")).to eq(true) + expect(@cookbook.fully_qualified_recipe_names.include?("openldap::gigantor")).to eq(true) + expect(@cookbook.fully_qualified_recipe_names.include?("openldap::return")).to eq(true) end it "should raise an ArgumentException if you try to load a bad recipe name" do diff --git a/spec/unit/cookbook_uploader_spec.rb b/spec/unit/cookbook_uploader_spec.rb index c30df71e34..2c36c2c9c7 100644 --- a/spec/unit/cookbook_uploader_spec.rb +++ b/spec/unit/cookbook_uploader_spec.rb @@ -65,7 +65,7 @@ describe Chef::CookbookUploader do it "creates an HTTP client with default configuration when not initialized with one" do default_http_client = double("Chef::ServerAPI") - expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url]).and_return(default_http_client) + expect(Chef::ServerAPI).to receive(:new).with(Chef::Config[:chef_server_url], version_class: Chef::CookbookManifestVersions).and_return(default_http_client) uploader = described_class.new(cookbooks_to_upload) expect(uploader.rest).to eq(default_http_client) end diff --git a/spec/unit/cookbook_version_file_specificity_spec.rb b/spec/unit/cookbook_version_file_specificity_spec.rb index 3b5450cb2d..ba7aaa59f5 100644 --- a/spec/unit/cookbook_version_file_specificity_spec.rb +++ b/spec/unit/cookbook_version_file_specificity_spec.rb @@ -23,177 +23,204 @@ describe Chef::CookbookVersion, "file specificity" do before(:each) do @cookbook = Chef::CookbookVersion.new("test-cookbook", "/cookbook-folder") @cookbook.manifest = { - "files" => + "all_files" => [ # afile.rb { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/host-examplehost.example.org/afile.rb", + :full_path => "/cookbook-folder/files/host-examplehost.example.org/afile.rb", :checksum => "csum-host", :specificity => "host-examplehost.example.org", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/ubuntu-9.10/afile.rb", + :full_path => "/cookbook-folder/files/ubuntu-9.10/afile.rb", :checksum => "csum-platver-full", :specificity => "ubuntu-9.10", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/newubuntu-9/afile.rb", + :full_path => "/cookbook-folder/files/newubuntu-9/afile.rb", :checksum => "csum-platver-partial", :specificity => "newubuntu-9", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/ubuntu/afile.rb", + :full_path => "/cookbook-folder/files/ubuntu/afile.rb", :checksum => "csum-plat", :specificity => "ubuntu", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/default/afile.rb", + :full_path => "/cookbook-folder/files/default/afile.rb", :checksum => "csum-default", :specificity => "default", }, # for different/odd platform_versions { - :name => "bfile.rb", + :name => "files/bfile.rb", :path => "files/fakeos-2.0.rc.1/bfile.rb", + :full_path => "/cookbook-folder/files/fakeos-2.0.rc.1/bfile.rb", :checksum => "csum2-platver-full", :specificity => "fakeos-2.0.rc.1", }, { - :name => "bfile.rb", + :name => "files/bfile.rb", :path => "files/newfakeos-2.0.rc/bfile.rb", + :full_path => "/cookbook-folder/files/newfakeos-2.0.rc/bfile.rb", :checksum => "csum2-platver-partial", :specificity => "newfakeos-2.0.rc", }, { - :name => "bfile.rb", + :name => "files/bfile.rb", :path => "files/fakeos-maple tree/bfile.rb", + :full_path => "/cookbook-folder/files/fakeos-maple tree/bfile.rb", :checksum => "csum3-platver-full", :specificity => "maple tree", }, { - :name => "bfile.rb", + :name => "files/bfile.rb", :path => "files/fakeos-1/bfile.rb", + :full_path => "/cookbook-folder/files/fakeos-1/bfile.rb", :checksum => "csum4-platver-full", :specificity => "fakeos-1", }, # directory adirectory { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/host-examplehost.example.org/adirectory/anotherfile1.rb.host", + :full_path => "/cookbook-folder/files/host-examplehost.example.org/adirectory/anotherfile1.rb.host", :checksum => "csum-host-1", :specificity => "host-examplehost.example.org", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/host-examplehost.example.org/adirectory/anotherfile2.rb.host", + :full_path => "/cookbook-folder/files/host-examplehost.example.org/adirectory/anotherfile2.rb.host", :checksum => "csum-host-2", :specificity => "host-examplehost.example.org", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/ubuntu-9.10/adirectory/anotherfile1.rb.platform-full-version", + :full_path => "/cookbook-folder/files/ubuntu-9.10/adirectory/anotherfile1.rb.platform-full-version", :checksum => "csum-platver-full-1", :specificity => "ubuntu-9.10", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/ubuntu-9.10/adirectory/anotherfile2.rb.platform-full-version", + :full_path => "/cookbook-folder/files/ubuntu-9.10/adirectory/anotherfile2.rb.platform-full-version", :checksum => "csum-platver-full-2", :specificity => "ubuntu-9.10", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/newubuntu-9/adirectory/anotherfile1.rb.platform-partial-version", + :full_path => "/cookbook-folder/files/newubuntu-9/adirectory/anotherfile1.rb.platform-partial-version", :checksum => "csum-platver-partial-1", :specificity => "newubuntu-9", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/newubuntu-9/adirectory/anotherfile2.rb.platform-partial-version", + :full_path => "/cookbook-folder/files/newubuntu-9/adirectory/anotherfile2.rb.platform-partial-version", :checksum => "csum-platver-partial-2", :specificity => "nweubuntu-9", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/ubuntu/adirectory/anotherfile1.rb.platform", + :full_path => "/cookbook-folder/files/ubuntu/adirectory/anotherfile1.rb.platform", :checksum => "csum-plat-1", :specificity => "ubuntu", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/ubuntu/adirectory/anotherfile2.rb.platform", + :full_path => "/cookbook-folder/files/ubuntu/adirectory/anotherfile2.rb.platform", :checksum => "csum-plat-2", :specificity => "ubuntu", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/default/adirectory/anotherfile1.rb.default", + :full_path => "/cookbook-folder/files/default/adirectory/anotherfile1.rb.default", :checksum => "csum-default-1", :specificity => "default", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/default/adirectory/anotherfile2.rb.default", + :full_path => "/cookbook-folder/files/default/adirectory/anotherfile2.rb.default", :checksum => "csum-default-2", :specificity => "default", }, # for different/odd platform_versions { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/fakeos-2.0.rc.1/adirectory/anotherfile1.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-2.0.rc.1/adirectory/anotherfile1.rb.platform-full-version", :checksum => "csum2-platver-full-1", :specificity => "fakeos-2.0.rc.1", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/fakeos-2.0.rc.1/adirectory/anotherfile2.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-2.0.rc.1/adirectory/anotherfile2.rb.platform-full-version", :checksum => "csum2-platver-full-2", :specificity => "fakeos-2.0.rc.1", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/newfakeos-2.0.rc.1/adirectory/anotherfile1.rb.platform-partial-version", + :full_path => "/cookbook-folder/files/newfakeos-2.0.rc.1/adirectory/anotherfile1.rb.platform-partial-version", :checksum => "csum2-platver-partial-1", :specificity => "newfakeos-2.0.rc", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/newfakeos-2.0.rc.1/adirectory/anotherfile2.rb.platform-partial-version", + :full_path => "/cookbook-folder/files/newfakeos-2.0.rc.1/adirectory/anotherfile2.rb.platform-partial-version", :checksum => "csum2-platver-partial-2", :specificity => "newfakeos-2.0.rc", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/fakeos-maple tree/adirectory/anotherfile1.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-maple tree/adirectory/anotherfile1.rb.platform-full-version", :checksum => "csum3-platver-full-1", :specificity => "fakeos-maple tree", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/fakeos-maple tree/adirectory/anotherfile2.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-maple tree/adirectory/anotherfile2.rb.platform-full-version", :checksum => "csum3-platver-full-2", :specificity => "fakeos-maple tree", }, { - :name => "anotherfile1.rb", + :name => "files/anotherfile1.rb", :path => "files/fakeos-1/adirectory/anotherfile1.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-1/adirectory/anotherfile1.rb.platform-full-version", :checksum => "csum4-platver-full-1", :specificity => "fakeos-1", }, { - :name => "anotherfile2.rb", + :name => "files/anotherfile2.rb", :path => "files/fakeos-1/adirectory/anotherfile2.rb.platform-full-version", + :full_path => "/cookbook-folder/files/fakeos-1/adirectory/anotherfile2.rb.platform-full-version", :checksum => "csum4-platver-full-2", :specificity => "fakeos-1", }, diff --git a/spec/unit/cookbook_version_spec.rb b/spec/unit/cookbook_version_spec.rb index 81ea161bfe..83fb3f578f 100644 --- a/spec/unit/cookbook_version_spec.rb +++ b/spec/unit/cookbook_version_spec.rb @@ -25,38 +25,6 @@ describe Chef::CookbookVersion do expect(cookbook_version.name).to eq("tatft") end - it "has no attribute files" do - expect(cookbook_version.attribute_filenames).to be_empty - end - - it "has no resource definition files" do - expect(cookbook_version.definition_filenames).to be_empty - end - - it "has no cookbook files" do - expect(cookbook_version.file_filenames).to be_empty - end - - it "has no recipe files" do - expect(cookbook_version.recipe_filenames).to be_empty - end - - it "has no library files" do - expect(cookbook_version.library_filenames).to be_empty - end - - it "has no LWRP resource files" do - expect(cookbook_version.resource_filenames).to be_empty - end - - it "has no LWRP provider files" do - expect(cookbook_version.provider_filenames).to be_empty - end - - it "has no metadata files" do - expect(cookbook_version.metadata_filenames).to be_empty - end - it "has an empty set of all_files" do expect(cookbook_version.all_files).to be_empty end @@ -82,17 +50,7 @@ describe Chef::CookbookVersion do let(:cookbook_paths_by_type) do { # Dunno if the paths here are representitive of what is set by CookbookLoader... - all_files: Dir[File.join(cookbook_root, "**", "*.rb")], - attribute_filenames: Dir[File.join(cookbook_root, "attributes", "**", "*.rb")], - definition_filenames: Dir[File.join(cookbook_root, "definitions", "**", "*.rb")], - file_filenames: Dir[File.join(cookbook_root, "files", "**", "*.tgz")], - recipe_filenames: Dir[File.join(cookbook_root, "recipes", "**", "*.rb")], - template_filenames: Dir[File.join(cookbook_root, "templates", "**", "*.erb")], - library_filenames: Dir[File.join(cookbook_root, "libraries", "**", "*.rb")], - resource_filenames: Dir[File.join(cookbook_root, "resources", "**", "*.rb")], - provider_filenames: Dir[File.join(cookbook_root, "providers", "**", "*.rb")], - root_filenames: Array(File.join(cookbook_root, "README.rdoc")), - metadata_filenames: Array(File.join(cookbook_root, "metadata.json")), + all_files: Dir[File.join(cookbook_root, "**", "**")], } end @@ -102,18 +60,9 @@ describe Chef::CookbookVersion do let(:cookbook_version) do Chef::CookbookVersion.new("tatft", cookbook_root).tap do |c| - # Currently the cookbook loader finds all the files then tells CookbookVersion - # where they are. - c.attribute_filenames = cookbook_paths_by_type[:attribute_filenames] - c.definition_filenames = cookbook_paths_by_type[:definition_filenames] - c.recipe_filenames = cookbook_paths_by_type[:recipe_filenames] - c.template_filenames = cookbook_paths_by_type[:template_filenames] - c.file_filenames = cookbook_paths_by_type[:file_filenames] - c.library_filenames = cookbook_paths_by_type[:library_filenames] - c.resource_filenames = cookbook_paths_by_type[:resource_filenames] - c.provider_filenames = cookbook_paths_by_type[:provider_filenames] - c.root_filenames = cookbook_paths_by_type[:root_filenames] - c.metadata_filenames = cookbook_paths_by_type[:metadata_filenames] + # Currently the cookbook loader finds all the files then tells CookbookVersion + # where they are. + c.all_files = cookbook_paths_by_type[:all_files] end end @@ -168,18 +117,7 @@ describe Chef::CookbookVersion do let(:cookbook_paths_by_type) do { - # Dunno if the paths here are representitive of what is set by CookbookLoader... - all_files: Dir[File.join(cookbook_root, "**", "*.rb")], - attribute_filenames: Dir[File.join(cookbook_root, "attributes", "**", "*.rb")], - definition_filenames: Dir[File.join(cookbook_root, "definitions", "**", "*.rb")], - file_filenames: Dir[File.join(cookbook_root, "files", "**", "*.*")], - recipe_filenames: Dir[File.join(cookbook_root, "recipes", "**", "*.rb")], - template_filenames: Dir[File.join(cookbook_root, "templates", "**", "*.*")], - library_filenames: Dir[File.join(cookbook_root, "libraries", "**", "*.rb")], - resource_filenames: Dir[File.join(cookbook_root, "resources", "**", "*.rb")], - provider_filenames: Dir[File.join(cookbook_root, "providers", "**", "*.rb")], - root_filenames: Array(File.join(cookbook_root, "README.rdoc")), - metadata_filenames: Array(File.join(cookbook_root, "metadata.json")), + all_files: Dir[File.join(cookbook_root, "**", "**")], } end @@ -187,16 +125,7 @@ describe Chef::CookbookVersion do let(:cookbook_version) do Chef::CookbookVersion.new("cookbook2", cookbook_root).tap do |c| - c.attribute_filenames = cookbook_paths_by_type[:attribute_filenames] - c.definition_filenames = cookbook_paths_by_type[:definition_filenames] - c.recipe_filenames = cookbook_paths_by_type[:recipe_filenames] - c.template_filenames = cookbook_paths_by_type[:template_filenames] - c.file_filenames = cookbook_paths_by_type[:file_filenames] - c.library_filenames = cookbook_paths_by_type[:library_filenames] - c.resource_filenames = cookbook_paths_by_type[:resource_filenames] - c.provider_filenames = cookbook_paths_by_type[:provider_filenames] - c.root_filenames = cookbook_paths_by_type[:root_filenames] - c.metadata_filenames = cookbook_paths_by_type[:metadata_filenames] + c.all_files = cookbook_paths_by_type[:all_files] end end @@ -255,19 +184,19 @@ describe Chef::CookbookVersion do it "should sort based on the version number" do examples = [ - # smaller, larger - ["1.0", "2.0"], - ["1.2.3", "1.2.4"], - ["1.2.3", "1.3.0"], - ["1.2.3", "1.3"], - ["1.2.3", "2.1.1"], - ["1.2.3", "2.1"], - ["1.2", "1.2.4"], - ["1.2", "1.3.0"], - ["1.2", "1.3"], - ["1.2", "2.1.1"], - ["1.2", "2.1"], - ] + # smaller, larger + ["1.0", "2.0"], + ["1.2.3", "1.2.4"], + ["1.2.3", "1.3.0"], + ["1.2.3", "1.3"], + ["1.2.3", "2.1.1"], + ["1.2.3", "2.1"], + ["1.2", "1.2.4"], + ["1.2", "1.3.0"], + ["1.2", "1.3"], + ["1.2", "2.1.1"], + ["1.2", "2.1"], + ] examples.each do |smaller, larger| sm = Chef::CookbookVersion.new("foo", "/tmp/blah") lg = Chef::CookbookVersion.new("foo", "/tmp/blah") @@ -318,42 +247,4 @@ describe Chef::CookbookVersion do end - describe "when deprecation warnings are errors" do - - subject(:cbv) { Chef::CookbookVersion.new("version validation", "/tmp/blah") } - - it "errors on #status and #status=" do - expect { cbv.status = :wat }.to raise_error(Chef::Exceptions::DeprecatedFeatureError) - expect { cbv.status }.to raise_error(Chef::Exceptions::DeprecatedFeatureError) - end - - end - - describe "deprecated features" do - - subject(:cbv) { Chef::CookbookVersion.new("tatft", "/tmp/blah").tap { |c| c.version = "1.2.3" } } - - before do - Chef::Config[:treat_deprecation_warnings_as_errors] = false - end - - it "gives a save URL for the standard cookbook API" do - expect(cbv.save_url).to eq("cookbooks/tatft/1.2.3") - end - - it "gives a force save URL for the standard cookbook API" do - expect(cbv.force_save_url).to eq("cookbooks/tatft/1.2.3?force=true") - end - - it "is \"ready\"" do - # WTF is this? what are the valid states? and why aren't they set with encapsulating methods? - # [Dan 15-Jul-2010] - expect(cbv.status).to eq(:ready) - end - - include_examples "to_json equivalent to Chef::JSONCompat.to_json" do - let(:jsonable) { Chef::CookbookVersion.new("tatft", "/tmp/blah") } - end - - end end diff --git a/spec/unit/http/socketless_chef_zero_client_spec.rb b/spec/unit/http/socketless_chef_zero_client_spec.rb index 637e562799..4f3aed13c5 100644 --- a/spec/unit/http/socketless_chef_zero_client_spec.rb +++ b/spec/unit/http/socketless_chef_zero_client_spec.rb @@ -132,7 +132,7 @@ describe Chef::HTTP::SocketlessChefZeroClient do let(:method) { :GET } let(:relative_url) { "clients" } - let(:headers) { { "Accept" => "application/json" } } + let(:headers) { { "Accept" => "application/json", "X-Ops-Server-API-Version" => "2" } } let(:body) { false } let(:expected_rack_req) do @@ -144,6 +144,7 @@ describe Chef::HTTP::SocketlessChefZeroClient do "QUERY_STRING" => uri.query, "SERVER_PORT" => uri.port, "HTTP_HOST" => "localhost:#{uri.port}", + "HTTP_X_OPS_SERVER_API_VERSION" => "2", "rack.url_scheme" => "chefzero", "rack.input" => an_instance_of(StringIO), } diff --git a/spec/unit/knife/cookbook_download_spec.rb b/spec/unit/knife/cookbook_download_spec.rb index 38a4974774..1fb995f71d 100644 --- a/spec/unit/knife/cookbook_download_spec.rb +++ b/spec/unit/knife/cookbook_download_spec.rb @@ -47,44 +47,62 @@ describe Chef::Knife::CookbookDownload do @rest_mock = double("rest") allow(@knife).to receive(:rest).and_return(@rest_mock) - @manifest_data = { - :recipes => [ - { "path" => "recipes/foo.rb", - "url" => "http://example.org/files/foo.rb" }, - { "path" => "recipes/bar.rb", - "url" => "http://example.org/files/bar.rb" }, - ], - :templates => [ - { "path" => "templates/default/foo.erb", - "url" => "http://example.org/files/foo.erb" }, - { "path" => "templates/default/bar.erb", - "url" => "http://example.org/files/bar.erb" }, - ], - :attributes => [ - { "path" => "attributes/default.rb", - "url" => "http://example.org/files/default.rb" }, + expect(Chef::CookbookVersion).to receive(:load).with("foobar", "1.0.0"). + and_return(cookbook) + end + + let(:manifest_data) do + { + :all_files => [ + { + "path" => "recipes/foo.rb", + "name" => "recipes/foo.rb", + "url" => "http://example.org/files/foo.rb", + }, + { + "path" => "recipes/bar.rb", + "name" => "recipes/bar.rb", + "url" => "http://example.org/files/bar.rb", + }, + { + "path" => "templates/default/foo.erb", + "name" => "templates/foo.erb", + "url" => "http://example.org/files/foo.erb", + }, + { + "path" => "templates/default/bar.erb", + "name" => "templates/bar.erb", + "url" => "http://example.org/files/bar.erb", + }, + { + "path" => "attributes/default.rb", + "name" => "attributes/default.rb", + "url" => "http://example.org/files/default.rb", + }, ], } + end - @cookbook_mock = double("cookbook") - allow(@cookbook_mock).to receive(:version).and_return("1.0.0") - allow(@cookbook_mock).to receive(:manifest).and_return(@manifest_data) - expect(Chef::CookbookVersion).to receive(:load).with("foobar", "1.0.0"). - and_return(@cookbook_mock) + let (:cookbook) do + cb = Chef::CookbookVersion.new("foobar") + cb.version = "1.0.0" + cb.manifest = manifest_data + cb end - it "should determine which version if one was not explicitly specified" do - allow(@cookbook_mock).to receive(:manifest).and_return({}) - expect(@knife).to receive(:determine_version).and_return("1.0.0") - expect(File).to receive(:exists?).with("/var/tmp/chef/foobar-1.0.0").and_return(false) - allow(Chef::CookbookVersion).to receive(:COOKBOOK_SEGEMENTS).and_return([]) - @knife.run + describe "and no version" do + let (:manifest_data) { { all_files: [] } } + it "should determine which version to download" do + expect(@knife).to receive(:determine_version).and_return("1.0.0") + expect(File).to receive(:exists?).with("/var/tmp/chef/foobar-1.0.0").and_return(false) + @knife.run + end end describe "and a version" do before(:each) do @knife.name_args << "1.0.0" - @files = @manifest_data.values.map { |v| v.map { |i| i["path"] } }.flatten.uniq + @files = manifest_data.values.map { |v| v.map { |i| i["path"] } }.flatten.uniq @files_mocks = {} @files.map { |f| File.basename(f) }.flatten.uniq.each do |f| @files_mocks[f] = double("#{f}_mock") diff --git a/spec/unit/knife/cookbook_show_spec.rb b/spec/unit/knife/cookbook_show_spec.rb index 749e50c647..1e8ea836d7 100644 --- a/spec/unit/knife/cookbook_show_spec.rb +++ b/spec/unit/knife/cookbook_show_spec.rb @@ -47,9 +47,9 @@ describe Chef::Knife::CookbookShow do let (:manifest) do { - "recipes" => [ + "all_files" => [ { - :name => "default.rb", + :name => "recipes/default.rb", :path => "recipes/default.rb", :checksum => "1234", :url => "http://example.org/files/default.rb", @@ -101,9 +101,42 @@ describe Chef::Knife::CookbookShow do knife.name_args << "0.1.0" end + let(:output) do + { "cookbook_name" => "cookbook_name", + "name" => "cookbook_name-0.0.0", + "frozen?" => false, + "version" => "0.0.0", + "metadata" => { + "name" => nil, + "description" => "", + "long_description" => "", + "maintainer" => nil, + "maintainer_email" => nil, + "license" => "All rights reserved", + "platforms" => {}, + "dependencies" => {}, + "providing" => {}, + "attributes" => {}, + "recipes" => {}, + "version" => "0.0.0", + "source_url" => "", + "issues_url" => "", + "privacy" => false, + "chef_versions" => [], + "ohai_versions" => [], + "gems" => [], + }, + "recipes" => + [{ "name" => "recipes/default.rb", + "path" => "recipes/default.rb", + "checksum" => "1234", + "url" => "http://example.org/files/default.rb" }], + } + end + it "should show the specific part of a cookbook" do expect(Chef::CookbookVersion).to receive(:load).with("cookbook_name", "0.1.0").and_return(cb) - expect(knife).to receive(:output).with(cb) + expect(knife).to receive(:output).with(output) knife.run end end @@ -115,7 +148,7 @@ describe Chef::Knife::CookbookShow do it "should print the json of the part" do expect(Chef::CookbookVersion).to receive(:load).with("cookbook_name", "0.1.0").and_return(cb) - expect(knife).to receive(:output).with(cb.manifest["recipes"]) + expect(knife).to receive(:output).with(cb.files_for("recipes")) knife.run end end @@ -137,30 +170,30 @@ describe Chef::Knife::CookbookShow do before(:each) do knife.name_args = [ "cookbook_name", "0.1.0", "files", "afile.rb" ] cb.manifest = { - "files" => [ + "all_files" => [ { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/host-examplehost.example.org/afile.rb", :checksum => "1111", :specificity => "host-examplehost.example.org", :url => "http://example.org/files/1111", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/ubuntu-9.10/afile.rb", :checksum => "2222", :specificity => "ubuntu-9.10", :url => "http://example.org/files/2222", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/ubuntu/afile.rb", :checksum => "3333", :specificity => "ubuntu", :url => "http://example.org/files/3333", }, { - :name => "afile.rb", + :name => "files/afile.rb", :path => "files/default/afile.rb", :checksum => "4444", :specificity => "default", diff --git a/spec/unit/policy_builder/policyfile_spec.rb b/spec/unit/policy_builder/policyfile_spec.rb index 307bd45c18..c9086c2f63 100644 --- a/spec/unit/policy_builder/policyfile_spec.rb +++ b/spec/unit/policy_builder/policyfile_spec.rb @@ -100,8 +100,8 @@ describe Chef::PolicyBuilder::Policyfile do http = double("Chef::ServerAPI") server_url = "https://api.opscode.com/organizations/example" Chef::Config[:chef_server_url] = server_url - expect(Chef::ServerAPI).to receive(:new).with(server_url).and_return(http) - expect(policy_builder.http_api).to eq(http) + expect(Chef::ServerAPI).to receive(:new).with(server_url, version_class: Chef::CookbookManifestVersions).and_return(http) + expect(policy_builder.api_service).to eq(http) end describe "reporting unsupported features" do @@ -150,7 +150,7 @@ describe Chef::PolicyBuilder::Policyfile do describe "loading policy data" do - let(:http_api) { double("Chef::ServerAPI") } + let(:api_service) { double("Chef::ServerAPI") } let(:configured_environment) { nil } @@ -172,7 +172,7 @@ describe Chef::PolicyBuilder::Policyfile do before do Chef::Config[:policy_document_native_api] = false Chef::Config[:deployment_group] = "example-policy-stage" - allow(policy_builder).to receive(:http_api).and_return(http_api) + allow(policy_builder).to receive(:api_service).and_return(api_service) end describe "when using compatibility mode (policy_document_native_api == false)" do @@ -185,7 +185,7 @@ describe Chef::PolicyBuilder::Policyfile do let(:error404) { Net::HTTPServerException.new("404 message", :body) } before do - expect(http_api).to receive(:get). + expect(api_service).to receive(:get). with("data/policyfiles/example-policy-stage"). and_raise(error404) end @@ -212,7 +212,7 @@ describe Chef::PolicyBuilder::Policyfile do let(:policy_relative_url) { "data/policyfiles/example-policy-stage" } before do - expect(http_api).to receive(:get).with(policy_relative_url).and_return(parsed_policyfile_json) + expect(api_service).to receive(:get).with(policy_relative_url).and_return(parsed_policyfile_json) end it "fetches the policy file from a data bag item" do @@ -253,7 +253,7 @@ describe Chef::PolicyBuilder::Policyfile do let(:policy_relative_url) { "policy_groups/policy-stage/policies/example" } before do - expect(http_api).to receive(:get).with(policy_relative_url).and_return(parsed_policyfile_json) + expect(api_service).to receive(:get).with(policy_relative_url).and_return(parsed_policyfile_json) end it "fetches the policy file from a data bag item" do @@ -617,7 +617,7 @@ describe Chef::PolicyBuilder::Policyfile do policy_builder.finish_load_node(node) policy_builder.build_node - expect(http_api).to receive(:get).with(cookbook1_url). + expect(api_service).to receive(:get).with(cookbook1_url). and_raise(error404) end @@ -687,9 +687,9 @@ describe Chef::PolicyBuilder::Policyfile do context "when the cookbooks exist on the server" do before do - expect(http_api).to receive(:get).with(cookbook1_url). + expect(api_service).to receive(:get).with(cookbook1_url). and_return(example1_cookbook_data) - expect(http_api).to receive(:get).with(cookbook2_url). + expect(api_service).to receive(:get).with(cookbook2_url). and_return(example2_cookbook_data) expect(Chef::CookbookVersion).to receive(:from_cb_artifact_data).with(example1_cookbook_data). @@ -720,9 +720,9 @@ describe Chef::PolicyBuilder::Policyfile do context "when the cookbooks exist on the server" do before do - expect(http_api).to receive(:get).with(cookbook1_url). + expect(api_service).to receive(:get).with(cookbook1_url). and_return(example1_cookbook_data) - expect(http_api).to receive(:get).with(cookbook2_url). + expect(api_service).to receive(:get).with(cookbook2_url). and_return(example2_cookbook_data) expect(Chef::CookbookVersion).to receive(:from_cb_artifact_data).with(example1_cookbook_data). diff --git a/spec/unit/run_context/cookbook_compiler_spec.rb b/spec/unit/run_context/cookbook_compiler_spec.rb index feb39615b6..e93088cd5f 100644 --- a/spec/unit/run_context/cookbook_compiler_spec.rb +++ b/spec/unit/run_context/cookbook_compiler_spec.rb @@ -163,9 +163,7 @@ describe Chef::RunContext::CookbookCompiler do describe "event dispatch" do let(:recipe) { "dependency1::default" } let(:recipe_path) do - File.expand_path("../../../data/run_context/cookbooks/dependency1/recipes/default.rb", __FILE__).tap do |path| - path.gsub!(File::SEPARATOR, File::ALT_SEPARATOR) if File::ALT_SEPARATOR - end + File.expand_path("../../../data/run_context/cookbooks/dependency1/recipes/default.rb", __FILE__) end before do node.run_list(recipe) -- cgit v1.2.1