RFC 67: Remove cookbook segments

This implements RFC 67, which removes cookbook segments, and moves to a
single list of all the files contained in a cookbook. This allows us to
move forward with better audit modes and also proper shipping of ohai
plugins.

Signed-off-by: Thom May <thom@chef.io>

6 files changed, 34 insertions, 36 deletions

diff --git a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
index 5030a0733f..63ce71ef40 100644
--- a/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/chef_server_root_dir.rb
@@ -103,11 +103,11 @@ class Chef
           end
 
           def get_json(path)
-            Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key, :api_version => "0").get(path)
+            chef_rest.get(path)
           end
 
           def chef_rest
-            Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key)
+            Chef::ServerAPI.new(chef_server_url, :client_name => chef_username, :signing_key_filename => chef_private_key, :api_version => "0")
           end
 
           def api_path
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb
index 0b82a64a0a..e4df7858a7 100644
--- a/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_artifacts_dir.rb
@@ -56,7 +56,7 @@ class Chef
           # to make this work. So instead, we make a temporary cookbook
           # symlinking back to real cookbook, and upload the proxy.
           def upload_cookbook(other, options)
-            cookbook_name, dash, identifier = other.name.rpartition("-")
+            cookbook_name, _, identifier = other.name.rpartition("-")
 
             Dir.mktmpdir do |temp_cookbooks_path|
               proxy_cookbook_path = "#{temp_cookbooks_path}/#{cookbook_name}"
@@ -73,7 +73,7 @@ class Chef
               cookbook_to_upload.freeze_version if options[:freeze]
 
               # Instantiate a new uploader based on the proxy loader
-              uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: root.chef_rest, policy_mode: true)
+              uploader = Chef::CookbookUploader.new(cookbook_to_upload, force: options[:force], rest: chef_rest, policy_mode: true)
 
               with_actual_cookbooks_dir(temp_cookbooks_path) do
                 uploader.upload_cookbooks
@@ -92,6 +92,10 @@ class Chef
             end
           end
 
+          def chef_rest
+            Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions))
+          end
+
           def can_have_child?(name, is_dir)
             is_dir && name.include?("-")
           end
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb
index 8f5faf2183..64488ed705 100644
--- a/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbook_dir.rb
@@ -49,18 +49,6 @@ class Chef
 
           attr_reader :cookbook_name, :version
 
-          COOKBOOK_SEGMENT_INFO = {
-            :attributes => { :ruby_only => true },
-            :definitions => { :ruby_only => true },
-            :recipes => { :ruby_only => true },
-            :libraries => { :recursive => true },
-            :templates => { :recursive => true },
-            :files => { :recursive => true },
-            :resources => { :ruby_only => true, :recursive => true },
-            :providers => { :ruby_only => true, :recursive => true },
-            :root_files => {},
-          }
-
           def add_child(child)
             @children << child
           end
@@ -80,34 +68,29 @@ class Chef
           end
 
           def can_have_child?(name, is_dir)
-            # A cookbook's root may not have directories unless they are segment directories
-            return name != "root_files" && COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) if is_dir
+            return name != "root_files" if is_dir
             true
           end
 
           def children
             if @children.nil?
               @children = []
-              manifest = chef_object.manifest
-              COOKBOOK_SEGMENT_INFO.each do |segment, segment_info|
-                next unless manifest.has_key?(segment)
-
-                # Go through each file in the manifest for the segment, and
-                # add cookbook subdirs and files for it.
-                manifest[segment].each do |segment_file|
-                  parts = segment_file[:path].split("/")
+              manifest = chef_object.cookbook_manifest
+              manifest.by_parent_directory.each do |segment, files|
+                files.each do |file|
+                  parts = file[:path].split("/")
                   # Get or create the path to the file
                   container = self
                   parts[0, parts.length - 1].each do |part|
                     old_container = container
                     container = old_container.children.find { |child| part == child.name }
                     if !container
-                      container = CookbookSubdir.new(part, old_container, segment_info[:ruby_only], segment_info[:recursive])
+                      container = CookbookSubdir.new(part, old_container, false, true)
                       old_container.add_child(container)
                     end
                   end
                   # Create the file itself
-                  container.add_child(CookbookFile.new(parts[parts.length - 1], container, segment_file))
+                  container.add_child(CookbookFile.new(parts[parts.length - 1], container, file))
                 end
               end
               @children = @children.sort_by { |c| c.name }
@@ -165,7 +148,11 @@ class Chef
           end
 
           def rest
-            parent.rest
+            Chef::ServerAPI.new(parent.rest.url, parent.rest.options.merge(version_class: Chef::CookbookManifestVersions))
+          end
+
+          def chef_rest
+            Chef::ServerAPI.new(parent.chef_rest.url, parent.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions))
           end
 
           def chef_object
@@ -187,7 +174,7 @@ class Chef
               old_retry_count = Chef::Config[:http_retry_count]
               begin
                 Chef::Config[:http_retry_count] = 0
-                @chef_object ||= Chef::CookbookVersion.from_hash(root.get_json(api_path))
+                @chef_object ||= Chef::CookbookVersion.from_hash(chef_rest.get(api_path))
               ensure
                 Chef::Config[:http_retry_count] = old_retry_count
               end
diff --git a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
index 631562d7ef..4e8e68e364 100644
--- a/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/cookbooks_dir.rb
@@ -74,13 +74,17 @@ class Chef
           def upload_cookbook(other, options)
             cookbook_to_upload = other.chef_object
             cookbook_to_upload.freeze_version if options[:freeze]
-            uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest)
+            uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => chef_rest)
 
             with_actual_cookbooks_dir(other.parent.file_path) do
               uploader.upload_cookbooks
             end
           end
 
+          def chef_rest
+            Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions))
+          end
+
           # Work around the fact that CookbookUploader doesn't understand chef_repo_path (yet)
           def with_actual_cookbooks_dir(actual_cookbook_path)
             old_cookbook_path = Chef::Config.cookbook_path
diff --git a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
index 172405763a..8da3718136 100644
--- a/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server/versioned_cookbooks_dir.rb
@@ -78,7 +78,7 @@ class Chef
               cookbook_to_upload.freeze_version if options[:freeze]
 
               # Instantiate a new uploader based on the proxy loader
-              uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest)
+              uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => chef_rest)
 
               with_actual_cookbooks_dir(temp_cookbooks_path) do
                 uploader.upload_cookbooks
@@ -97,6 +97,10 @@ class Chef
             end
           end
 
+          def chef_rest
+            Chef::ServerAPI.new(root.chef_rest.url, root.chef_rest.options.merge(version_class: Chef::CookbookManifestVersions))
+          end
+
           def can_have_child?(name, is_dir)
             is_dir && name =~ Chef::ChefFS::FileSystem::ChefServer::VersionedCookbookDir::VALID_VERSIONED_COOKBOOK_NAME
           end
diff --git a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb
index 31b538b9ce..b296901dd1 100644
--- a/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb
+++ b/lib/chef/chef_fs/file_system/repository/chef_repository_file_system_cookbook_dir.rb
@@ -97,9 +97,9 @@ class Chef
           end
 
           def can_have_child?(name, is_dir)
-            if is_dir
+            if is_dir && !%w{ root_files .. . }.include?(name)
               # Only the given directories will be uploaded.
-              return Chef::ChefFS::FileSystem::ChefServer::CookbookDir::COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) && name != "root_files"
+              return true
             elsif name == Chef::Cookbook::CookbookVersionLoader::UPLOADED_COOKBOOK_VERSION_FILE
               return false
             end
@@ -128,8 +128,7 @@ class Chef
           protected
 
           def make_child_entry(child_name)
-            segment_info = Chef::ChefFS::FileSystem::ChefServer::CookbookDir::COOKBOOK_SEGMENT_INFO[child_name.to_sym] || {}
-            ChefRepositoryFileSystemCookbookEntry.new(child_name, self, nil, segment_info[:ruby_only], segment_info[:recursive])
+            ChefRepositoryFileSystemCookbookEntry.new(child_name, self, nil, false, true)
           end
 
           def cookbook_version