diff options
-rw-r--r-- | lib/chef/chef_fs/file_system.rb | 40 | ||||
-rw-r--r-- | lib/chef/chef_fs/file_system/memory_dir.rb | 42 | ||||
-rw-r--r-- | lib/chef/chef_fs/file_system/memory_file.rb | 17 | ||||
-rw-r--r-- | lib/chef/chef_fs/file_system/memory_root.rb | 21 | ||||
-rw-r--r-- | lib/chef/knife/zero.rb | 304 | ||||
-rw-r--r-- | spec/support/shared/unit/file_system_support.rb | 54 |
6 files changed, 340 insertions, 138 deletions
diff --git a/lib/chef/chef_fs/file_system.rb b/lib/chef/chef_fs/file_system.rb index 000a5cf90f..2c50d9537e 100644 --- a/lib/chef/chef_fs/file_system.rb +++ b/lib/chef/chef_fs/file_system.rb @@ -136,7 +136,7 @@ class Chef # puts message # end # - def self.copy_to(pattern, src_root, dest_root, recurse_depth, options, ui, format_path) + def self.copy_to(pattern, src_root, dest_root, recurse_depth, options, ui = nil, format_path = nil) found_result = false error = false parallel_do(list_pairs(pattern, src_root, dest_root)) do |src, dest| @@ -146,7 +146,7 @@ class Chef error ||= child_error end if !found_result && pattern.exact_path - ui.error "#{pattern}: No such file or directory on remote or local" + ui.error "#{pattern}: No such file or directory on remote or local" if ui error = true end error @@ -283,13 +283,13 @@ class Chef # If we would not have uploaded it, we will not purge it. if src_entry.parent.can_have_child?(dest_entry.name, dest_entry.dir?) if options[:dry_run] - ui.output "Would delete #{dest_path}" + ui.output "Would delete #{dest_path}" if ui else dest_entry.delete(true) - ui.output "Deleted extra entry #{dest_path} (purge is on)" + ui.output "Deleted extra entry #{dest_path} (purge is on)" if ui end else - Chef::Log.info("Not deleting extra entry #{dest_path} (purge is off)") + Chef::Log.info("Not deleting extra entry #{dest_path} (purge is off)") if ui end end @@ -298,21 +298,21 @@ class Chef # If the entry can do a copy directly from filesystem, do that. if new_dest_parent.respond_to?(:create_child_from) if options[:dry_run] - ui.output "Would create #{dest_path}" + ui.output "Would create #{dest_path}" if ui else new_dest_parent.create_child_from(src_entry) - ui.output "Created #{dest_path}" + ui.output "Created #{dest_path}" if ui end return end if src_entry.dir? if options[:dry_run] - ui.output "Would create #{dest_path}" + ui.output "Would create #{dest_path}" if ui new_dest_dir = new_dest_parent.child(src_entry.name) else new_dest_dir = new_dest_parent.create_child(src_entry.name, nil) - ui.output "Created #{dest_path}" + ui.output "Created #{dest_path}" if ui end # Directory creation is recursive. if recurse_depth != 0 @@ -324,10 +324,10 @@ class Chef end else if options[:dry_run] - ui.output "Would create #{dest_path}" + ui.output "Would create #{dest_path}" if ui else new_dest_parent.create_child(src_entry.name, src_entry.read) - ui.output "Created #{dest_path}" + ui.output "Created #{dest_path}" if ui end end end @@ -339,10 +339,10 @@ class Chef if dest_entry.respond_to?(:copy_from) if options[:force] || compare(src_entry, dest_entry)[0] == false if options[:dry_run] - ui.output "Would update #{dest_path}" + ui.output "Would update #{dest_path}" if ui else dest_entry.copy_from(src_entry) - ui.output "Updated #{dest_path}" + ui.output "Updated #{dest_path}" if ui end end return @@ -360,12 +360,12 @@ class Chef end else # If they are different types. - ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") + ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") if ui return end else if dest_entry.dir? - ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") + ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") if ui return else @@ -381,23 +381,23 @@ class Chef end if should_copy if options[:dry_run] - ui.output "Would update #{dest_path}" + ui.output "Would update #{dest_path}" if ui else src_value = src_entry.read if src_value.nil? dest_entry.write(src_value) - ui.output "Updated #{dest_path}" + ui.output "Updated #{dest_path}" if ui end end end end end rescue DefaultEnvironmentCannotBeModifiedError => e - ui.warn "#{format_path.call(e.entry)} #{e.reason}." + ui.warn "#{format_path.call(e.entry)} #{e.reason}." if ui rescue OperationFailedError => e - ui.error "#{format_path.call(e.entry)} failed to #{e.operation}: #{e.message}" + ui.error "#{format_path.call(e.entry)} failed to #{e.operation}: #{e.message}" if ui error = true rescue OperationNotAllowedError => e - ui.error "#{format_path.call(e.entry)} #{e.reason}." + ui.error "#{format_path.call(e.entry)} #{e.reason}." if ui error = true end error diff --git a/lib/chef/chef_fs/file_system/memory_dir.rb b/lib/chef/chef_fs/file_system/memory_dir.rb new file mode 100644 index 0000000000..c0ae1a2027 --- /dev/null +++ b/lib/chef/chef_fs/file_system/memory_dir.rb @@ -0,0 +1,42 @@ +require 'chef/chef_fs/file_system/base_fs_dir' +require 'chef/chef_fs/file_system/nonexistent_fs_object' +require 'chef/chef_fs/file_system/memory_file' + +class Chef + module ChefFS + module FileSystem + class MemoryDir < Chef::ChefFS::FileSystem::BaseFSDir + def initialize(name, parent) + super(name, parent) + @children = [] + end + + attr_reader :children + + def child(name) + @children.select { |child| child.name == name }.first || Chef::ChefFS::FileSystem::NonexistentFSObject.new(name, self) + end + + def add_child(child) + @children.push(child) + end + + def can_have_child?(name, is_dir) + root.cannot_be_in_regex ? (name !~ root.cannot_be_in_regex) : true + end + + def add_file(path, value) + path_parts = path.split('/') + if path_parts.length == 1 + add_child(MemoryFile.new(path_parts[0], self, value)) + else + if !child(path_parts[0]).exists? + add_child(MemoryDir.new(path_parts[0], self)) + end + child(path_parts[0]).add_file(path_parts[1..-1], value) + end + end + end + end + end +end diff --git a/lib/chef/chef_fs/file_system/memory_file.rb b/lib/chef/chef_fs/file_system/memory_file.rb new file mode 100644 index 0000000000..0c44e703f1 --- /dev/null +++ b/lib/chef/chef_fs/file_system/memory_file.rb @@ -0,0 +1,17 @@ +require 'chef/chef_fs/file_system/base_fs_object' + +class Chef + module ChefFS + module FileSystem + class MemoryFile < Chef::ChefFS::FileSystem::BaseFSObject + def initialize(name, parent, value) + super(name, parent) + @value = value + end + def read + return @value + end + end + end + end +end diff --git a/lib/chef/chef_fs/file_system/memory_root.rb b/lib/chef/chef_fs/file_system/memory_root.rb new file mode 100644 index 0000000000..4a83830946 --- /dev/null +++ b/lib/chef/chef_fs/file_system/memory_root.rb @@ -0,0 +1,21 @@ +require 'chef/chef_fs/file_system/memory_dir' + +class Chef + module ChefFS + module FileSystem + class MemoryRoot < MemoryDir + def initialize(pretty_name, cannot_be_in_regex = nil) + super('', nil) + @pretty_name = pretty_name + @cannot_be_in_regex = cannot_be_in_regex + end + + attr_reader :cannot_be_in_regex + + def path_for_printing + @pretty_name + end + end + end + end +end diff --git a/lib/chef/knife/zero.rb b/lib/chef/knife/zero.rb index 6068202759..066eeae0ec 100644 --- a/lib/chef/knife/zero.rb +++ b/lib/chef/knife/zero.rb @@ -3,8 +3,10 @@ require 'chef_zero/server' require 'chef_zero/data_store/memory_store' # For ChefFSStore +require 'chef/chef_fs/file_pattern' require 'chef/chef_fs/file_system' require 'chef/chef_fs/file_system/not_found_error' +require 'chef/chef_fs/file_system/memory_root' require 'chef_zero/data_store/data_already_exists_error' require 'chef_zero/data_store/data_not_found_error' @@ -25,146 +27,308 @@ class Chef ChefZero::Server.new(:data_store => data_store, :log_level => Chef::Log.level).start(:publish => true) end - class ChefFSDataStore def initialize(chef_fs) @chef_fs = chef_fs - @memory_fs = ChefZero::DataStore::MemoryStore.new + @memory_store = ChefZero::DataStore::MemoryStore.new end attr_reader :chef_fs - MEMORY_PATHS = %w(sandboxes file_store cookbooks) - - # TODO carve out a space for cookbooks + MEMORY_PATHS = %w(sandboxes file_store) def create_dir(path, name, *options) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.create_dir(path, name, *options) + if is_memory_store(path) + @memory_store.create_dir(path, name, *options) else - path = fix_path(path) - - parent = get_dir(path, options.include?(:create_dir)) - parent.create_child(name, nil) + with_dir(path) do |parent| + parent.create_child(chef_fs_filename(path + [name]), nil) + end end end def create(path, name, data, *options) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.create(path, name, data, *options) - else - path = fix_path(path) + if is_memory_store(path) + @memory_store.create(path, name, data, *options) + elsif path[0] == 'cookbooks' && path.length == 2 + # Do nothing. The entry gets created when the cookbook is created. + + else if !data.is_a?(String) raise "set only works with strings" end - parent = get_dir(path, options.include?(:create_dir)) - parent.create_child("#{name}.json", data) + with_dir(path) do |parent| + parent.create_child(chef_fs_filename(path + [name]), data) + end end end - def get(path) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.get(path) - else - path = fix_path(path) + def get(path, request=nil) + if is_memory_store(path) + @memory_store.get(path) + elsif path[0] == 'file_store' && path[1] == 'repo' + entry = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[2..-1].join('/')) begin - Chef::ChefFS::FileSystem.resolve_path(chef_fs, "#{path.join('/')}.json").read + entry.read rescue Chef::ChefFS::FileSystem::NotFoundError => e - raise ChefZero::DataStore::DataNotFoundError.new(path, e) + raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e) + end + + else + with_entry(path) do |entry| + if path[0] == 'cookbooks' && path.length == 3 + # get /cookbooks/NAME/version + result = entry.chef_object.to_hash + result.each_pair do |key, value| + if value.is_a?(Array) + value.each do |file| + if file.is_a?(Hash) && file.has_key?('checksum') + relative = ['file_store', 'repo', 'cookbooks'] + if Chef::Config.versioned_cookbooks + relative << "#{path[1]}-#{path[2]}" + else + relative << path[1] + end + relative = relative + file[:path].split('/') + file['url'] = ChefZero::RestBase::build_uri(request.base_uri, relative) + end + end + end + end + JSON.pretty_generate(result) + + else + entry.read + end end end end def set(path, data, *options) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.set(path, data, *options) + if is_memory_store(path) + @memory_store.set(path, data, *options) else - path = fix_path(path) - if !data.is_a?(String) raise "set only works with strings: #{path} = #{data.inspect}" end - parent = get_dir(path[0..-2], options.include?(:create_dir)) - parent.create_child("#{path[-1]}.json", data) + # Write out the files! + if path[0] == 'cookbooks' && path.length == 3 + write_cookbook(path, data, *options) + else + with_dir(path[0..-2]) do |parent| + parent.create_child(chef_fs_filename(path), data) + end + end end end def delete(path) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.delete(path) + if is_memory_store(path) + @memory_store.delete(path) else - path = fix_path(path) - - begin - Chef::ChefFS::FileSystem.resolve_path(chef_fs, "#{path.join('/')}.json").delete - rescue Chef::ChefFS::FileSystem::NotFoundError => e - raise ChefZero::DataStore::DataNotFoundError.new(path, e) + with_entry(path) do |entry| + if path[0] == 'cookbooks' && path.length >= 3 + entry.delete(true) + else + entry.delete + end end end end def delete_dir(path, *options) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.delete_dir(path, *options) + if is_memory_store(path) + @memory_store.delete_dir(path, *options) else - path = fix_path(path) - - begin - Chef::ChefFS::FileSystem.resolve_path(chef_fs, path.join('/')).delete(options.include?(:recursive)) - rescue Chef::ChefFS::FileSystem::NotFoundError => e - raise ChefZero::DataStore::DataNotFoundError.new(path, e) + with_entry(path) do |entry| + entry.delete(options.include?(:recursive)) end end end def list(path) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.list(path) - else - path = fix_path(path) + if is_memory_store(path) + @memory_store.list(path) + + elsif path[0] == 'cookbooks' && path.length == 1 + with_entry(path) do |entry| + if Chef::Config.versioned_cookbooks + # /cookbooks/name-version -> /cookbooks/name + entry.children.map { |child| split_name_version(child.name)[0] }.uniq + else + entry.children.map { |child| child.name } + end + end - begin - Chef::ChefFS::FileSystem.resolve_path(chef_fs, path.join('/')).children.map { |c| remove_dot_json(c) }.sort - rescue Chef::ChefFS::FileSystem::NotFoundError => e - raise ChefZero::DataStore::DataNotFoundError.new(path, e) + elsif path[0] == 'cookbooks' && path.length == 2 + if Chef::Config.versioned_cookbooks + # list /cookbooks/name = filter /cookbooks/name-version down to name + entry.children.map { |child| split_name_version(child.name) }. + select { |name, version| name == path[1] }. + map { |name, version| version }.to_a + else + # list /cookbooks/name = <single version> + version = get_single_cookbook_version(path) + [version] + end + + else + with_entry(path) do |entry| + entry.children.map { |c| to_leaf_name(c) }.sort end end end def exists?(path) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.exists?(path) + if is_memory_store(path) + @memory_store.exists?(path) else - path = fix_path(path) - - Chef::ChefFS::FileSystem.resolve_path(chef_fs, "#{path.join('/')}.json").exists? + Chef::ChefFS::FileSystem.resolve_path(chef_fs, path_to_chef_fs(path)).exists? end end def exists_dir?(path) - if MEMORY_PATHS.include?(path[0]) - @memory_fs.exists_dir?(path) + if is_memory_store(path) + @memory_store.exists_dir?(path) + elsif path[0] == 'cookbooks' && path.length == 2 + list([ path[0] ]).include?(path[1]) else - path = fix_path(path) - - Chef::ChefFS::FileSystem.resolve_path(chef_fs, path.join('/')).exists? + Chef::ChefFS::FileSystem.resolve_path(chef_fs, path_to_chef_fs(path)).exists? end end private - def fix_path(path) + def is_memory_store(path) + return path[0] == 'sandboxes' || path[0] == 'file_store' && path[1] == 'checksums' + end + + def write_cookbook(path, data, *options) + # Create a little Chef::ChefFS memory filesystem with the data + if Chef::Config.versioned_cookbooks + cookbook_path = "cookbooks/#{path[1]}-#{path[2]}" + else + cookbook_path = "cookbooks/#{path[1]}" + end + cookbook_fs = Chef::ChefFS::FileSystem::MemoryRoot.new('uploading') + cookbook = JSON.parse(data, :create_additions => false) + cookbook.each_pair do |key, value| + if value.is_a?(Array) + value.each do |file| + if file.is_a?(Hash) && file.has_key?('checksum') + file_data = @memory_store.get(['file_store', 'checksums', file['checksum']]) + cookbook_fs.add_file("#{cookbook_path}/#{file['path']}", file_data) + end + end + end + end + + # Use the copy/diff algorithm to copy it down so we don't destroy + # chefignored data. This is terribly un-thread-safe. + Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new(cookbook_path), cookbook_fs, chef_fs, nil, {:purge => true}) + end + + def split_name_version(entry_name) + name_version = entry_name.split('-') + name = name_version[0..-2].join('-') + version = name_version[-1] + [name,version] + end + + def path_to_chef_fs(path) + _path_to_chef_fs(path).join('/') + end + + def chef_fs_filename(path) + _path_to_chef_fs(path)[-1] + end + + def _path_to_chef_fs(path) if path[0] == 'data' path = path.dup path[0] = 'data_bags' + if path.length >= 3 + path[2] = "#{path[2]}.json" + end + elsif path[0] == 'cookbooks' + if path.length == 2 + raise ChefZero::DataStore::DataNotFoundError.new(path) + elsif Chef::Config.versioned_cookbooks + if path.length >= 3 + # cookbooks/name/version -> cookbooks/name-version + path = [ path[0], "#{path[1]}-#{path[2]}" ] + path[3..-1] + end + else + if path.length >= 3 + # cookbooks/name/version/... -> /cookbooks/name/... iff metadata says so + version = get_single_cookbook_version(path) + if path[2] == version + path = path[0..1] + path[3..-1] + else + raise ChefZero::DataStore::DataNotFoundError.new(path) + end + end + end + elsif path.length == 2 + path = path.dup + path[1] = "#{path[1]}.json" end path end + def to_zero_path(entry) + path = entry.path.split('/') + if path[0] == 'data' + path = path.dup + path[0] = 'data_bags' + if path.length >= 3 + path[2] = path[2][0..-6] + end + elsif path[0] == 'cookbooks' + if Chef::Config.versioned_cookbooks + # cookbooks/name-version/... -> cookbooks/name/version/... + if path.length >= 2 + name, version = split_name_version(path[1]) + path = [ path[0], name, version ] + path[2..-1] + end + else + if path.length >= 2 + # cookbooks/name/... -> cookbooks/name/version/... + version = get_single_cookbook_version(path) + path = path[0..1] + version + path[2..-1] + end + end + elsif path.length == 2 && path[0] != 'cookbooks' + path = path.dup + path[1] = path[1][0..-6] + end + path + end + + def to_leaf_name(entry) + to_zero_path(entry)[-1] + end + + def with_entry(path) + begin + yield Chef::ChefFS::FileSystem.resolve_path(chef_fs, path_to_chef_fs(path)) + rescue Chef::ChefFS::FileSystem::NotFoundError => e + raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e) + end + end + + def with_dir(path) + begin + yield get_dir(_path_to_chef_fs(path), true) + rescue Chef::ChefFS::FileSystem::NotFoundError => e + raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e) + end + end + def get_dir(path, create=false) result = Chef::FileSystem.resolve_path(chef_fs, path.join('/')) if result.exists? @@ -176,12 +340,10 @@ class Chef end end - def remove_dot_json(entry) - if entry.dir? - entry.name - else - entry.name[0..-6] - end + def get_single_cookbook_version(path) + dir = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[0..1].join('/')) + metadata = ChefZero::CookbookData.metadata_from(dir, path[1], nil, []) + metadata[:version] || '0.0.0' end end end diff --git a/spec/support/shared/unit/file_system_support.rb b/spec/support/shared/unit/file_system_support.rb index 2040e69d63..3e771dd187 100644 --- a/spec/support/shared/unit/file_system_support.rb +++ b/spec/support/shared/unit/file_system_support.rb @@ -17,56 +17,16 @@ # require 'chef/chef_fs/file_system' -require 'chef/chef_fs/file_system/base_fs_dir' -require 'chef/chef_fs/file_system/base_fs_object' +require 'chef/chef_fs/file_system/memory_root' +require 'chef/chef_fs/file_system/memory_dir' +require 'chef/chef_fs/file_system/memory_file' module FileSystemSupport - class MemoryFile < Chef::ChefFS::FileSystem::BaseFSObject - def initialize(name, parent, value) - super(name, parent) - @value = value - end - def read - return @value - end - end - - class MemoryDir < Chef::ChefFS::FileSystem::BaseFSDir - def initialize(name, parent) - super(name, parent) - @children = [] - end - attr_reader :children - def child(name) - @children.select { |child| child.name == name }.first || Chef::ChefFS::FileSystem::NonexistentFSObject.new(name, self) - end - def add_child(child) - @children.push(child) - end - def can_have_child?(name, is_dir) - root.cannot_be_in_regex ? (name !~ root.cannot_be_in_regex) : true - end - end - - class MemoryRoot < MemoryDir - def initialize(pretty_name, cannot_be_in_regex = nil) - super('', nil) - @pretty_name = pretty_name - @cannot_be_in_regex = cannot_be_in_regex - end - - attr_reader :cannot_be_in_regex - - def path_for_printing - @pretty_name - end - end - def memory_fs(pretty_name, value, cannot_be_in_regex = nil) if !value.is_a?(Hash) raise "memory_fs() must take a Hash" end - dir = MemoryRoot.new(pretty_name, cannot_be_in_regex) + dir = Chef::ChefFS::FileSystem::MemoryRoot.new(pretty_name, cannot_be_in_regex) value.each do |key, child| dir.add_child(memory_fs_value(child, key.to_s, dir)) end @@ -75,13 +35,13 @@ module FileSystemSupport def memory_fs_value(value, name = '', parent = nil) if value.is_a?(Hash) - dir = MemoryDir.new(name, parent) + dir = Chef::ChefFS::FileSystem::MemoryDir.new(name, parent) value.each do |key, child| dir.add_child(memory_fs_value(child, key.to_s, dir)) end dir else - MemoryFile.new(name, parent, value || "#{name}\n") + Chef::ChefFS::FileSystem::MemoryFile.new(name, parent, value || "#{name}\n") end end @@ -94,7 +54,7 @@ module FileSystemSupport end def no_blocking_calls_allowed - [ MemoryFile, MemoryDir ].each do |c| + [ Chef::ChefFS::FileSystem::MemoryFile, Chef::ChefFS::FileSystem::MemoryDir ].each do |c| [ :children, :exists?, :read ].each do |m| c.any_instance.stub(m).and_raise("#{m.to_s} should not be called") end |